cloud-hypervisor: clean up the short-circuiting logic

Yeah, yeah, I know, it's stupid, adding add_argument() statements
manually, et c. I'll throw the whole Python thing away whenever I might
have the time

examples/dummy.nix

@@ -5,41 +5,49 @@
   modulesPath,
   ...
 }:
+let
+  uvmsPkgs = pkgs.callPackage ../pkgs { };
+in
 {
   imports = [
-    ../profiles/all.nix
+    ../profiles/ch-runner.nix
+    ../profiles/baseImage.nix
     (modulesPath + "/profiles/minimal.nix")
   ];
 
   system.stateVersion = "25.11";
 
+  networking.hostName = "dummy";
+
   vmapps.enable = true;
   _module.args.inputs = import ../npins;
 
-  # following microvm.nix:
+  # boot.isContainer = true;
   boot.loader.grub.enable = false;
   boot.initrd.systemd.enable = true;
-  fileSystems."/" = lib.mkDefault {
+
-    device = "rootfs"; # how does this work? does this assign a label to the tmpfs?
+  boot.initrd.systemd.settings.Manager.DefaultTimeoutStartSec = 5;
-    fsType = "tmpfs";
-    options = [ "size=20%,mode=0755" ];
-    neededForBoot = true;
-  };
-  boot.initrd.systemd.settings.Manager.DefaultTimeoutStartSec = 30;
   systemd.settings.Manager.DefaultTimeoutStopSec = 10;
-  networking.useNetworkd = true;
-  networking.nftables.enable = true;
 
   uvms.cloud-hypervisor.enable = true;
 
-  users.mutableUsers = false;
-  users.users.root.password = "hacktheplanet!";
-  services.getty.autologinUser = "root";
-
   systemd.services."suid-sgid-wrappers".serviceConfig = {
     StandardOutput = "journal+console";
     StandardError = "journal+console";
   };
 
+  systemd.services."terminal" = {
+    wantedBy = [ "multi-user.target" ];
+    wants = [ "wayland-proxy.service" ];
+    after = [ "wayland-proxy.service" ];
+    serviceConfig = {
+      User = "user";
+      WorkingDirectory = "/home/user";
+      ExecStart = lib.getExe pkgs.alacritty;
+      StandardOutput = "journal+console";
+      StandardError = "journal+console";
+    };
+  };
+
   # TODO: cmdline, kernel, initrd, fileSystems
 }

pkgs/baseImage.nix

@@ -0,0 +1,11 @@
+{ nixos, overrideScope }:
+
+nixos {
+  imports = [
+    ../profiles/baseImage.nix
+  ];
+  _module.args.uvmsPkgs = builtins.removeAttrs (overrideScope (fi: _: { _self = fi; }))._self [
+    "_self"
+  ];
+  _file = __curPos.file;
+}

pkgs/ch-proxy/meson.build

@@ -1,3 +1,9 @@
 project('ch-proxy', 'c')
 
-executable('ch-proxy', 'proxy.c', install: true)
+pkg = import('pkgconfig')
+
+sendfd = library('sendfd', [ 'sendfd.c', 'sendfd.h' ], install: true)
+pkg.generate(sendfd)
+install_headers('sendfd.h')
+
+executable('ch-proxy', 'proxy.c', link_with: [sendfd], install: true)

pkgs/ch-proxy/package.nix

@@ -8,6 +8,7 @@
 stdenv.mkDerivation {
   pname = "ch-proxy";
   version = "0.0.0";
+  outputs = [ "out" "lib" ];
   nativeBuildInputs = [
     meson
     ninja
@@ -19,6 +20,8 @@ stdenv.mkDerivation {
     fs.toSource {
       fileset = fs.unions [
         ./proxy.c
+        ./sendfd.c
+        ./sendfd.h
         ./meson.build
       ];
       root = ./.;

pkgs/ch-proxy/proxy.c

@@ -9,9 +9,9 @@
 
 #include <linux/vm_sockets.h>
 
-struct msghdr mk_msghdr();
+#include "sendfd.h"
+
 int ch_connect(const char*, const char*);
-ssize_t send_fd(int, int);
 
 #define _WRITE_CONFIRM(fd, buf, buflen) {if (write((fd), (buf), (buflen)) != (buflen)) { perror("ch-proxy/write/partial write"); exit(EXIT_FAILURE); }}
 
@@ -168,19 +168,13 @@ int main(int argc, char** argv) {
         exit(EXIT_FAILURE);
     }
 
-    if (send_fd(1, s) == -1) {
+    if (send_fd(1, s, NULL) == -1) {
         perror("ssh-vsock-proxy/main/send_fd");
         return EXIT_FAILURE;
     }
     return 0;
 }
 
-struct msghdr mk_msghdr() {
-    struct msghdr msg;
-    memset(&msg, 0, sizeof(msg));
-
-    return msg;
-}
 
 int ch_connect(const char *path, const char *port) {
     int s = socket(AF_UNIX, SOCK_STREAM, 0);
@@ -212,38 +206,3 @@ int ch_connect(const char *path, const char *port) {
 
     return s;
 }
-
-ssize_t send_fd(int dst_fd, int fd) {
-    struct msghdr msg = mk_msghdr();
-
-    /* openssh expects to receive a dummy length=1 iovec? */
-    char ch;
-    struct iovec vec;
-    vec.iov_base = &ch;
-    vec.iov_len = 1;
-    msg.msg_iov = &vec;
-    msg.msg_iovlen = 1;
-
-    union {
-        struct cmsghdr align;
-        char buf[CMSG_SPACE(sizeof(int))];
-    } u;
-
-    msg.msg_control = u.buf;
-    msg.msg_controllen = sizeof(u.buf);
-
-    struct cmsghdr *cmptr;
-    cmptr = CMSG_FIRSTHDR(&msg);
-
-    if (cmptr == NULL) {
-        fprintf(stderr, "ch-proxy/send_fd/CMSG_FIRSTHDR: failed to initialize msg_control\n");
-        exit(EXIT_FAILURE);
-    }
-
-    cmptr->cmsg_len = CMSG_LEN(sizeof(int));
-    cmptr->cmsg_level = SOL_SOCKET;
-    cmptr->cmsg_type = SCM_RIGHTS;
-    *((int*) CMSG_DATA(cmptr)) = fd;
-
-    return (sendmsg(dst_fd, &msg, 0));
-}

pkgs/ch-proxy/sendfd.c

@@ -0,0 +1,74 @@
+#include "sendfd.h"
+#include "sys/socket.h" /* cmsghdr */
+#include "stdio.h" /* perror */
+
+
+ssize_t send_fd(int dst_fd, int fd, const struct iovec *iov) {
+    struct msghdr msg = { 0 };
+
+    /* openssh expects to receive a dummy length=1 iovec? */
+    char ch = 0;
+    struct iovec vecDefault = { 0 };
+    vecDefault.iov_base = &ch;
+    vecDefault.iov_len = 1;
+    msg.msg_iov = iov == NULL ? &vecDefault : iov;
+    msg.msg_iovlen = 1;
+
+    union {
+        struct cmsghdr align;
+        char buf[CMSG_SPACE(sizeof(int))];
+    } u;
+
+    msg.msg_control = u.buf;
+    msg.msg_controllen = sizeof(u.buf);
+
+    struct cmsghdr *cmptr;
+    cmptr = CMSG_FIRSTHDR(&msg);
+
+    if (cmptr == NULL) {
+        perror("ch-proxy/send_fd/CMSG_FIRSTHDR: failed to initialize msg_control\n");
+    }
+
+    cmptr->cmsg_len = CMSG_LEN(sizeof(int));
+    cmptr->cmsg_level = SOL_SOCKET;
+    cmptr->cmsg_type = SCM_RIGHTS;
+    *((int*) CMSG_DATA(cmptr)) = fd;
+
+    return (sendmsg(dst_fd, &msg, 0));
+}
+
+int recv_fd(int sock, int flags) {
+    int out = -1;
+
+    struct msghdr msg = { 0 };
+    struct cmsghdr *cmsg = NULL;
+    struct iovec iov = { 0 };
+    char dummy = 0;
+
+    msg.msg_iov = &iov;
+    msg.msg_iovlen = 1;
+
+    iov.iov_base = &dummy;
+    iov.iov_len = sizeof(dummy);
+
+    union {
+        struct cmsghdr align;
+        char buf[CMSG_SPACE(sizeof(int))];
+    } u;
+
+    msg.msg_control = u.buf;
+    msg.msg_controllen = sizeof(u.buf);
+
+    int bytes = 0;
+    if ((bytes = recvmsg(sock, &msg, flags)) < 0) {
+        perror("recv_fd: recvmsg");
+        return -1;
+    }
+    for (cmsg = CMSG_FIRSTHDR(&msg); cmsg != NULL; cmsg = CMSG_NXTHDR(&msg, cmsg)) {
+        if (cmsg->cmsg_level != SOL_SOCKET) { continue; }
+        if (cmsg->cmsg_type != SCM_RIGHTS) { continue; }
+        if (CMSG_LEN(cmsg) < sizeof(out)) { continue; }
+        out = *(int*)CMSG_DATA(cmsg);
+    }
+    return out;
+}

pkgs/ch-proxy/sendfd.h

@@ -0,0 +1,27 @@
+#ifndef _CH_PROXY_SENFD
+#define _CH_PROXY_SENFD
+
+#include <stddef.h> /* size_t */
+#include <sys/types.h> /* ssize_t */
+#include <sys/uio.h> /* iovec */
+
+
+/* send_fd(chanFd, fd, *iov)
+ *
+ * chanFd: fd to sendmsg over;
+ * fd: fd to send;
+ * iov: extra data to send or NULL;
+ *
+ * returns: result of sendmsg,
+ * i.e. the number of bytes sent */
+ssize_t send_fd(int chanFd, int fd, const struct iovec *);
+
+/* recv_fd(chanFd, flags)
+ *
+ * chanFd: fd to recvmsg from;
+ * flags: recvmsg flags e.g. 0, or MSG_CMSG_CLOEXEC?
+ *
+ * returns: the received fd or -1 */
+int recv_fd(int chanFd, int flags);
+
+#endif /* _CH_PROXY_SENFD */

pkgs/cloud-hypervisor-gpu.nix

@@ -0,0 +1,79 @@
+{
+  lib,
+  cloud-hypervisor,
+  fetchFromGitHub,
+  rustPlatform,
+  enableDebug ? true,
+}:
+
+let
+  inherit (lib) optionals optionalAttrs optionalString;
+  spectrum = builtins.fetchTree {
+    url = "https://spectrum-os.org/git/spectrum";
+    type = "git";
+    rev = "0f3388f0191d9a03c7bf471c269a34a79f22018b";
+  };
+in
+cloud-hypervisor.overrideAttrs (
+  finalAttrs: oldAttrs:
+  let
+    patchesFromDir =
+      root:
+      builtins.concatMap (name: lib.optionals (lib.hasSuffix ".patch" name) [ (root + "/${name}") ]) (
+        builtins.attrNames (builtins.readDir root)
+      );
+    spectrumPatches = {
+      version = "2025-12-20";
+      vhost = fetchFromGitHub {
+        name = "vhost";
+        owner = "rust-vmm";
+        repo = "vhost";
+        rev = "vhost-user-backend-v0.20.0";
+        hash = "sha256-KK1+mwYQr7YkyGT9+51v7TJael9D0lle2JXfRoTqYq8=";
+      };
+      patches = patchesFromDir (spectrum + "/pkgs/cloud-hypervisor");
+      vhostPatches = patchesFromDir (spectrum + "/pkgs/cloud-hypervisor/vhost");
+    };
+    oldPatchesStruct = oldAttrs.passthru.spectrumPatches or { };
+    missingPatchPhases = oldPatchesStruct != { };
+    isNewer = lib.versionOlder oldPatchesStruct.version or "2000-00-00" spectrumPatches.version;
+    oldPatches = oldPatchesStruct.patches or [ ];
+    removeAll = removeElts: lst: builtins.filter (x: !(builtins.elem x removeElts)) lst;
+  in
+  optionalAttrs isNewer {
+    passthru = oldAttrs.passthru or { } // {
+      inherit spectrumPatches;
+    };
+    # Verbatim from spectrum
+    postUnpack =
+      oldAttrs.postUnpack or ""
+      + optionalString missingPatchPhases ''
+        unpackFile $vhost
+        chmod -R +w vhost
+      '';
+    inherit (spectrumPatches) vhost;
+
+    patches = removeAll oldPatches (oldAttrs.patches or [ ]) ++ spectrumPatches.patches;
+    inherit (spectrumPatches) vhostPatches;
+    # Verbatim copy from spectrum
+    postPatch =
+      oldAttrs.postPatch or ""
+      + optionalString missingPatchPhases ''
+        pushd ../vhost
+        for patch in $vhostPatches; do
+            echo applying patch $patch
+            patch -p1 < $patch
+        done
+        popd
+      '';
+    cargoDeps = rustPlatform.fetchCargoVendor {
+      inherit (finalAttrs) patches;
+      inherit (oldAttrs) src;
+      hash = "sha256-wGtsyKDg1z1QK9mJ1Q43NSjoPbm3m81p++DoD8ipIUI=";
+    };
+  }
+  // lib.optionalAttrs enableDebug {
+    buildType = "debug";
+    dontStrip = true;
+  }
+)

pkgs/default.nix

@@ -4,6 +4,14 @@ let
 in
 lib.makeScope newScope (
   self:
+  let
+    callPackage =
+      fun: overrides:
+      let
+        result = self.callPackage fun overrides;
+      in
+      result // { override = result.__originalOverride or result.override; };
+  in
   dirToAttrs ./.
     [
       (
@@ -14,9 +22,9 @@ lib.makeScope newScope (
     (
       name: fpath: typ:
       if typ == "regular" then
-        self.callPackage fpath { }
+        callPackage fpath { }
       else if typ == "directory" && builtins.pathExists (fpath + "/package.nix") then
-        self.callPackage (fpath + "/package.nix") { }
+        callPackage (fpath + "/package.nix") { }
       else
         null
     )

pkgs/linux-uvm.nix

@@ -0,0 +1,326 @@
+{
+  lib,
+  linux_latest,
+}:
+
+let
+  inherit (lib.kernel) yes no unset;
+  inherit (lib) mkForce;
+  result = linux_latest.override {
+    structuredExtraConfig = {
+      # From spectrum
+      VT = no;
+      DRM_FBDEV_EMULATION = lib.mkForce no;
+      FONTS = mkForce unset;
+      FONT_8x8 = mkForce unset;
+      FONT_TER16x32 = mkForce unset;
+      FRAMEBUFFER_CONSOLE = mkForce unset;
+      FRAMEBUFFER_CONSOLE_DEFERRED_TAKEOVER = mkForce unset;
+      FRAMEBUFFER_CONSOLE_DETECT_PRIMARY = mkForce unset;
+      FRAMEBUFFER_CONSOLE_ROTATION = mkForce unset;
+      RC_CORE = mkForce unset;
+
+      # Manually specified:
+      EROFS_FS = yes;
+      # TSI = yes; # Needs external patches
+      DAX = yes;
+      FS_DAX = yes;
+      FUSE_DAX = yes;
+      OVERLAY_FS = yes;
+      OVERLAY_FS_REDIRECT_ALWAYS_FOLLOW = yes;
+      VIRTIO_BALLOON = yes;
+      VIRTIO_BLK = yes;
+      VIRTIO_CONSOLE = yes;
+      VIRTIO_FS = yes;
+      VIRTIO_MMIO = yes;
+      VIRTIO_PCI = yes;
+      VIRTIO = yes;
+      FUSE_FS = yes;
+      VSOCKETS = yes;
+
+      # # libkurnfw a70c65d97eda1d53a55602bdef984aba82383097
+      DRM = yes;
+      DRM_KMS_HELPER = yes;
+      DRM_GEM_SHMEM_HELPER = yes;
+      # Can't select???
+      # VIRTIO_GPU = yes;
+      # VIRTIO_GPU_KMS = yes;
+
+      DRM_VIRTIO_GPU = yes; # Based on compiled diff?
+
+      BASE_SMALL = yes; # libkurnfw 33a72344da4938c41616d200372542b3f7eb4412
+
+      # # Based on compiled config diff with libkurnfw
+      NO_HZ_COMMON = mkForce unset;
+      NO_HZ_IDLE = mkForce yes;
+      NO_HZ_FULL = mkForce unset;
+      HZ_1000 = unset;
+      HZ_250 = yes; # NixOS default: 1000
+
+      LSM = lib.kernel.freeform "lockdown,yama,loadpin,safesetid,integrity,bpf";
+
+      EXT4_FS = yes;
+      # # EXT4_USE_FOR_EXT2 = yes;
+      XFS_FS = yes;
+      DEFAULT_SECURITY_APPARMOR = mkForce unset;
+
+      HW_RANDOM = yes;
+      HW_RANDOM_VIRTIO = yes;
+
+      # # libkurnfw
+      POSIX_MQUEUE = mkForce unset;
+
+      # # libkurnfw 0ad58f60dc061e61f088a8b9b2758bea4a3bd41d
+      HAVE_RELIABLE_STACKTRACE = mkForce unset;
+      STACKDEPOT = mkForce unset;
+      ARCH_WANT_FRAME_POINTERS = mkForce unset;
+      FRAME_POINTER = mkForce unset;
+      STACK_VALIDATION = mkForce unset;
+      SLUB_DEBUG = mkForce unset;
+      UNWINDER_FRAME_POINTER = mkForce unset;
+      # UNWINDER_GUESS = mkForce yes;
+      UNWINDER_ORC = mkForce unset;
+
+      # # libkurnfw a5094ce1633889250482b812265c15d9a252f6a7
+      PROFILING = mkForce no;
+      KEXEC_CORE = mkForce unset; # Still selected?
+      KEXEC_FILE = mkForce unset; # Still selected?
+      DEBUG_MISC = mkForce no;
+      MAGIC_SYSRQ = mkForce no;
+      DEBUG_FS = mkForce unset; # mkForce no;
+      DEBUG_INFO_NONE = mkForce yes;
+
+      KEXEC_JUMP = mkForce unset;
+      DEBUG_INFO = mkForce unset;
+      DEBUG_INFO_REDUCED = mkForce unset;
+      DEBUG_LIST = mkForce unset;
+      DEBUG_INFO_BTF = mkForce unset;
+      DYNAMIC_DEBUG = mkForce unset;
+      SUNRPC_DEBUG = mkForce unset;
+      MAC80211_DEBUGFS = mkForce unset;
+      CFG80211_DEBUGFS = mkForce unset;
+      ACPI_DEBUG = mkForce unset;
+      DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT = mkForce unset;
+      MODULE_ALLOW_BTF_MISMATCH = mkForce unset;
+      CRC32_SELFTEST = mkForce unset;
+      BLK_DEBUG_FS = mkForce unset;
+
+      # Conflicts:
+      MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT = mkForce unset;
+      KEXEC_HANDOVER = mkForce unset;
+      MEM_ALLOC_PROFILING = mkForce unset;
+
+      # # libkurnfw d28d2632b704335c4065648afa490b63cfe49393
+      X86_MSR = mkForce no;
+      PERF_EVENTS_AMD_UNCORE = mkForce unset;
+      ARCH_MEMORY_PROBE = mkForce unset;
+      X86_CHECK_BIOS_CORRUPTION = mkForce unset;
+      X86_REROUTE_FOR_BROKEN_BOOT_IRQS = mkForce unset;
+      MTRR = mkForce no;
+      MTRR_SANITIZER = mkForce unset;
+      X86_PAT = mkForce unset;
+      X86_SGX = mkForce unset;
+      MODIFY_LDT_SYSCALL = mkForce unset;
+      ARCH_HAS_PKEYS = mkForce unset;
+      ARCH_USES_HIGH_VMA_FLAGS = mkForce unset;
+
+      X86_INTEL_PSTATE = mkForce unset;
+      X86_AMD_PSTATE = mkForce unset;
+      X86_SGX_KVM = mkForce unset;
+      UCLAMP_TASK = mkForce unset;
+
+      # # libkurnfw 8043cecb92f7384f648bfb2fde9e19653fe877e8
+      PM = mkForce unset; # want "no" but can't select?
+      CPU_FREQ = mkForce unset;
+      CPU_IDLE_GOV_LADDER = mkForce unset;
+      INTEL_IDLE = mkForce unset;
+      SCHED_MC_PRIO = mkForce unset;
+      BLK_PM = mkForce unset;
+      CPU_IDLE = yes;
+      CPU_IDLE_GOV_HALTPOLL = yes;
+      HALTPOLL_CPUIDLE = yes;
+
+      # # Conflicts:
+      PM_DEBUG = mkForce unset;
+      PM_ADVANCED_DEBUG = mkForce unset;
+      PM_WAKELOCKS = mkForce unset;
+      TPMI = mkForce unset;
+      INTEL_TPMI = mkForce unset;
+
+      # # libkurnfw 63c4d29cd28ab73ea1c3a85c9df0c71a6101dd41
+      IA32_EMULATION = mkForce no;
+      COMPAT_32BIT_TIME = mkForce no;
+
+      # # libkurnfw b085fa0f8958ea37f0e3c16dfec6cd7cc58c6b41
+      HIBERNATION = mkForce unset;
+      SWAP = mkForce no;
+      ZSWAP = mkForce unset;
+      ZSWAP_COMPRESSOR_DEFAULT_ZSTD = mkForce unset;
+      ZONE_DMA = mkForce no;
+      VM_EVENT_COUNTERS = mkForce unset; # Still selected as a dep?
+      PERCPU_STATS = mkForce no;
+
+      # # libkurnfw 123090b6960cd0beb7fcb19e2fc383d24c6b74e9
+      XFRM_USER = mkForce unset;
+      SYN_COOKIES = mkForce unset;
+      TCP_CONG_ADVANCED = mkForce no;
+      TCP_MD5SIG = mkForce no;
+      NETLABEL = mkForce no;
+      ETHTOOL_NETLINK = mkForce no;
+
+      # # libkurnfw b31a4ab84f13bde66497fd21b277503f7ad4b541
+      FW_LOADER = mkForce unset;
+
+      # SOUND = mkForce unset;
+      USB_SUPPORT = mkForce unset;
+      MEMSTICK = mkForce unset;
+      NEW_LEDS = mkForce unset;
+      ACCESSIBILITY = mkForce unset;
+      INFINIBAND = mkForce unset;
+      INFINIBAND_ADDR_TRANS = mkForce unset;
+
+      # UIO = mkForce unset;
+      VFIO = mkForce unset;
+      VIRTIO_PMEM = mkForce unset;
+      VDPA = mkForce unset;
+      HYPERV = mkForce unset;
+      FB_HYPERV = mkForce unset;
+      DRM_HYPERV = mkForce unset;
+      KVM_HYPERV = mkForce unset;
+      COMMON_CLK = mkForce unset;
+      EXT2_FS = mkForce unset;
+      EXT3_FS = mkForce unset;
+
+      EXT3_FS_POSIX_ACL = mkForce unset;
+      EXT3_FS_SECURITY = mkForce unset;
+
+      # libkrunfw e2fd98beb10f2fd9b827e188cc55ec0f90d44932
+      PAGE_SIZE_LESS_THAN_64KB = yes;
+      PAGE_SIZE_LESS_THAN_256KB = yes;
+      CHROME_PLATFORMS = mkForce unset;
+      X86_PLATFORM_DEVICES = mkForce unset;
+      SURFACE_PLATFORMS = mkForce unset;
+      INTEL_UNCORE_FREQ_CONTROL = mkForce unset;
+      INTEL_TURBO_MAX_3 = mkForce unset;
+
+      CHROMEOS_PSTORE = mkForce unset;
+      CHROMEOS_LAPTOP = mkForce unset;
+      CHROMEOS_TBMC = mkForce unset;
+
+      CROS_EC = mkForce unset;
+      CRYPTO_TEST = mkForce unset;
+      NFS_SWAP = mkForce unset;
+      CROS_EC_I2C = mkForce unset;
+      CROS_EC_ISHTP = mkForce unset;
+      CROS_EC_LPC = mkForce unset;
+      CROS_EC_SPI = mkForce unset;
+
+      NET_SCH_BPF = mkForce unset;
+
+      UCLAMP_TASK_GROUP = mkForce unset;
+      SCHED_CLASS_EXT = mkForce unset;
+      DRM_NOUVEAU_SVM = mkForce unset;
+      CROS_KBD_LED_BACKLIGHT = mkForce unset;
+
+      # Based on compiled config diff with libkurnfw
+      XEN = mkForce unset;
+      XEN_EFI = mkForce unset;
+      HVC_XEN = mkForce unset;
+      HVC_XEN_FRONTEND = mkForce unset;
+      XEN_BACKEND = mkForce unset;
+      XEN_BALLOON = mkForce unset;
+      XEN_BALLOON_MEMORY_HOTPLUG = mkForce unset;
+      XEN_DOM0 = mkForce unset;
+      XEN_HAVE_PVMMU = mkForce unset;
+      XEN_MCE_LOG = mkForce unset;
+      XEN_PVH = mkForce unset;
+      XEN_PVHVM = mkForce unset;
+      XEN_SAVE_RESTORE = mkForce unset;
+      XEN_SYS_HYPERVISOR = mkForce unset;
+      PCI_XEN = mkForce unset;
+      POWER_RESET_GPIO = mkForce unset;
+      POWER_RESET_GPIO_RESTART = mkForce unset;
+      RCU_LAZY = mkForce unset;
+      REISERFS_FS_POSIX_ACL = mkForce unset;
+      REISERFS_FS_SECURITY = mkForce unset;
+      REISERFS_FS_XATTR = mkForce unset;
+      SWIOTLB_XEN = mkForce unset;
+      SUSPEND = mkForce unset;
+      ACPI = mkForce unset;
+      ISA_DMA_API = mkForce unset;
+      COMPAT = mkForce unset;
+      COMPAT_32 = mkForce unset;
+      KVM = mkForce unset;
+      BLOCK_LEGACY_AUTOLOAD = mkForce unset;
+      CMA = mkForce unset;
+      FB = mkForce unset;
+      FB_EFI = mkForce unset;
+      FB_VESA = mkForce unset;
+      SECURITY_APPARMOR = mkForce unset;
+      SERIAL_8250 = mkForce unset;
+      ISA_BUS = unset;
+      X86_X32_ABI = unset;
+      SCHED_AUTOGROUP = mkForce unset; # libkurnfw
+      RAID6_PQ = mkForce unset; # libkurnfw: yes
+      ARCH_HAS_FAST_MULTIPLIER = yes;
+      GENERIC_NET_UTILS = yes;
+      BITREVERSE = yes;
+
+      # Conflicts
+      IIO = mkForce unset;
+      IIO_TRIGGERED_EVENT = mkForce unset;
+      IIO_CONFIGFS = mkForce unset;
+      IIO_KFIFO_BUF = mkForce unset;
+      IIO_TRIGGER = mkForce unset;
+      STAGING = mkForce unset;
+      STAGING_MEDIA = mkForce unset;
+
+      LEGACY_PTYS = mkForce unset;
+      NULL_TTY = mkForce unset;
+      N_GSM = mkForce unset;
+      SERIAL_NONSTANDARD = mkForce unset;
+      TTY_PRINTK = mkForce unset;
+      IPMI_HANDLER = mkForce unset;
+      TELCLOCK = mkForce unset;
+      TCG_TPM = mkForce unset;
+      SERIAL_UARTLITE = mkForce unset;
+      SERIAL_LANTIQ = mkForce unset;
+      MWAVE = mkForce unset;
+      SERIAL_CORE = mkForce unset;
+      SERIAL_MCTRL_GPIO = mkForce unset;
+      I2C = mkForce unset;
+
+      # Interactively solving conflicts...
+
+      COMEDI = mkForce unset;
+
+      NLS = mkForce yes;
+      NLS_CODEPAGE_437 = mkForce yes;
+      NLS_CODEPAGE_737 = mkForce unset;
+      NLS_CODEPAGE_775 = mkForce unset;
+      NLS_CODEPAGE_850 = mkForce unset;
+      NLS_CODEPAGE_852 = mkForce unset;
+      NLS_CODEPAGE_855 = mkForce unset;
+      NLS_CODEPAGE_857 = mkForce unset;
+      NLS_CODEPAGE_860 = mkForce unset;
+      NLS_CODEPAGE_861 = mkForce unset;
+      NLS_CODEPAGE_862 = mkForce unset;
+      NLS_CODEPAGE_863 = mkForce unset;
+      NLS_CODEPAGE_864 = mkForce unset;
+      NLS_CODEPAGE_865 = mkForce unset;
+      NLS_CODEPAGE_866 = mkForce unset;
+      NLS_CODEPAGE_869 = mkForce unset;
+      NLS_CODEPAGE_936 = mkForce unset;
+      NLS_CODEPAGE_950 = mkForce unset;
+      NLS_CODEPAGE_932 = mkForce unset;
+      NLS_CODEPAGE_949 = mkForce unset;
+      NLS_CODEPAGE_874 = mkForce unset;
+      NLS_ISO8859_8 = mkForce unset;
+      NLS_CODEPAGE_1250 = mkForce unset;
+      NLS_CODEPAGE_1251 = mkForce unset;
+
+      ZSMALLOC = mkForce unset;
+    };
+  };
+in
+result // { __originalOverride = result.override; }

pkgs/mkSystemdDropin.nix

@@ -0,0 +1,39 @@
+{
+  lib,
+  runCommand,
+  writeShellScriptBin,
+}:
+{
+  name,
+  prefix ? "10-all-",
+  dirs ? [
+    "service"
+    "mount"
+    "socket"
+    "timer"
+    "target"
+  ],
+
+  dropinText ? null,
+  extraCommands ? "",
+  ...
+}@args:
+
+runCommand "${name}-dropin"
+  (
+    lib.removeAttrs args [
+      "name"
+    ]
+    // {
+      inherit dirs dropinText extraCommands;
+    }
+  )
+  ''
+    set -euo pipefail
+    root=$out/lib/systemd/system
+    for dir in $dirs ; do
+      mkdir -p "$root/$dir".d
+      printf "%s" "$dropinText" > "$root/$dir.d/${prefix}${name}.conf"
+    done
+    runHook extraCommands
+  ''

pkgs/taps/.envrc

@@ -0,0 +1 @@
+use nix ../../ -A pkgs.taps

pkgs/taps/main.c

@@ -0,0 +1,342 @@
+#define _GNU_SOURCE
+
+#include <stdio.h>
+#include <stdlib.h> /* secure_getenv */
+#include <sys/socket.h>
+#include <unistd.h>
+#include <sys/un.h>
+#include <string.h>
+#include <stdbool.h>
+#include <ctype.h>
+#include <pwd.h>
+#include <fnmatch.h>
+#include <fcntl.h> /* open, O_NONBLOCK, &c */
+#include <err.h>
+#include <error.h>
+#include <errno.h>
+#include <sys/ioctl.h>
+#include <signal.h>
+#include <fcntl.h>
+
+#define __UAPI_DEF_IF_IFNAMSIZ 1
+#include <linux/if_tun.h>
+#include <linux/if.h>
+#include <linux/virtio_net.h>
+
+#include "sendfd.h"
+
+// From `man unix`
+#define SUN_PATH_SZ 108
+#define N_CONNS 16
+
+char *TEMP_PATHS[1024] = { 0 }; 
+int LAST_TEMP_PATH = -1;
+
+#define IFR_FLAGS_ALLOWED (IFF_NO_PI | IFF_TAP | IFF_TUN | IFF_VNET_HDR | IFF_MULTI_QUEUE | IFF_PERSIST)
+#define IFR_FLAGS_DEFAULT (IFF_NO_PI | IFF_TAP | IFF_VNET_HDR | IFF_PERSIST)
+
+#define PTR_OR_DIE(expr) TRUE_OR_DIE((expr) != NULL)
+#define DO_OR_DIE(expr) TRUE_OR_DIE((expr) != -1)
+#define TRUE_OR_DIE(expr, ...) TRUE_OR_(EXIT_FAILURE, expr, __VA_ARGS__)
+#define TRUE_OR_WARN(expr, ...) TRUE_OR_(0, expr, __VA_ARGS__)
+#define TRUE_OR_(status, expr, ...) \
+    do if (!(expr)) { \
+        error(status, errno, "Failed assertion: " #expr "." __VA_ARGS__); \
+    } while(false)
+
+struct allow_pattern {
+    // enum { USER = 1, GROUP = 2 } type;
+    // union { uid_t uid, gid_t gid } xid;
+    char *name;
+};
+struct allow_patterns {
+    size_t n;
+    struct allow_pattern *patterns;
+};
+
+/* Running on the same host, not caring for alignment */
+struct tap_request {
+    short ifrFlags; /* 0 to use defaults: IFF_TAP | IFF_NO_PI | IFF_VNET_HDR */
+    char name[IFNAMSIZ];
+};
+
+struct tap_reply {
+    enum { OK = 0, AUTH_ERROR = 1 } status;
+    char name[IFNAMSIZ];
+};
+
+int tuntap_alloc(char *dev, short openFlags, short ifrFlags, int *out_fd);
+
+bool match_mask(const char *test_addr, const char *expected_addr, const char *mask, int n) {
+    for (int octet = 0; octet < n; ++octet) {
+        if ((test_addr[octet] & mask[octet]) != expected_addr[octet]) {
+            return false;
+        }
+    }
+    return true;
+}
+
+/* 
+ * Adapted from spectrum's `mktuntap.c` (2019 Alyssa Ross <hi@alyssa.is>
+ * GPL-2.0-only), which in turn adapts `tun_alloc` from
+ * `linux/Documentation/networking/tuntap.rst`.
+ *
+ * ifrFlags: IFF_TUN   - TUN device (no Ethernet headers)
+ *           IFF_TAP   - TAP device
+ *
+ *           IFF_NO_PI - Do not provide packet information
+ */
+int tuntap_alloc(char *dev, short openFlags, short ifrFlags, int *out_fd) {
+  struct ifreq ifr = { 0 };
+  int fd = -1, err = 0;
+
+  DO_OR_DIE(fd = open("/dev/net/tun", openFlags));
+
+  if (dev != NULL) {
+    int devLen = strlen(dev);
+    if (devLen >= IFNAMSIZ) {
+      /* If client requests a name, we do want the entire name to fit */
+      errno = EINVAL;
+      return EINVAL;
+    }
+    strncpy(ifr.ifr_name, dev, IFNAMSIZ - 1);
+  }
+  ifr.ifr_flags = ifrFlags;
+
+  TRUE_OR_WARN((err = ioctl(fd, TUNSETIFF, (void *)&ifr)) == 0);
+  if (err != 0) {
+    close(fd);
+    return err;
+  }
+
+  strncpy(dev, ifr.ifr_name, IFNAMSIZ);
+  *out_fd = fd;
+
+  {
+      int sz = sizeof(struct virtio_net_hdr_v1);
+      DO_OR_DIE(ioctl(fd, TUNSETVNETHDRSZ, &sz));
+  }
+  return 0;
+}
+
+int acceptRequests(const char *requestsPath, const struct allow_patterns *patterns) {
+    int listener;
+    struct sockaddr_un addr;
+    const int t = 1;
+
+    DO_OR_DIE(listener = socket(AF_UNIX, SOCK_SEQPACKET, 0));
+    DO_OR_DIE(setsockopt(listener, SOL_SOCKET, SO_PASSCRED, &t, sizeof(t)));
+
+    addr.sun_family = AF_UNIX;
+    strncpy(addr.sun_path, requestsPath, SUN_PATH_SZ - 1);
+    DO_OR_DIE (bind(listener, &addr, sizeof(addr)));
+    PTR_OR_DIE(TEMP_PATHS[++LAST_TEMP_PATH] = strdup(requestsPath));
+
+    DO_OR_DIE(listen(listener, N_CONNS));
+
+    for (;;) {
+        /* Already changed my mind about looking at ucred, but keeping the code around for now */
+        int sock = -1;
+        struct ucred cred = { 0 };
+        struct msghdr msg = { 0 };
+        struct cmsghdr *cmsg = NULL;
+        struct iovec iov = { 0 };
+        struct tap_request req = { 0 };
+
+        msg.msg_iov = &iov;
+        msg.msg_iovlen = 1;
+
+        iov.iov_base = &req;
+        iov.iov_len = sizeof(struct tap_request);
+
+        DO_OR_DIE((sock = accept(listener, NULL, NULL)));
+
+        TRUE_OR_DIE(recvmsg(sock, &msg, 0) > 0);
+        req.name[IFNAMSIZ] = 0;
+        for (cmsg = CMSG_FIRSTHDR(&msg); cmsg != NULL; cmsg = CMSG_NXTHDR(&msg, cmsg)) {
+            if (cmsg->cmsg_level != SOL_SOCKET) { continue; }
+            if (cmsg->cmsg_type != SCM_CREDENTIALS) { continue; }
+            if (CMSG_LEN(cmsg) < sizeof(struct ucred)) { continue; }
+            memcpy(&cred, CMSG_DATA(cmsg), sizeof(struct ucred));
+            break;
+        }
+
+        if (req.ifrFlags == 0) {
+            req.ifrFlags = IFR_FLAGS_DEFAULT;
+        }
+
+        bool allowed = false;
+        for (int i = 0; !allowed && i < patterns->n; ++i) {
+            bool ifnameOk = fnmatch(patterns->patterns[i].name, req.name, 0) == 0;
+            bool flagsOk = (req.ifrFlags & IFR_FLAGS_ALLOWED) == req.ifrFlags;
+            allowed = ifnameOk && flagsOk;
+        }
+
+        struct tap_reply reply = { 0 };
+        if (!allowed) { reply.status = AUTH_ERROR; }
+        if (allowed) {
+            /* O_CLOEXEC? */
+            int fd = -1;
+            TRUE_OR_DIE(tuntap_alloc(req.name, O_RDWR | O_NONBLOCK, req.ifrFlags, &fd) == 0);
+            struct iovec iov = { 0 };
+            iov.iov_base = &reply;
+            iov.iov_len = sizeof(struct tap_reply);
+            TRUE_OR_DIE(send_fd(sock, fd, &iov) > 0);
+            close(fd); 
+        }
+        close(sock);
+    }
+    close(listener);
+}
+
+struct allow_patterns parsePatterns(const char *raw) {
+    const size_t rawLen = strlen(raw);
+
+    size_t nPatterns = 0;
+    for (int i = 0; i < rawLen; ++i) {
+        const int start = i;
+        if (isspace(raw[i])) { continue; }
+        for (; i < rawLen && !isspace(raw[i]); ++i) { }
+        if (start < i) { ++nPatterns; }
+    }
+
+    struct allow_pattern *patterns = NULL;
+    PTR_OR_DIE(patterns = calloc(nPatterns, sizeof(struct allow_pattern)));
+
+    int iPattern = 0;
+    for (int i = 0; i < rawLen; ++i) {
+        if (isspace(raw[i])) { continue; }
+        /* used to have per-group/per-user patterns, "u:$username:$pattern", &c - gone */
+        {
+            const int start = i;
+            for (; i < rawLen && !isspace(raw[i]); ++i) { }
+            if (start < i) {
+                PTR_OR_DIE(patterns[iPattern].name = strndup(&raw[start], i - start));
+                iPattern += 1;
+            }
+        }
+    }
+    struct allow_patterns out = {
+        .n = nPatterns,
+        .patterns = patterns
+    };
+    return out;
+}
+
+int get(const char *servePath, const char *ifname, short ifrFlags) {
+    /* TODO: sock: move out */
+    int sock;
+    struct sockaddr_un addr;
+
+    DO_OR_DIE(sock = socket(AF_UNIX, SOCK_SEQPACKET, 0));
+
+    addr.sun_family = AF_UNIX;
+    strncpy(addr.sun_path, servePath, SUN_PATH_SZ - 1);
+    DO_OR_DIE (connect(sock, &addr, sizeof(addr)));
+
+    struct msghdr msg = { 0 };
+    struct cmsghdr *cmsg = NULL;
+    struct iovec iov = { 0 };
+    struct tap_request req = { 0 };
+    strncpy(req.name, ifname, IFNAMSIZ - 1);
+    req.ifrFlags = ifrFlags;
+
+    msg.msg_iov = &iov;
+    msg.msg_iovlen = 1;
+
+    iov.iov_base = &req;
+    iov.iov_len = sizeof(struct tap_request);
+
+    TRUE_OR_DIE(sendmsg(sock, &msg, 0) > 0);
+
+    int tunFd = -1;
+    DO_OR_DIE(tunFd = recv_fd(sock, 0));
+    close(sock);
+    return tunFd;
+}
+
+void cleanup(int signo, siginfo_t *info, void *_context) {
+    for (int i = 0; i <= LAST_TEMP_PATH; ++i) {
+        TRUE_OR_DIE(unlink(TEMP_PATHS[i]) != -1 || errno == ENOENT);
+    }
+    if (signo == SIGINT) {
+        exit(EXIT_SUCCESS);
+    }
+    errx(EXIT_FAILURE, "Exiting with signal %d", signo);
+}
+
+/* skarlibs under ISC */
+int uncoe (int fd)
+{
+  int flags = fcntl(fd, F_GETFD, 0) ;
+  return flags < 0 ? flags : flags & FD_CLOEXEC ? fcntl(fd, F_SETFD, flags & ~FD_CLOEXEC) : 0 ;
+}
+
+int main(int argc, char **argv) {
+    struct sigaction act = { 0 };
+    act.sa_flags = SA_SIGINFO;
+    act.sa_sigaction = cleanup;
+    DO_OR_DIE(sigaction(SIGINT, &act, NULL));
+    DO_OR_DIE(sigaction(SIGSEGV, &act, NULL));
+
+    bool cmdServe = false;
+    bool cmdPass = false;
+    char *ifname = "vt-%d";
+
+    char **rest = argv + 1;
+    char **end = argv + argc;
+
+    TRUE_OR_DIE(argc > 1);
+    if (strcmp(rest[0], "serve") == 0) {
+        cmdServe = true;
+        ++rest;
+    } else if (strcmp(rest[0], "pass") == 0) {
+        cmdPass = true;
+        ++rest;
+        for (; rest != end && rest[0][0] == '-'; ++rest) {
+            if (strcmp(rest[0], "--")) { break; }
+            else if (strncmp(rest[0], "--ifname=", sizeof("--ifname="))) {
+                ifname = rest[0] + sizeof("--ifname=");
+            }
+        }
+    } else {
+        error(EINVAL, EINVAL, "no subcommand \"%s\"", rest[0]);
+    }
+
+    int nextArgc = argc - (rest - argv);
+    char * const* nextArgv = rest;
+
+    const char *patternsRaw = secure_getenv("TAPS_ALLOW");
+    if (patternsRaw == NULL) {
+        patternsRaw = "*";
+    }
+
+    struct allow_patterns patterns = { 0 };
+    if (cmdServe) {
+        PTR_OR_DIE((patterns = parsePatterns(patternsRaw)).patterns);
+    }
+
+    const char *servePath = secure_getenv("TAPS_SOCK");
+    if (servePath == NULL) {
+        servePath = "/run/taps/taps.sock";
+    }
+
+    if (cmdServe) {
+        acceptRequests(servePath, &patterns);
+    } else if (cmdPass) {
+        TRUE_OR_DIE(nextArgc > 0);
+        int fd = -1;
+        DO_OR_DIE(fd = get(servePath, ifname, 0));
+        if (fd != 3) {
+            DO_OR_DIE(dup2(fd, 3));
+            close(fd);
+            fd = 3;
+        }
+        uncoe(fd);
+        DO_OR_DIE(execvp(nextArgv[0], nextArgv));
+    } else {
+        error(EINVAL, EINVAL, "subcommand args");
+    }
+
+    return 0;
+}

pkgs/taps/meson.build

@@ -0,0 +1,4 @@
+project('taps', 'c')
+
+sendfd = dependency('sendfd')
+executable('taps', 'main.c', dependencies: [sendfd], install: true)

pkgs/taps/package.nix

@@ -0,0 +1,46 @@
+{
+  lib,
+  stdenv,
+  meson,
+  pkg-config,
+  rustc,
+  ninja,
+  ch-proxy,
+}:
+
+stdenv.mkDerivation {
+  pname = "taps";
+  version = "0.0.0";
+  src =
+    let
+      fs = lib.fileset;
+    in
+    fs.toSource {
+      root = ./.;
+      fileset = fs.unions [
+        ./meson.build
+        ./main.c
+      ];
+    };
+
+  nativeBuildInputs = [
+    ninja
+    meson
+    pkg-config
+    rustc
+  ];
+  buildInputs = [ ch-proxy ];
+
+  meta.mainProgram = "taps";
+}
+# { lib, rustPlatform }:
+#
+# rustPlatform.buildRustPackage {
+#     pname = "taps";
+#     version = "0.0.0";
+#     src = let fs = lib.filesystem; in fs.toSource {
+#         root = ./.;
+#         fileset = fs.unions [
+#         ];
+#     };
+# };

pkgs/uvms/package.nix

@@ -0,0 +1,62 @@
+{
+  lib,
+  symlinkJoin,
+  writers,
+  writeClosure,
+  replaceVars,
+  bubblewrap,
+  cloud-hypervisor-gpu,
+  crosvm,
+  effective-cloud-hypervisor ? cloud-hypervisor-gpu,
+  execline,
+  s6,
+  strace,
+  util-linux,
+  virtiofsd,
+  python3Packages,
+  uvmslib,
+
+  taps,
+  baseImage,
+}:
+
+let
+  tools = map lib.getBin [
+    execline
+    s6
+    effective-cloud-hypervisor
+    virtiofsd
+    bubblewrap
+    strace
+    crosvm
+    taps
+    util-linux
+  ];
+  toolsFarm = symlinkJoin {
+    name = "tools";
+    paths = tools;
+  };
+  toolsClosure = writeClosure toolsFarm;
+in
+writers.writePython3Bin "uvms"
+  {
+    libraries = [ uvmslib ];
+  }
+  (
+    replaceVars ./uvms.py {
+      BWRAP = "${lib.getExe bubblewrap}";
+      TOOLS = "${toolsFarm}/bin";
+      TOOLS_CLOSURE = toolsClosure;
+      CROSVM = lib.getExe crosvm;
+      STRACE = lib.getExe strace;
+      TAPS = "${lib.getExe taps}";
+      VIRTIOFSD = "${lib.getExe virtiofsd}";
+
+      BASE_CONFIG = baseImage.config.system.build.ch;
+      SYSTEM = baseImage.config.system.build.toplevel;
+      SYSTEM_CLOSURE = writeClosure [
+        baseImage.config.system.build.toplevel
+        baseImage.config.system.build.ch
+      ];
+    }
+  )

pkgs/uvms/uvms.py

@@ -0,0 +1,739 @@
+# NOTE: This would have been bash,
+# and this was execlineb previously,
+# but it was just easier to reason in terms of context managers
+# and try-except-finally branches for the cleanup bit,
+# than in terms of traps or such.
+# Treat this as bash.
+# Treat this as throwaway shitcode.
+
+import os
+import subprocess
+import socket
+import json
+import re
+from argparse import ArgumentParser
+from contextlib import contextmanager, closing, ExitStack
+from urllib.parse import urlparse, parse_qs
+
+
+parser = ArgumentParser("supervise-vm")
+parser.add_argument("--vm", default=None)
+parser.add_argument("--prefix", default="$HOME/uvms/$VM")
+parser.add_argument("--vm-config", default="@BASE_CONFIG@")  # noqa: E501
+parser.add_argument("--persist-home", action="store_true")
+parser.add_argument("--run", action="append")
+parser.add_argument("--mem", default=None)
+parser.add_argument("app", nargs="*", default=())
+
+TOOLS_DIR = "@TOOLS@"  # noqa: E501
+SOCKETBINDER = TOOLS_DIR + "/s6-ipcserver-socketbinder"  # noqa: E501
+CH = TOOLS_DIR + "/cloud-hypervisor"
+CHR = TOOLS_DIR + "/ch-remote"
+TAPS = "@TAPS@"  # noqa: E501
+VIRTIOFSD = "@VIRTIOFSD@"  # noqa: E501
+BWRAP = "@BWRAP@"  # noqa: E501
+
+with open("@TOOLS_CLOSURE@", mode="r") as f:  # noqa: E501
+    TOOLS_CLOSURE = [
+        *(ln.rstrip() for ln in f.readlines()),
+        os.path.dirname(__file__),
+    ]
+
+BASE_SYSTEM = "@SYSTEM@"  # noqa: E501
+with open("@SYSTEM_CLOSURE@", mode="r") as f:  # noqa: E501
+    BASE_SYSTEM_CLOSURE = [
+        *(ln.rstrip() for ln in f.readlines()),
+    ]
+
+PASSTHRU_PATH = ":".join([TOOLS_DIR, *os.environ.get("PATH", "").split(":")])
+PASSTHRU_ENV = {
+    **{
+        k: v
+        for k, v in os.environ.items()
+        if k.startswith("RUST_")
+        or k.startswith("WAYLAND")
+        or k.startswith("XDG_")
+        or k.startswith("DBUS_")
+        or k.startswith("NIX_")
+        or k
+        in [
+            "TAPS_SOCK",
+        ]
+    },
+    "HOME": os.environ.get("HOME", os.getcwd()),
+    "PATH": PASSTHRU_PATH,
+}
+
+
+def preprocess_args(args_mut):
+    if not args_mut.app and args_mut.run:
+        args_mut.app = [*args_mut.run]
+    if not args_mut.vm:
+        args_mut.vm = args_mut.run[0]
+    keys = [k for k, v in args_mut._get_kwargs() if isinstance(v, str)]
+    for k in keys:
+        v = getattr(args_mut, k)
+        if "$HOME" in v:
+            setattr(args_mut, k, v.replace("$HOME", PASSTHRU_ENV["HOME"]))
+    for k in keys:
+        v = getattr(args_mut, k)
+        if "$VM" in v:
+            setattr(args_mut, k, v.replace("$VM", args.vm))
+    for k in keys:
+        v = getattr(args_mut, k)
+        if "$PREFIX" in v:
+            setattr(args_mut, k, v.replace("$PREFIX", args.prefix))
+    return args_mut
+
+
+def alive_after(proc, timeout):
+    if proc is None:
+        return False
+    if proc.returncode is not None:
+        return False
+    try:
+        proc.wait(timeout)
+    except subprocess.TimeoutExpired:
+        return True
+    return False
+
+
+class Processes:
+    def __init__(self, prefix, vm, check=True, **defaults):
+        self.prefix = prefix
+        self.vm = vm
+        self.check = check
+        self.defaults = defaults
+        self.processes = []
+
+    def make_env(self):
+        return {
+            **PASSTHRU_ENV,
+            "PATH": PASSTHRU_PATH,
+            "PREFIX": self.prefix,
+            "VM": self.vm,
+        }
+
+    def exec(self, *args, **kwargs):
+        kwargs["cwd"] = kwargs.get("cwd", self.prefix)
+        kwargs["check"] = kwargs.get("check", self.check)
+        kwargs["env"] = kwargs.get("env", self.make_env())
+        return subprocess.run([*args], **self.defaults, **kwargs)
+
+    def execline(self, *args, **kwargs):
+        return exec(
+            "execlineb",
+            "-c",
+            "\n".join(args),
+            **self.defaults,
+            executable=TOOLS_DIR + "/execlineb",
+            **{
+                "env": self.make_env(),
+                "check": self.check,
+                "cwd": self.prefix,
+                **kwargs,
+            },
+        )
+
+    @contextmanager
+    def popen(self, *args, **kwargs):
+        kwargs["pass_fds"] = kwargs.get("pass_fds", ())
+        kwargs["env"] = kwargs.get("env", self.make_env())
+        kwargs["cwd"] = kwargs.get("cwd", self.prefix)
+        kwargs["text"] = kwargs.get("text", True)
+        kwargs["stdin"] = kwargs.get("stdin", subprocess.DEVNULL)
+        kwargs["stdout"] = kwargs.get("stdout", subprocess.DEVNULL)
+        kwargs["stderr"] = kwargs.get("stderr", subprocess.DEVNULL)
+        proc = None
+        try:
+            proc = subprocess.Popen(
+                args,
+                **kwargs,
+            )
+            if not alive_after(proc, 0.125):
+                raise RuntimeError("Failed to start", args)
+            print(f"Started {args}")
+            self.processes.append(proc)
+            yield proc
+            print(f"Releasing {args}")
+        finally:
+            if subprocess.PIPE in (kwargs["stderr"], kwargs["stdout"]):
+                print(proc.communicate(timeout=0.125))
+            while alive_after(proc, 0.125):
+                try:
+                    proc.terminate()
+                    proc.wait()
+                except Exception as e:
+                    print(f"Cleanup failing: {e}")
+
+    @contextmanager
+    def bwrap(
+        self,
+        *bwrap_args,
+        die_with_parent=True,
+        # Based on the args from
+        # `host/rootfs/image/usr/bin/run-vmm`
+        unshare_all=True,
+        uid=1000,
+        gid=100,
+        unshare_user=True,
+        unshare_ipc=None,
+        unshare_pid=None,
+        unshare_net=None,
+        unshare_uts=None,
+        unshare_cgroup_try=True,
+        bind=(),
+        dev_bind=(),
+        dev_bind_implicit=("/dev/kvm", "/dev/vfio"),
+        dev="/dev",
+        proc="/proc",
+        ro_bind_implicit=(
+            "/etc",
+            "/sys",
+            "/proc/sys",
+            "/dev/null",
+            "/proc/kallsyms",
+            *sorted(set([*TOOLS_CLOSURE, *BASE_SYSTEM_CLOSURE])),
+        ),
+        ro_bind=(),
+        remount_ro=("/proc/fs", "/proc/irq"),
+        tmpfs_implicit=(
+            "/dev/shm",
+            "/tmp",
+            "/var/tmp",
+            "/proc/fs",
+            "/proc/irq",
+        ),
+        tmpfs=(),
+        pass_fds=(2,),
+        **popen_kwargs,
+    ):
+
+        bwrap_args_sock, remote = socket.socketpair()
+        remote.set_inheritable(True)
+        bwrap_args_f = bwrap_args_sock.makefile("w")
+
+        def print_arg(*args):
+            print(*args, file=bwrap_args_f, sep="\0", end="\0")
+
+        if unshare_all:
+            print_arg("--unshare-all")
+        if unshare_user:
+            print_arg("--unshare-user")
+        if uid is not None:
+            assert unshare_user
+            print_arg("--uid", uid)
+        if gid is not None:
+            assert unshare_user
+            print_arg("--gid", gid)
+        if unshare_ipc:
+            print_arg("--unshare-ipc")
+        if unshare_pid:
+            print_arg("--unshare-pid")
+        if unshare_net:
+            print_arg("--unshare-net")
+        elif unshare_net is False:
+            print_arg("--share-net")
+        if unshare_uts:
+            print_arg("--unshare-uts")
+        if unshare_cgroup_try:
+            print_arg("--unshare-cgroup-try")
+        if die_with_parent:
+            print_arg("--die-with-parent")
+        if dev:
+            print_arg("--dev", dev)
+        if proc:
+            print_arg("--proc", proc)
+
+        for p in bind:
+            assert isinstance(p, (str, tuple)), p
+            p1, p2 = (p, p) if isinstance(p, str) else p
+            print_arg("--bind", p1, p2)
+        for p in (*ro_bind, *ro_bind_implicit):
+            assert isinstance(p, (str, tuple)), p
+            p1, p2 = (p, p) if isinstance(p, str) else p
+            print_arg("--ro-bind", p1, p2)
+        for p in (*dev_bind, *dev_bind_implicit):
+            assert isinstance(p, (str, tuple)), p
+            p1, p2 = (p, p) if isinstance(p, str) else p
+            print_arg("--dev-bind", p1, p2)
+        for p in (*tmpfs, *tmpfs_implicit):
+            print_arg("--tmpfs", p)
+        # Hunch: order might matter...
+        for p in remount_ro:
+            print_arg("--remount-ro", p)
+
+        bwrap_args_f.flush()
+
+        try:
+            with ExitStack() as proc_es:
+                with ExitStack() as es:
+                    es.enter_context(closing(remote))
+                    es.enter_context(closing(bwrap_args_sock))
+                    es.enter_context(closing(bwrap_args_f))
+                    proc = proc_es.enter_context(
+                        self.popen(
+                            "bwrap",
+                            "--args",
+                            str(remote.fileno()),
+                            *bwrap_args,
+                            **popen_kwargs,
+                            executable=BWRAP,
+                            pass_fds=(*pass_fds, remote.fileno()),
+                        )
+                    )
+                yield proc
+        finally:
+            assert proc.returncode is not None, proc
+
+    @contextmanager
+    def run_ch(self):
+        # s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM, 0)
+        # s.set_inheritable(True)
+        # s.setblocking(True)
+        # s.bind(self.prefix + "/vmm.sock")
+        args = [
+            SOCKETBINDER,
+            "-B",
+            self.prefix + "/vmm.sock",
+            # "@STRACE@",  # noqa: E501
+            # "-Z",
+            # "-ff",
+            CH,
+            # "-v",
+            "--api-socket",
+            "fd=0",
+            # f"fd={s.fileno()}"
+        ]
+        cleanup_paths = [
+            self.prefix + "/vmm.sock",
+            self.prefix + "/vmm.sock.lock",
+            self.prefix + "/vsock.sock",
+        ]
+        new_paths = [p for p in cleanup_paths if not os.path.exists(p)]
+        old_paths = [p for p in cleanup_paths if p not in new_paths]
+        with ExitStack() as cleanup:
+            cleanup.enter_context(removing(*new_paths))
+            proc = cleanup.enter_context(
+                self.bwrap(
+                    *args,
+                    bind=[self.prefix],
+                    # Probably just need the path to vmlinux
+                    # ro_bind=["/nix/store"],  # I give up
+                    unshare_net=False,
+                    shell=False,
+                    stdout=None,
+                    stderr=None,
+                    # pass_fds=(s.fileno(),)
+                )
+            )
+            # s.close()
+            cleanup.enter_context(removing(*old_paths))
+            assert alive_after(proc, 1.0), proc
+            if not os.path.exists(self.prefix + "/vmm.sock"):
+                raise RuntimeError(
+                    f"{self.prefix}/vmm.sock should exist by now",
+                )
+            if proc.returncode is not None:
+                raise RuntimeError("CH exited early")
+            yield proc
+
+    @contextmanager
+    def start_gpu(
+        self,
+    ):
+        sock_path = self.prefix + "/gpu.sock"
+        args = [
+            "@CROSVM@",  # noqa: E501
+            "--no-syslog",
+            "--log-level",
+            "debug",
+            "device",
+            "gpu",
+            "--socket-path",
+            sock_path,
+            "--wayland-sock",
+            f'{PASSTHRU_ENV["XDG_RUNTIME_DIR"]}/{PASSTHRU_ENV["WAYLAND_DISPLAY"]}',  # noqa: E501
+            "--params",
+            '{ "context-types": "cross-domain:virgl2" }',
+        ]
+        with self.popen(
+            *args,
+            stderr=None,
+            stdout=None,
+        ) as proc, removing(sock_path, sock_path + ".lock"):
+            yield proc, sock_path
+
+    @contextmanager
+    def start_virtiofsd(
+        self,
+        root_dir,
+        tag,
+        ro=True,
+        subdirs=None,
+        extra_flags=("--posix-acl", "--xattr"),
+    ):
+
+        assert os.path.exists(root_dir)
+
+        sock_path = self.prefix + f"/virtiofsd-{tag}.sock"
+        # s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+        # NOTE: Nope. Virtiofsd actually expects a blocking socket
+        # s.setblocking(True)
+        # s.set_inheritable(True)
+
+        with ExitStack() as cleanup:  # noqa: F841
+            # s.bind(sock_path.encode("utf8"))
+            # cleanup.enter_context(closing(s))
+            cleanup.enter_context(removing(sock_path, sock_path + ".pid"))
+
+            args = [
+                # If using bwrap():
+                # "--argv0", "virtiofsd",
+                # "--uid", "1000",
+                # "--gid", "100",
+                # "--",
+                "unshare",
+                "-rUm",
+                "unshare",
+                "--map-user",
+                "1000",
+                "--map-group",
+                "100",
+                VIRTIOFSD,
+                "--shared-dir",
+                root_dir,
+                "--tag",
+                tag,
+                # "--fd",
+                # str(s.fileno()),
+                "--socket-path",
+                sock_path,
+                # If relying on bwrap():
+                # "--sandbox",
+                # "none",
+            ]
+            if ro:
+                args.append("--readonly")
+            kwargs = {
+                # If bwrap():
+                # "bind": [],
+                # ("ro_bind" if ro else "bind"):
+                #     [*subdirs]
+                #     if subdirs is not None
+                #     else [root_dir],
+                # "pass_fds": (2, s.fileno()),
+                "stdout": subprocess.PIPE,
+                "stderr": subprocess.PIPE,
+            }
+            try:
+                with self.popen(*args, **kwargs) as p:
+                    yield p, sock_path
+            finally:
+                if os.path.exists(sock_path):
+                    os.remove(sock_path)
+
+
+@contextmanager
+def defer(f):
+    try:
+        yield
+    finally:
+        f()
+
+
+@contextmanager
+def removing(*paths):
+    try:
+        yield
+    finally:
+        for p in paths:
+            if os.path.exists(p):
+                os.remove(p)
+
+
+@contextmanager
+def connect_ch_vsock(
+    vsock_sock_path,
+    port: int,
+    type=socket.SOCK_STREAM,
+    blocking=True,
+) -> socket.socket:
+    os.makedirs(os.path.dirname(vsock_sock_path), exist_ok=True)
+    s = socket.socket(socket.AF_UNIX, type, 0)
+    s.setblocking(blocking)
+    s.connect(vsock_sock_path)
+
+    with closing(s):
+        s.send(b"CONNECT %d\n" % port)
+        yield s
+
+
+BYTES_PATTERN = re.compile(r"^([0-9]+)([MmGgKk]?)$")
+BYTES_UNITS = {
+    "k": 1024,
+    "m": 1048576,
+    "g": 1024 * 1048576,
+}
+
+
+def parse_bytes(s):
+    m = BYTES_PATTERN.match(s)
+    assert m, s
+    size, unit = m.groups()
+    return int(size) * BYTES_UNITS.get(unit.lower(), 1)
+
+
+@contextmanager
+def listen_ch_vsock(
+    vsock_sock_path,
+    port: int,
+    type=socket.SOCK_STREAM,
+    blocking=True,
+) -> socket.socket:
+    os.makedirs(os.path.dirname(vsock_sock_path), exist_ok=True)
+    listen_path = vsock_sock_path + "_%d" % port
+    s = socket.socket(socket.AF_UNIX, type, 0)
+    s.setblocking(blocking)
+    s.bind(listen_path)
+    s.listen()
+    with removing(listen_path):
+        yield s
+
+
+def main(args, args_next, cleanup, ps):
+    send_dir = PASSTHRU_ENV["HOME"] + f"/send/{args.vm}"
+
+    os.makedirs(send_dir, exist_ok=True)
+    os.makedirs(args.prefix, exist_ok=True)
+
+    ch_remote = [
+        "ch-remote",
+        "--api-socket",
+        args.prefix + "/vmm.sock",
+    ]
+
+    with open(args.vm_config) as f:
+        config = json.load(f)
+
+    app_paths = []
+    for a in args.app:
+        a = urlparse(a)
+        nix_file = "./."
+        attr = None
+        if a.scheme == "":
+            nix_file = "<nixpkgs>"
+            attr = a.path
+        else:
+            assert a.fragment, a
+            attr = a.fragment
+            nix_file = a.path or "./."
+        arglist = []
+        for k, v in parse_qs(a.query).items():
+            arglist.append("--arg")
+            arglist.append(k)
+            arglist.append(v)
+
+        assert nix_file is not None, a
+        assert attr is not None, a
+        out_path = ps.exec(
+            "nix-build",
+            nix_file,
+            "-A",
+            attr,
+            *arglist,
+            "--no-out-link",
+            cwd=os.getcwd(),
+            capture_output=True,
+            text=True,
+        ).stdout.strip()
+        assert out_path.startswith("/nix/store/")
+        app_paths.append(out_path)
+    apps_closure = ps.exec(  # noqa: F841
+        "nix-store",
+        "-qR",
+        *app_paths,
+        capture_output=True,
+        text=True,
+    ).stdout.split()
+
+    ready_sock = cleanup.enter_context(
+        listen_ch_vsock(ps.prefix + "/vsock.sock", 8888),
+    )
+
+    virtiofs_socks = []
+    _, sock_path = cleanup.enter_context(
+        ps.start_virtiofsd(
+            send_dir,
+            tag="send",
+            ro=False,
+        )
+    )
+    virtiofs_socks.append(("send", sock_path))
+    _, sock_path = cleanup.enter_context(
+        ps.start_virtiofsd(
+            "/nix/store",
+            subdirs=apps_closure,
+            tag="apps",
+        )
+    )
+    virtiofs_socks.append(("apps", sock_path))
+    _, sock_path = cleanup.enter_context(
+        ps.start_virtiofsd(
+            "/nix/store",
+            subdirs=BASE_SYSTEM_CLOSURE,
+            tag="system",
+        )
+    )
+    virtiofs_socks.append(("system", sock_path))
+
+    if args.persist_home:
+        os.makedirs(args.prefix + "/home", exist_ok=True)
+        _, sock_path = cleanup.enter_context(
+            ps.start_virtiofsd(
+                args.prefix + "/home",
+                subdirs=BASE_SYSTEM_CLOSURE,
+                tag="home",
+                ro=False,
+            )
+        )
+        virtiofs_socks.append(("home", sock_path))
+        config["payload"]["cmdline"] += " uvms.persist-home=1"
+    if args.mem is not None:
+        config["memory"]["size"] = parse_bytes(args.mem)
+        config["memory"]["hotplug_size"] = parse_bytes(args.mem)
+
+    if "platform" not in config:
+        config["platform"] = {}
+    config["platform"]["oem_strings"] = [
+        "io.systemd.credential:vmm.notify_socket=vsock-stream:2:8888",
+        *config["platform"].get("oem_strings", []),
+    ]
+
+    gpud, gpud_path = cleanup.enter_context(ps.start_gpu())
+
+    ch = cleanup.enter_context(ps.run_ch())
+
+    ps.exec(
+        *ch_remote,
+        "create",
+        input=json.dumps(config),
+        text=True,
+    )
+    ps.exec(
+        TAPS,
+        "pass",
+        *ch_remote,
+        "add-net",
+        "id=wan,fd=3,mac=00:00:00:00:00:01",
+    )
+
+    # TODO: add-fs apps closure separately
+    for tag, sock_path in virtiofs_socks:
+        ps.exec(*ch_remote, "add-fs", f"tag={tag},socket={sock_path},id={tag}")
+    ps.exec(*ch_remote, "add-gpu", f"socket={gpud_path}")
+    ps.exec(*ch_remote, "boot")
+    ps.exec(*ch_remote, "info")
+
+    ready = False
+    with ready_sock:
+        ready_sock.settimeout(8.0)
+        for _ in range(1048576):
+            if ready:
+                break
+            try:
+                con, _ = ready_sock.accept()
+            except:  # noqa: E722
+                print(
+                    "WARNING: CH didn't try connecting to the readiness notification socket"  # noqa: E501
+                )
+                ready = True
+                break
+            else:
+                with con:
+                    msg = con.recv(1024)
+                    for ln in msg.split(b"\n"):
+                        ln = ln.strip()
+                        print(ln)
+                        # if ln.startswith(b"X_SYSTEMD_UNIT_ACTIVE=uvms-guest.service"):  # noqa: E501
+                        if ln.startswith(b"READY=1"):  # noqa: E501
+                            ready = True
+                            break
+
+    assert ready
+
+    cleanup.enter_context(removing(ps.prefix + "/vsock.sock"))
+    with connect_ch_vsock(ps.prefix + "/vsock.sock", 24601) as guest:
+        for r in args.run:
+            res = {}
+            for _ in range(1):
+                if "status" in res:
+                    break
+                try:
+                    guest.send(
+                        json.dumps(
+                            {
+                                "run": {
+                                    "argv": [r],
+                                    "EXTRA_PATH": [
+                                        f"{a}/bin" for a in app_paths
+                                    ],  # noqa: E501
+                                    "EXTRA_XDG_DATA_DIRS": [
+                                        f"{a}/share" for a in app_paths
+                                    ],  # noqa: E501
+                                }
+                            }
+                        ).encode("utf8")
+                    )
+                    res = guest.recv(8192)
+                    try:
+                        res = json.loads(guest.recv(8192))
+                    except json.JSONDecodeError as e:
+                        print(
+                            f"Couldn't interpret --run {r} response: {e} {res}"
+                        )  # noqa: E501
+                        res = {}
+                        # res = {"status": "failed"}
+                except Exception as e:
+                    print(f"Couldn't --run {r}: {repr(e)}")
+            if "status" not in res:
+                res["status"] = "fail"
+            adverb = (
+                "Successfully"
+                if res["status"] == "exec succeeded"
+                else "Failed to"  # noqa: E501
+            )
+            print(f"{adverb} --run {r}: {res}")
+    try:
+        ch.wait()
+    except KeyboardInterrupt:
+        pass
+
+
+if __name__ == "__main__":
+    args, args_next = parser.parse_known_args()
+    preprocess_args(args)
+    ps = Processes(
+        prefix=args.prefix,
+        vm=args.vm,
+    )
+
+    try:
+        with ExitStack() as cleanup:
+            main(args, args_next, cleanup, ps)
+    finally:
+        for p in ps.processes:
+            if p.returncode is not None:
+                continue
+            try:
+                print(f"Cleanup failed. Re-trying the killing of {p}")
+                p.terminate()
+            except:  # noqa: E722
+                pass
+        for p in ps.processes:
+            if p.returncode is not None:
+                continue
+            try:
+                p.wait()
+            except:  # noqa: E722
+                pass

pkgs/uvmslib/package.nix

@@ -0,0 +1,21 @@
+{
+  lib,
+  python3Packages,
+}:
+python3Packages.buildPythonPackage {
+  pname = "uvmslib";
+  version = "0.0.0";
+  pyproject = true;
+  src =
+    let
+      fs = lib.fileset;
+    in
+    fs.toSource {
+      root = ./.;
+      fileset = fs.unions [
+        ./pyproject.toml
+        ./uvmslib.py
+      ];
+    };
+  build-system = [ python3Packages.setuptools ];
+}

pkgs/uvmslib/pyproject.toml

@@ -0,0 +1,10 @@
+[build-system]
+build-backend = "setuptools.build_meta"
+requires = [ "setuptools" ]
+
+[project]
+name = "uvms"
+version = "0.0.0"
+
+[project.scripts]
+"uvms-guest" = "uvmslib:guest_main"

pkgs/uvmslib/uvmslib.py

@@ -0,0 +1,151 @@
+import json
+import os
+import select
+import socket
+import subprocess
+import sys
+
+
+class Processes:
+    def __init__(self):
+        self.processes = []
+        self.sources = []
+        self.liveness_fds = dict()
+        self.client_fds = set()
+
+    def popen(self, *args, **kwargs):
+        a, b = socket.socketpair()
+        pass_fds = [*kwargs.get("pass_fds", ()), b.fileno()]
+        proc = subprocess.Popen(*args, **kwargs, pass_fds=pass_fds)
+        self.processes.append(proc)
+        self.sources.append(a)
+        assert a.fileno() not in self.liveness_fds
+        self.liveness_fds[a.fileno()] = proc
+        b.close()
+        return proc
+
+    def handle_run(self, run: dict) -> dict:
+        res = {}
+        text = run.get("text", False)
+        env = {
+            **{
+                k: v
+                for k, v in os.environ.items()
+                # if any(
+                #     k.startswith(prefix)
+                #     for prefix in (
+                #         "XDG_",
+                #         "NIX_",
+                #         "RUST_",
+                #         "WAYLAND_",
+                #         "DBUS_",
+                #         "HOME",
+                #         "PS",
+                #     )
+                # )
+            },
+            "PATH": ":".join(
+                [
+                    *os.environ.get("PATH", "").split(":"),
+                    *run.get(
+                        "EXTRA_PATH",
+                        [],
+                    ),
+                    "/run/wrappers/bin",
+                    "/run/current-system/sw/bin",
+                ],
+            ),
+            "XDG_DATA_DIRS": ":".join(
+                [
+                    *os.environ.get("XDG_DATA_DIRS", "").split(":"),
+                    *run.get(
+                        "EXTRA_XDG_DATA_DIRS",
+                        [],
+                    ),
+                    "/run/current-system/sw/share",
+                ],
+            ),
+        }
+        proc = None
+        try:
+            proc = self.popen(
+                run["argv"],
+                text=text,
+                env=env,
+                cwd="/home/user",
+                stdin=None,
+                stdout=None,
+            )
+            res["status"] = "exec succeeded"
+        except Exception as e:
+            print(e)
+            res["status"] = "exec failed"
+            res["exception"] = repr(e)
+        res["pid"] = getattr(proc, "pid", None)
+        try:
+            if proc is not None:
+                proc.wait(0.125)
+            res["long_running"] = False
+            res["returncode"] = getattr(proc, "returncode", None)
+        except subprocess.TimeoutExpired:
+            res["long_running"] = True
+        return res, proc
+
+    def accept_vsock(self, s):
+        con, (cid, port) = s.accept()
+        assert cid == 2, cid
+        self.sources.append(con)
+        self.client_fds.add(con.fileno())
+        return con, (cid, port)
+
+
+def guest_main():
+    ps = Processes()
+    serv = socket.fromfd(3, socket.AF_VSOCK, socket.SOCK_STREAM)
+    ps.sources.append(serv)
+
+    while True:
+        rr, rw, xs = select.select(ps.sources, [], ps.sources)
+
+        for con in (*rr, *xs):
+            if con.fileno() in ps.liveness_fds:
+                assert con.recv(128) == b""
+                proc = ps.liveness_fds[con.fileno()]
+                proc.wait()
+                assert proc.returncode is not None, proc
+                print(f"{proc} has terminated, shutting down")
+                sys.exit(proc.returncode)
+        for con in rr:
+            if con is serv:
+                con, _ = ps.accept_vsock(serv)
+                print(f"Open [{con.fileno()}]")
+            if con.fileno() in ps.liveness_fds:
+                assert False, "Must already be processed"
+            elif con.fileno() in ps.client_fds:
+                req = con.recv(8192)
+                # IDK why but I keep getting empty messages
+                if req == b"":
+                    print(f"Lost [{con.fileno()}]")
+                    ps.sources = [s for s in ps.sources if s.fileno() != con.fileno()]
+                    continue
+                try:
+                    req = json.loads(req)
+                    print(f"Received {req=}")
+                except json.JSONDecodeError as e:
+                    print(f"Couldn't interpret {req=}: {e}")
+                    continue
+                if "run" in req:
+                    res, proc = ps.handle_run(req["run"])
+                else:
+                    res = {"status": "unknown command"}
+                _, rw, _ = select.select([], [con], [])
+                assert rw, rw
+                res = json.dumps(res).encode("utf8")
+                print(f"Responding with {res=}")
+                con.send(res)
+            else:
+                assert False, con.fileno()
+
+
+if __name__ == "__main__":
+    guest_main()

profiles/baseImage.nix

@@ -0,0 +1,455 @@
+{
+  lib,
+  config,
+  modulesPath,
+  pkgs,
+  uvmsPkgs,
+  ...
+}:
+let
+  inherit (lib) mkOption types concatStringsSep;
+  jsonType = (pkgs.formats.json { }).type;
+
+  inherit (config.system.build) initialRamdisk;
+  inherit (config.system.boot.loader) initrdFile;
+  inherit (config.boot.kernelPackages) kernel;
+  kernelTarget = pkgs.stdenv.hostPlatform.linux-kernel.target;
+  waylandSock = "/run/user/1000/wayland-1";
+  env = {
+    DBUS_SESSION_BUS_ADDRESS = "unix:path=/run/user/1000/bus";
+    XDG_RUNTIME_DIR = "/run/user/1000";
+    WAYLAND_DISPLAY = "wayland-1";
+
+    # MESA_LOADER_DRIVER_OVERRIDE = "zink";
+
+    ELECTRON_OZONE_PLATFORM_HINT = "wayland";
+    MOZ_ENABLE_WAYLAND = "1";
+    QT_QPA_PLATFORM = "wayland"; # Qt Applications
+    GDK_BACKEND = "wayland"; # GTK Applications
+    XDG_SESSION_TYPE = "wayland"; # Electron Applications
+    SDL_VIDEODRIVER = "wayland";
+    CLUTTER_BACKEND = "wayland";
+    NIXOS_OZONE_WL = "1";
+  };
+in
+{
+  imports = [
+    (modulesPath + "/profiles/minimal.nix")
+    ./debug-closure.nix
+    ./minimal.nix
+    ./on-failure.nix
+  ];
+  config = {
+    _module.args.uvmsPkgs = lib.mkDefault (pkgs.callPackage ../pkgs { });
+    # some.failure-handler.enable = true;
+    hardware.graphics.enable = true;
+    # boot.kernelPackages = pkgs.linuxPackagesFor uvmsPkgs.linux-uvm;
+    # boot.isContainer = true;
+    boot.kernel.sysctl = {
+        "vm.overcommit_memory" = 1; # "always"
+        # "vm.overcommit_memory" = 2; # "never"
+        "vm.panic_on_oom" = 1;
+    };
+    boot.initrd.kernelModules = [
+      "drm"
+      "virtio_blk"
+      "virtiofs"
+      "virtio_gpu"
+      "virtio_mmio"
+      "virtio_pci"
+      "overlay"
+    ];
+    boot.kernelModules = [
+      "drm"
+      "erofs"
+      "overlay"
+      "virtio_blk"
+      "virtiofs"
+      "virtio_gpu"
+      "virtio_mmio"
+      "virtio_pci"
+    ];
+    boot.initrd.systemd.initrdBin = [
+      pkgs.fuse
+      pkgs.fuse3
+    ];
+    fileSystems = {
+      "/" = lib.mkDefault {
+        device = "rootfs"; # how does this work? does this assign a label to the tmpfs?
+        fsType = "tmpfs";
+        options = [ "size=20%,mode=0755" ];
+        neededForBoot = true;
+      };
+      "/nix/store" = {
+        fsType = "overlay";
+        overlay.lowerdir = [
+          "/nix/.ro-stores/system"
+          "/nix/.ro-stores/apps"
+        ];
+        neededForBoot = true;
+      };
+      "/nix/.ro-stores/system" = {
+        device = "system";
+        fsType = "virtiofs";
+        options = [
+          "defaults"
+          "ro"
+          "x-systemd.requires=systemd-modules-load.service"
+        ];
+        neededForBoot = true;
+      };
+      "/nix/.ro-stores/apps" = {
+        device = "apps";
+        fsType = "virtiofs";
+        options = [
+          "defaults"
+          "ro"
+          "x-systemd.requires=systemd-modules-load.service"
+        ];
+        neededForBoot = true;
+      };
+    };
+
+    programs.dconf.enable = true;
+    systemd.mounts = [
+      {
+        type = "virtiofs";
+        where = "/home/user";
+        what = "home";
+        after = [ "systemd-modules-load.service" ];
+        wantedBy = [ "local-fs.target" ];
+        before = [ "local-fs.target" ];
+        requires = [ "systemd-modules-load.service" ];
+        options = lib.concatStringsSep "," [
+          "defaults"
+          "rw"
+          "X-mount.owner=1000"
+          "X-mount.group=100"
+        ];
+        unitConfig = {
+          ConditionKernelCommandLine = "uvms.persist-home=1";
+        };
+      }
+      {
+        type = "virtiofs";
+        where = "/home/user/send";
+        what = "send";
+        wants = [
+          "home-user.mount"
+          "-.mount"
+        ];
+        after = [
+          "systemd-modules-load.service"
+          "home-user.mount"
+          "-.mount"
+        ];
+        wantedBy = [ "local-fs.target" ];
+        before = [ "local-fs.target" ];
+        options = lib.concatStringsSep "," [
+          "defaults"
+          "rw"
+          "X-mount.owner=1000"
+          "X-mount.group=100"
+        ];
+        unitConfig = {
+          DefaultDependencies = false;
+        };
+      }
+    ];
+    # systemd.services."mount-home-user-send" = {
+    #   wants = [ "home-user.mount" ];
+    #   after = [
+    #     "systemd-modules-load.service"
+    #     "home-user.mount"
+    #     "-.mount"
+    #   ];
+    #   wantedBy = [ "local-fs.target" ];
+    #   before = [ "local-fs.target" ];
+    #   unitConfig = {
+    #     DefaultDependencies = false;
+    #   };
+    #   environment.PATH = lib.mkForce (
+    #     lib.makeBinPath [
+    #       pkgs.fuse
+    #       pkgs.fuse3
+    #       pkgs.coreutils
+    #     ]
+    #   );
+    #   serviceConfig = {
+    #       Type = "oneshot";
+    #       RemainsAfterExit = true;
+    #     ExecStart = [
+    #       "/run/current-system/sw/bin/mkdir -p /home/user/send"
+    #       "/run/current-system/sw/bin/chown user /home/user/send"
+    #       "/run/current-system/sw/sbin/mount -t virtiofs -o defaults,rw send /home/user/send"
+    #     ];
+    #     StandardOutput = "journal+console";
+    #     StandardError = "journal+console";
+    #   };
+    # };
+
+    systemd.network.enable = true;
+    networking.useNetworkd = true;
+    networking.nftables.enable = true;
+    networking.useDHCP = true;
+    networking.nameservers = [ "1.1.1.1" ];
+    services.resolved.enable = lib.mkForce true;
+
+    system.activationScripts.specialfs = lib.mkForce "";
+    # networking.firewall.enable = false;
+    console.enable = false;
+    services.udev.packages = lib.mkDefault [ ];
+    systemd.services."systemd-oomd".enable = false;
+
+    users.mutableUsers = false;
+    users.users.root.password = "hacktheplanet!";
+    users.groups.users = { };
+    users.users.user = {
+      uid = 1000;
+      isNormalUser = true;
+      password = "hacktheplanet!";
+      extraGroups = [
+        "video"
+        "render"
+        "users"
+        "wheel"
+      ];
+    };
+
+    environment.profileRelativeSessionVariables.PATH = lib.mkForce [ "/bin\${PATH:+:}$PATH" ];
+    environment.profileRelativeSessionVariables.XDG_DATA_DIRS = lib.mkForce [ "/run/current-system/sw/share/\${XDG_DATA_DIRS:+:}$XDG_DATA_DIRS" ];
+    environment.sessionVariables = env;
+    environment.variables = env;
+    systemd.globalEnvironment = env;
+
+    systemd.tmpfiles.settings."10-xdg" = {
+      ${env.XDG_RUNTIME_DIR}.d = {
+        user = "user";
+        group = "users";
+        mode = "0755";
+      };
+    };
+
+    systemd.sockets."wayland-proxy" = {
+      listenStreams = [
+        waylandSock
+      ];
+      socketConfig = {
+        SocketUser = "user";
+        SocketGroup = "users";
+        FileDescriptorName = "wayland";
+      };
+      wantedBy = [ "sockets.target" ];
+      partOf = [ "wayland-proxy.service" ];
+    };
+    systemd.services."wayland-proxy" = {
+      wantedBy = [ "default.target" ];
+      serviceConfig = {
+        User = "user";
+        Group = "users";
+        ExecStart = "${lib.getExe pkgs.wayland-proxy-virtwl} --virtio-gpu";
+        # ExecStart = "${lib.getExe uvmsPkgs.wl-cross-domain-proxy}  --listen-fd --filter-global wp_presentation";
+        ExecStartPre = [
+          "+/run/current-system/sw/bin/chmod 0666 /dev/dri/card0 /dev/dri/renderD128"
+        ];
+        StandardOutput = "journal+console";
+        StandardError = "journal+console";
+        Restart = "on-failure";
+        RestartSec = 5;
+      };
+    };
+
+    systemd.sockets."uvms-guest" = {
+      requiredBy = [ "multi-user.target" ];
+      before = [ "multi-user.target" ];
+      listenStreams = [
+        "vsock::24601"
+      ];
+      partOf = [ "uvms-guest.service" ];
+    };
+    systemd.services."uvms-guest" = {
+      before = [ "multi-user.target" ];
+      onFailure = [ "shutdown.service" ];
+      serviceConfig = {
+        User = "user";
+        Group = "users";
+        ExecStart = "${lib.getExe' uvmsPkgs.uvmslib "uvms-guest"}";
+        ExecStop = [
+          "/run/current-system/sw/bin/echo GUEST DOWN"
+          "/run/current-system/sw/bin/systemctl poweroff"
+        ];
+        StandardOutput = "journal+console";
+        StandardError = "journal+console";
+        Restart = "no";
+      };
+    };
+    systemd.services."shutdown" = {
+      serviceConfig = {
+        ExecStart = [ "/run/current-system/sw/bin/systemctl poweroff" ];
+        StandardOutput = "journal+console";
+        StandardError = "journal+console";
+      };
+    };
+
+    fonts.enableDefaultPackages = true;
+
+    boot.kernelParams = [
+      "earlyprintk=ttyS0"
+      "console=ttyS0"
+      "reboot=t"
+      "panic=-1"
+      # "rootfstype=virtiofs"
+      # "root=rootstore"
+    ];
+  };
+
+  options = {
+    system.build.ch = mkOption {
+      type = types.package;
+      default = (pkgs.formats.json { }).generate "vm.json" config.uvms.ch.settings;
+    };
+    uvms.ch.settings = mkOption {
+      default = { };
+      type = types.submodule (
+        let
+          osConfig = config;
+        in
+        { config, ... }:
+        {
+          freeformType = jsonType;
+          options = {
+            payload = {
+              cmdline = mkOption {
+                type = types.str;
+                default = concatStringsSep " " (
+                  osConfig.boot.kernelParams
+                  ++ [
+                    # "init=${lib.removePrefix "/nix/store" "${osConfig.system.build.toplevel}"}/init"
+                    "init=${osConfig.system.build.toplevel}/init"
+                  ]
+                );
+                defaultText = ''concatStringsSep " " ${osConfig.boot.kernelParams}'';
+              };
+              kernel = mkOption {
+                type = types.str;
+                default = "${kernel}/${kernelTarget}";
+              };
+              initramfs = mkOption {
+                type = types.nullOr types.str;
+                default = "${initialRamdisk}/${initrdFile}";
+              };
+            };
+            vsock = {
+              cid = mkOption {
+                type = types.int;
+                default = 4;
+              };
+              socket = mkOption {
+                type = types.str;
+                default = "vsock.sock";
+              };
+            };
+            "api-socket" = mkOption {
+              type = types.str;
+              default = "vmm.sock";
+            };
+            "serial".mode = mkOption {
+              type = types.str;
+              default = "File";
+            };
+            "serial".file = mkOption {
+              type = types.nullOr types.str;
+              default = "serial";
+            };
+            "console".mode = mkOption {
+              type = types.str;
+              default = "Pty";
+            };
+            "console".file = mkOption {
+              type = types.nullOr types.str;
+              default = null;
+            };
+            # "watchdog" = true;
+            # "seccomp" = true;
+            disks = mkOption {
+              default = [ ];
+              type = types.listOf (
+                types.submodule {
+                  freeformType = jsonType;
+                  options = {
+                    path = mkOption {
+                      type = types.oneOf [
+                        types.path
+                        types.str
+                      ];
+                    };
+                    readonly = mkOption {
+                      type = types.bool;
+                      default = true;
+                    };
+                    id = mkOption { type = types.str; };
+                  };
+                }
+              );
+            };
+            memory = mkOption {
+              default = { };
+              type = types.submodule {
+                freeformType = jsonType;
+                options = {
+                  size = mkOption {
+                    type = types.int;
+                    default = 4 * 1024 * 1048576;
+                  };
+                  shared = mkOption {
+                    type = types.bool;
+                    default = true;
+                  };
+                  mergeable = mkOption {
+                    type = types.bool;
+                    default = true;
+                  };
+                  hotplug_method = mkOption {
+                    default = "VirtioMem";
+                    type = types.enum [
+                      "Acpi"
+                      "VirtioMem"
+                    ];
+                  };
+                  hotplugged_size = mkOption {
+                    type = types.int;
+                    default = 512 * 1048576;
+                  };
+                  hotplug_size = mkOption {
+                    type = types.int;
+                    default = config.memory.size;
+                  };
+                  # hugepages = mkOption {
+                  #   type = types.bool;
+                  #   default = true;
+                  # };
+                };
+              };
+            };
+            cpus = mkOption {
+              default = { };
+              type = types.submodule {
+                freeformType = jsonType;
+                options = {
+                  boot_vcpus = mkOption {
+                    type = types.int;
+                    default = 4;
+                  };
+                  max_vcpus = mkOption {
+                    type = types.int;
+                    default = 4;
+                  };
+                };
+              };
+            };
+          };
+        }
+      );
+    };
+  };
+}

profiles/ch-runner.nix

@@ -9,22 +9,59 @@
 # but we shall begin by reproducing at least some of their work.
 
 let
-  cfg = config.uvms.cloud-hypervisor;
+  cfg = config.uvms.ch;
+
+  inherit (config.networking) hostName;
   inherit (config.debug.closure.erofs) layers;
+  inherit (lib)
+    mkOption
+    types
+    concatMapStringsSep
+    getExe
+    getExe'
+    getBin
+    ;
+
+  package = uvmsPkgs.cloud-hypervisor-gpu;
+  uvmsPkgs = pkgs.callPackage ../pkgs { };
+
+  chSettingsFile = (pkgs.formats.json { }).generate "vm.json" cfg.settings;
+
+  uvmPrefix = "\${HOME}/uvms/${hostName}";
+  vmmSock = "${uvmPrefix}/vmm.sock";
+  elbPrefix = "${lib.getBin pkgs.execline}/bin";
+  s6Prefix = "${lib.getBin pkgs.s6}/bin";
+  writeElb = name: text: writeElb' name "-W" text;
+  writeElb' =
+    name: elArgs: text:
+    pkgs.writeTextFile {
+      inherit name;
+      destination = "/bin/${name}";
+      executable = true;
+      text = ''
+        #!${getExe' pkgs.execline "execlineb"}${lib.optionalString (elArgs != null) " "}${elArgs}
+        importas OLDPATH PATH
+        export PATH "${elbPrefix}:${s6Prefix}:''${OLDPATH}"
+        ${text}
+      '';
+    };
 in
 {
   options = {
-    uvms.cloud-hypervisor.enable = lib.mkEnableOption "Configure guest (e.g. fileSystems)";
+    uvms.ch.enable = lib.mkEnableOption "Configure guest (e.g. fileSystems)";
-    uvms.cloud-hypervisor.runner = lib.mkOption {
+    uvms.ch.runner = mkOption {
-      type = lib.types.package;
+      type = types.package;
       description = "A naive script for running this system in cloud-hypervisor";
     };
-    uvms.cloud-hypervisor.extraArgv = lib.mkOption {
+    uvms.ch.debugger = mkOption {
-      type = lib.types.listOf lib.types.str;
+      type = types.lazyAttrsOf types.anything;
-      default = [ ];
+      description = "Same but you can debug the kernel";
     };
-    uvms.cloud-hypervisor.argv = lib.mkOption {
+    uvms.ch.settingsFile = mkOption {
-      type = lib.types.listOf lib.types.str;
+      type = types.package;
+      default = chSettingsFile;
+      defaultText = "...";
+      readOnly = true;
     };
     uvms.cloud-hypervisor.extraCmdline = lib.mkOption {
       type = lib.types.listOf lib.types.str;
@@ -33,82 +70,136 @@ in
     uvms.cloud-hypervisor.cmdline = lib.mkOption {
       type = lib.types.listOf lib.types.str;
       default = [
-        "earlyprintk=ttyS0"
-        "console=ttyS0"
-        "reboot=t"
-        "panic=-1"
-        "init=${config.system.build.toplevel}/init"
       ]
       ++ config.boot.kernelParams
       ++ config.uvms.cloud-hypervisor.extraCmdline;
     };
   };
+  imports = [ ./baseImage.nix ];
   config = lib.mkMerge [
     {
-      uvms.cloud-hypervisor.argv = lib.mkBefore (
+      # boot.kernelPackages = pkgs.linuxPackagesFor (uvmsPkgs.linux-uvm);
-        [
+      uvms.ch.settings = {
-          (lib.getExe pkgs.cloud-hypervisor)
+        memory = {
-          "--cmdline=${lib.concatStringsSep " " cfg.cmdline}"
+          # hotplugged_size = 512 * 1048576;
-          "--kernel=${config.boot.kernelPackages.kernel}/${pkgs.stdenv.hostPlatform.linux-kernel.target}"
+          # hotplugd_size = 1536 * 1048576;
-          "--initramfs=${config.system.build.initialRamdisk}/${config.system.boot.loader.initrdFile}"
+          # hotplug_method = "virtio-mem"
-          "--vsock=cid=4,socket=vsock.sock"
+        };
-          "--api-socket=vmm.sock"
+      };
-          "--serial=tty"
+
-          "--console=null"
+      uvms.ch.debugger = pkgs.testers.runNixOSTest (
-          "--watchdog"
+        { config, ... }:
-          "--seccomp=true"
+        {
-        ]
+          name = "test-run-${hostName}";
-        ++ cfg.extraArgv
+          passthru = rec {
-      );
+            inherit (config.nodes.machine.system.build) gdbScript;
-      uvms.cloud-hypervisor.runner = pkgs.writeShellScriptBin "run-${config.networking.hostName}" ''
+            inherit (config.nodes.machine.boot.kernelPackages) kernel;
-        set -euo pipefail
+            kernelSrc = pkgs.srcOnly kernel;
-        GUESTNAME=${config.networking.hostName}
+          };
-        args=(
+          nodes.machine =
-        ${lib.concatMapStringsSep "\n" lib.escapeShellArg cfg.argv}
+            { config, ... }:
-        )
+            let
-        mkdir -p "$HOME/uvms/$GUESTNAME"
+              kernel = config.boot.kernelPackages.kernel;
-        cd "$HOME/uvms/$GUESTNAME"
+              kernelSrc = pkgs.srcOnly kernel;
-        cleanup() {
+              gdbScript = writeElb "attach-gdb" ''
-            rm "$HOME/uvms/$GUESTNAME"/{vmm,vsock}.sock
+                if { rm -rf /tmp/gdb }
+                if { mkdir -p /tmp/gdb/kos }
+                cd /tmp/gdb
+                if {
+                    elglob -0 files ${kernelSrc}/*
+                    forx -E f { $files }
+                    ln -s $f ./
                 }
-        exec -a "uuvm/$GUESTNAME" "''${args[@]}"
+                if { mkdir -p build }
+                cd build
+                if {
+                    forx -E pattern { 
+                        ${kernel.modules}/lib/modules/*/kernel/drivers/net/tun*
+                        ${kernel.modules}/lib/modules/*/kernel/drivers/net/tap*
+                    }
+                    elglob -0 files $pattern
+                    forx -E f { $files }
+                    if { cp $f . }
+                    backtick -E COMPRESSED { basename $f }
+                    xz -d $COMPRESSED
+                }
+                elglob -0 GDB_SCRIPT_DIR ${lib.getDev kernel}/lib/modules/*/build/scripts/gdb
+                if {
+                    if { cp -r --no-preserve=all $GDB_SCRIPT_DIR gdb_scripts }
+                    mv gdb_scripts/linux/constants.py.in gdb_scripts/linux/constants.py
+                }
+                ${getExe pkgs.gdb}
+                -ex "python import sys; sys.path.insert(0, \"''${GDB_SCRIPT_DIR}\")"
+                -ex "target remote :1234"
+                -ex "source ''${GDB_SCRIPT_DIR}/vmlinux-gdb.py"
+                -ex "lx-symbols"
+                ${kernel.dev}/vmlinux
               '';
-    }
+            in
-    (lib.mkIf cfg.enable {
+            {
-      boot.initrd.availableKernelModules = [
+              boot.kernelPackages = pkgs.linuxPackagesFor (
-        "erofs"
+                (pkgs.linux.override (oldArgs: {
-        "overlay"
+                  # extraMakeFlags = oldArgs.extraMakeFlags or [ ] ++ [
-        "virtio_mmio"
+                  #   "scripts_gdb"
-        "virtio_pci"
+                  # ];
-        "virtio_blk"
+                  kernelPatches = oldArgs.kernelPatches or [ ] ++ [
-        # "9pnet_virtio"
+                    {
-        # "9p"
+                      name = "debug";
-        "virtiofs"
+                      patch = null;
-      ];
+                      structuredExtraConfig = {
-      boot.initrd.systemd.enable = lib.mkDefault true;
+                        GDB_SCRIPTS = lib.kernel.yes;
-      fileSystems = {
+                        DEBUG_INFO = lib.kernel.yes;
-        "/nix/store" = {
+                        DEBUG_INFO_REDUCED = lib.kernel.no;
-          fsType = "overlay";
+                        # FRAME_POINTER = lib.kernel.yes; # "unused option"???
-          overlay.lowerdir = map (img: "/nix/.ro-stores/${toString img.seq}") layers;
+                        KALLSYMS = lib.kernel.yes;
-          neededForBoot = true;
+                        KGDB = lib.kernel.yes;
                       };
                     }
-      // lib.listToAttrs (
+                  ];
-        map (
+                })).overrideAttrs
-          img:
+                  (oldAttrs: {
-          lib.nameValuePair "/nix/.ro-stores/${toString img.seq}" {
+                    dontStrip = true;
-            device = "/dev/disk/by-label/${img.label}";
+                    postInstall = oldAttrs.postInstall or "" + ''
-            neededForBoot = true;
+                      cp "$buildRoot/scripts/gdb/linux/constants.py" $dev/lib/modules/*/build/scripts/gdb/linux/ || echo "$buildRoot/scripts/gdb/linux/constants.py doesn't exist"
-            options = [ "x-systemd.device-timeout=5" ];
+                    '';
-          }
-        ) layers
-      );
-      uvms.cloud-hypervisor.argv = [
-        "--memory=size=1536M,hotplug_size=1536M,hotplugged_size=512M,hotplug_method=virtio-mem,mergeable=on,shared=on"
-        "--cpus=boot=4"
-        "--disk"
-      ]
-      ++ map (img: "path=${img},readonly=true,id=${toString img.label}") layers;
                   })
+              );
+              boot.kernelParams = [ "nokaslr" ];
+              networking.useNetworkd = true;
+              virtualisation.qemu.options = [ "-s" ];
+              environment.systemPackages = [
+                pkgs.gdb
+                package # CH
+                cfg.runner
+                uvmsPkgs.taps
+              ];
+              system.build.gdbScript = gdbScript;
+              systemd.services.taps = {
+                wantedBy = [ "multi-user.target" ];
+                environment.TAPS_SOCK = "/run/taps/taps.sock";
+                serviceConfig = {
+                  UMask = "0007";
+                  ExecStart = "${getExe uvmsPkgs.taps} serve";
+                  RuntimeDirectory = "taps";
+                  DynamicUser = true;
+                  AmbientCapabilities = [
+                    "CAP_NET_BIND_SERVICE"
+                    "CAP_NET_ADMIN"
+                  ];
+                  NoNewPrivileges = true;
+                };
+              };
+            };
+          testScript = ''
+            machine.succeed("${getExe cfg.runner}")
+          '';
+        }
+      );
+
+      # NOTE: Used to be an even uglier bash script, but, for now, execline makes for easier comparisons against spectrum
+      uvms.ch.runner = writeElb "run-${hostName}" ''
+        ${lib.getExe uvmsPkgs.uvms} --vm-config=${chSettingsFile} --vm=${hostName}
+      '';
+    }
   ];
 }

profiles/debug-closure.nix

@@ -15,15 +15,9 @@ let
   inherit (ps) writeErofsLayers;
   emptySystem = import (pkgs.path + "/nixos/lib/eval-config.nix") {
     modules = [
-      (modulesPath + "/profiles/minimal.nix")
+      ./minimal.nix
       {
         system.stateVersion = config.system.stateVersion;
-        fileSystems."/".fsType = "tmpfs";
-        boot.loader.grub.enable = false;
-        networking.hostName = "base";
-        networking.nftables.enable = true;
-        networking.useNetworkd = true;
-        systemd.network.enable = true;
       }
     ];
   };

profiles/minimal.nix

@@ -0,0 +1,36 @@
+{
+  lib,
+  config,
+  modulesPath,
+  ...
+}:
+{
+  imports = [
+    (modulesPath + "/profiles/minimal.nix")
+  ];
+  boot.loader.grub.enable = false;
+  boot.initrd.systemd.enable = true;
+  networking.useNetworkd = true;
+  networking.nftables.enable = config.networking.firewall.enable || config.networking.nat.enable;
+  fileSystems."/".fsType = lib.mkDefault "tmpfs";
+  networking.hostName = lib.mkDefault "base";
+
+  systemd.sysusers.enable = false;
+  services.userborn.enable = true; # nikstur it
+
+  nix.enable = false;
+  services.logrotate.enable = false;
+  services.udisks2.enable = false;
+  system.tools.nixos-generate-config.enable = false;
+  systemd.coredump.enable = false;
+  powerManagement.enable = false;
+  boot.kexec.enable = false;
+  system.switch.enable = false;
+  services.resolved.enable = false;
+
+  systemd.services.generate-shutdown-ramfs.enable = lib.mkForce false;
+  systemd.services.systemd-remount-fs.enable = lib.mkForce false;
+  systemd.services.systemd-pstore.enable = lib.mkForce false;
+  systemd.services.lastlog2-import.enable = lib.mkForce false;
+  # systemd.services.suid-sgid-wrappers.enable = lib.mkForce false;
+}

profiles/on-failure.nix

@@ -0,0 +1,72 @@
+{
+  lib,
+  config,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.some.failure-handler;
+  jobScript = pkgs.writeShellScriptBin "show-status" ''
+    set -euo pipefail
+
+    export PATH=${lib.getBin config.boot.initrd.systemd.package}/bin''${PATH:+:}$PATH
+    export PATH=${lib.getBin pkgs.util-linux}/bin''${PATH:+:}$PATH
+    export PATH=${lib.getBin pkgs.gnugrep}/bin''${PATH:+:}$PATH
+
+    unit="$1"
+    shift
+
+    systemctl status "$unit" >&2 || true
+    patterns=$unit$'\n'error
+    dmesg | grep -Fi "$patterns" || true
+  '';
+  mkSystemdDropin = pkgs.callPackage ../pkgs/mkSystemdDropin.nix { };
+in
+{
+  options.some.failure-handler = {
+    enable = lib.mkEnableOption "Set up show-status@.service as a default OnFailure dependency";
+    stage-1.enable =
+      lib.mkEnableOption "Set up show-status@.service as a default OnFailure dependency in initramfs/initrd"
+      // {
+        default = cfg.enable;
+      };
+    package = lib.mkOption {
+      type = lib.types.package;
+      readOnly = true;
+      description = "The internal package with the drop-ins";
+    };
+  };
+  config = {
+    some.failure-handler.package = mkSystemdDropin {
+      name = "status-on-failure";
+      inherit jobScript;
+      dropinText = ''
+        [Unit]
+        OnFailure=status@%n.service
+      '';
+      serviceText = ''
+        [Unit]
+        DefaultDependencies=no
+        Description=Show status for %i
+
+        [Service]
+        Type=oneshot
+        StandardOutput=journal+console
+        StandardError=journal+console
+        ExecStart=${lib.getExe jobScript} "%i"
+        JoinsNamespaceOf=
+        DelegateNamespaces=
+      '';
+      extraCommands = ''
+        printf "%s" "$serviceText" > "$root/status@.service"
+      '';
+    };
+    boot.initrd.systemd.packages = lib.optionals cfg.stage-1.enable [ cfg.package ];
+    boot.initrd.systemd.storePaths = lib.optionals cfg.stage-1.enable [
+      jobScript
+      pkgs.util-linux
+      pkgs.gnugrep
+    ];
+    systemd.packages = lib.optionals cfg.enable [ cfg.package ];
+  };
+}

profiles/uvms-guest.nix

@@ -41,6 +41,7 @@ in
         volumes = [
           {
             image = "swapfile.img";
+            serial = "swapfiles";
             mountPoint = "/var/swapfiles";
             size = 1024;
           }

profiles/uvms-users.nix

@@ -29,7 +29,6 @@ in
   };
   config = mergeIf cfg.enable [
     {
-      services.getty.autologinUser = "user";
       security.sudo.wheelNeedsPassword = false;
       users.mutableUsers = false;
       users.users.user = {

profiles/vmapp-demo.nix

@@ -212,9 +212,9 @@ in
         ''} %i";
       };
 
-      boot.initrd.systemd.settings.Manager.DefaultTimeoutStartSec = 30;
+      boot.initrd.systemd.settings.Manager.DefaultTimeoutStartSec = lib.mkDefault 30;
-      systemd.settings.Manager.DefaultTimeoutStopSec = 10;
+      systemd.settings.Manager.DefaultTimeoutStopSec = lib.mkDefault 10;
-      systemd.services."user@".serviceConfig.TimeoutStopSec = 10;
+      systemd.services."user@".serviceConfig.TimeoutStopSec = lib.mkDefault 10;
 
       services.openssh.enable = true;
 

shell.nix

@@ -0,0 +1,31 @@
+with import <nixpkgs> { };
+
+let
+  uvmPkgs = callPackage ./pkgs { };
+in
+mkShell.override { stdenv = stdenvNoCC; } {
+  inputsFrom = with uvmPkgs; [
+    ch-proxy
+    taps
+    writeErofsLayers
+    request-usb
+  ];
+  packages =
+    map lib.getBin [
+      uvmPkgs.cloud-hypervisor-gpu
+      virtiofsd
+      crosvm # virtio-gpu
+      npins
+      strace
+      bubblewrap
+      python3
+      execline
+      s6
+      wayland-proxy-virtwl
+      uvmPkgs.taps
+    ]
+    ++ [
+      man-pages
+      linux-manual
+    ];
+}