From 17bde34c967dad65019e6a2f767b307bb394d555 Mon Sep 17 00:00:00 2001 From: Else Someone Date: Thu, 12 Feb 2026 20:23:55 +0200 Subject: [PATCH 1/6] taps: MVE EIO crash in CH was caused by CH trying to writev before the interface has been assigned any addresses; adding an address prior to passing the FD (via 80-vm-vt.network) solves the issue. It is still unclear what causes the interrupt=None in the vhost_user/passt branch --- examples/dummy.nix | 2 + pkgs/taps/main.c | 20 ++- profiles/ch-runner.nix | 303 +++++++++++++++++++++++++++++++++++------ 3 files changed, 284 insertions(+), 41 deletions(-) diff --git a/examples/dummy.nix b/examples/dummy.nix index 4689af1..7d8c813 100644 --- a/examples/dummy.nix +++ b/examples/dummy.nix @@ -13,6 +13,8 @@ system.stateVersion = "25.11"; + networking.hostName = "dummy"; + vmapps.enable = true; _module.args.inputs = import ../npins; diff --git a/pkgs/taps/main.c b/pkgs/taps/main.c index 05edb64..3b276df 100644 --- a/pkgs/taps/main.c +++ b/pkgs/taps/main.c @@ -1,5 +1,6 @@ #define _GNU_SOURCE +#include #include /* secure_getenv */ #include #include @@ -15,10 +16,12 @@ #include #include #include +#include #define __UAPI_DEF_IF_IFNAMSIZ 1 #include #include +#include #include "sendfd.h" @@ -108,6 +111,11 @@ int tuntap_alloc(char *dev, short openFlags, short ifrFlags, int *out_fd) { strncpy(dev, ifr.ifr_name, IFNAMSIZ); *out_fd = fd; + + { + int sz = sizeof(struct virtio_net_hdr_v1); + DO_OR_DIE(ioctl(fd, TUNSETVNETHDRSZ, &sz)); + } return 0; } @@ -257,6 +265,13 @@ void cleanup(int signo, siginfo_t *info, void *_context) { errx(EXIT_FAILURE, "Exiting with signal %d", signo); } +/* skarlibs under ISC */ +int uncoe (int fd) +{ + int flags = fcntl(fd, F_GETFD, 0) ; + return flags < 0 ? flags : flags & FD_CLOEXEC ? fcntl(fd, F_SETFD, flags & ~FD_CLOEXEC) : 0 ; +} + int main(int argc, char **argv) { struct sigaction act = { 0 }; act.sa_flags = SA_SIGINFO; @@ -266,7 +281,7 @@ int main(int argc, char **argv) { bool cmdServe = false; bool cmdPass = false; - char *ifname = "vt%d"; + char *ifname = "vt-%d"; char **rest = argv + 1; char **end = argv + argc; @@ -303,7 +318,7 @@ int main(int argc, char **argv) { const char *servePath = secure_getenv("TAPS_SOCK"); if (servePath == NULL) { - servePath = "taps.sock"; + servePath = "/run/taps/taps.sock"; } if (cmdServe) { @@ -317,6 +332,7 @@ int main(int argc, char **argv) { close(fd); fd = 3; } + uncoe(fd); DO_OR_DIE(execvp(nextArgv[0], nextArgv)); } else { error(EINVAL, EINVAL, "subcommand args"); diff --git a/profiles/ch-runner.nix b/profiles/ch-runner.nix index fbb09a4..f680354 100644 --- a/profiles/ch-runner.nix +++ b/profiles/ch-runner.nix @@ -10,21 +10,107 @@ let cfg = config.uvms.cloud-hypervisor; + + inherit (config.networking) hostName; inherit (config.debug.closure.erofs) layers; + inherit (lib) mkOption types; + + package = pkgs.cloud-hypervisor.overrideAttrs (oldAttrs: { + patches = oldAttrs.patches or [ ] ++ [ + # ../patches/ch.patch + ]; + buildType = "debug"; + dontStrip = true; + }); + uvmsPkgs = pkgs.callPackage ../pkgs { }; + + chSettingsFile = (pkgs.formats.json { }).generate "vm.json" cfg.settings; + + uvmPrefix = "\${HOME}/uvms/${hostName}"; + vmmSock = "${uvmPrefix}/vmm.sock"; + pastaSock = "${uvmPrefix}/pasta.sock"; + pastaPidPath = "${uvmPrefix}/pasta.pid"; + elbPrefix = "${lib.getBin pkgs.execline}/bin"; + s6Prefix = "${lib.getBin pkgs.s6}/bin"; + writeElb = name: text: writeElb' name "-W" text; + writeElb' = + name: elArgs: text: + pkgs.writeTextFile { + inherit name; + destination = "/bin/${name}"; + executable = true; + text = '' + #!${lib.getExe' pkgs.execline "execlineb"}${lib.optionalString (elArgs != null) " "}${elArgs} + importas OLDPATH PATH + export PATH "${elbPrefix}:${s6Prefix}:''${OLDPATH}" + ${text} + ''; + }; in { options = { uvms.cloud-hypervisor.enable = lib.mkEnableOption "Configure guest (e.g. fileSystems)"; - uvms.cloud-hypervisor.runner = lib.mkOption { - type = lib.types.package; + uvms.cloud-hypervisor.runner = mkOption { + type = types.package; description = "A naive script for running this system in cloud-hypervisor"; }; - uvms.cloud-hypervisor.extraArgv = lib.mkOption { - type = lib.types.listOf lib.types.str; - default = [ ]; + uvms.cloud-hypervisor.debugger = mkOption { + type = types.lazyAttrsOf types.anything; + description = "Same but you can debug the kernel"; }; - uvms.cloud-hypervisor.argv = lib.mkOption { - type = lib.types.listOf lib.types.str; + uvms.cloud-hypervisor.settingsFile = mkOption { + type = types.package; + default = chSettingsFile; + defaultText = "..."; + readOnly = true; + }; + uvms.cloud-hypervisor.settings = mkOption { + default = { }; + type = types.submodule { + freeformType = (pkgs.formats.json { }).type; + options = { + payload = { + cmdline = mkOption { type = types.str; }; + kernel = mkOption { type = types.str; }; + initramfs = mkOption { + type = types.str; + default = "${config.system.build.initialRamdisk}/${config.system.boot.loader.initrdFile}"; + }; + }; + vsock = { + cid = mkOption { + type = types.int; + default = 4; + }; + socket = mkOption { + type = types.str; + default = "vsock.sock"; + }; + }; + "api-socket" = mkOption { + type = types.str; + default = "vmm.sock"; + }; + "serial".mode = mkOption { + type = types.str; + default = "File"; + }; + "serial".file = mkOption { + type = types.nullOr types.str; + default = "serial"; + }; + "console".mode = mkOption { + type = types.str; + default = "Pty"; + }; + "console".file = mkOption { + type = types.nullOr types.str; + default = null; + }; + # "watchdog" = true; + # "seccomp" = true; + }; + }; }; uvms.cloud-hypervisor.extraCmdline = lib.mkOption { type = lib.types.listOf lib.types.str; @@ -45,33 +131,178 @@ in }; config = lib.mkMerge [ { - uvms.cloud-hypervisor.argv = lib.mkBefore ( - [ - (lib.getExe pkgs.cloud-hypervisor) - "--cmdline=${lib.concatStringsSep " " cfg.cmdline}" - "--kernel=${config.boot.kernelPackages.kernel}/${pkgs.stdenv.hostPlatform.linux-kernel.target}" - "--initramfs=${config.system.build.initialRamdisk}/${config.system.boot.loader.initrdFile}" - "--vsock=cid=4,socket=vsock.sock" - "--api-socket=vmm.sock" - "--serial=tty" - "--console=null" - "--watchdog" - "--seccomp=true" - ] - ++ cfg.extraArgv - ); - uvms.cloud-hypervisor.runner = pkgs.writeShellScriptBin "run-${config.networking.hostName}" '' - set -euo pipefail - GUESTNAME=${config.networking.hostName} - args=( - ${lib.concatMapStringsSep "\n" lib.escapeShellArg cfg.argv} - ) - mkdir -p "$HOME/uvms/$GUESTNAME" - cd "$HOME/uvms/$GUESTNAME" - cleanup() { - rm "$HOME/uvms/$GUESTNAME"/{vmm,vsock}.sock + uvms.cloud-hypervisor.settings = { + payload = { + cmdline = lib.concatStringsSep " " cfg.cmdline; + kernel = "${config.boot.kernelPackages.kernel}/${pkgs.stdenv.hostPlatform.linux-kernel.target}"; + }; + disks = map (img: { + path = img; + readonly = true; + id = toString img.label; + }) layers; + memory = { + size = 1536 * 1048576; + shared = true; + mergeable = true; + # hotplugged_size = 512 * 1048576; + # hotplugd_size = 1536 * 1048576; + # hotplug_method = "virtio-mem" + }; + cpus = { + boot_vcpus = 4; + max_vcpus = 4; + }; + }; + + uvms.cloud-hypervisor.debugger = pkgs.testers.runNixOSTest ( + { config, ... }: + { + name = "test-run-${hostName}"; + passthru = rec { + inherit (config.nodes.machine.system.build) gdbScript; + inherit (config.nodes.machine.boot.kernelPackages) kernel; + kernelSrc = pkgs.srcOnly kernel; + }; + nodes.machine = + { config, ... }: + let + kernel = config.boot.kernelPackages.kernel; + kernelSrc = pkgs.srcOnly kernel; + gdbScript = writeElb "attach-gdb" '' + if { rm -rf /tmp/gdb } + if { mkdir -p /tmp/gdb/kos } + cd /tmp/gdb + if { + elglob -0 files ${kernelSrc}/* + forx -E f { $files } + ln -s $f ./ + } + if { mkdir -p build } + cd build + if { + forx -E pattern { + ${kernel.modules}/lib/modules/*/kernel/drivers/net/tun* + ${kernel.modules}/lib/modules/*/kernel/drivers/net/tap* + } + elglob -0 files $pattern + forx -E f { $files } + if { cp $f . } + backtick -E COMPRESSED { basename $f } + xz -d $COMPRESSED + } + elglob -0 GDB_SCRIPT_DIR ${lib.getDev kernel}/lib/modules/*/build/scripts/gdb + if { + if { cp -r --no-preserve=all $GDB_SCRIPT_DIR gdb_scripts } + mv gdb_scripts/linux/constants.py.in gdb_scripts/linux/constants.py + } + ${lib.getExe pkgs.gdb} + -ex "python import sys; sys.path.insert(0, \"''${GDB_SCRIPT_DIR}\")" + -ex "target remote :1234" + -ex "source ''${GDB_SCRIPT_DIR}/vmlinux-gdb.py" + -ex "lx-symbols" + ${kernel.dev}/vmlinux + ''; + in + { + boot.kernelPackages = pkgs.linuxPackagesFor ( + (pkgs.linux.override (oldArgs: { + # extraMakeFlags = oldArgs.extraMakeFlags or [ ] ++ [ + # "scripts_gdb" + # ]; + kernelPatches = oldArgs.kernelPatches or [ ] ++ [ + { + name = "debug"; + patch = null; + structuredExtraConfig = { + GDB_SCRIPTS = lib.kernel.yes; + DEBUG_INFO = lib.kernel.yes; + DEBUG_INFO_REDUCED = lib.kernel.no; + # FRAME_POINTER = lib.kernel.yes; # "unused option"??? + KALLSYMS = lib.kernel.yes; + KGDB = lib.kernel.yes; + }; + } + ]; + })).overrideAttrs + (oldAttrs: { + dontStrip = true; + postInstall = oldAttrs.postInstall or "" + '' + cp "$buildRoot/scripts/gdb/linux/constants.py" $dev/lib/modules/*/build/scripts/gdb/linux/ || echo "$buildRoot/scripts/gdb/linux/constants.py doesn't exist" + ''; + }) + ); + boot.kernelParams = [ "nokaslr" ]; + networking.useNetworkd = true; + virtualisation.qemu.options = [ "-s" ]; + environment.systemPackages = [ + pkgs.gdb + package # CH + cfg.runner + uvmsPkgs.taps + ]; + system.build.gdbScript = gdbScript; + systemd.services.taps = { + wantedBy = [ "multi-user.target" ]; + environment.TAPS_SOCK = "/run/taps/taps.sock"; + serviceConfig = { + UMask = "0007"; + ExecStart = "${lib.getExe uvmsPkgs.taps} serve"; + RuntimeDirectory = "taps"; + DynamicUser = true; + AmbientCapabilities = [ + "CAP_NET_BIND_SERVICE" + "CAP_NET_ADMIN" + ]; + NoNewPrivileges = true; + }; + }; + }; + testScript = '' + machine.succeed("${lib.getExe cfg.runner}") + ''; } - exec -a "uuvm/$GUESTNAME" "''${args[@]}" + ); + + # NOTE: Used to be an even uglier bash script, but, for now, execline makes for easier comparisons against spectrum + uvms.cloud-hypervisor.runner = writeElb "run-${hostName}" '' + importas -i HOME HOME + importas -SsD ${lib.getExe' pkgs.passt "passt"} PASST + importas -SsD ${lib.getExe package} CH + importas -SsD "${lib.getExe' package "ch-remote"} --api-socket=${vmmSock}" CHR + foreground { mkdir -p "${uvmPrefix}" "${uvmPrefix}" } + cd "${uvmPrefix}" + background { + s6-ipcserver-socketbinder -B ${vmmSock} + trap { default { foreground { s6-ipcclient -q ${uvmPrefix}/cleanup.sock true } exit } } + fdmove -c 3 0 + redirfd -r 0 /dev/null + exec -a "uuvm/${hostName} cloud-hypervisor" $CH --api-socket fd=3 + } + importas CH_PID ! + background { + s6-ipcserver-socketbinder -B ${pastaSock} + s6-ipcserverd + trap { default { foreground { s6-ipcclient -q ${uvmPrefix}/cleanup.sock true } exit } } + exec -a "uuvm/${hostName} passt" $PASST --vhost-user -fF 0 + } + importas PASST_PID ! + background { + getpid -E CLEANUP_PID + s6-ipcserver-socketbinder -B "${uvmPrefix}/cleanup.sock" + s6-ipcserverd + foreground { kill $CH_PID $PASST_PID } + foreground { + rm -f ${vmmSock} ${uvmPrefix}/vsock.sock + } + kill $CLEANUP_PID + } + trap { default { foreground { s6-ipcclient -q ${uvmPrefix}/cleanup.sock true } exit } } + if { $CHR create ${chSettingsFile} } + # if { $CHR add-net "vhost_user=on,socket=${pastaSock}" } + if { ${lib.getExe uvmsPkgs.taps} pass $CHR add-net "id=wan,fd=3,mac=00:00:00:00:00:01" } + if { $CHR boot } + if { $CHR info } ''; } (lib.mkIf cfg.enable { @@ -103,12 +334,6 @@ in } ) layers ); - uvms.cloud-hypervisor.argv = [ - "--memory=size=1536M,hotplug_size=1536M,hotplugged_size=512M,hotplug_method=virtio-mem,mergeable=on,shared=on" - "--cpus=boot=4" - "--disk" - ] - ++ map (img: "path=${img},readonly=true,id=${toString img.label}") layers; }) ]; } From ee497cb7d6f6604dc13ad648ade0b3fe23c46bae Mon Sep 17 00:00:00 2001 From: Else Someone Date: Thu, 12 Feb 2026 22:22:06 +0200 Subject: [PATCH 2/6] ch-runner: drop passt from the PoC --- profiles/ch-runner.nix | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/profiles/ch-runner.nix b/profiles/ch-runner.nix index f680354..44f8205 100644 --- a/profiles/ch-runner.nix +++ b/profiles/ch-runner.nix @@ -28,8 +28,6 @@ let uvmPrefix = "\${HOME}/uvms/${hostName}"; vmmSock = "${uvmPrefix}/vmm.sock"; - pastaSock = "${uvmPrefix}/pasta.sock"; - pastaPidPath = "${uvmPrefix}/pasta.pid"; elbPrefix = "${lib.getBin pkgs.execline}/bin"; s6Prefix = "${lib.getBin pkgs.s6}/bin"; writeElb = name: text: writeElb' name "-W" text; @@ -280,13 +278,6 @@ in exec -a "uuvm/${hostName} cloud-hypervisor" $CH --api-socket fd=3 } importas CH_PID ! - background { - s6-ipcserver-socketbinder -B ${pastaSock} - s6-ipcserverd - trap { default { foreground { s6-ipcclient -q ${uvmPrefix}/cleanup.sock true } exit } } - exec -a "uuvm/${hostName} passt" $PASST --vhost-user -fF 0 - } - importas PASST_PID ! background { getpid -E CLEANUP_PID s6-ipcserver-socketbinder -B "${uvmPrefix}/cleanup.sock" @@ -299,7 +290,6 @@ in } trap { default { foreground { s6-ipcclient -q ${uvmPrefix}/cleanup.sock true } exit } } if { $CHR create ${chSettingsFile} } - # if { $CHR add-net "vhost_user=on,socket=${pastaSock}" } if { ${lib.getExe uvmsPkgs.taps} pass $CHR add-net "id=wan,fd=3,mac=00:00:00:00:00:01" } if { $CHR boot } if { $CHR info } From 0617d97ebf40ae188de20e6edbb0a354d020e3ba Mon Sep 17 00:00:00 2001 From: Else Someone Date: Wed, 18 Feb 2026 16:27:13 +0200 Subject: [PATCH 3/6] ch-runner: just move the cleanup bits out of elb Execlineb is insane. Skarnet is insane. POSIX is cancer, but elb seems to make it worse. Or it s a skill issue. I see no way to do any kind of error handling with elb, and that is even despite me only needing one kind of error handling: the cleaning up... --- profiles/ch-runner.nix | 251 +++++++++++++++++++++++++++++++++++------ 1 file changed, 215 insertions(+), 36 deletions(-) diff --git a/profiles/ch-runner.nix b/profiles/ch-runner.nix index 44f8205..8b69117 100644 --- a/profiles/ch-runner.nix +++ b/profiles/ch-runner.nix @@ -13,7 +13,14 @@ let inherit (config.networking) hostName; inherit (config.debug.closure.erofs) layers; - inherit (lib) mkOption types; + inherit (lib) + mkOption + types + concatMapStringsSep + getExe + getExe' + getBin + ; package = pkgs.cloud-hypervisor.overrideAttrs (oldAttrs: { patches = oldAttrs.patches or [ ] ++ [ @@ -38,7 +45,7 @@ let destination = "/bin/${name}"; executable = true; text = '' - #!${lib.getExe' pkgs.execline "execlineb"}${lib.optionalString (elArgs != null) " "}${elArgs} + #!${getExe' pkgs.execline "execlineb"}${lib.optionalString (elArgs != null) " "}${elArgs} importas OLDPATH PATH export PATH "${elbPrefix}:${s6Prefix}:''${OLDPATH}" ${text} @@ -194,7 +201,7 @@ in if { cp -r --no-preserve=all $GDB_SCRIPT_DIR gdb_scripts } mv gdb_scripts/linux/constants.py.in gdb_scripts/linux/constants.py } - ${lib.getExe pkgs.gdb} + ${getExe pkgs.gdb} -ex "python import sys; sys.path.insert(0, \"''${GDB_SCRIPT_DIR}\")" -ex "target remote :1234" -ex "source ''${GDB_SCRIPT_DIR}/vmlinux-gdb.py" @@ -245,7 +252,7 @@ in environment.TAPS_SOCK = "/run/taps/taps.sock"; serviceConfig = { UMask = "0007"; - ExecStart = "${lib.getExe uvmsPkgs.taps} serve"; + ExecStart = "${getExe uvmsPkgs.taps} serve"; RuntimeDirectory = "taps"; DynamicUser = true; AmbientCapabilities = [ @@ -257,43 +264,215 @@ in }; }; testScript = '' - machine.succeed("${lib.getExe cfg.runner}") + machine.succeed("${getExe cfg.runner}") ''; } ); # NOTE: Used to be an even uglier bash script, but, for now, execline makes for easier comparisons against spectrum - uvms.cloud-hypervisor.runner = writeElb "run-${hostName}" '' - importas -i HOME HOME - importas -SsD ${lib.getExe' pkgs.passt "passt"} PASST - importas -SsD ${lib.getExe package} CH - importas -SsD "${lib.getExe' package "ch-remote"} --api-socket=${vmmSock}" CHR - foreground { mkdir -p "${uvmPrefix}" "${uvmPrefix}" } - cd "${uvmPrefix}" - background { - s6-ipcserver-socketbinder -B ${vmmSock} - trap { default { foreground { s6-ipcclient -q ${uvmPrefix}/cleanup.sock true } exit } } - fdmove -c 3 0 - redirfd -r 0 /dev/null - exec -a "uuvm/${hostName} cloud-hypervisor" $CH --api-socket fd=3 - } - importas CH_PID ! - background { - getpid -E CLEANUP_PID - s6-ipcserver-socketbinder -B "${uvmPrefix}/cleanup.sock" - s6-ipcserverd - foreground { kill $CH_PID $PASST_PID } - foreground { - rm -f ${vmmSock} ${uvmPrefix}/vsock.sock - } - kill $CLEANUP_PID - } - trap { default { foreground { s6-ipcclient -q ${uvmPrefix}/cleanup.sock true } exit } } - if { $CHR create ${chSettingsFile} } - if { ${lib.getExe uvmsPkgs.taps} pass $CHR add-net "id=wan,fd=3,mac=00:00:00:00:00:01" } - if { $CHR boot } - if { $CHR info } - ''; + uvms.cloud-hypervisor.runner = + let + addProcess = getExe addProcess'; + addProcess' = pkgs.writers.writePython3Bin "add-process" { } '' + import os + import select + import socket + import subprocess + import sys + from argparse import ArgumentParser + from contextlib import contextmanager, ExitStack + from threading import Thread, Semaphore + + + parser = ArgumentParser() + parser.add_argument("events_path") + parser.add_argument("--then", action="append") + + MSG_SIZE = 16 + SHMEM = {} + + + def send(sock, msg): + assert len(msg) <= MSG_SIZE, len(msg) + return sock.send(msg.ljust(MSG_SIZE)) + + + def recv(sock): + msg = sock.recv(MSG_SIZE) + # assert len(msg) <= MSG_SIZE, len(msg) + assert len(msg) <= MSG_SIZE, len(msg) + return (msg.split() + [b""])[0] + + + def serve_impl(events_path, listener): + SHMEM["server"] = True + + cons = [] + state = "up" + while state == "up" or cons != []: + if state == "up": + rs, ws, es = select.select([listener, *cons], [], []) + else: + rs, ws, es = select.select(cons, cons, []) + events = [] + for r in rs: + if r is listener: + r, _ = r.accept() + cons.append(r) + else: + events.append(recv(r)) + if any(e == b"killall" for e in events): + state = "down" + if state == "down": + for w in ws: + with s_lock: + send(w, b"die") + w.close() + cons.remove(w) + for w in es: + w.close() + cons.remove(w) + + + def serve(events_path): + base_dir = os.path.dirname(events_path) + if base_dir: + os.makedirs(base_dir, exist_ok=True) + listener = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM, 0) + listener.setblocking(False) + + try: + listener.bind(events_path) + listener.listen() + return serve_impl(events_path, listener) + except OSError as e: + EADDRINUSE = 98 + if e.errno != EADDRINUSE: + raise + finally: + listener.close() + os.remove(events_path) + + + def register(events_path): + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM, 0) + sock.connect(events_path) + return sock + + + @contextmanager + def defer(f): + try: + yield + finally: + f() + + + if __name__ == "__main__": + args, args_next = parser.parse_known_args() + + with ExitStack() as cleanup: + if args_next: + p = subprocess.Popen( + args_next, + shell=False) + then_cmds = reversed(getattr(args, "then") or []) + if not args_next: + then_cmds = [] + try: + p.wait(0.5) + then_cmds = [] + except subprocess.TimeoutExpired: + pass + for f in then_cmds: + def run_f(): + subprocess.run(f) + cleanup.enter_context(defer(run_f)) + + maybe_server = Thread( + target=serve, + args=(args.events_path,), + daemon=True) + maybe_server.start() + maybe_server.join(0.5) + + assert ( + ("server" in SHMEM) == bool(maybe_server.is_alive) + ), (SHMEM, maybe_server) + + if args_next: + s = register(args.events_path) + s_lock = Semaphore() + + if args_next: + def watch_p(p, s): + p.wait() + with s_lock: + try: + send(s, b"killall") + except BrokenPipeError: + pass + + def watch_s(p, s): + while True: + if recv(s) == b"die": + p.terminate() + break + + s_watcher = Thread( + target=watch_s, + args=(p, s), + daemon=True) + s_watcher.start() + watch_p(p, s) + s_watcher.join() + s.close() + + if SHMEM.get("server", False): + maybe_server.join() + + exit_code = 0 + if args_next: + exit_code |= p.returncode + sys.exit(exit_code) + ''; + ch = getExe package; + chr = getExe' package "ch-remote"; + in + writeElb "run-${hostName}" '' + importas -i HOME HOME + importas -SsD "${chr} --api-socket=${vmmSock}" CHR + importas -SsD "${uvmPrefix}" PREFIX + define EVENTS ''${PREFIX}/events.sock + define -s ADD_PROC "${addProcess} ''${EVENTS}" + + cd $PREFIX + background { + $ADD_PROC --then ${getExe ( + writeElb "rm-vmmsock" '' + importas -i HOME HOME + rm -f ${vmmSock} + rm -f ${uvmPrefix}/vsock.sock + '' + )} ${getExe ( + writeElb "ch" '' + importas -Si 1 + importas -Si 2 + s6-ipcserver-socketbinder -B $1 + exec -a "uuvm/''${2} cloud-hypervisor" ${ch} --api-socket fd=0 + '' + )} ${vmmSock} ${hostName} + } + foreground { sleep 0.1 } + ifelse -n { test -S ${vmmSock} } { echo "Apparently ${vmmSock} does not exist" } + foreground { echo "Loading the configuration" } + if { $CHR create ${chSettingsFile} } + foreground { echo "Adding TAP" } + if { ${lib.getExe uvmsPkgs.taps} pass $CHR add-net "id=wan,fd=3,mac=00:00:00:00:00:01" } + foreground { echo "Booting" } + if { $CHR boot } + if { $CHR info } + ''; } (lib.mkIf cfg.enable { boot.initrd.availableKernelModules = [ From 97f2ba4c66f36a6c929250429cb6a6ccebcb3f7f Mon Sep 17 00:00:00 2001 From: Else Someone Date: Thu, 19 Feb 2026 19:24:44 +0200 Subject: [PATCH 4/6] ch-runner: move more logic out of elb --- profiles/ch-runner.nix | 302 ++++++++++++++++++----------------------- 1 file changed, 131 insertions(+), 171 deletions(-) diff --git a/profiles/ch-runner.nix b/profiles/ch-runner.nix index 8b69117..dee946b 100644 --- a/profiles/ch-runner.nix +++ b/profiles/ch-runner.nix @@ -272,92 +272,85 @@ in # NOTE: Used to be an even uglier bash script, but, for now, execline makes for easier comparisons against spectrum uvms.cloud-hypervisor.runner = let - addProcess = getExe addProcess'; - addProcess' = pkgs.writers.writePython3Bin "add-process" { } '' + superviseVm = getExe superviseVm'; + superviseVm' = pkgs.writers.writePython3Bin "supervise-vm" { } '' import os - import select - import socket import subprocess - import sys from argparse import ArgumentParser from contextlib import contextmanager, ExitStack - from threading import Thread, Semaphore - parser = ArgumentParser() - parser.add_argument("events_path") - parser.add_argument("--then", action="append") + parser = ArgumentParser("supervise-vm") + parser.add_argument("--vm") + parser.add_argument("--prefix", default="$HOME/uvms/$VM") + parser.add_argument("--sock", default="$PREFIX/supervisor.sock") + parser.add_argument("--vm-config") MSG_SIZE = 16 - SHMEM = {} + ELB_DIR = "${lib.getBin pkgs.execline}/bin" # noqa: E501 + S6_DIR = "${lib.getBin pkgs.s6}/bin" # noqa: E501 + CH_DIR = "${lib.getBin package}/bin" # noqa: E501 + SOCKETBINDER_PATH = S6_DIR + "/s6-ipcserver-socketbinder" # noqa: E501 + CH_PATH = CH_DIR + "/cloud-hypervisor" + CHR_PATH = CH_DIR + "/ch-remote" + TAPS_PATH = "${lib.getExe uvmsPkgs.taps}" # noqa: E501 + + PASSTHRU_PATH = ":".join([ELB_DIR, S6_DIR, CH_DIR]) + PASSTHRU_ENV = { + **{ + k: v + for k, v in os.environ.items() + if k.startswith("RUST") + or k.startswith("WAYLAND") + or k in [ + "TAPS_SOCK", + ] + }, + "HOME": os.environ.get("HOME", os.getcwd()), + "PATH": PASSTHRU_PATH, + } - def send(sock, msg): - assert len(msg) <= MSG_SIZE, len(msg) - return sock.send(msg.ljust(MSG_SIZE)) + def configure_execline(prefix, vm, check=True, **defaults): + def execline(*args, check=check, **kwargs): + return subprocess.run( + ["execlineb", "-c", "\n".join(args)], + **defaults, + executable=ELB_DIR + "/execlineb", + env={ + **PASSTHRU_ENV, + "PATH": PASSTHRU_PATH, + "PREFIX": prefix, + "VM": vm, + }, + check=check, + cwd=prefix, + **kwargs) + return execline - def recv(sock): - msg = sock.recv(MSG_SIZE) - # assert len(msg) <= MSG_SIZE, len(msg) - assert len(msg) <= MSG_SIZE, len(msg) - return (msg.split() + [b""])[0] - - - def serve_impl(events_path, listener): - SHMEM["server"] = True - - cons = [] - state = "up" - while state == "up" or cons != []: - if state == "up": - rs, ws, es = select.select([listener, *cons], [], []) - else: - rs, ws, es = select.select(cons, cons, []) - events = [] - for r in rs: - if r is listener: - r, _ = r.accept() - cons.append(r) - else: - events.append(recv(r)) - if any(e == b"killall" for e in events): - state = "down" - if state == "down": - for w in ws: - with s_lock: - send(w, b"die") - w.close() - cons.remove(w) - for w in es: - w.close() - cons.remove(w) - - - def serve(events_path): - base_dir = os.path.dirname(events_path) - if base_dir: - os.makedirs(base_dir, exist_ok=True) - listener = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM, 0) - listener.setblocking(False) - - try: - listener.bind(events_path) - listener.listen() - return serve_impl(events_path, listener) - except OSError as e: - EADDRINUSE = 98 - if e.errno != EADDRINUSE: - raise - finally: - listener.close() - os.remove(events_path) - - - def register(events_path): - sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM, 0) - sock.connect(events_path) - return sock + def preprocess_args(args_mut): + keys = [ + k + for k, v + in args_mut._get_kwargs() + if isinstance(v, str)] + for k in keys: + v = getattr(args_mut, k) + if "$HOME" in v: + setattr( + args_mut, + k, + v.replace("$HOME", PASSTHRU_ENV["HOME"])) + for k in keys: + v = getattr(args_mut, k) + if "$VM" in v: + setattr(args_mut, k, v.replace("$VM", args.vm)) + for k in keys: + v = getattr(args_mut, k) + if "$PREFIX" in v: + setattr(args_mut, k, v.replace("$PREFIX", args.prefix)) + return args_mut @contextmanager @@ -368,110 +361,77 @@ in f() + @contextmanager + def run_ch(vm_prefix): + args = [ + SOCKETBINDER_PATH, + "-B", + vm_prefix + "/vmm.sock", + CH_PATH, + "--api-socket", + "fd=0", + ] + p = subprocess.Popen( + args, + shell=False) + try: + p.wait(1.0) + needs_cleanup = False + except subprocess.TimeoutExpired: + needs_cleanup = True + if not os.path.exists(vm_prefix + "/vmm.sock"): + raise RuntimeError(f"{vm_prefix}/vmm.sock should exist by now") + if p.returncode is not None: + raise RuntimeError("CH exited early") + try: + yield p + finally: + try: + p.poll() + except: # noqa: E722 + pass + if p.returncode is None: + p.terminate() # CH handles SIG{INT,TERM}? + p.wait() + unlink_paths = [ + vm_prefix + "/vmm.sock", + vm_prefix + "/vmm.sock.lock", + vm_prefix + "/vsock.sock", + ] if needs_cleanup else [] + for p in unlink_paths: + if os.path.exists(p): + os.remove(p) + + if __name__ == "__main__": args, args_next = parser.parse_known_args() + preprocess_args(args) + + os.makedirs(args.prefix, exist_ok=True) + execline = configure_execline( + prefix=args.prefix, + vm=args.vm) + + ch_remote = [ + "ch-remote", + "--api-socket", + args.prefix + "/vmm.sock", + ] with ExitStack() as cleanup: - if args_next: - p = subprocess.Popen( - args_next, - shell=False) - then_cmds = reversed(getattr(args, "then") or []) - if not args_next: - then_cmds = [] - try: - p.wait(0.5) - then_cmds = [] - except subprocess.TimeoutExpired: - pass - for f in then_cmds: - def run_f(): - subprocess.run(f) - cleanup.enter_context(defer(run_f)) - - maybe_server = Thread( - target=serve, - args=(args.events_path,), - daemon=True) - maybe_server.start() - maybe_server.join(0.5) - - assert ( - ("server" in SHMEM) == bool(maybe_server.is_alive) - ), (SHMEM, maybe_server) - - if args_next: - s = register(args.events_path) - s_lock = Semaphore() - - if args_next: - def watch_p(p, s): - p.wait() - with s_lock: - try: - send(s, b"killall") - except BrokenPipeError: - pass - - def watch_s(p, s): - while True: - if recv(s) == b"die": - p.terminate() - break - - s_watcher = Thread( - target=watch_s, - args=(p, s), - daemon=True) - s_watcher.start() - watch_p(p, s) - s_watcher.join() - s.close() - - if SHMEM.get("server", False): - maybe_server.join() - - exit_code = 0 - if args_next: - exit_code |= p.returncode - sys.exit(exit_code) + ch = cleanup.enter_context(run_ch(args.prefix)) + execline(*ch_remote, "create", args.vm_config) + execline( + TAPS_PATH, "pass", + *ch_remote, "add-net", + "id=wan,fd=3,mac=00:00:00:00:00:01") + execline(*ch_remote, "boot") + execline(*ch_remote, "info") + ch.wait() ''; - ch = getExe package; - chr = getExe' package "ch-remote"; in writeElb "run-${hostName}" '' - importas -i HOME HOME - importas -SsD "${chr} --api-socket=${vmmSock}" CHR - importas -SsD "${uvmPrefix}" PREFIX - define EVENTS ''${PREFIX}/events.sock - define -s ADD_PROC "${addProcess} ''${EVENTS}" - - cd $PREFIX - background { - $ADD_PROC --then ${getExe ( - writeElb "rm-vmmsock" '' - importas -i HOME HOME - rm -f ${vmmSock} - rm -f ${uvmPrefix}/vsock.sock - '' - )} ${getExe ( - writeElb "ch" '' - importas -Si 1 - importas -Si 2 - s6-ipcserver-socketbinder -B $1 - exec -a "uuvm/''${2} cloud-hypervisor" ${ch} --api-socket fd=0 - '' - )} ${vmmSock} ${hostName} - } - foreground { sleep 0.1 } - ifelse -n { test -S ${vmmSock} } { echo "Apparently ${vmmSock} does not exist" } - foreground { echo "Loading the configuration" } - if { $CHR create ${chSettingsFile} } - foreground { echo "Adding TAP" } - if { ${lib.getExe uvmsPkgs.taps} pass $CHR add-net "id=wan,fd=3,mac=00:00:00:00:00:01" } - foreground { echo "Booting" } - if { $CHR boot } - if { $CHR info } + ${superviseVm} --vm-config=${chSettingsFile} --vm=${hostName} ''; } (lib.mkIf cfg.enable { From 342a1dfe3a788ad524692f95be140aef9b0f7ff5 Mon Sep 17 00:00:00 2001 From: Else Someone Date: Fri, 20 Feb 2026 10:50:31 +0200 Subject: [PATCH 5/6] ch-runner: another iter of cleanup prior to moving things around --- profiles/ch-runner.nix | 42 +++++++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/profiles/ch-runner.nix b/profiles/ch-runner.nix index dee946b..b3d0385 100644 --- a/profiles/ch-runner.nix +++ b/profiles/ch-runner.nix @@ -311,10 +311,25 @@ in } - def configure_execline(prefix, vm, check=True, **defaults): - def execline(*args, check=check, **kwargs): + def configure_exec(prefix, vm, check=True, **defaults): + + def exec(*args, check=check, **kwargs): return subprocess.run( - ["execlineb", "-c", "\n".join(args)], + [*args], + **defaults, + env={ + **PASSTHRU_ENV, + "PATH": PASSTHRU_PATH, + "PREFIX": prefix, + "VM": vm, + }, + check=check, + cwd=prefix, + **kwargs) + + def execline(*args, check=check, **kwargs): + return exec( + "execlineb", "-c", "\n".join(args), **defaults, executable=ELB_DIR + "/execlineb", env={ @@ -326,7 +341,8 @@ in check=check, cwd=prefix, **kwargs) - return execline + + return exec, execline def preprocess_args(args_mut): @@ -373,7 +389,8 @@ in ] p = subprocess.Popen( args, - shell=False) + shell=False, + pass_fds=(2,)) try: p.wait(1.0) needs_cleanup = False @@ -408,7 +425,7 @@ in preprocess_args(args) os.makedirs(args.prefix, exist_ok=True) - execline = configure_execline( + exec, _ = configure_exec( prefix=args.prefix, vm=args.vm) @@ -420,14 +437,17 @@ in with ExitStack() as cleanup: ch = cleanup.enter_context(run_ch(args.prefix)) - execline(*ch_remote, "create", args.vm_config) - execline( + exec(*ch_remote, "create", args.vm_config) + exec( TAPS_PATH, "pass", *ch_remote, "add-net", "id=wan,fd=3,mac=00:00:00:00:00:01") - execline(*ch_remote, "boot") - execline(*ch_remote, "info") - ch.wait() + exec(*ch_remote, "boot") + exec(*ch_remote, "info") + try: + ch.wait() + except KeyboardInterrupt: + pass ''; in writeElb "run-${hostName}" '' From 7516eae7a6a83b1253441b56ceb3370248ac7fdd Mon Sep 17 00:00:00 2001 From: Else Someone Date: Sat, 21 Feb 2026 18:11:37 +0200 Subject: [PATCH 6/6] ch-runner: add minimal virtiofsd support Somehow it doesnt work if I open the socket in python and try to inherit it. Also didnt work with bwrap. And theres a bunch of warnings that virtiofsd keeps printing. Need investigation --- profiles/ch-runner.nix | 388 ++++++++++++++++++++++++++++++++--------- 1 file changed, 302 insertions(+), 86 deletions(-) diff --git a/profiles/ch-runner.nix b/profiles/ch-runner.nix index b3d0385..f156705 100644 --- a/profiles/ch-runner.nix +++ b/profiles/ch-runner.nix @@ -272,12 +272,22 @@ in # NOTE: Used to be an even uglier bash script, but, for now, execline makes for easier comparisons against spectrum uvms.cloud-hypervisor.runner = let + toolsClosure = pkgs.writeClosure [ + (lib.getBin pkgs.execline) + (lib.getBin pkgs.s6) + (lib.getBin package) + (lib.getBin pkgs.virtiofsd) + (lib.getBin pkgs.bubblewrap) + uvmsPkgs.taps + ]; + superviseVm = getExe superviseVm'; superviseVm' = pkgs.writers.writePython3Bin "supervise-vm" { } '' import os import subprocess + import socket from argparse import ArgumentParser - from contextlib import contextmanager, ExitStack + from contextlib import contextmanager, closing, ExitStack parser = ArgumentParser("supervise-vm") @@ -290,12 +300,21 @@ in ELB_DIR = "${lib.getBin pkgs.execline}/bin" # noqa: E501 S6_DIR = "${lib.getBin pkgs.s6}/bin" # noqa: E501 CH_DIR = "${lib.getBin package}/bin" # noqa: E501 + UTIL_LINUX_DIR = "${lib.getBin pkgs.util-linux}/bin" # noqa: E501 SOCKETBINDER_PATH = S6_DIR + "/s6-ipcserver-socketbinder" # noqa: E501 CH_PATH = CH_DIR + "/cloud-hypervisor" CHR_PATH = CH_DIR + "/ch-remote" TAPS_PATH = "${lib.getExe uvmsPkgs.taps}" # noqa: E501 + VIRTIOFSD_PATH = "${lib.getExe pkgs.virtiofsd}" # noqa: E501 + BWRAP_PATH = "${lib.getExe pkgs.bubblewrap}" # noqa: E501 - PASSTHRU_PATH = ":".join([ELB_DIR, S6_DIR, CH_DIR]) + with open("${toolsClosure}", mode="r") as f: # noqa: E501 + CLOSURE = [ + *(ln.rstrip() for ln in f.readlines()), + "${placeholder "out"}", # noqa: E501 + ] + + PASSTHRU_PATH = ":".join([ELB_DIR, S6_DIR, CH_DIR, UTIL_LINUX_DIR]) PASSTHRU_ENV = { **{ k: v @@ -311,40 +330,6 @@ in } - def configure_exec(prefix, vm, check=True, **defaults): - - def exec(*args, check=check, **kwargs): - return subprocess.run( - [*args], - **defaults, - env={ - **PASSTHRU_ENV, - "PATH": PASSTHRU_PATH, - "PREFIX": prefix, - "VM": vm, - }, - check=check, - cwd=prefix, - **kwargs) - - def execline(*args, check=check, **kwargs): - return exec( - "execlineb", "-c", "\n".join(args), - **defaults, - executable=ELB_DIR + "/execlineb", - env={ - **PASSTHRU_ENV, - "PATH": PASSTHRU_PATH, - "PREFIX": prefix, - "VM": vm, - }, - check=check, - cwd=prefix, - **kwargs) - - return exec, execline - - def preprocess_args(args_mut): keys = [ k @@ -369,6 +354,270 @@ in return args_mut + class Processes: + def __init__(self, prefix, vm, check=True, **defaults): + self.prefix = prefix + self.vm = vm + self.check = check + self.defaults = defaults + + def make_env(self): + return { + **PASSTHRU_ENV, + "PATH": PASSTHRU_PATH, + "PREFIX": self.prefix, + "VM": self.vm, + } + + def exec(self, *args, **kwargs): + kwargs["cwd"] = kwargs.get("cwd", self.prefix) + kwargs["check"] = kwargs.get("check", self.check) + kwargs["env"] = kwargs.get("env", self.make_env()) + return subprocess.run( + [*args], + **self.defaults, + **kwargs) + + def execline(self, *args, **kwargs): + return exec( + "execlineb", "-c", "\n".join(args), + **self.defaults, + executable=ELB_DIR + "/execlineb", + **{ + "env": self.make_env(), + "check": self.check, + "cwd": self.prefix, + **kwargs, + }, + ) + + def popen(self, *args, **kwargs): + kwargs["pass_fds"] = kwargs.get("pass_fds", ()) + kwargs["env"] = kwargs.get("env", self.make_env()) + kwargs["cwd"] = kwargs.get("cwd", self.prefix) + return subprocess.Popen( + args, + **kwargs, + ) + + @contextmanager + def bwrap( + self, + *bwrap_args, + + die_with_parent=True, + + # Based on the args from + # `host/rootfs/image/usr/bin/run-vmm` + unshare_all=True, + unshare_user=True, + unshare_ipc=None, + unshare_pid=None, + unshare_net=None, + unshare_uts=None, + unshare_cgroup_try=True, + bind=(), + dev_bind=("/dev/kvm", "/dev/vfio"), + dev="/dev", + proc="/proc", + ro_bind=( + "/etc", + "/sys", + "/proc/sys", + "/dev/null", + "/proc/kallsyms", + *CLOSURE), + ro_bind_extra=(), + remount_ro=("/proc/fs", "/proc/irq"), + tmpfs=("/dev/shm", "/tmp", "/var/tmp", "/proc/fs", "/proc/irq"), + tmpfs_extra=(), + + pass_fds=(2,), + **popen_kwargs): + + bwrap_args_sock, remote = socket.socketpair() + remote.set_inheritable(True) + bwrap_args_f = bwrap_args_sock.makefile("w") + with closing(bwrap_args_sock), closing(bwrap_args_f): + def print_arg(*args): + print(*args, file=bwrap_args_f, sep="\0", end="\0") + + if unshare_all: + print_arg("--unshare-all") + if unshare_user: + print_arg("--unshare-user") + if unshare_ipc: + print_arg("--unshare-ipc") + if unshare_pid: + print_arg("--unshare-pid") + if unshare_net: + print_arg("--unshare-net") + if unshare_uts: + print_arg("--unshare-uts") + if unshare_cgroup_try: + print_arg("--unshare-cgroup-try") + if die_with_parent: + print_arg("--die-with-parent") + + for p in bind: + p1, p2 = (p, p) if isinstance(p, str) else p + print_arg("--bind", p1, p2) + for p in (*ro_bind, *ro_bind_extra): + p1, p2 = (p, p) if isinstance(p, str) else p + print_arg("--ro-bind", p1, p2) + for p in dev_bind: + p1, p2 = (p, p) if isinstance(p, str) else p + print_arg("--dev-bind", p1, p2) + for p in (*tmpfs, *tmpfs_extra): + print_arg("--tmpfs", p) + # Hunch: order might matter... + for p in remount_ro: + print_arg("--remount-ro", p) + + bwrap_args_f.flush() + + with closing(remote): + proc = self.popen( + "bwrap", "--args", str(remote.fileno()), *bwrap_args, + **popen_kwargs, + executable=BWRAP_PATH, + pass_fds=(*pass_fds, remote.fileno()), + ) + + with proc as p: + try: + yield p + finally: + try: + p.poll() + except: # noqa: E722 + pass + if p.returncode is None: + p.terminate() + p.wait() + + @contextmanager + def run_ch(self): + args = [ + SOCKETBINDER_PATH, + "-B", + self.prefix + "/vmm.sock", + CH_PATH, + "--api-socket", + "fd=0", + ] + p = self.popen( + *args, + shell=False, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + pass_fds=(2,)) + try: + p.wait(0.125) + needs_cleanup = False + except subprocess.TimeoutExpired: + needs_cleanup = True + if not os.path.exists(self.prefix + "/vmm.sock"): + raise RuntimeError(f"{self.prefix}/vmm.sock should exist by now") + if p.returncode is not None: + raise RuntimeError("CH exited early") + try: + yield p + finally: + try: + p.poll() + except: # noqa: E722 + pass + if p.returncode is None: + p.terminate() # CH handles SIG{INT,TERM}? + p.wait() + unlink_paths = [ + self.prefix + "/vmm.sock", + self.prefix + "/vmm.sock.lock", + self.prefix + "/vsock.sock", + ] if needs_cleanup else [] + for p in unlink_paths: + if os.path.exists(p): + os.remove(p) + + @contextmanager + def add_virtiofsd( + self, + root_dir, + tag, + ro=False, + subdirs=None, + extra_flags=("--posix-acl",)): + + assert os.path.exists(root_dir) + + sock_path = self.prefix + f"/virtiofsd-{tag}.sock" + # s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + # NOTE: Nope. Virtiofsd actually expects a blocking socket + # s.setblocking(True) + + def rm_sock(): + if os.path.exists(sock_path): + os.remove(sock_path) + + with ExitStack() as cleanup: # noqa: F841 + # s.bind(sock_path.encode("utf8")) + # cleanup.enter_context(closing(s)) + cleanup.enter_context(defer(rm_sock)) + + args = [ + # If using bwrap(): + # "--argv0", "virtiofsd", + # "--uid", "1000", + # "--gid", "1000", + # "--", + "unshare", "-rUm", + "unshare", "--map-user", "1000", "--map-group", "1000", + VIRTIOFSD_PATH, + "--shared-dir", + root_dir, + "--tag", + tag, + + # "--fd", + # str(s.fileno()), + "--socket-path", + sock_path, + + # If relying on bwrap(): + # "--sandbox", + # "none", + ] + if ro: + args.append("--readonly") + kwargs = { + # If bwrap(): + # "bind": [], + # ("ro_bind_extra" if ro else "bind"): + # [*subdirs] + # if subdirs is not None + # else [root_dir], + + # "pass_fds": (2, s.fileno()), + } + proc_ctx = self.popen(*args, **kwargs) + with proc_ctx as p: + try: + try: + p.wait(0.125) + except subprocess.TimeoutExpired: + pass + if p.returncode is not None: + raise RuntimeError("virtiofsd exited too early") + yield p, sock_path + finally: + if p.returncode is None: + p.kill() + p.wait() + if os.path.exists(sock_path): + os.remove(sock_path) + + @contextmanager def defer(f): try: @@ -377,57 +626,15 @@ in f() - @contextmanager - def run_ch(vm_prefix): - args = [ - SOCKETBINDER_PATH, - "-B", - vm_prefix + "/vmm.sock", - CH_PATH, - "--api-socket", - "fd=0", - ] - p = subprocess.Popen( - args, - shell=False, - pass_fds=(2,)) - try: - p.wait(1.0) - needs_cleanup = False - except subprocess.TimeoutExpired: - needs_cleanup = True - if not os.path.exists(vm_prefix + "/vmm.sock"): - raise RuntimeError(f"{vm_prefix}/vmm.sock should exist by now") - if p.returncode is not None: - raise RuntimeError("CH exited early") - try: - yield p - finally: - try: - p.poll() - except: # noqa: E722 - pass - if p.returncode is None: - p.terminate() # CH handles SIG{INT,TERM}? - p.wait() - unlink_paths = [ - vm_prefix + "/vmm.sock", - vm_prefix + "/vmm.sock.lock", - vm_prefix + "/vsock.sock", - ] if needs_cleanup else [] - for p in unlink_paths: - if os.path.exists(p): - os.remove(p) - - if __name__ == "__main__": args, args_next = parser.parse_known_args() preprocess_args(args) os.makedirs(args.prefix, exist_ok=True) - exec, _ = configure_exec( + ps = Processes( prefix=args.prefix, - vm=args.vm) + vm=args.vm, + ) ch_remote = [ "ch-remote", @@ -436,14 +643,23 @@ in ] with ExitStack() as cleanup: - ch = cleanup.enter_context(run_ch(args.prefix)) - exec(*ch_remote, "create", args.vm_config) - exec( + ch = cleanup.enter_context(ps.run_ch()) + ps.exec(*ch_remote, "create", args.vm_config) + ps.exec( TAPS_PATH, "pass", *ch_remote, "add-net", "id=wan,fd=3,mac=00:00:00:00:00:01") - exec(*ch_remote, "boot") - exec(*ch_remote, "info") + + send_dir = PASSTHRU_ENV["HOME"] + f"/send/{args.vm}" + os.makedirs(send_dir, exist_ok=True) + vfsd, vfsd_path = cleanup.enter_context( + ps.add_virtiofsd( + send_dir, + tag="send", + )) + ps.exec(*ch_remote, "add-fs", f"tag=send,socket={vfsd_path},id=send") + ps.exec(*ch_remote, "boot") + ps.exec(*ch_remote, "info") try: ch.wait() except KeyboardInterrupt: