diff --git a/profiles/ch-runner.nix b/profiles/ch-runner.nix index 44f8205..8b69117 100644 --- a/profiles/ch-runner.nix +++ b/profiles/ch-runner.nix @@ -13,7 +13,14 @@ let inherit (config.networking) hostName; inherit (config.debug.closure.erofs) layers; - inherit (lib) mkOption types; + inherit (lib) + mkOption + types + concatMapStringsSep + getExe + getExe' + getBin + ; package = pkgs.cloud-hypervisor.overrideAttrs (oldAttrs: { patches = oldAttrs.patches or [ ] ++ [ @@ -38,7 +45,7 @@ let destination = "/bin/${name}"; executable = true; text = '' - #!${lib.getExe' pkgs.execline "execlineb"}${lib.optionalString (elArgs != null) " "}${elArgs} + #!${getExe' pkgs.execline "execlineb"}${lib.optionalString (elArgs != null) " "}${elArgs} importas OLDPATH PATH export PATH "${elbPrefix}:${s6Prefix}:''${OLDPATH}" ${text} @@ -194,7 +201,7 @@ in if { cp -r --no-preserve=all $GDB_SCRIPT_DIR gdb_scripts } mv gdb_scripts/linux/constants.py.in gdb_scripts/linux/constants.py } - ${lib.getExe pkgs.gdb} + ${getExe pkgs.gdb} -ex "python import sys; sys.path.insert(0, \"''${GDB_SCRIPT_DIR}\")" -ex "target remote :1234" -ex "source ''${GDB_SCRIPT_DIR}/vmlinux-gdb.py" @@ -245,7 +252,7 @@ in environment.TAPS_SOCK = "/run/taps/taps.sock"; serviceConfig = { UMask = "0007"; - ExecStart = "${lib.getExe uvmsPkgs.taps} serve"; + ExecStart = "${getExe uvmsPkgs.taps} serve"; RuntimeDirectory = "taps"; DynamicUser = true; AmbientCapabilities = [ @@ -257,43 +264,215 @@ in }; }; testScript = '' - machine.succeed("${lib.getExe cfg.runner}") + machine.succeed("${getExe cfg.runner}") ''; } ); # NOTE: Used to be an even uglier bash script, but, for now, execline makes for easier comparisons against spectrum - uvms.cloud-hypervisor.runner = writeElb "run-${hostName}" '' - importas -i HOME HOME - importas -SsD ${lib.getExe' pkgs.passt "passt"} PASST - importas -SsD ${lib.getExe package} CH - importas -SsD "${lib.getExe' package "ch-remote"} --api-socket=${vmmSock}" CHR - foreground { mkdir -p "${uvmPrefix}" "${uvmPrefix}" } - cd "${uvmPrefix}" - background { - s6-ipcserver-socketbinder -B ${vmmSock} - trap { default { foreground { s6-ipcclient -q ${uvmPrefix}/cleanup.sock true } exit } } - fdmove -c 3 0 - redirfd -r 0 /dev/null - exec -a "uuvm/${hostName} cloud-hypervisor" $CH --api-socket fd=3 - } - importas CH_PID ! - background { - getpid -E CLEANUP_PID - s6-ipcserver-socketbinder -B "${uvmPrefix}/cleanup.sock" - s6-ipcserverd - foreground { kill $CH_PID $PASST_PID } - foreground { - rm -f ${vmmSock} ${uvmPrefix}/vsock.sock - } - kill $CLEANUP_PID - } - trap { default { foreground { s6-ipcclient -q ${uvmPrefix}/cleanup.sock true } exit } } - if { $CHR create ${chSettingsFile} } - if { ${lib.getExe uvmsPkgs.taps} pass $CHR add-net "id=wan,fd=3,mac=00:00:00:00:00:01" } - if { $CHR boot } - if { $CHR info } - ''; + uvms.cloud-hypervisor.runner = + let + addProcess = getExe addProcess'; + addProcess' = pkgs.writers.writePython3Bin "add-process" { } '' + import os + import select + import socket + import subprocess + import sys + from argparse import ArgumentParser + from contextlib import contextmanager, ExitStack + from threading import Thread, Semaphore + + + parser = ArgumentParser() + parser.add_argument("events_path") + parser.add_argument("--then", action="append") + + MSG_SIZE = 16 + SHMEM = {} + + + def send(sock, msg): + assert len(msg) <= MSG_SIZE, len(msg) + return sock.send(msg.ljust(MSG_SIZE)) + + + def recv(sock): + msg = sock.recv(MSG_SIZE) + # assert len(msg) <= MSG_SIZE, len(msg) + assert len(msg) <= MSG_SIZE, len(msg) + return (msg.split() + [b""])[0] + + + def serve_impl(events_path, listener): + SHMEM["server"] = True + + cons = [] + state = "up" + while state == "up" or cons != []: + if state == "up": + rs, ws, es = select.select([listener, *cons], [], []) + else: + rs, ws, es = select.select(cons, cons, []) + events = [] + for r in rs: + if r is listener: + r, _ = r.accept() + cons.append(r) + else: + events.append(recv(r)) + if any(e == b"killall" for e in events): + state = "down" + if state == "down": + for w in ws: + with s_lock: + send(w, b"die") + w.close() + cons.remove(w) + for w in es: + w.close() + cons.remove(w) + + + def serve(events_path): + base_dir = os.path.dirname(events_path) + if base_dir: + os.makedirs(base_dir, exist_ok=True) + listener = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM, 0) + listener.setblocking(False) + + try: + listener.bind(events_path) + listener.listen() + return serve_impl(events_path, listener) + except OSError as e: + EADDRINUSE = 98 + if e.errno != EADDRINUSE: + raise + finally: + listener.close() + os.remove(events_path) + + + def register(events_path): + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM, 0) + sock.connect(events_path) + return sock + + + @contextmanager + def defer(f): + try: + yield + finally: + f() + + + if __name__ == "__main__": + args, args_next = parser.parse_known_args() + + with ExitStack() as cleanup: + if args_next: + p = subprocess.Popen( + args_next, + shell=False) + then_cmds = reversed(getattr(args, "then") or []) + if not args_next: + then_cmds = [] + try: + p.wait(0.5) + then_cmds = [] + except subprocess.TimeoutExpired: + pass + for f in then_cmds: + def run_f(): + subprocess.run(f) + cleanup.enter_context(defer(run_f)) + + maybe_server = Thread( + target=serve, + args=(args.events_path,), + daemon=True) + maybe_server.start() + maybe_server.join(0.5) + + assert ( + ("server" in SHMEM) == bool(maybe_server.is_alive) + ), (SHMEM, maybe_server) + + if args_next: + s = register(args.events_path) + s_lock = Semaphore() + + if args_next: + def watch_p(p, s): + p.wait() + with s_lock: + try: + send(s, b"killall") + except BrokenPipeError: + pass + + def watch_s(p, s): + while True: + if recv(s) == b"die": + p.terminate() + break + + s_watcher = Thread( + target=watch_s, + args=(p, s), + daemon=True) + s_watcher.start() + watch_p(p, s) + s_watcher.join() + s.close() + + if SHMEM.get("server", False): + maybe_server.join() + + exit_code = 0 + if args_next: + exit_code |= p.returncode + sys.exit(exit_code) + ''; + ch = getExe package; + chr = getExe' package "ch-remote"; + in + writeElb "run-${hostName}" '' + importas -i HOME HOME + importas -SsD "${chr} --api-socket=${vmmSock}" CHR + importas -SsD "${uvmPrefix}" PREFIX + define EVENTS ''${PREFIX}/events.sock + define -s ADD_PROC "${addProcess} ''${EVENTS}" + + cd $PREFIX + background { + $ADD_PROC --then ${getExe ( + writeElb "rm-vmmsock" '' + importas -i HOME HOME + rm -f ${vmmSock} + rm -f ${uvmPrefix}/vsock.sock + '' + )} ${getExe ( + writeElb "ch" '' + importas -Si 1 + importas -Si 2 + s6-ipcserver-socketbinder -B $1 + exec -a "uuvm/''${2} cloud-hypervisor" ${ch} --api-socket fd=0 + '' + )} ${vmmSock} ${hostName} + } + foreground { sleep 0.1 } + ifelse -n { test -S ${vmmSock} } { echo "Apparently ${vmmSock} does not exist" } + foreground { echo "Loading the configuration" } + if { $CHR create ${chSettingsFile} } + foreground { echo "Adding TAP" } + if { ${lib.getExe uvmsPkgs.taps} pass $CHR add-net "id=wan,fd=3,mac=00:00:00:00:00:01" } + foreground { echo "Booting" } + if { $CHR boot } + if { $CHR info } + ''; } (lib.mkIf cfg.enable { boot.initrd.availableKernelModules = [