ch-runner: just move the cleanup bits out of elb

Execlineb is insane. Skarnet is insane. POSIX is cancer,
but elb seems to make it worse. Or it s a skill issue.
I see no way to do any kind of error handling with elb,
and that is even despite me only needing one kind of error handling:
the cleaning up...

profiles/ch-runner.nix

@@ -13,7 +13,14 @@ let
 
   inherit (config.networking) hostName;
   inherit (config.debug.closure.erofs) layers;
-  inherit (lib) mkOption types;
+  inherit (lib)
+    mkOption
+    types
+    concatMapStringsSep
+    getExe
+    getExe'
+    getBin
+    ;
 
   package = pkgs.cloud-hypervisor.overrideAttrs (oldAttrs: {
     patches = oldAttrs.patches or [ ] ++ [
@@ -38,7 +45,7 @@ let
       destination = "/bin/${name}";
       executable = true;
       text = ''
-        #!${lib.getExe' pkgs.execline "execlineb"}${lib.optionalString (elArgs != null) " "}${elArgs}
+        #!${getExe' pkgs.execline "execlineb"}${lib.optionalString (elArgs != null) " "}${elArgs}
         importas OLDPATH PATH
         export PATH "${elbPrefix}:${s6Prefix}:''${OLDPATH}"
         ${text}
@@ -194,7 +201,7 @@ in
                     if { cp -r --no-preserve=all $GDB_SCRIPT_DIR gdb_scripts }
                     mv gdb_scripts/linux/constants.py.in gdb_scripts/linux/constants.py
                 }
-                ${lib.getExe pkgs.gdb}
+                ${getExe pkgs.gdb}
                 -ex "python import sys; sys.path.insert(0, \"''${GDB_SCRIPT_DIR}\")"
                 -ex "target remote :1234"
                 -ex "source ''${GDB_SCRIPT_DIR}/vmlinux-gdb.py"
@@ -245,7 +252,7 @@ in
                 environment.TAPS_SOCK = "/run/taps/taps.sock";
                 serviceConfig = {
                   UMask = "0007";
-                  ExecStart = "${lib.getExe uvmsPkgs.taps} serve";
+                  ExecStart = "${getExe uvmsPkgs.taps} serve";
                   RuntimeDirectory = "taps";
                   DynamicUser = true;
                   AmbientCapabilities = [
@@ -257,43 +264,215 @@ in
               };
             };
           testScript = ''
-            machine.succeed("${lib.getExe cfg.runner}")
+            machine.succeed("${getExe cfg.runner}")
           '';
         }
       );
 
       # NOTE: Used to be an even uglier bash script, but, for now, execline makes for easier comparisons against spectrum
-      uvms.cloud-hypervisor.runner = writeElb "run-${hostName}" ''
+      uvms.cloud-hypervisor.runner =
-        importas -i HOME HOME
+        let
-        importas -SsD ${lib.getExe' pkgs.passt "passt"} PASST
+          addProcess = getExe addProcess';
-        importas -SsD ${lib.getExe package} CH
+          addProcess' = pkgs.writers.writePython3Bin "add-process" { } ''
-        importas -SsD "${lib.getExe' package "ch-remote"} --api-socket=${vmmSock}" CHR
+            import os
-        foreground { mkdir -p "${uvmPrefix}" "${uvmPrefix}" }
+            import select
-        cd "${uvmPrefix}"
+            import socket
-        background {
+            import subprocess
-            s6-ipcserver-socketbinder -B ${vmmSock}
+            import sys
-            trap { default { foreground { s6-ipcclient -q ${uvmPrefix}/cleanup.sock true } exit } }
+            from argparse import ArgumentParser
-            fdmove -c 3 0
+            from contextlib import contextmanager, ExitStack
-            redirfd -r 0 /dev/null
+            from threading import Thread, Semaphore
-            exec -a "uuvm/${hostName} cloud-hypervisor" $CH --api-socket fd=3
+
-        }
+
-        importas CH_PID !
+            parser = ArgumentParser()
-        background {
+            parser.add_argument("events_path")
-            getpid -E CLEANUP_PID
+            parser.add_argument("--then", action="append")
-            s6-ipcserver-socketbinder -B "${uvmPrefix}/cleanup.sock"
+
-            s6-ipcserverd
+            MSG_SIZE = 16
-            foreground { kill $CH_PID $PASST_PID }
+            SHMEM = {}
-            foreground {
+
-                rm -f ${vmmSock} ${uvmPrefix}/vsock.sock
+
-            }
+            def send(sock, msg):
-            kill $CLEANUP_PID
+                assert len(msg) <= MSG_SIZE, len(msg)
-        }
+                return sock.send(msg.ljust(MSG_SIZE))
-        trap { default { foreground { s6-ipcclient -q ${uvmPrefix}/cleanup.sock true } exit } }
+
-        if { $CHR create ${chSettingsFile} }
+
-        if { ${lib.getExe uvmsPkgs.taps} pass $CHR add-net "id=wan,fd=3,mac=00:00:00:00:00:01" }
+            def recv(sock):
-        if { $CHR boot }
+                msg = sock.recv(MSG_SIZE)
-        if { $CHR info }
+                # assert len(msg) <= MSG_SIZE, len(msg)
-      '';
+                assert len(msg) <= MSG_SIZE, len(msg)
+                return (msg.split() + [b""])[0]
+
+
+            def serve_impl(events_path, listener):
+                SHMEM["server"] = True
+
+                cons = []
+                state = "up"
+                while state == "up" or cons != []:
+                    if state == "up":
+                        rs, ws, es = select.select([listener, *cons], [], [])
+                    else:
+                        rs, ws, es = select.select(cons, cons, [])
+                    events = []
+                    for r in rs:
+                        if r is listener:
+                            r, _ = r.accept()
+                            cons.append(r)
+                        else:
+                            events.append(recv(r))
+                    if any(e == b"killall" for e in events):
+                        state = "down"
+                    if state == "down":
+                        for w in ws:
+                            with s_lock:
+                                send(w, b"die")
+                            w.close()
+                            cons.remove(w)
+                    for w in es:
+                        w.close()
+                        cons.remove(w)
+
+
+            def serve(events_path):
+                base_dir = os.path.dirname(events_path)
+                if base_dir:
+                    os.makedirs(base_dir, exist_ok=True)
+                listener = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM, 0)
+                listener.setblocking(False)
+
+                try:
+                    listener.bind(events_path)
+                    listener.listen()
+                    return serve_impl(events_path, listener)
+                except OSError as e:
+                    EADDRINUSE = 98
+                    if e.errno != EADDRINUSE:
+                        raise
+                finally:
+                    listener.close()
+                    os.remove(events_path)
+
+
+            def register(events_path):
+                sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM, 0)
+                sock.connect(events_path)
+                return sock
+
+
+            @contextmanager
+            def defer(f):
+                try:
+                    yield
+                finally:
+                    f()
+
+
+            if __name__ == "__main__":
+                args, args_next = parser.parse_known_args()
+
+                with ExitStack() as cleanup:
+                    if args_next:
+                        p = subprocess.Popen(
+                            args_next,
+                            shell=False)
+                        then_cmds = reversed(getattr(args, "then") or [])
+                        if not args_next:
+                            then_cmds = []
+                        try:
+                            p.wait(0.5)
+                            then_cmds = []
+                        except subprocess.TimeoutExpired:
+                            pass
+                        for f in then_cmds:
+                            def run_f():
+                                subprocess.run(f)
+                            cleanup.enter_context(defer(run_f))
+
+                    maybe_server = Thread(
+                        target=serve,
+                        args=(args.events_path,),
+                        daemon=True)
+                    maybe_server.start()
+                    maybe_server.join(0.5)
+
+                    assert (
+                        ("server" in SHMEM) == bool(maybe_server.is_alive)
+                    ), (SHMEM, maybe_server)
+
+                    if args_next:
+                        s = register(args.events_path)
+                        s_lock = Semaphore()
+
+                    if args_next:
+                        def watch_p(p, s):
+                            p.wait()
+                            with s_lock:
+                                try:
+                                    send(s, b"killall")
+                                except BrokenPipeError:
+                                    pass
+
+                        def watch_s(p, s):
+                            while True:
+                                if recv(s) == b"die":
+                                    p.terminate()
+                                    break
+
+                        s_watcher = Thread(
+                            target=watch_s,
+                            args=(p, s),
+                            daemon=True)
+                        s_watcher.start()
+                        watch_p(p, s)
+                        s_watcher.join()
+                        s.close()
+
+                    if SHMEM.get("server", False):
+                        maybe_server.join()
+
+                    exit_code = 0
+                    if args_next:
+                        exit_code |= p.returncode
+                    sys.exit(exit_code)
+          '';
+          ch = getExe package;
+          chr = getExe' package "ch-remote";
+        in
+        writeElb "run-${hostName}" ''
+          importas -i HOME HOME
+          importas -SsD "${chr} --api-socket=${vmmSock}" CHR
+          importas -SsD "${uvmPrefix}" PREFIX
+          define EVENTS ''${PREFIX}/events.sock
+          define -s ADD_PROC "${addProcess} ''${EVENTS}"
+
+          cd $PREFIX
+          background {
+              $ADD_PROC --then ${getExe (
+                writeElb "rm-vmmsock" ''
+                  importas -i HOME HOME
+                  rm -f ${vmmSock}
+                  rm -f ${uvmPrefix}/vsock.sock
+                ''
+              )} ${getExe (
+                writeElb "ch" ''
+                  importas -Si 1
+                  importas -Si 2
+                  s6-ipcserver-socketbinder -B $1
+                  exec -a "uuvm/''${2} cloud-hypervisor" ${ch} --api-socket fd=0 
+                ''
+              )} ${vmmSock} ${hostName}
+          }
+          foreground { sleep 0.1 }
+          ifelse -n { test -S ${vmmSock} } { echo "Apparently ${vmmSock} does not exist" }
+          foreground { echo "Loading the configuration" }
+          if { $CHR create ${chSettingsFile} }
+          foreground { echo "Adding TAP" }
+          if { ${lib.getExe uvmsPkgs.taps} pass $CHR add-net "id=wan,fd=3,mac=00:00:00:00:00:01" }
+          foreground { echo "Booting" }
+          if { $CHR boot }
+          if { $CHR info }
+        '';
     }
     (lib.mkIf cfg.enable {
       boot.initrd.availableKernelModules = [