ch-runner: just move the cleanup bits out of elb
Execlineb is insane. Skarnet is insane. POSIX is cancer, but elb seems to make it worse. Or it s a skill issue. I see no way to do any kind of error handling with elb, and that is even despite me only needing one kind of error handling: the cleaning up...
This commit is contained in:
Else Someone
parent
ee497cb7d6
commit
0617d97ebf
1 changed files with 215 additions and 36 deletions
|
|
@ -13,7 +13,14 @@ let
|
|||
|
||||
inherit (config.networking) hostName;
|
||||
inherit (config.debug.closure.erofs) layers;
|
||||
inherit (lib) mkOption types;
|
||||
inherit (lib)
|
||||
mkOption
|
||||
types
|
||||
concatMapStringsSep
|
||||
getExe
|
||||
getExe'
|
||||
getBin
|
||||
;
|
||||
|
||||
package = pkgs.cloud-hypervisor.overrideAttrs (oldAttrs: {
|
||||
patches = oldAttrs.patches or [ ] ++ [
|
||||
|
|
@ -38,7 +45,7 @@ let
|
|||
destination = "/bin/${name}";
|
||||
executable = true;
|
||||
text = ''
|
||||
#!${lib.getExe' pkgs.execline "execlineb"}${lib.optionalString (elArgs != null) " "}${elArgs}
|
||||
#!${getExe' pkgs.execline "execlineb"}${lib.optionalString (elArgs != null) " "}${elArgs}
|
||||
importas OLDPATH PATH
|
||||
export PATH "${elbPrefix}:${s6Prefix}:''${OLDPATH}"
|
||||
${text}
|
||||
|
|
@ -194,7 +201,7 @@ in
|
|||
if { cp -r --no-preserve=all $GDB_SCRIPT_DIR gdb_scripts }
|
||||
mv gdb_scripts/linux/constants.py.in gdb_scripts/linux/constants.py
|
||||
}
|
||||
${lib.getExe pkgs.gdb}
|
||||
${getExe pkgs.gdb}
|
||||
-ex "python import sys; sys.path.insert(0, \"''${GDB_SCRIPT_DIR}\")"
|
||||
-ex "target remote :1234"
|
||||
-ex "source ''${GDB_SCRIPT_DIR}/vmlinux-gdb.py"
|
||||
|
|
@ -245,7 +252,7 @@ in
|
|||
environment.TAPS_SOCK = "/run/taps/taps.sock";
|
||||
serviceConfig = {
|
||||
UMask = "0007";
|
||||
ExecStart = "${lib.getExe uvmsPkgs.taps} serve";
|
||||
ExecStart = "${getExe uvmsPkgs.taps} serve";
|
||||
RuntimeDirectory = "taps";
|
||||
DynamicUser = true;
|
||||
AmbientCapabilities = [
|
||||
|
|
@ -257,43 +264,215 @@ in
|
|||
};
|
||||
};
|
||||
testScript = ''
|
||||
machine.succeed("${lib.getExe cfg.runner}")
|
||||
machine.succeed("${getExe cfg.runner}")
|
||||
'';
|
||||
}
|
||||
);
|
||||
|
||||
# NOTE: Used to be an even uglier bash script, but, for now, execline makes for easier comparisons against spectrum
|
||||
uvms.cloud-hypervisor.runner = writeElb "run-${hostName}" ''
|
||||
importas -i HOME HOME
|
||||
importas -SsD ${lib.getExe' pkgs.passt "passt"} PASST
|
||||
importas -SsD ${lib.getExe package} CH
|
||||
importas -SsD "${lib.getExe' package "ch-remote"} --api-socket=${vmmSock}" CHR
|
||||
foreground { mkdir -p "${uvmPrefix}" "${uvmPrefix}" }
|
||||
cd "${uvmPrefix}"
|
||||
background {
|
||||
s6-ipcserver-socketbinder -B ${vmmSock}
|
||||
trap { default { foreground { s6-ipcclient -q ${uvmPrefix}/cleanup.sock true } exit } }
|
||||
fdmove -c 3 0
|
||||
redirfd -r 0 /dev/null
|
||||
exec -a "uuvm/${hostName} cloud-hypervisor" $CH --api-socket fd=3
|
||||
}
|
||||
importas CH_PID !
|
||||
background {
|
||||
getpid -E CLEANUP_PID
|
||||
s6-ipcserver-socketbinder -B "${uvmPrefix}/cleanup.sock"
|
||||
s6-ipcserverd
|
||||
foreground { kill $CH_PID $PASST_PID }
|
||||
foreground {
|
||||
rm -f ${vmmSock} ${uvmPrefix}/vsock.sock
|
||||
}
|
||||
kill $CLEANUP_PID
|
||||
}
|
||||
trap { default { foreground { s6-ipcclient -q ${uvmPrefix}/cleanup.sock true } exit } }
|
||||
if { $CHR create ${chSettingsFile} }
|
||||
if { ${lib.getExe uvmsPkgs.taps} pass $CHR add-net "id=wan,fd=3,mac=00:00:00:00:00:01" }
|
||||
if { $CHR boot }
|
||||
if { $CHR info }
|
||||
'';
|
||||
uvms.cloud-hypervisor.runner =
|
||||
let
|
||||
addProcess = getExe addProcess';
|
||||
addProcess' = pkgs.writers.writePython3Bin "add-process" { } ''
|
||||
import os
|
||||
import select
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from contextlib import contextmanager, ExitStack
|
||||
from threading import Thread, Semaphore
|
||||
|
||||
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("events_path")
|
||||
parser.add_argument("--then", action="append")
|
||||
|
||||
MSG_SIZE = 16
|
||||
SHMEM = {}
|
||||
|
||||
|
||||
def send(sock, msg):
|
||||
assert len(msg) <= MSG_SIZE, len(msg)
|
||||
return sock.send(msg.ljust(MSG_SIZE))
|
||||
|
||||
|
||||
def recv(sock):
|
||||
msg = sock.recv(MSG_SIZE)
|
||||
# assert len(msg) <= MSG_SIZE, len(msg)
|
||||
assert len(msg) <= MSG_SIZE, len(msg)
|
||||
return (msg.split() + [b""])[0]
|
||||
|
||||
|
||||
def serve_impl(events_path, listener):
|
||||
SHMEM["server"] = True
|
||||
|
||||
cons = []
|
||||
state = "up"
|
||||
while state == "up" or cons != []:
|
||||
if state == "up":
|
||||
rs, ws, es = select.select([listener, *cons], [], [])
|
||||
else:
|
||||
rs, ws, es = select.select(cons, cons, [])
|
||||
events = []
|
||||
for r in rs:
|
||||
if r is listener:
|
||||
r, _ = r.accept()
|
||||
cons.append(r)
|
||||
else:
|
||||
events.append(recv(r))
|
||||
if any(e == b"killall" for e in events):
|
||||
state = "down"
|
||||
if state == "down":
|
||||
for w in ws:
|
||||
with s_lock:
|
||||
send(w, b"die")
|
||||
w.close()
|
||||
cons.remove(w)
|
||||
for w in es:
|
||||
w.close()
|
||||
cons.remove(w)
|
||||
|
||||
|
||||
def serve(events_path):
|
||||
base_dir = os.path.dirname(events_path)
|
||||
if base_dir:
|
||||
os.makedirs(base_dir, exist_ok=True)
|
||||
listener = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM, 0)
|
||||
listener.setblocking(False)
|
||||
|
||||
try:
|
||||
listener.bind(events_path)
|
||||
listener.listen()
|
||||
return serve_impl(events_path, listener)
|
||||
except OSError as e:
|
||||
EADDRINUSE = 98
|
||||
if e.errno != EADDRINUSE:
|
||||
raise
|
||||
finally:
|
||||
listener.close()
|
||||
os.remove(events_path)
|
||||
|
||||
|
||||
def register(events_path):
|
||||
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM, 0)
|
||||
sock.connect(events_path)
|
||||
return sock
|
||||
|
||||
|
||||
@contextmanager
|
||||
def defer(f):
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
f()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
args, args_next = parser.parse_known_args()
|
||||
|
||||
with ExitStack() as cleanup:
|
||||
if args_next:
|
||||
p = subprocess.Popen(
|
||||
args_next,
|
||||
shell=False)
|
||||
then_cmds = reversed(getattr(args, "then") or [])
|
||||
if not args_next:
|
||||
then_cmds = []
|
||||
try:
|
||||
p.wait(0.5)
|
||||
then_cmds = []
|
||||
except subprocess.TimeoutExpired:
|
||||
pass
|
||||
for f in then_cmds:
|
||||
def run_f():
|
||||
subprocess.run(f)
|
||||
cleanup.enter_context(defer(run_f))
|
||||
|
||||
maybe_server = Thread(
|
||||
target=serve,
|
||||
args=(args.events_path,),
|
||||
daemon=True)
|
||||
maybe_server.start()
|
||||
maybe_server.join(0.5)
|
||||
|
||||
assert (
|
||||
("server" in SHMEM) == bool(maybe_server.is_alive)
|
||||
), (SHMEM, maybe_server)
|
||||
|
||||
if args_next:
|
||||
s = register(args.events_path)
|
||||
s_lock = Semaphore()
|
||||
|
||||
if args_next:
|
||||
def watch_p(p, s):
|
||||
p.wait()
|
||||
with s_lock:
|
||||
try:
|
||||
send(s, b"killall")
|
||||
except BrokenPipeError:
|
||||
pass
|
||||
|
||||
def watch_s(p, s):
|
||||
while True:
|
||||
if recv(s) == b"die":
|
||||
p.terminate()
|
||||
break
|
||||
|
||||
s_watcher = Thread(
|
||||
target=watch_s,
|
||||
args=(p, s),
|
||||
daemon=True)
|
||||
s_watcher.start()
|
||||
watch_p(p, s)
|
||||
s_watcher.join()
|
||||
s.close()
|
||||
|
||||
if SHMEM.get("server", False):
|
||||
maybe_server.join()
|
||||
|
||||
exit_code = 0
|
||||
if args_next:
|
||||
exit_code |= p.returncode
|
||||
sys.exit(exit_code)
|
||||
'';
|
||||
ch = getExe package;
|
||||
chr = getExe' package "ch-remote";
|
||||
in
|
||||
writeElb "run-${hostName}" ''
|
||||
importas -i HOME HOME
|
||||
importas -SsD "${chr} --api-socket=${vmmSock}" CHR
|
||||
importas -SsD "${uvmPrefix}" PREFIX
|
||||
define EVENTS ''${PREFIX}/events.sock
|
||||
define -s ADD_PROC "${addProcess} ''${EVENTS}"
|
||||
|
||||
cd $PREFIX
|
||||
background {
|
||||
$ADD_PROC --then ${getExe (
|
||||
writeElb "rm-vmmsock" ''
|
||||
importas -i HOME HOME
|
||||
rm -f ${vmmSock}
|
||||
rm -f ${uvmPrefix}/vsock.sock
|
||||
''
|
||||
)} ${getExe (
|
||||
writeElb "ch" ''
|
||||
importas -Si 1
|
||||
importas -Si 2
|
||||
s6-ipcserver-socketbinder -B $1
|
||||
exec -a "uuvm/''${2} cloud-hypervisor" ${ch} --api-socket fd=0
|
||||
''
|
||||
)} ${vmmSock} ${hostName}
|
||||
}
|
||||
foreground { sleep 0.1 }
|
||||
ifelse -n { test -S ${vmmSock} } { echo "Apparently ${vmmSock} does not exist" }
|
||||
foreground { echo "Loading the configuration" }
|
||||
if { $CHR create ${chSettingsFile} }
|
||||
foreground { echo "Adding TAP" }
|
||||
if { ${lib.getExe uvmsPkgs.taps} pass $CHR add-net "id=wan,fd=3,mac=00:00:00:00:00:01" }
|
||||
foreground { echo "Booting" }
|
||||
if { $CHR boot }
|
||||
if { $CHR info }
|
||||
'';
|
||||
}
|
||||
(lib.mkIf cfg.enable {
|
||||
boot.initrd.availableKernelModules = [
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue