From 7516eae7a6a83b1253441b56ceb3370248ac7fdd Mon Sep 17 00:00:00 2001 From: Else Someone Date: Sat, 21 Feb 2026 18:11:37 +0200 Subject: [PATCH] ch-runner: add minimal virtiofsd support Somehow it doesnt work if I open the socket in python and try to inherit it. Also didnt work with bwrap. And theres a bunch of warnings that virtiofsd keeps printing. Need investigation --- profiles/ch-runner.nix | 388 ++++++++++++++++++++++++++++++++--------- 1 file changed, 302 insertions(+), 86 deletions(-) diff --git a/profiles/ch-runner.nix b/profiles/ch-runner.nix index b3d0385..f156705 100644 --- a/profiles/ch-runner.nix +++ b/profiles/ch-runner.nix @@ -272,12 +272,22 @@ in # NOTE: Used to be an even uglier bash script, but, for now, execline makes for easier comparisons against spectrum uvms.cloud-hypervisor.runner = let + toolsClosure = pkgs.writeClosure [ + (lib.getBin pkgs.execline) + (lib.getBin pkgs.s6) + (lib.getBin package) + (lib.getBin pkgs.virtiofsd) + (lib.getBin pkgs.bubblewrap) + uvmsPkgs.taps + ]; + superviseVm = getExe superviseVm'; superviseVm' = pkgs.writers.writePython3Bin "supervise-vm" { } '' import os import subprocess + import socket from argparse import ArgumentParser - from contextlib import contextmanager, ExitStack + from contextlib import contextmanager, closing, ExitStack parser = ArgumentParser("supervise-vm") @@ -290,12 +300,21 @@ in ELB_DIR = "${lib.getBin pkgs.execline}/bin" # noqa: E501 S6_DIR = "${lib.getBin pkgs.s6}/bin" # noqa: E501 CH_DIR = "${lib.getBin package}/bin" # noqa: E501 + UTIL_LINUX_DIR = "${lib.getBin pkgs.util-linux}/bin" # noqa: E501 SOCKETBINDER_PATH = S6_DIR + "/s6-ipcserver-socketbinder" # noqa: E501 CH_PATH = CH_DIR + "/cloud-hypervisor" CHR_PATH = CH_DIR + "/ch-remote" TAPS_PATH = "${lib.getExe uvmsPkgs.taps}" # noqa: E501 + VIRTIOFSD_PATH = "${lib.getExe pkgs.virtiofsd}" # noqa: E501 + BWRAP_PATH = "${lib.getExe pkgs.bubblewrap}" # noqa: E501 - PASSTHRU_PATH = ":".join([ELB_DIR, S6_DIR, CH_DIR]) + with open("${toolsClosure}", mode="r") as f: # noqa: E501 + CLOSURE = [ + *(ln.rstrip() for ln in f.readlines()), + "${placeholder "out"}", # noqa: E501 + ] + + PASSTHRU_PATH = ":".join([ELB_DIR, S6_DIR, CH_DIR, UTIL_LINUX_DIR]) PASSTHRU_ENV = { **{ k: v @@ -311,40 +330,6 @@ in } - def configure_exec(prefix, vm, check=True, **defaults): - - def exec(*args, check=check, **kwargs): - return subprocess.run( - [*args], - **defaults, - env={ - **PASSTHRU_ENV, - "PATH": PASSTHRU_PATH, - "PREFIX": prefix, - "VM": vm, - }, - check=check, - cwd=prefix, - **kwargs) - - def execline(*args, check=check, **kwargs): - return exec( - "execlineb", "-c", "\n".join(args), - **defaults, - executable=ELB_DIR + "/execlineb", - env={ - **PASSTHRU_ENV, - "PATH": PASSTHRU_PATH, - "PREFIX": prefix, - "VM": vm, - }, - check=check, - cwd=prefix, - **kwargs) - - return exec, execline - - def preprocess_args(args_mut): keys = [ k @@ -369,6 +354,270 @@ in return args_mut + class Processes: + def __init__(self, prefix, vm, check=True, **defaults): + self.prefix = prefix + self.vm = vm + self.check = check + self.defaults = defaults + + def make_env(self): + return { + **PASSTHRU_ENV, + "PATH": PASSTHRU_PATH, + "PREFIX": self.prefix, + "VM": self.vm, + } + + def exec(self, *args, **kwargs): + kwargs["cwd"] = kwargs.get("cwd", self.prefix) + kwargs["check"] = kwargs.get("check", self.check) + kwargs["env"] = kwargs.get("env", self.make_env()) + return subprocess.run( + [*args], + **self.defaults, + **kwargs) + + def execline(self, *args, **kwargs): + return exec( + "execlineb", "-c", "\n".join(args), + **self.defaults, + executable=ELB_DIR + "/execlineb", + **{ + "env": self.make_env(), + "check": self.check, + "cwd": self.prefix, + **kwargs, + }, + ) + + def popen(self, *args, **kwargs): + kwargs["pass_fds"] = kwargs.get("pass_fds", ()) + kwargs["env"] = kwargs.get("env", self.make_env()) + kwargs["cwd"] = kwargs.get("cwd", self.prefix) + return subprocess.Popen( + args, + **kwargs, + ) + + @contextmanager + def bwrap( + self, + *bwrap_args, + + die_with_parent=True, + + # Based on the args from + # `host/rootfs/image/usr/bin/run-vmm` + unshare_all=True, + unshare_user=True, + unshare_ipc=None, + unshare_pid=None, + unshare_net=None, + unshare_uts=None, + unshare_cgroup_try=True, + bind=(), + dev_bind=("/dev/kvm", "/dev/vfio"), + dev="/dev", + proc="/proc", + ro_bind=( + "/etc", + "/sys", + "/proc/sys", + "/dev/null", + "/proc/kallsyms", + *CLOSURE), + ro_bind_extra=(), + remount_ro=("/proc/fs", "/proc/irq"), + tmpfs=("/dev/shm", "/tmp", "/var/tmp", "/proc/fs", "/proc/irq"), + tmpfs_extra=(), + + pass_fds=(2,), + **popen_kwargs): + + bwrap_args_sock, remote = socket.socketpair() + remote.set_inheritable(True) + bwrap_args_f = bwrap_args_sock.makefile("w") + with closing(bwrap_args_sock), closing(bwrap_args_f): + def print_arg(*args): + print(*args, file=bwrap_args_f, sep="\0", end="\0") + + if unshare_all: + print_arg("--unshare-all") + if unshare_user: + print_arg("--unshare-user") + if unshare_ipc: + print_arg("--unshare-ipc") + if unshare_pid: + print_arg("--unshare-pid") + if unshare_net: + print_arg("--unshare-net") + if unshare_uts: + print_arg("--unshare-uts") + if unshare_cgroup_try: + print_arg("--unshare-cgroup-try") + if die_with_parent: + print_arg("--die-with-parent") + + for p in bind: + p1, p2 = (p, p) if isinstance(p, str) else p + print_arg("--bind", p1, p2) + for p in (*ro_bind, *ro_bind_extra): + p1, p2 = (p, p) if isinstance(p, str) else p + print_arg("--ro-bind", p1, p2) + for p in dev_bind: + p1, p2 = (p, p) if isinstance(p, str) else p + print_arg("--dev-bind", p1, p2) + for p in (*tmpfs, *tmpfs_extra): + print_arg("--tmpfs", p) + # Hunch: order might matter... + for p in remount_ro: + print_arg("--remount-ro", p) + + bwrap_args_f.flush() + + with closing(remote): + proc = self.popen( + "bwrap", "--args", str(remote.fileno()), *bwrap_args, + **popen_kwargs, + executable=BWRAP_PATH, + pass_fds=(*pass_fds, remote.fileno()), + ) + + with proc as p: + try: + yield p + finally: + try: + p.poll() + except: # noqa: E722 + pass + if p.returncode is None: + p.terminate() + p.wait() + + @contextmanager + def run_ch(self): + args = [ + SOCKETBINDER_PATH, + "-B", + self.prefix + "/vmm.sock", + CH_PATH, + "--api-socket", + "fd=0", + ] + p = self.popen( + *args, + shell=False, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + pass_fds=(2,)) + try: + p.wait(0.125) + needs_cleanup = False + except subprocess.TimeoutExpired: + needs_cleanup = True + if not os.path.exists(self.prefix + "/vmm.sock"): + raise RuntimeError(f"{self.prefix}/vmm.sock should exist by now") + if p.returncode is not None: + raise RuntimeError("CH exited early") + try: + yield p + finally: + try: + p.poll() + except: # noqa: E722 + pass + if p.returncode is None: + p.terminate() # CH handles SIG{INT,TERM}? + p.wait() + unlink_paths = [ + self.prefix + "/vmm.sock", + self.prefix + "/vmm.sock.lock", + self.prefix + "/vsock.sock", + ] if needs_cleanup else [] + for p in unlink_paths: + if os.path.exists(p): + os.remove(p) + + @contextmanager + def add_virtiofsd( + self, + root_dir, + tag, + ro=False, + subdirs=None, + extra_flags=("--posix-acl",)): + + assert os.path.exists(root_dir) + + sock_path = self.prefix + f"/virtiofsd-{tag}.sock" + # s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + # NOTE: Nope. Virtiofsd actually expects a blocking socket + # s.setblocking(True) + + def rm_sock(): + if os.path.exists(sock_path): + os.remove(sock_path) + + with ExitStack() as cleanup: # noqa: F841 + # s.bind(sock_path.encode("utf8")) + # cleanup.enter_context(closing(s)) + cleanup.enter_context(defer(rm_sock)) + + args = [ + # If using bwrap(): + # "--argv0", "virtiofsd", + # "--uid", "1000", + # "--gid", "1000", + # "--", + "unshare", "-rUm", + "unshare", "--map-user", "1000", "--map-group", "1000", + VIRTIOFSD_PATH, + "--shared-dir", + root_dir, + "--tag", + tag, + + # "--fd", + # str(s.fileno()), + "--socket-path", + sock_path, + + # If relying on bwrap(): + # "--sandbox", + # "none", + ] + if ro: + args.append("--readonly") + kwargs = { + # If bwrap(): + # "bind": [], + # ("ro_bind_extra" if ro else "bind"): + # [*subdirs] + # if subdirs is not None + # else [root_dir], + + # "pass_fds": (2, s.fileno()), + } + proc_ctx = self.popen(*args, **kwargs) + with proc_ctx as p: + try: + try: + p.wait(0.125) + except subprocess.TimeoutExpired: + pass + if p.returncode is not None: + raise RuntimeError("virtiofsd exited too early") + yield p, sock_path + finally: + if p.returncode is None: + p.kill() + p.wait() + if os.path.exists(sock_path): + os.remove(sock_path) + + @contextmanager def defer(f): try: @@ -377,57 +626,15 @@ in f() - @contextmanager - def run_ch(vm_prefix): - args = [ - SOCKETBINDER_PATH, - "-B", - vm_prefix + "/vmm.sock", - CH_PATH, - "--api-socket", - "fd=0", - ] - p = subprocess.Popen( - args, - shell=False, - pass_fds=(2,)) - try: - p.wait(1.0) - needs_cleanup = False - except subprocess.TimeoutExpired: - needs_cleanup = True - if not os.path.exists(vm_prefix + "/vmm.sock"): - raise RuntimeError(f"{vm_prefix}/vmm.sock should exist by now") - if p.returncode is not None: - raise RuntimeError("CH exited early") - try: - yield p - finally: - try: - p.poll() - except: # noqa: E722 - pass - if p.returncode is None: - p.terminate() # CH handles SIG{INT,TERM}? - p.wait() - unlink_paths = [ - vm_prefix + "/vmm.sock", - vm_prefix + "/vmm.sock.lock", - vm_prefix + "/vsock.sock", - ] if needs_cleanup else [] - for p in unlink_paths: - if os.path.exists(p): - os.remove(p) - - if __name__ == "__main__": args, args_next = parser.parse_known_args() preprocess_args(args) os.makedirs(args.prefix, exist_ok=True) - exec, _ = configure_exec( + ps = Processes( prefix=args.prefix, - vm=args.vm) + vm=args.vm, + ) ch_remote = [ "ch-remote", @@ -436,14 +643,23 @@ in ] with ExitStack() as cleanup: - ch = cleanup.enter_context(run_ch(args.prefix)) - exec(*ch_remote, "create", args.vm_config) - exec( + ch = cleanup.enter_context(ps.run_ch()) + ps.exec(*ch_remote, "create", args.vm_config) + ps.exec( TAPS_PATH, "pass", *ch_remote, "add-net", "id=wan,fd=3,mac=00:00:00:00:00:01") - exec(*ch_remote, "boot") - exec(*ch_remote, "info") + + send_dir = PASSTHRU_ENV["HOME"] + f"/send/{args.vm}" + os.makedirs(send_dir, exist_ok=True) + vfsd, vfsd_path = cleanup.enter_context( + ps.add_virtiofsd( + send_dir, + tag="send", + )) + ps.exec(*ch_remote, "add-fs", f"tag=send,socket={vfsd_path},id=send") + ps.exec(*ch_remote, "boot") + ps.exec(*ch_remote, "info") try: ch.wait() except KeyboardInterrupt: