ch-runner: add minimal virtiofsd support

Somehow it doesnt work if I open the socket in python
and try to inherit it. Also didnt work with bwrap.
And theres a bunch of warnings that virtiofsd keeps printing.
Need investigation

profiles/ch-runner.nix

@@ -272,12 +272,22 @@ in
       # NOTE: Used to be an even uglier bash script, but, for now, execline makes for easier comparisons against spectrum
       uvms.cloud-hypervisor.runner =
         let
+          toolsClosure = pkgs.writeClosure [
+            (lib.getBin pkgs.execline)
+            (lib.getBin pkgs.s6)
+            (lib.getBin package)
+            (lib.getBin pkgs.virtiofsd)
+            (lib.getBin pkgs.bubblewrap)
+            uvmsPkgs.taps
+          ];
+
           superviseVm = getExe superviseVm';
           superviseVm' = pkgs.writers.writePython3Bin "supervise-vm" { } ''
             import os
             import subprocess
+            import socket
             from argparse import ArgumentParser
-            from contextlib import contextmanager, ExitStack
+            from contextlib import contextmanager, closing, ExitStack
 
 
             parser = ArgumentParser("supervise-vm")
@@ -290,12 +300,21 @@ in
             ELB_DIR = "${lib.getBin pkgs.execline}/bin"  # noqa: E501
             S6_DIR = "${lib.getBin pkgs.s6}/bin"  # noqa: E501
             CH_DIR = "${lib.getBin package}/bin"  # noqa: E501
+            UTIL_LINUX_DIR = "${lib.getBin pkgs.util-linux}/bin"  # noqa: E501
             SOCKETBINDER_PATH = S6_DIR + "/s6-ipcserver-socketbinder"  # noqa: E501
             CH_PATH = CH_DIR + "/cloud-hypervisor"
             CHR_PATH = CH_DIR + "/ch-remote"
             TAPS_PATH = "${lib.getExe uvmsPkgs.taps}"  # noqa: E501
+            VIRTIOFSD_PATH = "${lib.getExe pkgs.virtiofsd}"  # noqa: E501
+            BWRAP_PATH = "${lib.getExe pkgs.bubblewrap}"  # noqa: E501
 
-            PASSTHRU_PATH = ":".join([ELB_DIR, S6_DIR, CH_DIR])
+            with open("${toolsClosure}", mode="r") as f:  # noqa: E501
+                CLOSURE = [
+                    *(ln.rstrip() for ln in f.readlines()),
+                    "${placeholder "out"}",  # noqa: E501
+                ]
+
+            PASSTHRU_PATH = ":".join([ELB_DIR, S6_DIR, CH_DIR, UTIL_LINUX_DIR])
             PASSTHRU_ENV = {
                 **{
                     k: v
@@ -311,40 +330,6 @@ in
             }
 
 
-            def configure_exec(prefix, vm, check=True, **defaults):
-
-                def exec(*args, check=check, **kwargs):
-                    return subprocess.run(
-                        [*args],
-                        **defaults,
-                        env={
-                            **PASSTHRU_ENV,
-                            "PATH": PASSTHRU_PATH,
-                            "PREFIX": prefix,
-                            "VM": vm,
-                        },
-                        check=check,
-                        cwd=prefix,
-                        **kwargs)
-
-                def execline(*args, check=check, **kwargs):
-                    return exec(
-                        "execlineb", "-c", "\n".join(args),
-                        **defaults,
-                        executable=ELB_DIR + "/execlineb",
-                        env={
-                            **PASSTHRU_ENV,
-                            "PATH": PASSTHRU_PATH,
-                            "PREFIX": prefix,
-                            "VM": vm,
-                        },
-                        check=check,
-                        cwd=prefix,
-                        **kwargs)
-
-                return exec, execline
-
-
             def preprocess_args(args_mut):
                 keys = [
                     k
@@ -369,6 +354,270 @@ in
                 return args_mut
 
 
+            class Processes:
+                def __init__(self, prefix, vm, check=True, **defaults):
+                    self.prefix = prefix
+                    self.vm = vm
+                    self.check = check
+                    self.defaults = defaults
+
+                def make_env(self):
+                    return {
+                        **PASSTHRU_ENV,
+                        "PATH": PASSTHRU_PATH,
+                        "PREFIX": self.prefix,
+                        "VM": self.vm,
+                    }
+
+                def exec(self, *args, **kwargs):
+                    kwargs["cwd"] = kwargs.get("cwd", self.prefix)
+                    kwargs["check"] = kwargs.get("check", self.check)
+                    kwargs["env"] = kwargs.get("env", self.make_env())
+                    return subprocess.run(
+                        [*args],
+                        **self.defaults,
+                        **kwargs)
+
+                def execline(self, *args, **kwargs):
+                    return exec(
+                        "execlineb", "-c", "\n".join(args),
+                        **self.defaults,
+                        executable=ELB_DIR + "/execlineb",
+                        **{
+                            "env": self.make_env(),
+                            "check": self.check,
+                            "cwd": self.prefix,
+                            **kwargs,
+                        },
+                        )
+
+                def popen(self, *args, **kwargs):
+                    kwargs["pass_fds"] = kwargs.get("pass_fds", ())
+                    kwargs["env"] = kwargs.get("env", self.make_env())
+                    kwargs["cwd"] = kwargs.get("cwd", self.prefix)
+                    return subprocess.Popen(
+                        args,
+                        **kwargs,
+                    )
+
+                @contextmanager
+                def bwrap(
+                        self,
+                        *bwrap_args,
+
+                        die_with_parent=True,
+
+                        # Based on the args from
+                        # `host/rootfs/image/usr/bin/run-vmm`
+                        unshare_all=True,
+                        unshare_user=True,
+                        unshare_ipc=None,
+                        unshare_pid=None,
+                        unshare_net=None,
+                        unshare_uts=None,
+                        unshare_cgroup_try=True,
+                        bind=(),
+                        dev_bind=("/dev/kvm", "/dev/vfio"),
+                        dev="/dev",
+                        proc="/proc",
+                        ro_bind=(
+                            "/etc",
+                            "/sys",
+                            "/proc/sys",
+                            "/dev/null",
+                            "/proc/kallsyms",
+                            *CLOSURE),
+                        ro_bind_extra=(),
+                        remount_ro=("/proc/fs", "/proc/irq"),
+                        tmpfs=("/dev/shm", "/tmp", "/var/tmp", "/proc/fs", "/proc/irq"),
+                        tmpfs_extra=(),
+
+                        pass_fds=(2,),
+                        **popen_kwargs):
+
+                    bwrap_args_sock, remote = socket.socketpair()
+                    remote.set_inheritable(True)
+                    bwrap_args_f = bwrap_args_sock.makefile("w")
+                    with closing(bwrap_args_sock), closing(bwrap_args_f):
+                        def print_arg(*args):
+                            print(*args, file=bwrap_args_f, sep="\0", end="\0")
+
+                        if unshare_all:
+                            print_arg("--unshare-all")
+                        if unshare_user:
+                            print_arg("--unshare-user")
+                        if unshare_ipc:
+                            print_arg("--unshare-ipc")
+                        if unshare_pid:
+                            print_arg("--unshare-pid")
+                        if unshare_net:
+                            print_arg("--unshare-net")
+                        if unshare_uts:
+                            print_arg("--unshare-uts")
+                        if unshare_cgroup_try:
+                            print_arg("--unshare-cgroup-try")
+                        if die_with_parent:
+                            print_arg("--die-with-parent")
+
+                        for p in bind:
+                            p1, p2 = (p, p) if isinstance(p, str) else p
+                            print_arg("--bind", p1, p2)
+                        for p in (*ro_bind, *ro_bind_extra):
+                            p1, p2 = (p, p) if isinstance(p, str) else p
+                            print_arg("--ro-bind", p1, p2)
+                        for p in dev_bind:
+                            p1, p2 = (p, p) if isinstance(p, str) else p
+                            print_arg("--dev-bind", p1, p2)
+                        for p in (*tmpfs, *tmpfs_extra):
+                            print_arg("--tmpfs", p)
+                        # Hunch: order might matter...
+                        for p in remount_ro:
+                            print_arg("--remount-ro", p)
+
+                        bwrap_args_f.flush()
+
+                        with closing(remote):
+                            proc = self.popen(
+                                "bwrap", "--args", str(remote.fileno()), *bwrap_args,
+                                **popen_kwargs,
+                                executable=BWRAP_PATH,
+                                pass_fds=(*pass_fds, remote.fileno()),
+                            )
+
+                    with proc as p:
+                        try:
+                            yield p
+                        finally:
+                            try:
+                                p.poll()
+                            except:  # noqa: E722
+                                pass
+                            if p.returncode is None:
+                                p.terminate()
+                                p.wait()
+
+                @contextmanager
+                def run_ch(self):
+                    args = [
+                        SOCKETBINDER_PATH,
+                        "-B",
+                        self.prefix + "/vmm.sock",
+                        CH_PATH,
+                        "--api-socket",
+                        "fd=0",
+                    ]
+                    p = self.popen(
+                        *args,
+                        shell=False,
+                        stdin=subprocess.DEVNULL,
+                        stdout=subprocess.DEVNULL,
+                        pass_fds=(2,))
+                    try:
+                        p.wait(0.125)
+                        needs_cleanup = False
+                    except subprocess.TimeoutExpired:
+                        needs_cleanup = True
+                    if not os.path.exists(self.prefix + "/vmm.sock"):
+                        raise RuntimeError(f"{self.prefix}/vmm.sock should exist by now")
+                    if p.returncode is not None:
+                        raise RuntimeError("CH exited early")
+                    try:
+                        yield p
+                    finally:
+                        try:
+                            p.poll()
+                        except:  # noqa: E722
+                            pass
+                        if p.returncode is None:
+                            p.terminate()  # CH handles SIG{INT,TERM}?
+                            p.wait()
+                        unlink_paths = [
+                            self.prefix + "/vmm.sock",
+                            self.prefix + "/vmm.sock.lock",
+                            self.prefix + "/vsock.sock",
+                        ] if needs_cleanup else []
+                        for p in unlink_paths:
+                            if os.path.exists(p):
+                                os.remove(p)
+
+                @contextmanager
+                def add_virtiofsd(
+                        self,
+                        root_dir,
+                        tag,
+                        ro=False,
+                        subdirs=None,
+                        extra_flags=("--posix-acl",)):
+
+                    assert os.path.exists(root_dir)
+
+                    sock_path = self.prefix + f"/virtiofsd-{tag}.sock"
+                    # s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+                    # NOTE: Nope. Virtiofsd actually expects a blocking socket
+                    # s.setblocking(True)
+
+                    def rm_sock():
+                        if os.path.exists(sock_path):
+                            os.remove(sock_path)
+
+                    with ExitStack() as cleanup:  # noqa: F841
+                        # s.bind(sock_path.encode("utf8"))
+                        # cleanup.enter_context(closing(s))
+                        cleanup.enter_context(defer(rm_sock))
+
+                        args = [
+                            # If using bwrap():
+                            # "--argv0", "virtiofsd",
+                            # "--uid", "1000",
+                            # "--gid", "1000",
+                            # "--",
+                            "unshare", "-rUm",
+                            "unshare", "--map-user", "1000", "--map-group", "1000",
+                            VIRTIOFSD_PATH,
+                            "--shared-dir",
+                            root_dir,
+                            "--tag",
+                            tag,
+
+                            # "--fd",
+                            # str(s.fileno()),
+                            "--socket-path",
+                            sock_path,
+
+                            # If relying on bwrap():
+                            # "--sandbox",
+                            # "none",
+                        ]
+                        if ro:
+                            args.append("--readonly")
+                        kwargs = {
+                            # If bwrap():
+                            # "bind": [],
+                            # ("ro_bind_extra" if ro else "bind"):
+                            #     [*subdirs]
+                            #     if subdirs is not None
+                            #     else [root_dir],
+
+                            # "pass_fds": (2, s.fileno()),
+                        }
+                        proc_ctx = self.popen(*args, **kwargs)
+                        with proc_ctx as p:
+                            try:
+                                try:
+                                    p.wait(0.125)
+                                except subprocess.TimeoutExpired:
+                                    pass
+                                if p.returncode is not None:
+                                    raise RuntimeError("virtiofsd exited too early")
+                                yield p, sock_path
+                            finally:
+                                if p.returncode is None:
+                                    p.kill()
+                                    p.wait()
+                                if os.path.exists(sock_path):
+                                    os.remove(sock_path)
+
+
             @contextmanager
             def defer(f):
                 try:
@@ -377,57 +626,15 @@ in
                     f()
 
 
-            @contextmanager
-            def run_ch(vm_prefix):
-                args = [
-                    SOCKETBINDER_PATH,
-                    "-B",
-                    vm_prefix + "/vmm.sock",
-                    CH_PATH,
-                    "--api-socket",
-                    "fd=0",
-                ]
-                p = subprocess.Popen(
-                    args,
-                    shell=False,
-                    pass_fds=(2,))
-                try:
-                    p.wait(1.0)
-                    needs_cleanup = False
-                except subprocess.TimeoutExpired:
-                    needs_cleanup = True
-                if not os.path.exists(vm_prefix + "/vmm.sock"):
-                    raise RuntimeError(f"{vm_prefix}/vmm.sock should exist by now")
-                if p.returncode is not None:
-                    raise RuntimeError("CH exited early")
-                try:
-                    yield p
-                finally:
-                    try:
-                        p.poll()
-                    except:  # noqa: E722
-                        pass
-                    if p.returncode is None:
-                        p.terminate()  # CH handles SIG{INT,TERM}?
-                        p.wait()
-                    unlink_paths = [
-                        vm_prefix + "/vmm.sock",
-                        vm_prefix + "/vmm.sock.lock",
-                        vm_prefix + "/vsock.sock",
-                    ] if needs_cleanup else []
-                    for p in unlink_paths:
-                        if os.path.exists(p):
-                            os.remove(p)
-
-
             if __name__ == "__main__":
                 args, args_next = parser.parse_known_args()
                 preprocess_args(args)
 
                 os.makedirs(args.prefix, exist_ok=True)
-                exec, _ = configure_exec(
+                ps = Processes(
                     prefix=args.prefix,
-                    vm=args.vm)
+                    vm=args.vm,
+                )
 
                 ch_remote = [
                     "ch-remote",
@@ -436,14 +643,23 @@ in
                 ]
 
                 with ExitStack() as cleanup:
-                    ch = cleanup.enter_context(run_ch(args.prefix))
+                    ch = cleanup.enter_context(ps.run_ch())
-                    exec(*ch_remote, "create", args.vm_config)
+                    ps.exec(*ch_remote, "create", args.vm_config)
-                    exec(
+                    ps.exec(
                         TAPS_PATH, "pass",
                         *ch_remote, "add-net",
                         "id=wan,fd=3,mac=00:00:00:00:00:01")
-                    exec(*ch_remote, "boot")
+
-                    exec(*ch_remote, "info")
+                    send_dir = PASSTHRU_ENV["HOME"] + f"/send/{args.vm}"
+                    os.makedirs(send_dir, exist_ok=True)
+                    vfsd, vfsd_path = cleanup.enter_context(
+                        ps.add_virtiofsd(
+                            send_dir,
+                            tag="send",
+                        ))
+                    ps.exec(*ch_remote, "add-fs", f"tag=send,socket={vfsd_path},id=send")
+                    ps.exec(*ch_remote, "boot")
+                    ps.exec(*ch_remote, "info")
                     try:
                         ch.wait()
                     except KeyboardInterrupt: