diff --git a/pkgs/uvms-guest/package.nix b/pkgs/uvms-guest/package.nix deleted file mode 100644 index 66cfa2d..0000000 --- a/pkgs/uvms-guest/package.nix +++ /dev/null @@ -1,5 +0,0 @@ -{ - lib, - writers, -}: -writers.writePython3Bin "uvms-guest" { } ./guest.py diff --git a/pkgs/uvms/package.nix b/pkgs/uvms/package.nix index 109235e..5ebf056 100644 --- a/pkgs/uvms/package.nix +++ b/pkgs/uvms/package.nix @@ -13,6 +13,8 @@ strace, util-linux, virtiofsd, + python3Packages, + uvmslib, taps, baseImage, @@ -36,21 +38,25 @@ let }; toolsClosure = writeClosure toolsFarm; in -writers.writePython3Bin "uvms" { } ( - replaceVars ./uvms.py { - BWRAP = "${lib.getExe bubblewrap}"; - TOOLS = "${toolsFarm}/bin"; - TOOLS_CLOSURE = toolsClosure; - CROSVM = lib.getExe crosvm; - STRACE = lib.getExe strace; - TAPS = "${lib.getExe taps}"; - VIRTIOFSD = "${lib.getExe virtiofsd}"; - - BASE_CONFIG = baseImage.config.system.build.ch; - SYSTEM = baseImage.config.system.build.toplevel; - SYSTEM_CLOSURE = writeClosure [ - baseImage.config.system.build.toplevel - baseImage.config.system.build.ch - ]; +writers.writePython3Bin "uvms" + { + libraries = [ uvmslib ]; } -) + ( + replaceVars ./uvms.py { + BWRAP = "${lib.getExe bubblewrap}"; + TOOLS = "${toolsFarm}/bin"; + TOOLS_CLOSURE = toolsClosure; + CROSVM = lib.getExe crosvm; + STRACE = lib.getExe strace; + TAPS = "${lib.getExe taps}"; + VIRTIOFSD = "${lib.getExe virtiofsd}"; + + BASE_CONFIG = baseImage.config.system.build.ch; + SYSTEM = baseImage.config.system.build.toplevel; + SYSTEM_CLOSURE = writeClosure [ + baseImage.config.system.build.toplevel + baseImage.config.system.build.ch + ]; + } + ) diff --git a/pkgs/uvms/uvms.py b/pkgs/uvms/uvms.py index c473f11..ec4a139 100644 --- a/pkgs/uvms/uvms.py +++ b/pkgs/uvms/uvms.py @@ -10,6 +10,7 @@ import os import subprocess import socket import json +import re from argparse import ArgumentParser from contextlib import contextmanager, closing, ExitStack @@ -20,6 +21,7 @@ parser.add_argument("--prefix", default="$HOME/uvms/$VM") parser.add_argument("--vm-config", default="@BASE_CONFIG@") # noqa: E501 parser.add_argument("--persist-home", action="store_true") parser.add_argument("--run", action="append") +parser.add_argument("--mem", default=None) parser.add_argument("app", nargs="*", default=()) TOOLS_DIR = "@TOOLS@" # noqa: E501 @@ -297,6 +299,7 @@ class Processes: # "-Z", # "-ff", CH, + # "-v", "--api-socket", "fd=0", # f"fd={s.fileno()}" @@ -318,6 +321,8 @@ class Processes: # ro_bind=["/nix/store"], # I give up unshare_net=False, shell=False, + stdout=None, + stderr=None, # pass_fds=(s.fileno(),) ) ) @@ -338,29 +343,23 @@ class Processes: ): sock_path = self.prefix + "/gpu.sock" args = [ - SOCKETBINDER, - "-b", - "1", - sock_path, - "s6-ipcserverd", - "-1c1", - # "@STRACE@", # noqa: E501 - # "-Z", - # "-ff", "@CROSVM@", # noqa: E501 "--no-syslog", + "--log-level", + "debug", "device", "gpu", - "--fd", - "0", + "--socket-path", + sock_path, "--wayland-sock", f'{PASSTHRU_ENV["XDG_RUNTIME_DIR"]}/{PASSTHRU_ENV["WAYLAND_DISPLAY"]}', # noqa: E501 "--params", - '{ "context-types": "cross-domain:virgl2:venus" }', + '{ "context-types": "cross-domain" }', ] with self.popen( *args, stderr=None, + stdout=None, ) as proc, removing(sock_path, sock_path + ".lock"): yield proc, sock_path @@ -469,6 +468,21 @@ def connect_ch_vsock( yield s +BYTES_PATTERN = re.compile(r"^([0-9]+)([MmGgKk]?)$") +BYTES_UNITS = { + "k": 1024, + "m": 1048576, + "g": 1024 * 1048576, +} + + +def parse_bytes(s): + m = BYTES_PATTERN.match(s) + assert m, s + size, unit = m.groups() + return int(size) * BYTES_UNITS.get(unit.lower(), 1) + + @contextmanager def listen_ch_vsock( vsock_sock_path, @@ -564,6 +578,16 @@ def main(args, args_next, cleanup, ps): ) virtiofs_socks.append(("home", sock_path)) config["payload"]["cmdline"] += " uvms.persist-home=1" + if args.mem is not None: + config["memory"]["size"] = parse_bytes(args.mem) + config["memory"]["hotplug_size"] = parse_bytes(args.mem) + + if "platform" not in config: + config["platform"] = {} + config["platform"]["oem_strings"] = [ + "io.systemd.credential:vmm.notify_socket=vsock-stream:2:8888", + *config["platform"].get("oem_strings", []), + ] gpud, gpud_path = cleanup.enter_context(ps.start_gpu()) @@ -590,48 +614,71 @@ def main(args, args_next, cleanup, ps): ps.exec(*ch_remote, "boot") ps.exec(*ch_remote, "info") + ready = False with ready_sock: - ready_sock.settimeout(20.0) - try: - con, _ = ready_sock.accept() - except: # noqa: E722 - print( - "CH didn't try connecting to the readiness notification socket" - ) # noqa: E501 - else: - with con: - msg = con.recv(128) - assert msg.startswith(b"READY=1"), msg + ready_sock.settimeout(8.0) + for _ in range(1048576): + if ready: + break + try: + con, _ = ready_sock.accept() + except: # noqa: E722 + print( + "WARNING: CH didn't try connecting to the readiness notification socket" # noqa: E501 + ) + ready = True + break + else: + with con: + msg = con.recv(1024) + for ln in msg.split(b"\n"): + ln = ln.strip() + print(ln) + # if ln.startswith(b"X_SYSTEMD_UNIT_ACTIVE=uvms-guest.service"): # noqa: E501 + if ln.startswith(b"READY=1"): # noqa: E501 + ready = True + break + + assert ready with connect_ch_vsock(ps.prefix + "/vsock.sock", 24601) as guest: for r in args.run: - try: - guest.send( - json.dumps( - { - "run": { - "argv": [r], - "EXTRA_PATH": [ - f"{a}/bin" for a in app_paths - ], # noqa: E501 - } - } - ).encode("utf8") - ) - res = guest.recv(8192) + res = {} + for _ in range(1): + if "status" in res: + break try: - res = json.loads(guest.recv(8192)) - except json.JSONDecodeError as e: - print(f"Couldn't interpret --run {r} response: {e} {res}") - continue - adverb = ( - "Successfully" - if res["status"] == "exec succeeded" - else "Failed to" # noqa: E501 - ) - print(f"{adverb} --run {r}: {res}") - except Exception as e: - print(f"Couldn't --run {r}: {repr(e)}") + guest.send( + json.dumps( + { + "run": { + "argv": [r], + "EXTRA_PATH": [ + f"{a}/bin" for a in app_paths + ], # noqa: E501 + } + } + ).encode("utf8") + ) + res = guest.recv(8192) + try: + res = json.loads(guest.recv(8192)) + except json.JSONDecodeError as e: + print( + f"Couldn't interpret --run {r} response: {e} {res}" + ) # noqa: E501 + res = {} + # res = {"status": "failed"} + except Exception as e: + print(f"Couldn't --run {r}: {repr(e)}") + if "status" not in res: + res["status"] = "fail" + adverb = ( + "Successfully" + if res["status"] == "exec succeeded" + else "Failed to" # noqa: E501 + ) + print(f"{adverb} --run {r}: {res}") try: ch.wait() except KeyboardInterrupt: diff --git a/pkgs/uvmslib/package.nix b/pkgs/uvmslib/package.nix new file mode 100644 index 0000000..1bd51d1 --- /dev/null +++ b/pkgs/uvmslib/package.nix @@ -0,0 +1,21 @@ +{ + lib, + python3Packages, +}: +python3Packages.buildPythonPackage { + pname = "uvmslib"; + version = "0.0.0"; + pyproject = true; + src = + let + fs = lib.fileset; + in + fs.toSource { + root = ./.; + fileset = fs.unions [ + ./pyproject.toml + ./uvmslib.py + ]; + }; + build-system = [ python3Packages.setuptools ]; +} diff --git a/pkgs/uvmslib/pyproject.toml b/pkgs/uvmslib/pyproject.toml new file mode 100644 index 0000000..5995470 --- /dev/null +++ b/pkgs/uvmslib/pyproject.toml @@ -0,0 +1,10 @@ +[build-system] +build-backend = "setuptools.build_meta" +requires = [ "setuptools" ] + +[project] +name = "uvms" +version = "0.0.0" + +[project.scripts] +"uvms-guest" = "uvmslib:guest_main" diff --git a/pkgs/uvms-guest/guest.py b/pkgs/uvmslib/uvmslib.py similarity index 97% rename from pkgs/uvms-guest/guest.py rename to pkgs/uvmslib/uvmslib.py index 9fd9397..d6279ae 100644 --- a/pkgs/uvms-guest/guest.py +++ b/pkgs/uvmslib/uvmslib.py @@ -59,7 +59,7 @@ class Processes: proc = None try: proc = self.popen( - req["run"]["argv"], + run["argv"], text=text, env=env, cwd="/home/user", @@ -82,14 +82,14 @@ class Processes: return res, proc def accept_vsock(self, s): - con, (cid, port) = serv.accept() + con, (cid, port) = s.accept() assert cid == 2, cid self.sources.append(con) self.client_fds.add(con.fileno()) return con, (cid, port) -if __name__ == "__main__": +def guest_main(): ps = Processes() serv = socket.fromfd(3, socket.AF_VSOCK, socket.SOCK_STREAM) ps.sources.append(serv) @@ -134,3 +134,7 @@ if __name__ == "__main__": con.send(res) else: assert False, con.fileno() + + +if __name__ == "__main__": + guest_main() diff --git a/profiles/baseImage.nix b/profiles/baseImage.nix index 2f9029a..683f061 100644 --- a/profiles/baseImage.nix +++ b/profiles/baseImage.nix @@ -42,7 +42,7 @@ in _module.args.uvmsPkgs = lib.mkDefault (pkgs.callPackage ../pkgs { }); # some.failure-handler.enable = true; hardware.graphics.enable = true; - boot.kernelPackages = pkgs.linuxPackagesFor uvmsPkgs.linux-uvm; + # boot.kernelPackages = pkgs.linuxPackagesFor uvmsPkgs.linux-uvm; # boot.isContainer = true; boot.initrd.kernelModules = [ "drm" @@ -251,19 +251,20 @@ in }; systemd.sockets."uvms-guest" = { - wantedBy = [ "default.target" ]; + requiredBy = [ "multi-user.target" ]; + before = [ "multi-user.target" ]; listenStreams = [ "vsock::24601" ]; partOf = [ "uvms-guest.service" ]; }; systemd.services."uvms-guest" = { - requiredBy = [ "multi-user.target" ]; + before = [ "multi-user.target" ]; onFailure = [ "shutdown.service" ]; serviceConfig = { User = "user"; Group = "users"; - ExecStart = "${lib.getExe uvmsPkgs.uvms-guest}"; + ExecStart = "${lib.getExe' uvmsPkgs.uvmslib "uvms-guest"}"; ExecStop = [ "/run/current-system/sw/bin/echo GUEST DOWN" "/run/current-system/sw/bin/systemctl poweroff" @@ -288,7 +289,6 @@ in "console=ttyS0" "reboot=t" "panic=-1" - "io.systemd.credential:vmm.notify_socket=vsock-stream:2:8888" # "rootfstype=virtiofs" # "root=rootstore" ]; @@ -301,121 +301,146 @@ in }; uvms.ch.settings = mkOption { default = { }; - type = types.submodule { - freeformType = jsonType; - options = { - payload = { - cmdline = mkOption { - type = types.str; - default = concatStringsSep " " ( - config.boot.kernelParams - ++ [ - # "init=${lib.removePrefix "/nix/store" "${config.system.build.toplevel}"}/init" - "init=${config.system.build.toplevel}/init" - ] - ); - defaultText = ''concatStringsSep " " ${config.boot.kernelParams}''; + type = types.submodule ( + let + osConfig = config; + in + { config, ... }: + { + freeformType = jsonType; + options = { + payload = { + cmdline = mkOption { + type = types.str; + default = concatStringsSep " " ( + osConfig.boot.kernelParams + ++ [ + # "init=${lib.removePrefix "/nix/store" "${osConfig.system.build.toplevel}"}/init" + "init=${osConfig.system.build.toplevel}/init" + ] + ); + defaultText = ''concatStringsSep " " ${osConfig.boot.kernelParams}''; + }; + kernel = mkOption { + type = types.str; + default = "${kernel}/${kernelTarget}"; + }; + initramfs = mkOption { + type = types.nullOr types.str; + default = "${initialRamdisk}/${initrdFile}"; + }; }; - kernel = mkOption { - type = types.str; - default = "${kernel}/${kernelTarget}"; + vsock = { + cid = mkOption { + type = types.int; + default = 4; + }; + socket = mkOption { + type = types.str; + default = "vsock.sock"; + }; }; - initramfs = mkOption { + "api-socket" = mkOption { + type = types.str; + default = "vmm.sock"; + }; + "serial".mode = mkOption { + type = types.str; + default = "File"; + }; + "serial".file = mkOption { type = types.nullOr types.str; - default = "${initialRamdisk}/${initrdFile}"; + default = "serial"; }; - }; - vsock = { - cid = mkOption { - type = types.int; - default = 4; - }; - socket = mkOption { + "console".mode = mkOption { type = types.str; - default = "vsock.sock"; + default = "Pty"; }; - }; - "api-socket" = mkOption { - type = types.str; - default = "vmm.sock"; - }; - "serial".mode = mkOption { - type = types.str; - default = "File"; - }; - "serial".file = mkOption { - type = types.nullOr types.str; - default = "serial"; - }; - "console".mode = mkOption { - type = types.str; - default = "Pty"; - }; - "console".file = mkOption { - type = types.nullOr types.str; - default = null; - }; - # "watchdog" = true; - # "seccomp" = true; - disks = mkOption { - default = [ ]; - type = types.listOf ( - types.submodule { + "console".file = mkOption { + type = types.nullOr types.str; + default = null; + }; + # "watchdog" = true; + # "seccomp" = true; + disks = mkOption { + default = [ ]; + type = types.listOf ( + types.submodule { + freeformType = jsonType; + options = { + path = mkOption { + type = types.oneOf [ + types.path + types.str + ]; + }; + readonly = mkOption { + type = types.bool; + default = true; + }; + id = mkOption { type = types.str; }; + }; + } + ); + }; + memory = mkOption { + default = { }; + type = types.submodule { freeformType = jsonType; options = { - path = mkOption { - type = types.oneOf [ - types.path - types.str - ]; + size = mkOption { + type = types.int; + default = 4 * 1024 * 1048576; }; - readonly = mkOption { + shared = mkOption { type = types.bool; default = true; }; - id = mkOption { type = types.str; }; + mergeable = mkOption { + type = types.bool; + default = true; + }; + hotplug_method = mkOption { + default = "VirtioMem"; + type = types.enum [ + "Acpi" + "VirtioMem" + ]; + }; + hotplugged_size = mkOption { + type = types.int; + default = 512 * 1048576; + }; + hotplug_size = mkOption { + type = types.int; + default = config.memory.size; + }; + # hugepages = mkOption { + # type = types.bool; + # default = true; + # }; }; - } - ); - }; - memory = mkOption { - default = { }; - type = types.submodule { - freeformType = jsonType; - options = { - size = mkOption { - type = types.int; - default = 3 * 1024 * 1048576; - }; - shared = mkOption { - type = types.bool; - default = true; - }; - mergeable = mkOption { - type = types.bool; - default = true; + }; + }; + cpus = mkOption { + default = { }; + type = types.submodule { + freeformType = jsonType; + options = { + boot_vcpus = mkOption { + type = types.int; + default = 4; + }; + max_vcpus = mkOption { + type = types.int; + default = 4; + }; }; }; }; }; - cpus = mkOption { - default = { }; - type = types.submodule { - freeformType = jsonType; - options = { - boot_vcpus = mkOption { - type = types.int; - default = 4; - }; - max_vcpus = mkOption { - type = types.int; - default = 4; - }; - }; - }; - }; - }; - }; + } + ); }; }; }