uvms: churn

Yeah, yeah, I know, it's stupid, adding add_argument() statements
manually, et c. I'll throw the whole Python thing away whenever I might
have the time

pkgs/uvms-guest/package.nix

@@ -1,5 +0,0 @@
-{
-  lib,
-  writers,
-}:
-writers.writePython3Bin "uvms-guest" { } ./guest.py

pkgs/uvms/package.nix

@@ -13,6 +13,8 @@
   strace,
   util-linux,
   virtiofsd,
+  python3Packages,
+  uvmslib,
 
   taps,
   baseImage,
@@ -36,21 +38,25 @@ let
   };
   toolsClosure = writeClosure toolsFarm;
 in
-writers.writePython3Bin "uvms" { } (
-  replaceVars ./uvms.py {
-    BWRAP = "${lib.getExe bubblewrap}";
-    TOOLS = "${toolsFarm}/bin";
-    TOOLS_CLOSURE = toolsClosure;
-    CROSVM = lib.getExe crosvm;
-    STRACE = lib.getExe strace;
-    TAPS = "${lib.getExe taps}";
-    VIRTIOFSD = "${lib.getExe virtiofsd}";
-
-    BASE_CONFIG = baseImage.config.system.build.ch;
-    SYSTEM = baseImage.config.system.build.toplevel;
-    SYSTEM_CLOSURE = writeClosure [
-      baseImage.config.system.build.toplevel
-      baseImage.config.system.build.ch
-    ];
+writers.writePython3Bin "uvms"
+  {
+    libraries = [ uvmslib ];
   }
-)
+  (
+    replaceVars ./uvms.py {
+      BWRAP = "${lib.getExe bubblewrap}";
+      TOOLS = "${toolsFarm}/bin";
+      TOOLS_CLOSURE = toolsClosure;
+      CROSVM = lib.getExe crosvm;
+      STRACE = lib.getExe strace;
+      TAPS = "${lib.getExe taps}";
+      VIRTIOFSD = "${lib.getExe virtiofsd}";
+
+      BASE_CONFIG = baseImage.config.system.build.ch;
+      SYSTEM = baseImage.config.system.build.toplevel;
+      SYSTEM_CLOSURE = writeClosure [
+        baseImage.config.system.build.toplevel
+        baseImage.config.system.build.ch
+      ];
+    }
+  )

pkgs/uvms/uvms.py

@@ -10,6 +10,7 @@ import os
 import subprocess
 import socket
 import json
+import re
 from argparse import ArgumentParser
 from contextlib import contextmanager, closing, ExitStack
 
@@ -20,6 +21,7 @@ parser.add_argument("--prefix", default="$HOME/uvms/$VM")
 parser.add_argument("--vm-config", default="@BASE_CONFIG@")  # noqa: E501
 parser.add_argument("--persist-home", action="store_true")
 parser.add_argument("--run", action="append")
+parser.add_argument("--mem", default=None)
 parser.add_argument("app", nargs="*", default=())
 
 TOOLS_DIR = "@TOOLS@"  # noqa: E501
@@ -297,6 +299,7 @@ class Processes:
             # "-Z",
             # "-ff",
             CH,
+            # "-v",
             "--api-socket",
             "fd=0",
             # f"fd={s.fileno()}"
@@ -318,6 +321,8 @@ class Processes:
                     # ro_bind=["/nix/store"],  # I give up
                     unshare_net=False,
                     shell=False,
+                    stdout=None,
+                    stderr=None,
                     # pass_fds=(s.fileno(),)
                 )
             )
@@ -338,29 +343,23 @@ class Processes:
     ):
         sock_path = self.prefix + "/gpu.sock"
         args = [
-            SOCKETBINDER,
-            "-b",
-            "1",
-            sock_path,
-            "s6-ipcserverd",
-            "-1c1",
-            # "@STRACE@",  # noqa: E501
-            # "-Z",
-            # "-ff",
             "@CROSVM@",  # noqa: E501
             "--no-syslog",
+            "--log-level",
+            "debug",
             "device",
             "gpu",
-            "--fd",
-            "0",
+            "--socket-path",
+            sock_path,
             "--wayland-sock",
             f'{PASSTHRU_ENV["XDG_RUNTIME_DIR"]}/{PASSTHRU_ENV["WAYLAND_DISPLAY"]}',  # noqa: E501
             "--params",
-            '{ "context-types": "cross-domain:virgl2:venus" }',
+            '{ "context-types": "cross-domain" }',
         ]
         with self.popen(
             *args,
             stderr=None,
+            stdout=None,
         ) as proc, removing(sock_path, sock_path + ".lock"):
             yield proc, sock_path
 
@@ -469,6 +468,21 @@ def connect_ch_vsock(
         yield s
 
 
+BYTES_PATTERN = re.compile(r"^([0-9]+)([MmGgKk]?)$")
+BYTES_UNITS = {
+    "k": 1024,
+    "m": 1048576,
+    "g": 1024 * 1048576,
+}
+
+
+def parse_bytes(s):
+    m = BYTES_PATTERN.match(s)
+    assert m, s
+    size, unit = m.groups()
+    return int(size) * BYTES_UNITS.get(unit.lower(), 1)
+
+
 @contextmanager
 def listen_ch_vsock(
     vsock_sock_path,
@@ -564,6 +578,16 @@ def main(args, args_next, cleanup, ps):
         )
         virtiofs_socks.append(("home", sock_path))
         config["payload"]["cmdline"] += " uvms.persist-home=1"
+    if args.mem is not None:
+        config["memory"]["size"] = parse_bytes(args.mem)
+        config["memory"]["hotplug_size"] = parse_bytes(args.mem)
+
+    if "platform" not in config:
+        config["platform"] = {}
+    config["platform"]["oem_strings"] = [
+        "io.systemd.credential:vmm.notify_socket=vsock-stream:2:8888",
+        *config["platform"].get("oem_strings", []),
+    ]
 
     gpud, gpud_path = cleanup.enter_context(ps.start_gpu())
 
@@ -590,48 +614,71 @@ def main(args, args_next, cleanup, ps):
     ps.exec(*ch_remote, "boot")
     ps.exec(*ch_remote, "info")
 
+    ready = False
     with ready_sock:
-        ready_sock.settimeout(20.0)
-        try:
-            con, _ = ready_sock.accept()
-        except:  # noqa: E722
-            print(
-                "CH didn't try connecting to the readiness notification socket"
-            )  # noqa: E501
-        else:
-            with con:
-                msg = con.recv(128)
-                assert msg.startswith(b"READY=1"), msg
+        ready_sock.settimeout(8.0)
+        for _ in range(1048576):
+            if ready:
+                break
+            try:
+                con, _ = ready_sock.accept()
+            except:  # noqa: E722
+                print(
+                    "WARNING: CH didn't try connecting to the readiness notification socket"  # noqa: E501
+                )
+                ready = True
+                break
+            else:
+                with con:
+                    msg = con.recv(1024)
+                    for ln in msg.split(b"\n"):
+                        ln = ln.strip()
+                        print(ln)
+                        # if ln.startswith(b"X_SYSTEMD_UNIT_ACTIVE=uvms-guest.service"):  # noqa: E501
+                        if ln.startswith(b"READY=1"):  # noqa: E501
+                            ready = True
+                            break
+
+    assert ready
 
     with connect_ch_vsock(ps.prefix + "/vsock.sock", 24601) as guest:
         for r in args.run:
-            try:
-                guest.send(
-                    json.dumps(
-                        {
-                            "run": {
-                                "argv": [r],
-                                "EXTRA_PATH": [
-                                    f"{a}/bin" for a in app_paths
-                                ],  # noqa: E501
-                            }
-                        }
-                    ).encode("utf8")
-                )
-                res = guest.recv(8192)
+            res = {}
+            for _ in range(1):
+                if "status" in res:
+                    break
                 try:
-                    res = json.loads(guest.recv(8192))
-                except json.JSONDecodeError as e:
-                    print(f"Couldn't interpret --run {r} response: {e} {res}")
-                    continue
-                adverb = (
-                    "Successfully"
-                    if res["status"] == "exec succeeded"
-                    else "Failed to"  # noqa: E501
-                )
-                print(f"{adverb} --run {r}: {res}")
-            except Exception as e:
-                print(f"Couldn't --run {r}: {repr(e)}")
+                    guest.send(
+                        json.dumps(
+                            {
+                                "run": {
+                                    "argv": [r],
+                                    "EXTRA_PATH": [
+                                        f"{a}/bin" for a in app_paths
+                                    ],  # noqa: E501
+                                }
+                            }
+                        ).encode("utf8")
+                    )
+                    res = guest.recv(8192)
+                    try:
+                        res = json.loads(guest.recv(8192))
+                    except json.JSONDecodeError as e:
+                        print(
+                            f"Couldn't interpret --run {r} response: {e} {res}"
+                        )  # noqa: E501
+                        res = {}
+                        # res = {"status": "failed"}
+                except Exception as e:
+                    print(f"Couldn't --run {r}: {repr(e)}")
+            if "status" not in res:
+                res["status"] = "fail"
+            adverb = (
+                "Successfully"
+                if res["status"] == "exec succeeded"
+                else "Failed to"  # noqa: E501
+            )
+            print(f"{adverb} --run {r}: {res}")
     try:
         ch.wait()
     except KeyboardInterrupt:

pkgs/uvmslib/package.nix

@@ -0,0 +1,21 @@
+{
+  lib,
+  python3Packages,
+}:
+python3Packages.buildPythonPackage {
+  pname = "uvmslib";
+  version = "0.0.0";
+  pyproject = true;
+  src =
+    let
+      fs = lib.fileset;
+    in
+    fs.toSource {
+      root = ./.;
+      fileset = fs.unions [
+        ./pyproject.toml
+        ./uvmslib.py
+      ];
+    };
+  build-system = [ python3Packages.setuptools ];
+}

pkgs/uvmslib/pyproject.toml

@@ -0,0 +1,10 @@
+[build-system]
+build-backend = "setuptools.build_meta"
+requires = [ "setuptools" ]
+
+[project]
+name = "uvms"
+version = "0.0.0"
+
+[project.scripts]
+"uvms-guest" = "uvmslib:guest_main"

pkgs/uvmslib/uvmslib.py

@@ -59,7 +59,7 @@ class Processes:
         proc = None
         try:
             proc = self.popen(
-                req["run"]["argv"],
+                run["argv"],
                 text=text,
                 env=env,
                 cwd="/home/user",
@@ -82,14 +82,14 @@ class Processes:
         return res, proc
 
     def accept_vsock(self, s):
-        con, (cid, port) = serv.accept()
+        con, (cid, port) = s.accept()
         assert cid == 2, cid
         self.sources.append(con)
         self.client_fds.add(con.fileno())
         return con, (cid, port)
 
 
-if __name__ == "__main__":
+def guest_main():
     ps = Processes()
     serv = socket.fromfd(3, socket.AF_VSOCK, socket.SOCK_STREAM)
     ps.sources.append(serv)
@@ -134,3 +134,7 @@ if __name__ == "__main__":
                 con.send(res)
             else:
                 assert False, con.fileno()
+
+
+if __name__ == "__main__":
+    guest_main()

profiles/baseImage.nix

@@ -42,7 +42,7 @@ in
     _module.args.uvmsPkgs = lib.mkDefault (pkgs.callPackage ../pkgs { });
     # some.failure-handler.enable = true;
     hardware.graphics.enable = true;
-    boot.kernelPackages = pkgs.linuxPackagesFor uvmsPkgs.linux-uvm;
+    # boot.kernelPackages = pkgs.linuxPackagesFor uvmsPkgs.linux-uvm;
     # boot.isContainer = true;
     boot.initrd.kernelModules = [
       "drm"
@@ -251,19 +251,20 @@ in
     };
 
     systemd.sockets."uvms-guest" = {
-      wantedBy = [ "default.target" ];
+      requiredBy = [ "multi-user.target" ];
+      before = [ "multi-user.target" ];
       listenStreams = [
         "vsock::24601"
       ];
       partOf = [ "uvms-guest.service" ];
     };
     systemd.services."uvms-guest" = {
-      requiredBy = [ "multi-user.target" ];
+      before = [ "multi-user.target" ];
       onFailure = [ "shutdown.service" ];
       serviceConfig = {
         User = "user";
         Group = "users";
-        ExecStart = "${lib.getExe uvmsPkgs.uvms-guest}";
+        ExecStart = "${lib.getExe' uvmsPkgs.uvmslib "uvms-guest"}";
         ExecStop = [
           "/run/current-system/sw/bin/echo GUEST DOWN"
           "/run/current-system/sw/bin/systemctl poweroff"
@@ -288,7 +289,6 @@ in
       "console=ttyS0"
       "reboot=t"
       "panic=-1"
-      "io.systemd.credential:vmm.notify_socket=vsock-stream:2:8888"
       # "rootfstype=virtiofs"
       # "root=rootstore"
     ];
@@ -301,121 +301,146 @@ in
     };
     uvms.ch.settings = mkOption {
       default = { };
-      type = types.submodule {
-        freeformType = jsonType;
-        options = {
-          payload = {
-            cmdline = mkOption {
-              type = types.str;
-              default = concatStringsSep " " (
-                config.boot.kernelParams
-                ++ [
-                  # "init=${lib.removePrefix "/nix/store" "${config.system.build.toplevel}"}/init"
-                  "init=${config.system.build.toplevel}/init"
-                ]
-              );
-              defaultText = ''concatStringsSep " " ${config.boot.kernelParams}'';
+      type = types.submodule (
+      let
+      osConfig = config;
+      in
+        { config, ... }:
+        {
+          freeformType = jsonType;
+          options = {
+            payload = {
+              cmdline = mkOption {
+                type = types.str;
+                default = concatStringsSep " " (
+                  osConfig.boot.kernelParams
+                  ++ [
+                    # "init=${lib.removePrefix "/nix/store" "${osConfig.system.build.toplevel}"}/init"
+                    "init=${osConfig.system.build.toplevel}/init"
+                  ]
+                );
+                defaultText = ''concatStringsSep " " ${osConfig.boot.kernelParams}'';
+              };
+              kernel = mkOption {
+                type = types.str;
+                default = "${kernel}/${kernelTarget}";
+              };
+              initramfs = mkOption {
+                type = types.nullOr types.str;
+                default = "${initialRamdisk}/${initrdFile}";
+              };
             };
-            kernel = mkOption {
-              type = types.str;
-              default = "${kernel}/${kernelTarget}";
+            vsock = {
+              cid = mkOption {
+                type = types.int;
+                default = 4;
+              };
+              socket = mkOption {
+                type = types.str;
+                default = "vsock.sock";
+              };
             };
-            initramfs = mkOption {
+            "api-socket" = mkOption {
+              type = types.str;
+              default = "vmm.sock";
+            };
+            "serial".mode = mkOption {
+              type = types.str;
+              default = "File";
+            };
+            "serial".file = mkOption {
               type = types.nullOr types.str;
-              default = "${initialRamdisk}/${initrdFile}";
+              default = "serial";
             };
-          };
-          vsock = {
-            cid = mkOption {
-              type = types.int;
-              default = 4;
-            };
-            socket = mkOption {
+            "console".mode = mkOption {
               type = types.str;
-              default = "vsock.sock";
+              default = "Pty";
             };
-          };
-          "api-socket" = mkOption {
-            type = types.str;
-            default = "vmm.sock";
-          };
-          "serial".mode = mkOption {
-            type = types.str;
-            default = "File";
-          };
-          "serial".file = mkOption {
-            type = types.nullOr types.str;
-            default = "serial";
-          };
-          "console".mode = mkOption {
-            type = types.str;
-            default = "Pty";
-          };
-          "console".file = mkOption {
-            type = types.nullOr types.str;
-            default = null;
-          };
-          # "watchdog" = true;
-          # "seccomp" = true;
-          disks = mkOption {
-            default = [ ];
-            type = types.listOf (
-              types.submodule {
+            "console".file = mkOption {
+              type = types.nullOr types.str;
+              default = null;
+            };
+            # "watchdog" = true;
+            # "seccomp" = true;
+            disks = mkOption {
+              default = [ ];
+              type = types.listOf (
+                types.submodule {
+                  freeformType = jsonType;
+                  options = {
+                    path = mkOption {
+                      type = types.oneOf [
+                        types.path
+                        types.str
+                      ];
+                    };
+                    readonly = mkOption {
+                      type = types.bool;
+                      default = true;
+                    };
+                    id = mkOption { type = types.str; };
+                  };
+                }
+              );
+            };
+            memory = mkOption {
+              default = { };
+              type = types.submodule {
                 freeformType = jsonType;
                 options = {
-                  path = mkOption {
-                    type = types.oneOf [
-                      types.path
-                      types.str
-                    ];
+                  size = mkOption {
+                    type = types.int;
+                    default = 4 * 1024 * 1048576;
                   };
-                  readonly = mkOption {
+                  shared = mkOption {
                     type = types.bool;
                     default = true;
                   };
-                  id = mkOption { type = types.str; };
+                  mergeable = mkOption {
+                    type = types.bool;
+                    default = true;
+                  };
+                  hotplug_method = mkOption {
+                    default = "VirtioMem";
+                    type = types.enum [
+                      "Acpi"
+                      "VirtioMem"
+                    ];
+                  };
+                  hotplugged_size = mkOption {
+                    type = types.int;
+                    default = 512 * 1048576;
+                  };
+                  hotplug_size = mkOption {
+                    type = types.int;
+                    default = config.memory.size;
+                  };
+                  # hugepages = mkOption {
+                  #   type = types.bool;
+                  #   default = true;
+                  # };
                 };
-              }
-            );
-          };
-          memory = mkOption {
-            default = { };
-            type = types.submodule {
-              freeformType = jsonType;
-              options = {
-                size = mkOption {
-                  type = types.int;
-                  default = 3 * 1024 * 1048576;
-                };
-                shared = mkOption {
-                  type = types.bool;
-                  default = true;
-                };
-                mergeable = mkOption {
-                  type = types.bool;
-                  default = true;
+              };
+            };
+            cpus = mkOption {
+              default = { };
+              type = types.submodule {
+                freeformType = jsonType;
+                options = {
+                  boot_vcpus = mkOption {
+                    type = types.int;
+                    default = 4;
+                  };
+                  max_vcpus = mkOption {
+                    type = types.int;
+                    default = 4;
+                  };
                 };
               };
             };
           };
-          cpus = mkOption {
-            default = { };
-            type = types.submodule {
-              freeformType = jsonType;
-              options = {
-                boot_vcpus = mkOption {
-                  type = types.int;
-                  default = 4;
-                };
-                max_vcpus = mkOption {
-                  type = types.int;
-                  default = 4;
-                };
-              };
-            };
-          };
-        };
-      };
+        }
+      );
     };
   };
 }