pkgs/uvms-guest/guest.py

@@ -59,7 +59,7 @@ class Processes:
         proc = None
         try:
             proc = self.popen(
-                run["argv"],
+                req["run"]["argv"],
                 text=text,
                 env=env,
                 cwd="/home/user",
@@ -82,14 +82,14 @@ class Processes:
         return res, proc
 
     def accept_vsock(self, s):
-        con, (cid, port) = s.accept()
+        con, (cid, port) = serv.accept()
         assert cid == 2, cid
         self.sources.append(con)
         self.client_fds.add(con.fileno())
         return con, (cid, port)
 
 
-def guest_main():
+if __name__ == "__main__":
     ps = Processes()
     serv = socket.fromfd(3, socket.AF_VSOCK, socket.SOCK_STREAM)
     ps.sources.append(serv)
@@ -134,7 +134,3 @@ def guest_main():
                 con.send(res)
             else:
                 assert False, con.fileno()
-
-
-if __name__ == "__main__":
-    guest_main()

pkgs/uvms-guest/package.nix

@@ -0,0 +1,5 @@
+{
+  lib,
+  writers,
+}:
+writers.writePython3Bin "uvms-guest" { } ./guest.py

pkgs/uvms/package.nix

@@ -13,8 +13,6 @@
   strace,
   util-linux,
   virtiofsd,
-  python3Packages,
-  uvmslib,
 
   taps,
   baseImage,
@@ -38,25 +36,21 @@ let
   };
   toolsClosure = writeClosure toolsFarm;
 in
-writers.writePython3Bin "uvms"
-  {
-    libraries = [ uvmslib ];
-  }
-  (
-    replaceVars ./uvms.py {
-      BWRAP = "${lib.getExe bubblewrap}";
-      TOOLS = "${toolsFarm}/bin";
-      TOOLS_CLOSURE = toolsClosure;
-      CROSVM = lib.getExe crosvm;
-      STRACE = lib.getExe strace;
-      TAPS = "${lib.getExe taps}";
-      VIRTIOFSD = "${lib.getExe virtiofsd}";
+writers.writePython3Bin "uvms" { } (
+  replaceVars ./uvms.py {
+    BWRAP = "${lib.getExe bubblewrap}";
+    TOOLS = "${toolsFarm}/bin";
+    TOOLS_CLOSURE = toolsClosure;
+    CROSVM = lib.getExe crosvm;
+    STRACE = lib.getExe strace;
+    TAPS = "${lib.getExe taps}";
+    VIRTIOFSD = "${lib.getExe virtiofsd}";
 
-      BASE_CONFIG = baseImage.config.system.build.ch;
-      SYSTEM = baseImage.config.system.build.toplevel;
-      SYSTEM_CLOSURE = writeClosure [
-        baseImage.config.system.build.toplevel
-        baseImage.config.system.build.ch
-      ];
-    }
-  )
+    BASE_CONFIG = baseImage.config.system.build.ch;
+    SYSTEM = baseImage.config.system.build.toplevel;
+    SYSTEM_CLOSURE = writeClosure [
+      baseImage.config.system.build.toplevel
+      baseImage.config.system.build.ch
+    ];
+  }
+)

pkgs/uvms/uvms.py

@@ -10,7 +10,6 @@ import os
 import subprocess
 import socket
 import json
-import re
 from argparse import ArgumentParser
 from contextlib import contextmanager, closing, ExitStack
 
@@ -21,7 +20,6 @@ parser.add_argument("--prefix", default="$HOME/uvms/$VM")
 parser.add_argument("--vm-config", default="@BASE_CONFIG@")  # noqa: E501
 parser.add_argument("--persist-home", action="store_true")
 parser.add_argument("--run", action="append")
-parser.add_argument("--mem", default=None)
 parser.add_argument("app", nargs="*", default=())
 
 TOOLS_DIR = "@TOOLS@"  # noqa: E501
@@ -299,7 +297,6 @@ class Processes:
             # "-Z",
             # "-ff",
             CH,
-            # "-v",
             "--api-socket",
             "fd=0",
             # f"fd={s.fileno()}"
@@ -321,8 +318,6 @@ class Processes:
                     # ro_bind=["/nix/store"],  # I give up
                     unshare_net=False,
                     shell=False,
-                    stdout=None,
-                    stderr=None,
                     # pass_fds=(s.fileno(),)
                 )
             )
@@ -343,23 +338,29 @@ class Processes:
     ):
         sock_path = self.prefix + "/gpu.sock"
         args = [
+            SOCKETBINDER,
+            "-b",
+            "1",
+            sock_path,
+            "s6-ipcserverd",
+            "-1c1",
+            # "@STRACE@",  # noqa: E501
+            # "-Z",
+            # "-ff",
             "@CROSVM@",  # noqa: E501
             "--no-syslog",
-            "--log-level",
-            "debug",
             "device",
             "gpu",
-            "--socket-path",
-            sock_path,
+            "--fd",
+            "0",
             "--wayland-sock",
             f'{PASSTHRU_ENV["XDG_RUNTIME_DIR"]}/{PASSTHRU_ENV["WAYLAND_DISPLAY"]}',  # noqa: E501
             "--params",
-            '{ "context-types": "cross-domain" }',
+            '{ "context-types": "cross-domain:virgl2:venus" }',
         ]
         with self.popen(
             *args,
             stderr=None,
-            stdout=None,
         ) as proc, removing(sock_path, sock_path + ".lock"):
             yield proc, sock_path
 
@@ -468,21 +469,6 @@ def connect_ch_vsock(
         yield s
 
 
-BYTES_PATTERN = re.compile(r"^([0-9]+)([MmGgKk]?)$")
-BYTES_UNITS = {
-    "k": 1024,
-    "m": 1048576,
-    "g": 1024 * 1048576,
-}
-
-
-def parse_bytes(s):
-    m = BYTES_PATTERN.match(s)
-    assert m, s
-    size, unit = m.groups()
-    return int(size) * BYTES_UNITS.get(unit.lower(), 1)
-
-
 @contextmanager
 def listen_ch_vsock(
     vsock_sock_path,
@@ -578,16 +564,6 @@ def main(args, args_next, cleanup, ps):
         )
         virtiofs_socks.append(("home", sock_path))
         config["payload"]["cmdline"] += " uvms.persist-home=1"
-    if args.mem is not None:
-        config["memory"]["size"] = parse_bytes(args.mem)
-        config["memory"]["hotplug_size"] = parse_bytes(args.mem)
-
-    if "platform" not in config:
-        config["platform"] = {}
-    config["platform"]["oem_strings"] = [
-        "io.systemd.credential:vmm.notify_socket=vsock-stream:2:8888",
-        *config["platform"].get("oem_strings", []),
-    ]
 
     gpud, gpud_path = cleanup.enter_context(ps.start_gpu())
 
@@ -614,71 +590,48 @@ def main(args, args_next, cleanup, ps):
     ps.exec(*ch_remote, "boot")
     ps.exec(*ch_remote, "info")
 
-    ready = False
     with ready_sock:
-        ready_sock.settimeout(8.0)
-        for _ in range(1048576):
-            if ready:
-                break
-            try:
-                con, _ = ready_sock.accept()
-            except:  # noqa: E722
-                print(
-                    "WARNING: CH didn't try connecting to the readiness notification socket"  # noqa: E501
-                )
-                ready = True
-                break
-            else:
-                with con:
-                    msg = con.recv(1024)
-                    for ln in msg.split(b"\n"):
-                        ln = ln.strip()
-                        print(ln)
-                        # if ln.startswith(b"X_SYSTEMD_UNIT_ACTIVE=uvms-guest.service"):  # noqa: E501
-                        if ln.startswith(b"READY=1"):  # noqa: E501
-                            ready = True
-                            break
-
-    assert ready
+        ready_sock.settimeout(20.0)
+        try:
+            con, _ = ready_sock.accept()
+        except:  # noqa: E722
+            print(
+                "CH didn't try connecting to the readiness notification socket"
+            )  # noqa: E501
+        else:
+            with con:
+                msg = con.recv(128)
+                assert msg.startswith(b"READY=1"), msg
 
     with connect_ch_vsock(ps.prefix + "/vsock.sock", 24601) as guest:
         for r in args.run:
-            res = {}
-            for _ in range(1):
-                if "status" in res:
-                    break
-                try:
-                    guest.send(
-                        json.dumps(
-                            {
-                                "run": {
-                                    "argv": [r],
-                                    "EXTRA_PATH": [
-                                        f"{a}/bin" for a in app_paths
-                                    ],  # noqa: E501
-                                }
+            try:
+                guest.send(
+                    json.dumps(
+                        {
+                            "run": {
+                                "argv": [r],
+                                "EXTRA_PATH": [
+                                    f"{a}/bin" for a in app_paths
+                                ],  # noqa: E501
                             }
-                        ).encode("utf8")
-                    )
-                    res = guest.recv(8192)
-                    try:
-                        res = json.loads(guest.recv(8192))
-                    except json.JSONDecodeError as e:
-                        print(
-                            f"Couldn't interpret --run {r} response: {e} {res}"
-                        )  # noqa: E501
-                        res = {}
-                        # res = {"status": "failed"}
-                except Exception as e:
-                    print(f"Couldn't --run {r}: {repr(e)}")
-            if "status" not in res:
-                res["status"] = "fail"
-            adverb = (
-                "Successfully"
-                if res["status"] == "exec succeeded"
-                else "Failed to"  # noqa: E501
-            )
-            print(f"{adverb} --run {r}: {res}")
+                        }
+                    ).encode("utf8")
+                )
+                res = guest.recv(8192)
+                try:
+                    res = json.loads(guest.recv(8192))
+                except json.JSONDecodeError as e:
+                    print(f"Couldn't interpret --run {r} response: {e} {res}")
+                    continue
+                adverb = (
+                    "Successfully"
+                    if res["status"] == "exec succeeded"
+                    else "Failed to"  # noqa: E501
+                )
+                print(f"{adverb} --run {r}: {res}")
+            except Exception as e:
+                print(f"Couldn't --run {r}: {repr(e)}")
     try:
         ch.wait()
     except KeyboardInterrupt:

pkgs/uvmslib/package.nix

@@ -1,21 +0,0 @@
-{
-  lib,
-  python3Packages,
-}:
-python3Packages.buildPythonPackage {
-  pname = "uvmslib";
-  version = "0.0.0";
-  pyproject = true;
-  src =
-    let
-      fs = lib.fileset;
-    in
-    fs.toSource {
-      root = ./.;
-      fileset = fs.unions [
-        ./pyproject.toml
-        ./uvmslib.py
-      ];
-    };
-  build-system = [ python3Packages.setuptools ];
-}

pkgs/uvmslib/pyproject.toml

@@ -1,10 +0,0 @@
-[build-system]
-build-backend = "setuptools.build_meta"
-requires = [ "setuptools" ]
-
-[project]
-name = "uvms"
-version = "0.0.0"
-
-[project.scripts]
-"uvms-guest" = "uvmslib:guest_main"

profiles/baseImage.nix

@@ -42,7 +42,7 @@ in
     _module.args.uvmsPkgs = lib.mkDefault (pkgs.callPackage ../pkgs { });
     # some.failure-handler.enable = true;
     hardware.graphics.enable = true;
-    # boot.kernelPackages = pkgs.linuxPackagesFor uvmsPkgs.linux-uvm;
+    boot.kernelPackages = pkgs.linuxPackagesFor uvmsPkgs.linux-uvm;
     # boot.isContainer = true;
     boot.initrd.kernelModules = [
       "drm"
@@ -251,20 +251,19 @@ in
     };
 
     systemd.sockets."uvms-guest" = {
-      requiredBy = [ "multi-user.target" ];
-      before = [ "multi-user.target" ];
+      wantedBy = [ "default.target" ];
       listenStreams = [
         "vsock::24601"
       ];
       partOf = [ "uvms-guest.service" ];
     };
     systemd.services."uvms-guest" = {
-      before = [ "multi-user.target" ];
+      requiredBy = [ "multi-user.target" ];
       onFailure = [ "shutdown.service" ];
       serviceConfig = {
         User = "user";
         Group = "users";
-        ExecStart = "${lib.getExe' uvmsPkgs.uvmslib "uvms-guest"}";
+        ExecStart = "${lib.getExe uvmsPkgs.uvms-guest}";
         ExecStop = [
           "/run/current-system/sw/bin/echo GUEST DOWN"
           "/run/current-system/sw/bin/systemctl poweroff"
@@ -289,6 +288,7 @@ in
       "console=ttyS0"
       "reboot=t"
       "panic=-1"
+      "io.systemd.credential:vmm.notify_socket=vsock-stream:2:8888"
       # "rootfstype=virtiofs"
       # "root=rootstore"
     ];
@@ -301,146 +301,121 @@ in
     };
     uvms.ch.settings = mkOption {
       default = { };
-      type = types.submodule (
-      let
-      osConfig = config;
-      in
-        { config, ... }:
-        {
-          freeformType = jsonType;
-          options = {
-            payload = {
-              cmdline = mkOption {
-                type = types.str;
-                default = concatStringsSep " " (
-                  osConfig.boot.kernelParams
-                  ++ [
-                    # "init=${lib.removePrefix "/nix/store" "${osConfig.system.build.toplevel}"}/init"
-                    "init=${osConfig.system.build.toplevel}/init"
-                  ]
-                );
-                defaultText = ''concatStringsSep " " ${osConfig.boot.kernelParams}'';
-              };
-              kernel = mkOption {
-                type = types.str;
-                default = "${kernel}/${kernelTarget}";
-              };
-              initramfs = mkOption {
-                type = types.nullOr types.str;
-                default = "${initialRamdisk}/${initrdFile}";
-              };
-            };
-            vsock = {
-              cid = mkOption {
-                type = types.int;
-                default = 4;
-              };
-              socket = mkOption {
-                type = types.str;
-                default = "vsock.sock";
-              };
-            };
-            "api-socket" = mkOption {
+      type = types.submodule {
+        freeformType = jsonType;
+        options = {
+          payload = {
+            cmdline = mkOption {
               type = types.str;
-              default = "vmm.sock";
-            };
-            "serial".mode = mkOption {
-              type = types.str;
-              default = "File";
-            };
-            "serial".file = mkOption {
-              type = types.nullOr types.str;
-              default = "serial";
-            };
-            "console".mode = mkOption {
-              type = types.str;
-              default = "Pty";
-            };
-            "console".file = mkOption {
-              type = types.nullOr types.str;
-              default = null;
-            };
-            # "watchdog" = true;
-            # "seccomp" = true;
-            disks = mkOption {
-              default = [ ];
-              type = types.listOf (
-                types.submodule {
-                  freeformType = jsonType;
-                  options = {
-                    path = mkOption {
-                      type = types.oneOf [
-                        types.path
-                        types.str
-                      ];
-                    };
-                    readonly = mkOption {
-                      type = types.bool;
-                      default = true;
-                    };
-                    id = mkOption { type = types.str; };
-                  };
-                }
+              default = concatStringsSep " " (
+                config.boot.kernelParams
+                ++ [
+                  # "init=${lib.removePrefix "/nix/store" "${config.system.build.toplevel}"}/init"
+                  "init=${config.system.build.toplevel}/init"
+                ]
               );
+              defaultText = ''concatStringsSep " " ${config.boot.kernelParams}'';
             };
-            memory = mkOption {
-              default = { };
-              type = types.submodule {
+            kernel = mkOption {
+              type = types.str;
+              default = "${kernel}/${kernelTarget}";
+            };
+            initramfs = mkOption {
+              type = types.nullOr types.str;
+              default = "${initialRamdisk}/${initrdFile}";
+            };
+          };
+          vsock = {
+            cid = mkOption {
+              type = types.int;
+              default = 4;
+            };
+            socket = mkOption {
+              type = types.str;
+              default = "vsock.sock";
+            };
+          };
+          "api-socket" = mkOption {
+            type = types.str;
+            default = "vmm.sock";
+          };
+          "serial".mode = mkOption {
+            type = types.str;
+            default = "File";
+          };
+          "serial".file = mkOption {
+            type = types.nullOr types.str;
+            default = "serial";
+          };
+          "console".mode = mkOption {
+            type = types.str;
+            default = "Pty";
+          };
+          "console".file = mkOption {
+            type = types.nullOr types.str;
+            default = null;
+          };
+          # "watchdog" = true;
+          # "seccomp" = true;
+          disks = mkOption {
+            default = [ ];
+            type = types.listOf (
+              types.submodule {
                 freeformType = jsonType;
                 options = {
-                  size = mkOption {
-                    type = types.int;
-                    default = 4 * 1024 * 1048576;
-                  };
-                  shared = mkOption {
-                    type = types.bool;
-                    default = true;
-                  };
-                  mergeable = mkOption {
-                    type = types.bool;
-                    default = true;
-                  };
-                  hotplug_method = mkOption {
-                    default = "VirtioMem";
-                    type = types.enum [
-                      "Acpi"
-                      "VirtioMem"
+                  path = mkOption {
+                    type = types.oneOf [
+                      types.path
+                      types.str
                     ];
                   };
-                  hotplugged_size = mkOption {
-                    type = types.int;
-                    default = 512 * 1048576;
+                  readonly = mkOption {
+                    type = types.bool;
+                    default = true;
                   };
-                  hotplug_size = mkOption {
-                    type = types.int;
-                    default = config.memory.size;
-                  };
-                  # hugepages = mkOption {
-                  #   type = types.bool;
-                  #   default = true;
-                  # };
+                  id = mkOption { type = types.str; };
                 };
-              };
-            };
-            cpus = mkOption {
-              default = { };
-              type = types.submodule {
-                freeformType = jsonType;
-                options = {
-                  boot_vcpus = mkOption {
-                    type = types.int;
-                    default = 4;
-                  };
-                  max_vcpus = mkOption {
-                    type = types.int;
-                    default = 4;
-                  };
+              }
+            );
+          };
+          memory = mkOption {
+            default = { };
+            type = types.submodule {
+              freeformType = jsonType;
+              options = {
+                size = mkOption {
+                  type = types.int;
+                  default = 3 * 1024 * 1048576;
+                };
+                shared = mkOption {
+                  type = types.bool;
+                  default = true;
+                };
+                mergeable = mkOption {
+                  type = types.bool;
+                  default = true;
                 };
               };
             };
           };
-        }
-      );
+          cpus = mkOption {
+            default = { };
+            type = types.submodule {
+              freeformType = jsonType;
+              options = {
+                boot_vcpus = mkOption {
+                  type = types.int;
+                  default = 4;
+                };
+                max_vcpus = mkOption {
+                  type = types.int;
+                  default = 4;
+                };
+              };
+            };
+          };
+        };
+      };
     };
   };
 }