From 8bbafe0f0ac0a8131fb330bf82c32b68a144cdf5 Mon Sep 17 00:00:00 2001 From: "Else, Someone" Date: Thu, 15 Jan 2026 08:24:08 +0200 Subject: [PATCH 01/37] taps: wip: init --- pkgs/ch-proxy/sendfd.h | 9 ++ pkgs/taps/.envrc | 1 + pkgs/taps/main.c | 241 +++++++++++++++++++++++++++++++++++++++++ pkgs/taps/meson.build | 4 + pkgs/taps/package.nix | 44 ++++++++ shell.nix | 10 ++ 6 files changed, 309 insertions(+) create mode 100644 pkgs/ch-proxy/sendfd.h create mode 100644 pkgs/taps/.envrc create mode 100644 pkgs/taps/main.c create mode 100644 pkgs/taps/meson.build create mode 100644 pkgs/taps/package.nix create mode 100644 shell.nix diff --git a/pkgs/ch-proxy/sendfd.h b/pkgs/ch-proxy/sendfd.h new file mode 100644 index 0000000..65042d8 --- /dev/null +++ b/pkgs/ch-proxy/sendfd.h @@ -0,0 +1,9 @@ +#ifndef _CH_PROXY_SENFD +#define _CH_PROXY_SENFD + +#include + +ssize_t send_fd(int dst_fd, int fd); + +#endif _CH_PROXY_SENFD + diff --git a/pkgs/taps/.envrc b/pkgs/taps/.envrc new file mode 100644 index 0000000..35f8c10 --- /dev/null +++ b/pkgs/taps/.envrc @@ -0,0 +1 @@ +use nix ../../ -A pkgs.taps diff --git a/pkgs/taps/main.c b/pkgs/taps/main.c new file mode 100644 index 0000000..c921bd0 --- /dev/null +++ b/pkgs/taps/main.c @@ -0,0 +1,241 @@ +#define _GNU_SOURCE + +#include /* secure_getenv */ +#include +#include +#include +#include +#include +#include +#include +#include +#include /* open, O_NONBLOCK, &c */ +#include /* perror */ +#include +#include + + +#define __UAPI_DEF_IF_IFNAMSIZ 1 +#include +#include + +#include "sendfd.h" + +// From `man unix` +#define SUN_PATH_SZ 108 +#define N_CONNS 16 + +#define IFR_FLAGS_ALLOWED (IFF_NO_PI | IFF_TAP | IFF_TUN | IFF_VNET_HDR | IFF_MULTI_QUEUE) +#define IFR_FLAGS_DEFAULT (IFF_NO_PI | IFF_TAP | IFF_VNET_HDR) + +#define DO_OR_DIE(expr) DO_OR_DIE_X((expr) == 0) +#define DO_OR_DIE_X(expr) \ + do if (!(expr)) { \ + perror((#expr)); \ + exit(EXIT_FAILURE); \ + } while(false) + +struct allow_pattern { + // enum { USER = 1, GROUP = 2 } type; + // union { uid_t uid, gid_t gid } xid; + char *name; +}; +struct allow_patterns { + size_t n; + struct allow_pattern *patterns; +}; + +/* Running on the same host, not caring for alignment */ +struct tap_request { + short ifrFlags; /* 0 to use defaults: IFF_TAP | IFF_NO_PI | IFF_VNET_HDR */ + char name[IFNAMSIZ]; +}; + +struct tap_reply { + enum { OK = 0, AUTH_ERROR = 1 } status; + char name[IFNAMSIZ]; +}; + +int tuntap_alloc(char *dev, short openFlags, short ifrFlags, int *out_fd); + +bool match_mask(const char *test_addr, const char *expected_addr, const char *mask, int n) { + for (int octet = 0; octet < n; ++octet) { + if ((test_addr[octet] & mask[octet]) != expected_addr[octet]) { + return false; + } + } + return true; +} + +/* + * Adapted from spectrum's `mktuntap.c` (2019 Alyssa Ross + * GPL-2.0-only), which in turn adapts `tun_alloc` from + * `linux/Documentation/networking/tuntap.rst`. + * + * ifrFlags: IFF_TUN - TUN device (no Ethernet headers) + * IFF_TAP - TAP device + * + * IFF_NO_PI - Do not provide packet information + */ +int tuntap_alloc(char *dev, short openFlags, short ifrFlags, int *out_fd) { + struct ifreq ifr; + int fd, err; + + // if ((fd = open("/dev/net/tun", O_RDWR)) < 0) { + // return tun_alloc_old(dev); + // } + + DO_OR_DIE_X((fd = open("/dev/net/tun", openFlags)) >= 0); + + memset(&ifr, 0, sizeof(ifr)); + + if (*dev) { + int devLen = strlen(dev); + if (devLen >= IFNAMSIZ) { + /* If client requests a name, we do want the entire name to fit */ + errno = EINVAL; + return EINVAL; + } + strncpy(ifr.ifr_name, dev, IFNAMSIZ); + } + + if ((err = ioctl(fd, TUNSETIFF, (void *)&ifr)) < 0) { + close(fd); + return err; + } + + strcpy(dev, ifr.ifr_name); + *out_fd = fd; + return 0; +} + +int acceptRequests(const char *requestsPath, const struct allow_patterns *patterns) { + int listener; + struct sockaddr_un addr; + const bool t = 1; + + DO_OR_DIE(listener = socket(AF_UNIX, SOCK_SEQPACKET, 0)); + DO_OR_DIE(setsockopt(listener, SOL_SOCKET, SO_PASSCRED, &t, 1) != 0); + + addr.sun_family = AF_UNIX; + strncpy(addr.sun_path, requestsPath, SUN_PATH_SZ); + DO_OR_DIE (bind(listener, &addr, sizeof(addr)) == -1); + + DO_OR_DIE(listen(listener, N_CONNS)); + + for (;;) { + int sock = -1; + struct ucred cred = { 0 }; + struct msghdr msg = { 0 }; + struct cmsghdr *cmsg = NULL; + struct iovec iov = { 0 }; + struct tap_request req = { 0 }; + + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + + iov.iov_base = &req; + iov.iov_len = sizeof(struct tap_request); + + DO_OR_DIE(sock = accept(listener, NULL, NULL)); + + DO_OR_DIE_X(recvmsg(sock, &msg, 0) > 0); + for (cmsg = CMSG_FIRSTHDR(&msg); cmsg != NULL; cmsg = CMSG_NXTHDR(&msg, cmsg)) { + if (!(cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SCM_CREDENTIALS)) { + continue; + } + memcpy(&cred, CMSG_DATA(cmsg), sizeof(struct ucred)); + break; + } + + if (req.ifrFlags == 0) { + req.ifrFlags = IFR_FLAGS_DEFAULT; + } + + bool allowed = false; + for (int i = 0; !allowed && i < patterns->n; ++i) { + bool ifnameOk = fnmatch(patterns->patterns[i].name, req.name, 0) == 0; + bool flagsOk = (req.ifrFlags & IFR_FLAGS_ALLOWED) == req.ifrFlags; + allowed = ifnameOk && flagsOk; + } + + struct tap_reply reply = { 0 }; + if (!allowed) { reply.status = AUTH_ERROR; } + if (allowed) { + /* O_CLOEXEC? */ + int fd = 0; + DO_OR_DIE(tuntap_alloc(req.name, O_RDWR | O_NONBLOCK, req.ifrFlags, &fd)); + struct iovec iov = { 0 }; + iov.iov_base = &reply; + iov.iov_len = sizeof(struct tap_reply); + DO_OR_DIE_X(send_fd(sock, fd, &iov) > 0); + } + close(sock); + } + close(listener); +} + +struct allow_patterns parsePatterns(const char *raw) { + const size_t rawLen = strlen(raw); + + size_t nPatterns = 0; + for (int i = 0; i < rawLen; ++i) { + const int start = i; + if (isspace(raw[i])) { continue; } + for (; i < rawLen && !isspace(raw[i]); ++i) { } + if (start < i) { ++nPatterns; } + } + + struct allow_pattern *patterns = calloc(nPatterns, sizeof(struct allow_pattern)); + + int iPattern = 0; + for (int i = 0; i < rawLen; ++i) { + if (isspace(raw[i])) { continue; } + /* used to have per-group/per-user patterns, "u:$username:$pattern", &c - gone */ + { + const int start = i; + for (; i < rawLen && !isspace(raw[i]); ++i) { } + if (i < rawLen) { + patterns[iPattern].name = strndup(&raw[start], i - start); + iPattern += 1; + } + } + } + struct allow_patterns out = { + .n = nPatterns, + .patterns = patterns + }; + return out; +} + +int main(int argc, char **argv) { + bool cmdServe = false; + bool cmdGet = false; + + char **rest = argv + 1; + + if (strcmp(rest[0], "serve") == 0) { + cmdServe = true; + ++rest; + } else if (strcmp(rest[0], "get") == 0) { + cmdGet = true; + ++rest; + } else { + exit(EXIT_FAILURE); + } + + const char *patternsRaw = secure_getenv("TAPS_ALLOW"); + struct allow_patterns patterns = { 0 }; + if (cmdServe && patternsRaw != NULL) { patterns = parsePatterns(patternsRaw); } + DO_OR_DIE_X(patterns.patterns != NULL); + + const char *servePath = secure_getenv("TAPS_SOCK"); + + if (cmdServe && patterns.patterns != NULL) { + acceptRequests(servePath, &patterns); + } else if (cmdGet) { + } else { + exit(EXIT_FAILURE); } + + return 0; +} diff --git a/pkgs/taps/meson.build b/pkgs/taps/meson.build new file mode 100644 index 0000000..06057b4 --- /dev/null +++ b/pkgs/taps/meson.build @@ -0,0 +1,4 @@ +project('taps', 'c') + +sendfd = dependency('sendfd') +executable('taps', 'main.c', dependencies: [sendfd], install: true) diff --git a/pkgs/taps/package.nix b/pkgs/taps/package.nix new file mode 100644 index 0000000..c666cd9 --- /dev/null +++ b/pkgs/taps/package.nix @@ -0,0 +1,44 @@ +{ + lib, + stdenv, + meson, + pkg-config, + rustc, + ninja, + ch-proxy, +}: + +stdenv.mkDerivation { + pname = "taps"; + version = "0.0.0"; + src = + let + fs = lib.fileset; + in + fs.toSource { + root = ./.; + fileset = fs.unions [ + ./meson.build + ./main.c + ]; + }; + + nativeBuildInputs = [ + ninja + meson + pkg-config + rustc + ]; + buildInputs = [ ch-proxy ]; +} +# { lib, rustPlatform }: +# +# rustPlatform.buildRustPackage { +# pname = "taps"; +# version = "0.0.0"; +# src = let fs = lib.filesystem; in fs.toSource { +# root = ./.; +# fileset = fs.unions [ +# ]; +# }; +# }; diff --git a/shell.nix b/shell.nix new file mode 100644 index 0000000..497375b --- /dev/null +++ b/shell.nix @@ -0,0 +1,10 @@ +with import { }; + +mkShell.override { stdenv = stdenvNoCC; } { + packages = map lib.getBin [ + cloud-hypervisor + virtiofsd + crosvm # virtio-gpu + npins + ]; +} From 564913c123a87e9db2c9c133d1403823f925cdb0 Mon Sep 17 00:00:00 2001 From: "Else, Someone" Date: Thu, 22 Jan 2026 14:29:45 +0200 Subject: [PATCH 02/37] fixup! taps: wip: init --- pkgs/ch-proxy/meson.build | 8 ++++++- pkgs/ch-proxy/package.nix | 3 +++ pkgs/ch-proxy/proxy.c | 47 +++------------------------------------ pkgs/ch-proxy/sendfd.c | 38 +++++++++++++++++++++++++++++++ pkgs/ch-proxy/sendfd.h | 8 ++++--- profiles/uvms-guest.nix | 1 + profiles/uvms-users.nix | 1 - 7 files changed, 57 insertions(+), 49 deletions(-) create mode 100644 pkgs/ch-proxy/sendfd.c diff --git a/pkgs/ch-proxy/meson.build b/pkgs/ch-proxy/meson.build index e3976a2..379f96f 100644 --- a/pkgs/ch-proxy/meson.build +++ b/pkgs/ch-proxy/meson.build @@ -1,3 +1,9 @@ project('ch-proxy', 'c') -executable('ch-proxy', 'proxy.c', install: true) +pkg = import('pkgconfig') + +sendfd = library('sendfd', [ 'sendfd.c', 'sendfd.h' ], install: true) +pkg.generate(sendfd) +install_headers('sendfd.h') + +executable('ch-proxy', 'proxy.c', link_with: [sendfd], install: true) diff --git a/pkgs/ch-proxy/package.nix b/pkgs/ch-proxy/package.nix index 9fd3b21..e94eec8 100644 --- a/pkgs/ch-proxy/package.nix +++ b/pkgs/ch-proxy/package.nix @@ -8,6 +8,7 @@ stdenv.mkDerivation { pname = "ch-proxy"; version = "0.0.0"; + outputs = [ "out" "lib" ]; nativeBuildInputs = [ meson ninja @@ -19,6 +20,8 @@ stdenv.mkDerivation { fs.toSource { fileset = fs.unions [ ./proxy.c + ./sendfd.c + ./sendfd.h ./meson.build ]; root = ./.; diff --git a/pkgs/ch-proxy/proxy.c b/pkgs/ch-proxy/proxy.c index ed1dea0..46730e6 100644 --- a/pkgs/ch-proxy/proxy.c +++ b/pkgs/ch-proxy/proxy.c @@ -9,9 +9,9 @@ #include -struct msghdr mk_msghdr(); +#include "sendfd.h" + int ch_connect(const char*, const char*); -ssize_t send_fd(int, int); #define _WRITE_CONFIRM(fd, buf, buflen) {if (write((fd), (buf), (buflen)) != (buflen)) { perror("ch-proxy/write/partial write"); exit(EXIT_FAILURE); }} @@ -168,19 +168,13 @@ int main(int argc, char** argv) { exit(EXIT_FAILURE); } - if (send_fd(1, s) == -1) { + if (send_fd(1, s, NULL) == -1) { perror("ssh-vsock-proxy/main/send_fd"); return EXIT_FAILURE; } return 0; } -struct msghdr mk_msghdr() { - struct msghdr msg; - memset(&msg, 0, sizeof(msg)); - - return msg; -} int ch_connect(const char *path, const char *port) { int s = socket(AF_UNIX, SOCK_STREAM, 0); @@ -212,38 +206,3 @@ int ch_connect(const char *path, const char *port) { return s; } - -ssize_t send_fd(int dst_fd, int fd) { - struct msghdr msg = mk_msghdr(); - - /* openssh expects to receive a dummy length=1 iovec? */ - char ch; - struct iovec vec; - vec.iov_base = &ch; - vec.iov_len = 1; - msg.msg_iov = &vec; - msg.msg_iovlen = 1; - - union { - struct cmsghdr align; - char buf[CMSG_SPACE(sizeof(int))]; - } u; - - msg.msg_control = u.buf; - msg.msg_controllen = sizeof(u.buf); - - struct cmsghdr *cmptr; - cmptr = CMSG_FIRSTHDR(&msg); - - if (cmptr == NULL) { - fprintf(stderr, "ch-proxy/send_fd/CMSG_FIRSTHDR: failed to initialize msg_control\n"); - exit(EXIT_FAILURE); - } - - cmptr->cmsg_len = CMSG_LEN(sizeof(int)); - cmptr->cmsg_level = SOL_SOCKET; - cmptr->cmsg_type = SCM_RIGHTS; - *((int*) CMSG_DATA(cmptr)) = fd; - - return (sendmsg(dst_fd, &msg, 0)); -} diff --git a/pkgs/ch-proxy/sendfd.c b/pkgs/ch-proxy/sendfd.c new file mode 100644 index 0000000..c649316 --- /dev/null +++ b/pkgs/ch-proxy/sendfd.c @@ -0,0 +1,38 @@ +#include "sendfd.h" +#include "sys/socket.h" /* cmsghdr */ +#include "stdio.h" /* perror */ + + +ssize_t send_fd(int dst_fd, int fd, const struct iovec *iov) { + struct msghdr msg = { 0 }; + + /* openssh expects to receive a dummy length=1 iovec? */ + char ch = 0; + struct iovec vecDefault = { 0 }; + vecDefault.iov_base = &ch; + vecDefault.iov_len = 1; + msg.msg_iov = iov == NULL ? &vecDefault : iov; + msg.msg_iovlen = 1; + + union { + struct cmsghdr align; + char buf[CMSG_SPACE(sizeof(int))]; + } u; + + msg.msg_control = u.buf; + msg.msg_controllen = sizeof(u.buf); + + struct cmsghdr *cmptr; + cmptr = CMSG_FIRSTHDR(&msg); + + if (cmptr == NULL) { + perror("ch-proxy/send_fd/CMSG_FIRSTHDR: failed to initialize msg_control\n"); + } + + cmptr->cmsg_len = CMSG_LEN(sizeof(int)); + cmptr->cmsg_level = SOL_SOCKET; + cmptr->cmsg_type = SCM_RIGHTS; + *((int*) CMSG_DATA(cmptr)) = fd; + + return (sendmsg(dst_fd, &msg, 0)); +} diff --git a/pkgs/ch-proxy/sendfd.h b/pkgs/ch-proxy/sendfd.h index 65042d8..8c99389 100644 --- a/pkgs/ch-proxy/sendfd.h +++ b/pkgs/ch-proxy/sendfd.h @@ -1,9 +1,11 @@ #ifndef _CH_PROXY_SENFD #define _CH_PROXY_SENFD -#include +#include /* size_t */ +#include /* ssize_t */ +#include /* iovec */ -ssize_t send_fd(int dst_fd, int fd); +ssize_t send_fd(int dst_fd, int fd, const struct iovec *); -#endif _CH_PROXY_SENFD +#endif /* _CH_PROXY_SENFD */ diff --git a/profiles/uvms-guest.nix b/profiles/uvms-guest.nix index e8c307d..281f343 100644 --- a/profiles/uvms-guest.nix +++ b/profiles/uvms-guest.nix @@ -41,6 +41,7 @@ in volumes = [ { image = "swapfile.img"; + serial = "swapfiles"; mountPoint = "/var/swapfiles"; size = 1024; } diff --git a/profiles/uvms-users.nix b/profiles/uvms-users.nix index e75ac8f..e7bbacf 100644 --- a/profiles/uvms-users.nix +++ b/profiles/uvms-users.nix @@ -29,7 +29,6 @@ in }; config = mergeIf cfg.enable [ { - services.getty.autologinUser = "user"; security.sudo.wheelNeedsPassword = false; users.mutableUsers = false; users.users.user = { From 691a193bba4f2c9df5fe9aec049ca24ee887a866 Mon Sep 17 00:00:00 2001 From: Else Someone Date: Thu, 22 Jan 2026 15:08:38 +0200 Subject: [PATCH 03/37] taps: fix error messages --- pkgs/taps/main.c | 21 +++++++++++++-------- shell.nix | 3 +++ 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/pkgs/taps/main.c b/pkgs/taps/main.c index c921bd0..82a02cd 100644 --- a/pkgs/taps/main.c +++ b/pkgs/taps/main.c @@ -10,7 +10,7 @@ #include #include #include /* open, O_NONBLOCK, &c */ -#include /* perror */ +#include #include #include @@ -31,8 +31,7 @@ #define DO_OR_DIE(expr) DO_OR_DIE_X((expr) == 0) #define DO_OR_DIE_X(expr) \ do if (!(expr)) { \ - perror((#expr)); \ - exit(EXIT_FAILURE); \ + error(EXIT_FAILURE, errno, ("Failed assertion: " #expr)); \ } while(false) struct allow_pattern { @@ -214,6 +213,7 @@ int main(int argc, char **argv) { char **rest = argv + 1; + DO_OR_DIE_X(argc > 1); if (strcmp(rest[0], "serve") == 0) { cmdServe = true; ++rest; @@ -221,21 +221,26 @@ int main(int argc, char **argv) { cmdGet = true; ++rest; } else { - exit(EXIT_FAILURE); + error(EINVAL, EINVAL, "no subcommand \"%s\"", rest[0]); } const char *patternsRaw = secure_getenv("TAPS_ALLOW"); struct allow_patterns patterns = { 0 }; - if (cmdServe && patternsRaw != NULL) { patterns = parsePatterns(patternsRaw); } - DO_OR_DIE_X(patterns.patterns != NULL); + if (cmdServe && patternsRaw != NULL) { + patterns = parsePatterns(patternsRaw); + DO_OR_DIE_X(patterns.patterns != NULL); + } const char *servePath = secure_getenv("TAPS_SOCK"); - if (cmdServe && patterns.patterns != NULL) { + if (cmdServe) { + error(patterns.patterns == NULL, EINVAL, "TAPS_ALLOW"); acceptRequests(servePath, &patterns); } else if (cmdGet) { + error(ENOSYS, ENOSYS, "get"); } else { - exit(EXIT_FAILURE); } + error(EINVAL, EINVAL, "subcommand args"); + } return 0; } diff --git a/shell.nix b/shell.nix index 497375b..f8bb9a7 100644 --- a/shell.nix +++ b/shell.nix @@ -6,5 +6,8 @@ mkShell.override { stdenv = stdenvNoCC; } { virtiofsd crosvm # virtio-gpu npins + ] ++ [ + man-pages + linux-manual ]; } From 1c5e2b7e89930da572a10588a298fc9077c629b8 Mon Sep 17 00:00:00 2001 From: Else Someone Date: Wed, 28 Jan 2026 07:04:57 +0200 Subject: [PATCH 04/37] taps: MVE - TUNSETIFF works, the interface is indeed created. - An fd is indeed passed over the unix socket, - and is a valid (enough) fd because it can fed into `dup2`. - `nix run -f . --offline pkgs.taps -- pass sleep 5` works, the interface exists for 5 seconds and disappears - `nix run -f . --offline pkgs.taps -- pass ch-remote --api-socket=$HOME/uvms/nixos/vmm.sock add-net fd=3` obscurely fails, killing the VMM with: ```console [root@nixos:~]# cloud-hypervisor: 12.388270s: <_net1_qp0> ERROR:/build/source/net_util/src/queue_pair.rs:112 -- net: tx: failed writing to tap: Input/output error (os er ror 5) cloud-hypervisor: 12.388459s: <_net1_qp0> ERROR:virtio-devices/src/thread_helper.rs:54 -- Error running worker: HandleEvent(Error processing TX queue: NetQueuePair(Write Tap(Os { code: 5, kind: Uncategorized, message: "Input/output error" }))) ``` --- pkgs/ch-proxy/sendfd.c | 36 +++++++++ pkgs/ch-proxy/sendfd.h | 20 ++++- pkgs/taps/main.c | 172 ++++++++++++++++++++++++++++++----------- 3 files changed, 180 insertions(+), 48 deletions(-) diff --git a/pkgs/ch-proxy/sendfd.c b/pkgs/ch-proxy/sendfd.c index c649316..b20e284 100644 --- a/pkgs/ch-proxy/sendfd.c +++ b/pkgs/ch-proxy/sendfd.c @@ -36,3 +36,39 @@ ssize_t send_fd(int dst_fd, int fd, const struct iovec *iov) { return (sendmsg(dst_fd, &msg, 0)); } + +int recv_fd(int sock, int flags) { + int out = -1; + + struct msghdr msg = { 0 }; + struct cmsghdr *cmsg = NULL; + struct iovec iov = { 0 }; + char dummy = 0; + + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + + iov.iov_base = &dummy; + iov.iov_len = sizeof(dummy); + + union { + struct cmsghdr align; + char buf[CMSG_SPACE(sizeof(int))]; + } u; + + msg.msg_control = u.buf; + msg.msg_controllen = sizeof(u.buf); + + int bytes = 0; + if ((bytes = recvmsg(sock, &msg, flags)) < 0) { + perror("recv_fd: recvmsg"); + return -1; + } + for (cmsg = CMSG_FIRSTHDR(&msg); cmsg != NULL; cmsg = CMSG_NXTHDR(&msg, cmsg)) { + if (cmsg->cmsg_level != SOL_SOCKET) { continue; } + if (cmsg->cmsg_type != SCM_RIGHTS) { continue; } + if (CMSG_LEN(cmsg) < sizeof(out)) { continue; } + out = *(int*)CMSG_DATA(cmsg); + } + return out; +} diff --git a/pkgs/ch-proxy/sendfd.h b/pkgs/ch-proxy/sendfd.h index 8c99389..fc1d2f8 100644 --- a/pkgs/ch-proxy/sendfd.h +++ b/pkgs/ch-proxy/sendfd.h @@ -5,7 +5,23 @@ #include /* ssize_t */ #include /* iovec */ -ssize_t send_fd(int dst_fd, int fd, const struct iovec *); + +/* send_fd(chanFd, fd, *iov) + * + * chanFd: fd to sendmsg over; + * fd: fd to send; + * iov: extra data to send or NULL; + * + * returns: result of sendmsg, + * i.e. the number of bytes sent */ +ssize_t send_fd(int chanFd, int fd, const struct iovec *); + +/* recv_fd(chanFd, flags) + * + * chanFd: fd to recvmsg from; + * flags: recvmsg flags e.g. 0, or MSG_CMSG_CLOEXEC? + * + * returns: the received fd or -1 */ +int recv_fd(int chanFd, int flags); #endif /* _CH_PROXY_SENFD */ - diff --git a/pkgs/taps/main.c b/pkgs/taps/main.c index 82a02cd..05edb64 100644 --- a/pkgs/taps/main.c +++ b/pkgs/taps/main.c @@ -10,10 +10,11 @@ #include #include #include /* open, O_NONBLOCK, &c */ +#include #include #include #include - +#include #define __UAPI_DEF_IF_IFNAMSIZ 1 #include @@ -25,13 +26,19 @@ #define SUN_PATH_SZ 108 #define N_CONNS 16 -#define IFR_FLAGS_ALLOWED (IFF_NO_PI | IFF_TAP | IFF_TUN | IFF_VNET_HDR | IFF_MULTI_QUEUE) -#define IFR_FLAGS_DEFAULT (IFF_NO_PI | IFF_TAP | IFF_VNET_HDR) +char *TEMP_PATHS[1024] = { 0 }; +int LAST_TEMP_PATH = -1; -#define DO_OR_DIE(expr) DO_OR_DIE_X((expr) == 0) -#define DO_OR_DIE_X(expr) \ +#define IFR_FLAGS_ALLOWED (IFF_NO_PI | IFF_TAP | IFF_TUN | IFF_VNET_HDR | IFF_MULTI_QUEUE | IFF_PERSIST) +#define IFR_FLAGS_DEFAULT (IFF_NO_PI | IFF_TAP | IFF_VNET_HDR | IFF_PERSIST) + +#define PTR_OR_DIE(expr) TRUE_OR_DIE((expr) != NULL) +#define DO_OR_DIE(expr) TRUE_OR_DIE((expr) != -1) +#define TRUE_OR_DIE(expr, ...) TRUE_OR_(EXIT_FAILURE, expr, __VA_ARGS__) +#define TRUE_OR_WARN(expr, ...) TRUE_OR_(0, expr, __VA_ARGS__) +#define TRUE_OR_(status, expr, ...) \ do if (!(expr)) { \ - error(EXIT_FAILURE, errno, ("Failed assertion: " #expr)); \ + error(status, errno, "Failed assertion: " #expr "." __VA_ARGS__); \ } while(false) struct allow_pattern { @@ -72,38 +79,34 @@ bool match_mask(const char *test_addr, const char *expected_addr, const char *ma * `linux/Documentation/networking/tuntap.rst`. * * ifrFlags: IFF_TUN - TUN device (no Ethernet headers) - * IFF_TAP - TAP device + * IFF_TAP - TAP device * - * IFF_NO_PI - Do not provide packet information + * IFF_NO_PI - Do not provide packet information */ int tuntap_alloc(char *dev, short openFlags, short ifrFlags, int *out_fd) { - struct ifreq ifr; - int fd, err; + struct ifreq ifr = { 0 }; + int fd = -1, err = 0; - // if ((fd = open("/dev/net/tun", O_RDWR)) < 0) { - // return tun_alloc_old(dev); - // } + DO_OR_DIE(fd = open("/dev/net/tun", openFlags)); - DO_OR_DIE_X((fd = open("/dev/net/tun", openFlags)) >= 0); - - memset(&ifr, 0, sizeof(ifr)); - - if (*dev) { + if (dev != NULL) { int devLen = strlen(dev); if (devLen >= IFNAMSIZ) { /* If client requests a name, we do want the entire name to fit */ errno = EINVAL; return EINVAL; } - strncpy(ifr.ifr_name, dev, IFNAMSIZ); + strncpy(ifr.ifr_name, dev, IFNAMSIZ - 1); } + ifr.ifr_flags = ifrFlags; - if ((err = ioctl(fd, TUNSETIFF, (void *)&ifr)) < 0) { + TRUE_OR_WARN((err = ioctl(fd, TUNSETIFF, (void *)&ifr)) == 0); + if (err != 0) { close(fd); return err; } - strcpy(dev, ifr.ifr_name); + strncpy(dev, ifr.ifr_name, IFNAMSIZ); *out_fd = fd; return 0; } @@ -111,18 +114,20 @@ int tuntap_alloc(char *dev, short openFlags, short ifrFlags, int *out_fd) { int acceptRequests(const char *requestsPath, const struct allow_patterns *patterns) { int listener; struct sockaddr_un addr; - const bool t = 1; + const int t = 1; DO_OR_DIE(listener = socket(AF_UNIX, SOCK_SEQPACKET, 0)); - DO_OR_DIE(setsockopt(listener, SOL_SOCKET, SO_PASSCRED, &t, 1) != 0); + DO_OR_DIE(setsockopt(listener, SOL_SOCKET, SO_PASSCRED, &t, sizeof(t))); addr.sun_family = AF_UNIX; - strncpy(addr.sun_path, requestsPath, SUN_PATH_SZ); - DO_OR_DIE (bind(listener, &addr, sizeof(addr)) == -1); + strncpy(addr.sun_path, requestsPath, SUN_PATH_SZ - 1); + DO_OR_DIE (bind(listener, &addr, sizeof(addr))); + PTR_OR_DIE(TEMP_PATHS[++LAST_TEMP_PATH] = strdup(requestsPath)); DO_OR_DIE(listen(listener, N_CONNS)); for (;;) { + /* Already changed my mind about looking at ucred, but keeping the code around for now */ int sock = -1; struct ucred cred = { 0 }; struct msghdr msg = { 0 }; @@ -136,13 +141,14 @@ int acceptRequests(const char *requestsPath, const struct allow_patterns *patter iov.iov_base = &req; iov.iov_len = sizeof(struct tap_request); - DO_OR_DIE(sock = accept(listener, NULL, NULL)); + DO_OR_DIE((sock = accept(listener, NULL, NULL))); - DO_OR_DIE_X(recvmsg(sock, &msg, 0) > 0); + TRUE_OR_DIE(recvmsg(sock, &msg, 0) > 0); + req.name[IFNAMSIZ] = 0; for (cmsg = CMSG_FIRSTHDR(&msg); cmsg != NULL; cmsg = CMSG_NXTHDR(&msg, cmsg)) { - if (!(cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SCM_CREDENTIALS)) { - continue; - } + if (cmsg->cmsg_level != SOL_SOCKET) { continue; } + if (cmsg->cmsg_type != SCM_CREDENTIALS) { continue; } + if (CMSG_LEN(cmsg) < sizeof(struct ucred)) { continue; } memcpy(&cred, CMSG_DATA(cmsg), sizeof(struct ucred)); break; } @@ -162,12 +168,13 @@ int acceptRequests(const char *requestsPath, const struct allow_patterns *patter if (!allowed) { reply.status = AUTH_ERROR; } if (allowed) { /* O_CLOEXEC? */ - int fd = 0; - DO_OR_DIE(tuntap_alloc(req.name, O_RDWR | O_NONBLOCK, req.ifrFlags, &fd)); + int fd = -1; + TRUE_OR_DIE(tuntap_alloc(req.name, O_RDWR | O_NONBLOCK, req.ifrFlags, &fd) == 0); struct iovec iov = { 0 }; iov.iov_base = &reply; iov.iov_len = sizeof(struct tap_reply); - DO_OR_DIE_X(send_fd(sock, fd, &iov) > 0); + TRUE_OR_DIE(send_fd(sock, fd, &iov) > 0); + close(fd); } close(sock); } @@ -185,7 +192,8 @@ struct allow_patterns parsePatterns(const char *raw) { if (start < i) { ++nPatterns; } } - struct allow_pattern *patterns = calloc(nPatterns, sizeof(struct allow_pattern)); + struct allow_pattern *patterns = NULL; + PTR_OR_DIE(patterns = calloc(nPatterns, sizeof(struct allow_pattern))); int iPattern = 0; for (int i = 0; i < rawLen; ++i) { @@ -194,8 +202,8 @@ struct allow_patterns parsePatterns(const char *raw) { { const int start = i; for (; i < rawLen && !isspace(raw[i]); ++i) { } - if (i < rawLen) { - patterns[iPattern].name = strndup(&raw[start], i - start); + if (start < i) { + PTR_OR_DIE(patterns[iPattern].name = strndup(&raw[start], i - start)); iPattern += 1; } } @@ -207,37 +215,109 @@ struct allow_patterns parsePatterns(const char *raw) { return out; } +int get(const char *servePath, const char *ifname, short ifrFlags) { + /* TODO: sock: move out */ + int sock; + struct sockaddr_un addr; + + DO_OR_DIE(sock = socket(AF_UNIX, SOCK_SEQPACKET, 0)); + + addr.sun_family = AF_UNIX; + strncpy(addr.sun_path, servePath, SUN_PATH_SZ - 1); + DO_OR_DIE (connect(sock, &addr, sizeof(addr))); + + struct msghdr msg = { 0 }; + struct cmsghdr *cmsg = NULL; + struct iovec iov = { 0 }; + struct tap_request req = { 0 }; + strncpy(req.name, ifname, IFNAMSIZ - 1); + req.ifrFlags = ifrFlags; + + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + + iov.iov_base = &req; + iov.iov_len = sizeof(struct tap_request); + + TRUE_OR_DIE(sendmsg(sock, &msg, 0) > 0); + + int tunFd = -1; + DO_OR_DIE(tunFd = recv_fd(sock, 0)); + close(sock); + return tunFd; +} + +void cleanup(int signo, siginfo_t *info, void *_context) { + for (int i = 0; i <= LAST_TEMP_PATH; ++i) { + TRUE_OR_DIE(unlink(TEMP_PATHS[i]) != -1 || errno == ENOENT); + } + if (signo == SIGINT) { + exit(EXIT_SUCCESS); + } + errx(EXIT_FAILURE, "Exiting with signal %d", signo); +} + int main(int argc, char **argv) { + struct sigaction act = { 0 }; + act.sa_flags = SA_SIGINFO; + act.sa_sigaction = cleanup; + DO_OR_DIE(sigaction(SIGINT, &act, NULL)); + DO_OR_DIE(sigaction(SIGSEGV, &act, NULL)); + bool cmdServe = false; - bool cmdGet = false; + bool cmdPass = false; + char *ifname = "vt%d"; char **rest = argv + 1; + char **end = argv + argc; - DO_OR_DIE_X(argc > 1); + TRUE_OR_DIE(argc > 1); if (strcmp(rest[0], "serve") == 0) { cmdServe = true; ++rest; - } else if (strcmp(rest[0], "get") == 0) { - cmdGet = true; + } else if (strcmp(rest[0], "pass") == 0) { + cmdPass = true; ++rest; + for (; rest != end && rest[0][0] == '-'; ++rest) { + if (strcmp(rest[0], "--")) { break; } + else if (strncmp(rest[0], "--ifname=", sizeof("--ifname="))) { + ifname = rest[0] + sizeof("--ifname="); + } + } } else { error(EINVAL, EINVAL, "no subcommand \"%s\"", rest[0]); } + int nextArgc = argc - (rest - argv); + char * const* nextArgv = rest; + const char *patternsRaw = secure_getenv("TAPS_ALLOW"); + if (patternsRaw == NULL) { + patternsRaw = "*"; + } + struct allow_patterns patterns = { 0 }; - if (cmdServe && patternsRaw != NULL) { - patterns = parsePatterns(patternsRaw); - DO_OR_DIE_X(patterns.patterns != NULL); + if (cmdServe) { + PTR_OR_DIE((patterns = parsePatterns(patternsRaw)).patterns); } const char *servePath = secure_getenv("TAPS_SOCK"); + if (servePath == NULL) { + servePath = "taps.sock"; + } if (cmdServe) { - error(patterns.patterns == NULL, EINVAL, "TAPS_ALLOW"); acceptRequests(servePath, &patterns); - } else if (cmdGet) { - error(ENOSYS, ENOSYS, "get"); + } else if (cmdPass) { + TRUE_OR_DIE(nextArgc > 0); + int fd = -1; + DO_OR_DIE(fd = get(servePath, ifname, 0)); + if (fd != 3) { + DO_OR_DIE(dup2(fd, 3)); + close(fd); + fd = 3; + } + DO_OR_DIE(execvp(nextArgv[0], nextArgv)); } else { error(EINVAL, EINVAL, "subcommand args"); } From 17bde34c967dad65019e6a2f767b307bb394d555 Mon Sep 17 00:00:00 2001 From: Else Someone Date: Thu, 12 Feb 2026 20:23:55 +0200 Subject: [PATCH 05/37] taps: MVE EIO crash in CH was caused by CH trying to writev before the interface has been assigned any addresses; adding an address prior to passing the FD (via 80-vm-vt.network) solves the issue. It is still unclear what causes the interrupt=None in the vhost_user/passt branch --- examples/dummy.nix | 2 + pkgs/taps/main.c | 20 ++- profiles/ch-runner.nix | 303 +++++++++++++++++++++++++++++++++++------ 3 files changed, 284 insertions(+), 41 deletions(-) diff --git a/examples/dummy.nix b/examples/dummy.nix index 4689af1..7d8c813 100644 --- a/examples/dummy.nix +++ b/examples/dummy.nix @@ -13,6 +13,8 @@ system.stateVersion = "25.11"; + networking.hostName = "dummy"; + vmapps.enable = true; _module.args.inputs = import ../npins; diff --git a/pkgs/taps/main.c b/pkgs/taps/main.c index 05edb64..3b276df 100644 --- a/pkgs/taps/main.c +++ b/pkgs/taps/main.c @@ -1,5 +1,6 @@ #define _GNU_SOURCE +#include #include /* secure_getenv */ #include #include @@ -15,10 +16,12 @@ #include #include #include +#include #define __UAPI_DEF_IF_IFNAMSIZ 1 #include #include +#include #include "sendfd.h" @@ -108,6 +111,11 @@ int tuntap_alloc(char *dev, short openFlags, short ifrFlags, int *out_fd) { strncpy(dev, ifr.ifr_name, IFNAMSIZ); *out_fd = fd; + + { + int sz = sizeof(struct virtio_net_hdr_v1); + DO_OR_DIE(ioctl(fd, TUNSETVNETHDRSZ, &sz)); + } return 0; } @@ -257,6 +265,13 @@ void cleanup(int signo, siginfo_t *info, void *_context) { errx(EXIT_FAILURE, "Exiting with signal %d", signo); } +/* skarlibs under ISC */ +int uncoe (int fd) +{ + int flags = fcntl(fd, F_GETFD, 0) ; + return flags < 0 ? flags : flags & FD_CLOEXEC ? fcntl(fd, F_SETFD, flags & ~FD_CLOEXEC) : 0 ; +} + int main(int argc, char **argv) { struct sigaction act = { 0 }; act.sa_flags = SA_SIGINFO; @@ -266,7 +281,7 @@ int main(int argc, char **argv) { bool cmdServe = false; bool cmdPass = false; - char *ifname = "vt%d"; + char *ifname = "vt-%d"; char **rest = argv + 1; char **end = argv + argc; @@ -303,7 +318,7 @@ int main(int argc, char **argv) { const char *servePath = secure_getenv("TAPS_SOCK"); if (servePath == NULL) { - servePath = "taps.sock"; + servePath = "/run/taps/taps.sock"; } if (cmdServe) { @@ -317,6 +332,7 @@ int main(int argc, char **argv) { close(fd); fd = 3; } + uncoe(fd); DO_OR_DIE(execvp(nextArgv[0], nextArgv)); } else { error(EINVAL, EINVAL, "subcommand args"); diff --git a/profiles/ch-runner.nix b/profiles/ch-runner.nix index fbb09a4..f680354 100644 --- a/profiles/ch-runner.nix +++ b/profiles/ch-runner.nix @@ -10,21 +10,107 @@ let cfg = config.uvms.cloud-hypervisor; + + inherit (config.networking) hostName; inherit (config.debug.closure.erofs) layers; + inherit (lib) mkOption types; + + package = pkgs.cloud-hypervisor.overrideAttrs (oldAttrs: { + patches = oldAttrs.patches or [ ] ++ [ + # ../patches/ch.patch + ]; + buildType = "debug"; + dontStrip = true; + }); + uvmsPkgs = pkgs.callPackage ../pkgs { }; + + chSettingsFile = (pkgs.formats.json { }).generate "vm.json" cfg.settings; + + uvmPrefix = "\${HOME}/uvms/${hostName}"; + vmmSock = "${uvmPrefix}/vmm.sock"; + pastaSock = "${uvmPrefix}/pasta.sock"; + pastaPidPath = "${uvmPrefix}/pasta.pid"; + elbPrefix = "${lib.getBin pkgs.execline}/bin"; + s6Prefix = "${lib.getBin pkgs.s6}/bin"; + writeElb = name: text: writeElb' name "-W" text; + writeElb' = + name: elArgs: text: + pkgs.writeTextFile { + inherit name; + destination = "/bin/${name}"; + executable = true; + text = '' + #!${lib.getExe' pkgs.execline "execlineb"}${lib.optionalString (elArgs != null) " "}${elArgs} + importas OLDPATH PATH + export PATH "${elbPrefix}:${s6Prefix}:''${OLDPATH}" + ${text} + ''; + }; in { options = { uvms.cloud-hypervisor.enable = lib.mkEnableOption "Configure guest (e.g. fileSystems)"; - uvms.cloud-hypervisor.runner = lib.mkOption { - type = lib.types.package; + uvms.cloud-hypervisor.runner = mkOption { + type = types.package; description = "A naive script for running this system in cloud-hypervisor"; }; - uvms.cloud-hypervisor.extraArgv = lib.mkOption { - type = lib.types.listOf lib.types.str; - default = [ ]; + uvms.cloud-hypervisor.debugger = mkOption { + type = types.lazyAttrsOf types.anything; + description = "Same but you can debug the kernel"; }; - uvms.cloud-hypervisor.argv = lib.mkOption { - type = lib.types.listOf lib.types.str; + uvms.cloud-hypervisor.settingsFile = mkOption { + type = types.package; + default = chSettingsFile; + defaultText = "..."; + readOnly = true; + }; + uvms.cloud-hypervisor.settings = mkOption { + default = { }; + type = types.submodule { + freeformType = (pkgs.formats.json { }).type; + options = { + payload = { + cmdline = mkOption { type = types.str; }; + kernel = mkOption { type = types.str; }; + initramfs = mkOption { + type = types.str; + default = "${config.system.build.initialRamdisk}/${config.system.boot.loader.initrdFile}"; + }; + }; + vsock = { + cid = mkOption { + type = types.int; + default = 4; + }; + socket = mkOption { + type = types.str; + default = "vsock.sock"; + }; + }; + "api-socket" = mkOption { + type = types.str; + default = "vmm.sock"; + }; + "serial".mode = mkOption { + type = types.str; + default = "File"; + }; + "serial".file = mkOption { + type = types.nullOr types.str; + default = "serial"; + }; + "console".mode = mkOption { + type = types.str; + default = "Pty"; + }; + "console".file = mkOption { + type = types.nullOr types.str; + default = null; + }; + # "watchdog" = true; + # "seccomp" = true; + }; + }; }; uvms.cloud-hypervisor.extraCmdline = lib.mkOption { type = lib.types.listOf lib.types.str; @@ -45,33 +131,178 @@ in }; config = lib.mkMerge [ { - uvms.cloud-hypervisor.argv = lib.mkBefore ( - [ - (lib.getExe pkgs.cloud-hypervisor) - "--cmdline=${lib.concatStringsSep " " cfg.cmdline}" - "--kernel=${config.boot.kernelPackages.kernel}/${pkgs.stdenv.hostPlatform.linux-kernel.target}" - "--initramfs=${config.system.build.initialRamdisk}/${config.system.boot.loader.initrdFile}" - "--vsock=cid=4,socket=vsock.sock" - "--api-socket=vmm.sock" - "--serial=tty" - "--console=null" - "--watchdog" - "--seccomp=true" - ] - ++ cfg.extraArgv - ); - uvms.cloud-hypervisor.runner = pkgs.writeShellScriptBin "run-${config.networking.hostName}" '' - set -euo pipefail - GUESTNAME=${config.networking.hostName} - args=( - ${lib.concatMapStringsSep "\n" lib.escapeShellArg cfg.argv} - ) - mkdir -p "$HOME/uvms/$GUESTNAME" - cd "$HOME/uvms/$GUESTNAME" - cleanup() { - rm "$HOME/uvms/$GUESTNAME"/{vmm,vsock}.sock + uvms.cloud-hypervisor.settings = { + payload = { + cmdline = lib.concatStringsSep " " cfg.cmdline; + kernel = "${config.boot.kernelPackages.kernel}/${pkgs.stdenv.hostPlatform.linux-kernel.target}"; + }; + disks = map (img: { + path = img; + readonly = true; + id = toString img.label; + }) layers; + memory = { + size = 1536 * 1048576; + shared = true; + mergeable = true; + # hotplugged_size = 512 * 1048576; + # hotplugd_size = 1536 * 1048576; + # hotplug_method = "virtio-mem" + }; + cpus = { + boot_vcpus = 4; + max_vcpus = 4; + }; + }; + + uvms.cloud-hypervisor.debugger = pkgs.testers.runNixOSTest ( + { config, ... }: + { + name = "test-run-${hostName}"; + passthru = rec { + inherit (config.nodes.machine.system.build) gdbScript; + inherit (config.nodes.machine.boot.kernelPackages) kernel; + kernelSrc = pkgs.srcOnly kernel; + }; + nodes.machine = + { config, ... }: + let + kernel = config.boot.kernelPackages.kernel; + kernelSrc = pkgs.srcOnly kernel; + gdbScript = writeElb "attach-gdb" '' + if { rm -rf /tmp/gdb } + if { mkdir -p /tmp/gdb/kos } + cd /tmp/gdb + if { + elglob -0 files ${kernelSrc}/* + forx -E f { $files } + ln -s $f ./ + } + if { mkdir -p build } + cd build + if { + forx -E pattern { + ${kernel.modules}/lib/modules/*/kernel/drivers/net/tun* + ${kernel.modules}/lib/modules/*/kernel/drivers/net/tap* + } + elglob -0 files $pattern + forx -E f { $files } + if { cp $f . } + backtick -E COMPRESSED { basename $f } + xz -d $COMPRESSED + } + elglob -0 GDB_SCRIPT_DIR ${lib.getDev kernel}/lib/modules/*/build/scripts/gdb + if { + if { cp -r --no-preserve=all $GDB_SCRIPT_DIR gdb_scripts } + mv gdb_scripts/linux/constants.py.in gdb_scripts/linux/constants.py + } + ${lib.getExe pkgs.gdb} + -ex "python import sys; sys.path.insert(0, \"''${GDB_SCRIPT_DIR}\")" + -ex "target remote :1234" + -ex "source ''${GDB_SCRIPT_DIR}/vmlinux-gdb.py" + -ex "lx-symbols" + ${kernel.dev}/vmlinux + ''; + in + { + boot.kernelPackages = pkgs.linuxPackagesFor ( + (pkgs.linux.override (oldArgs: { + # extraMakeFlags = oldArgs.extraMakeFlags or [ ] ++ [ + # "scripts_gdb" + # ]; + kernelPatches = oldArgs.kernelPatches or [ ] ++ [ + { + name = "debug"; + patch = null; + structuredExtraConfig = { + GDB_SCRIPTS = lib.kernel.yes; + DEBUG_INFO = lib.kernel.yes; + DEBUG_INFO_REDUCED = lib.kernel.no; + # FRAME_POINTER = lib.kernel.yes; # "unused option"??? + KALLSYMS = lib.kernel.yes; + KGDB = lib.kernel.yes; + }; + } + ]; + })).overrideAttrs + (oldAttrs: { + dontStrip = true; + postInstall = oldAttrs.postInstall or "" + '' + cp "$buildRoot/scripts/gdb/linux/constants.py" $dev/lib/modules/*/build/scripts/gdb/linux/ || echo "$buildRoot/scripts/gdb/linux/constants.py doesn't exist" + ''; + }) + ); + boot.kernelParams = [ "nokaslr" ]; + networking.useNetworkd = true; + virtualisation.qemu.options = [ "-s" ]; + environment.systemPackages = [ + pkgs.gdb + package # CH + cfg.runner + uvmsPkgs.taps + ]; + system.build.gdbScript = gdbScript; + systemd.services.taps = { + wantedBy = [ "multi-user.target" ]; + environment.TAPS_SOCK = "/run/taps/taps.sock"; + serviceConfig = { + UMask = "0007"; + ExecStart = "${lib.getExe uvmsPkgs.taps} serve"; + RuntimeDirectory = "taps"; + DynamicUser = true; + AmbientCapabilities = [ + "CAP_NET_BIND_SERVICE" + "CAP_NET_ADMIN" + ]; + NoNewPrivileges = true; + }; + }; + }; + testScript = '' + machine.succeed("${lib.getExe cfg.runner}") + ''; } - exec -a "uuvm/$GUESTNAME" "''${args[@]}" + ); + + # NOTE: Used to be an even uglier bash script, but, for now, execline makes for easier comparisons against spectrum + uvms.cloud-hypervisor.runner = writeElb "run-${hostName}" '' + importas -i HOME HOME + importas -SsD ${lib.getExe' pkgs.passt "passt"} PASST + importas -SsD ${lib.getExe package} CH + importas -SsD "${lib.getExe' package "ch-remote"} --api-socket=${vmmSock}" CHR + foreground { mkdir -p "${uvmPrefix}" "${uvmPrefix}" } + cd "${uvmPrefix}" + background { + s6-ipcserver-socketbinder -B ${vmmSock} + trap { default { foreground { s6-ipcclient -q ${uvmPrefix}/cleanup.sock true } exit } } + fdmove -c 3 0 + redirfd -r 0 /dev/null + exec -a "uuvm/${hostName} cloud-hypervisor" $CH --api-socket fd=3 + } + importas CH_PID ! + background { + s6-ipcserver-socketbinder -B ${pastaSock} + s6-ipcserverd + trap { default { foreground { s6-ipcclient -q ${uvmPrefix}/cleanup.sock true } exit } } + exec -a "uuvm/${hostName} passt" $PASST --vhost-user -fF 0 + } + importas PASST_PID ! + background { + getpid -E CLEANUP_PID + s6-ipcserver-socketbinder -B "${uvmPrefix}/cleanup.sock" + s6-ipcserverd + foreground { kill $CH_PID $PASST_PID } + foreground { + rm -f ${vmmSock} ${uvmPrefix}/vsock.sock + } + kill $CLEANUP_PID + } + trap { default { foreground { s6-ipcclient -q ${uvmPrefix}/cleanup.sock true } exit } } + if { $CHR create ${chSettingsFile} } + # if { $CHR add-net "vhost_user=on,socket=${pastaSock}" } + if { ${lib.getExe uvmsPkgs.taps} pass $CHR add-net "id=wan,fd=3,mac=00:00:00:00:00:01" } + if { $CHR boot } + if { $CHR info } ''; } (lib.mkIf cfg.enable { @@ -103,12 +334,6 @@ in } ) layers ); - uvms.cloud-hypervisor.argv = [ - "--memory=size=1536M,hotplug_size=1536M,hotplugged_size=512M,hotplug_method=virtio-mem,mergeable=on,shared=on" - "--cpus=boot=4" - "--disk" - ] - ++ map (img: "path=${img},readonly=true,id=${toString img.label}") layers; }) ]; } From ee497cb7d6f6604dc13ad648ade0b3fe23c46bae Mon Sep 17 00:00:00 2001 From: Else Someone Date: Thu, 12 Feb 2026 22:22:06 +0200 Subject: [PATCH 06/37] ch-runner: drop passt from the PoC --- profiles/ch-runner.nix | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/profiles/ch-runner.nix b/profiles/ch-runner.nix index f680354..44f8205 100644 --- a/profiles/ch-runner.nix +++ b/profiles/ch-runner.nix @@ -28,8 +28,6 @@ let uvmPrefix = "\${HOME}/uvms/${hostName}"; vmmSock = "${uvmPrefix}/vmm.sock"; - pastaSock = "${uvmPrefix}/pasta.sock"; - pastaPidPath = "${uvmPrefix}/pasta.pid"; elbPrefix = "${lib.getBin pkgs.execline}/bin"; s6Prefix = "${lib.getBin pkgs.s6}/bin"; writeElb = name: text: writeElb' name "-W" text; @@ -280,13 +278,6 @@ in exec -a "uuvm/${hostName} cloud-hypervisor" $CH --api-socket fd=3 } importas CH_PID ! - background { - s6-ipcserver-socketbinder -B ${pastaSock} - s6-ipcserverd - trap { default { foreground { s6-ipcclient -q ${uvmPrefix}/cleanup.sock true } exit } } - exec -a "uuvm/${hostName} passt" $PASST --vhost-user -fF 0 - } - importas PASST_PID ! background { getpid -E CLEANUP_PID s6-ipcserver-socketbinder -B "${uvmPrefix}/cleanup.sock" @@ -299,7 +290,6 @@ in } trap { default { foreground { s6-ipcclient -q ${uvmPrefix}/cleanup.sock true } exit } } if { $CHR create ${chSettingsFile} } - # if { $CHR add-net "vhost_user=on,socket=${pastaSock}" } if { ${lib.getExe uvmsPkgs.taps} pass $CHR add-net "id=wan,fd=3,mac=00:00:00:00:00:01" } if { $CHR boot } if { $CHR info } From 0617d97ebf40ae188de20e6edbb0a354d020e3ba Mon Sep 17 00:00:00 2001 From: Else Someone Date: Wed, 18 Feb 2026 16:27:13 +0200 Subject: [PATCH 07/37] ch-runner: just move the cleanup bits out of elb Execlineb is insane. Skarnet is insane. POSIX is cancer, but elb seems to make it worse. Or it s a skill issue. I see no way to do any kind of error handling with elb, and that is even despite me only needing one kind of error handling: the cleaning up... --- profiles/ch-runner.nix | 251 +++++++++++++++++++++++++++++++++++------ 1 file changed, 215 insertions(+), 36 deletions(-) diff --git a/profiles/ch-runner.nix b/profiles/ch-runner.nix index 44f8205..8b69117 100644 --- a/profiles/ch-runner.nix +++ b/profiles/ch-runner.nix @@ -13,7 +13,14 @@ let inherit (config.networking) hostName; inherit (config.debug.closure.erofs) layers; - inherit (lib) mkOption types; + inherit (lib) + mkOption + types + concatMapStringsSep + getExe + getExe' + getBin + ; package = pkgs.cloud-hypervisor.overrideAttrs (oldAttrs: { patches = oldAttrs.patches or [ ] ++ [ @@ -38,7 +45,7 @@ let destination = "/bin/${name}"; executable = true; text = '' - #!${lib.getExe' pkgs.execline "execlineb"}${lib.optionalString (elArgs != null) " "}${elArgs} + #!${getExe' pkgs.execline "execlineb"}${lib.optionalString (elArgs != null) " "}${elArgs} importas OLDPATH PATH export PATH "${elbPrefix}:${s6Prefix}:''${OLDPATH}" ${text} @@ -194,7 +201,7 @@ in if { cp -r --no-preserve=all $GDB_SCRIPT_DIR gdb_scripts } mv gdb_scripts/linux/constants.py.in gdb_scripts/linux/constants.py } - ${lib.getExe pkgs.gdb} + ${getExe pkgs.gdb} -ex "python import sys; sys.path.insert(0, \"''${GDB_SCRIPT_DIR}\")" -ex "target remote :1234" -ex "source ''${GDB_SCRIPT_DIR}/vmlinux-gdb.py" @@ -245,7 +252,7 @@ in environment.TAPS_SOCK = "/run/taps/taps.sock"; serviceConfig = { UMask = "0007"; - ExecStart = "${lib.getExe uvmsPkgs.taps} serve"; + ExecStart = "${getExe uvmsPkgs.taps} serve"; RuntimeDirectory = "taps"; DynamicUser = true; AmbientCapabilities = [ @@ -257,43 +264,215 @@ in }; }; testScript = '' - machine.succeed("${lib.getExe cfg.runner}") + machine.succeed("${getExe cfg.runner}") ''; } ); # NOTE: Used to be an even uglier bash script, but, for now, execline makes for easier comparisons against spectrum - uvms.cloud-hypervisor.runner = writeElb "run-${hostName}" '' - importas -i HOME HOME - importas -SsD ${lib.getExe' pkgs.passt "passt"} PASST - importas -SsD ${lib.getExe package} CH - importas -SsD "${lib.getExe' package "ch-remote"} --api-socket=${vmmSock}" CHR - foreground { mkdir -p "${uvmPrefix}" "${uvmPrefix}" } - cd "${uvmPrefix}" - background { - s6-ipcserver-socketbinder -B ${vmmSock} - trap { default { foreground { s6-ipcclient -q ${uvmPrefix}/cleanup.sock true } exit } } - fdmove -c 3 0 - redirfd -r 0 /dev/null - exec -a "uuvm/${hostName} cloud-hypervisor" $CH --api-socket fd=3 - } - importas CH_PID ! - background { - getpid -E CLEANUP_PID - s6-ipcserver-socketbinder -B "${uvmPrefix}/cleanup.sock" - s6-ipcserverd - foreground { kill $CH_PID $PASST_PID } - foreground { - rm -f ${vmmSock} ${uvmPrefix}/vsock.sock - } - kill $CLEANUP_PID - } - trap { default { foreground { s6-ipcclient -q ${uvmPrefix}/cleanup.sock true } exit } } - if { $CHR create ${chSettingsFile} } - if { ${lib.getExe uvmsPkgs.taps} pass $CHR add-net "id=wan,fd=3,mac=00:00:00:00:00:01" } - if { $CHR boot } - if { $CHR info } - ''; + uvms.cloud-hypervisor.runner = + let + addProcess = getExe addProcess'; + addProcess' = pkgs.writers.writePython3Bin "add-process" { } '' + import os + import select + import socket + import subprocess + import sys + from argparse import ArgumentParser + from contextlib import contextmanager, ExitStack + from threading import Thread, Semaphore + + + parser = ArgumentParser() + parser.add_argument("events_path") + parser.add_argument("--then", action="append") + + MSG_SIZE = 16 + SHMEM = {} + + + def send(sock, msg): + assert len(msg) <= MSG_SIZE, len(msg) + return sock.send(msg.ljust(MSG_SIZE)) + + + def recv(sock): + msg = sock.recv(MSG_SIZE) + # assert len(msg) <= MSG_SIZE, len(msg) + assert len(msg) <= MSG_SIZE, len(msg) + return (msg.split() + [b""])[0] + + + def serve_impl(events_path, listener): + SHMEM["server"] = True + + cons = [] + state = "up" + while state == "up" or cons != []: + if state == "up": + rs, ws, es = select.select([listener, *cons], [], []) + else: + rs, ws, es = select.select(cons, cons, []) + events = [] + for r in rs: + if r is listener: + r, _ = r.accept() + cons.append(r) + else: + events.append(recv(r)) + if any(e == b"killall" for e in events): + state = "down" + if state == "down": + for w in ws: + with s_lock: + send(w, b"die") + w.close() + cons.remove(w) + for w in es: + w.close() + cons.remove(w) + + + def serve(events_path): + base_dir = os.path.dirname(events_path) + if base_dir: + os.makedirs(base_dir, exist_ok=True) + listener = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM, 0) + listener.setblocking(False) + + try: + listener.bind(events_path) + listener.listen() + return serve_impl(events_path, listener) + except OSError as e: + EADDRINUSE = 98 + if e.errno != EADDRINUSE: + raise + finally: + listener.close() + os.remove(events_path) + + + def register(events_path): + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM, 0) + sock.connect(events_path) + return sock + + + @contextmanager + def defer(f): + try: + yield + finally: + f() + + + if __name__ == "__main__": + args, args_next = parser.parse_known_args() + + with ExitStack() as cleanup: + if args_next: + p = subprocess.Popen( + args_next, + shell=False) + then_cmds = reversed(getattr(args, "then") or []) + if not args_next: + then_cmds = [] + try: + p.wait(0.5) + then_cmds = [] + except subprocess.TimeoutExpired: + pass + for f in then_cmds: + def run_f(): + subprocess.run(f) + cleanup.enter_context(defer(run_f)) + + maybe_server = Thread( + target=serve, + args=(args.events_path,), + daemon=True) + maybe_server.start() + maybe_server.join(0.5) + + assert ( + ("server" in SHMEM) == bool(maybe_server.is_alive) + ), (SHMEM, maybe_server) + + if args_next: + s = register(args.events_path) + s_lock = Semaphore() + + if args_next: + def watch_p(p, s): + p.wait() + with s_lock: + try: + send(s, b"killall") + except BrokenPipeError: + pass + + def watch_s(p, s): + while True: + if recv(s) == b"die": + p.terminate() + break + + s_watcher = Thread( + target=watch_s, + args=(p, s), + daemon=True) + s_watcher.start() + watch_p(p, s) + s_watcher.join() + s.close() + + if SHMEM.get("server", False): + maybe_server.join() + + exit_code = 0 + if args_next: + exit_code |= p.returncode + sys.exit(exit_code) + ''; + ch = getExe package; + chr = getExe' package "ch-remote"; + in + writeElb "run-${hostName}" '' + importas -i HOME HOME + importas -SsD "${chr} --api-socket=${vmmSock}" CHR + importas -SsD "${uvmPrefix}" PREFIX + define EVENTS ''${PREFIX}/events.sock + define -s ADD_PROC "${addProcess} ''${EVENTS}" + + cd $PREFIX + background { + $ADD_PROC --then ${getExe ( + writeElb "rm-vmmsock" '' + importas -i HOME HOME + rm -f ${vmmSock} + rm -f ${uvmPrefix}/vsock.sock + '' + )} ${getExe ( + writeElb "ch" '' + importas -Si 1 + importas -Si 2 + s6-ipcserver-socketbinder -B $1 + exec -a "uuvm/''${2} cloud-hypervisor" ${ch} --api-socket fd=0 + '' + )} ${vmmSock} ${hostName} + } + foreground { sleep 0.1 } + ifelse -n { test -S ${vmmSock} } { echo "Apparently ${vmmSock} does not exist" } + foreground { echo "Loading the configuration" } + if { $CHR create ${chSettingsFile} } + foreground { echo "Adding TAP" } + if { ${lib.getExe uvmsPkgs.taps} pass $CHR add-net "id=wan,fd=3,mac=00:00:00:00:00:01" } + foreground { echo "Booting" } + if { $CHR boot } + if { $CHR info } + ''; } (lib.mkIf cfg.enable { boot.initrd.availableKernelModules = [ From 97f2ba4c66f36a6c929250429cb6a6ccebcb3f7f Mon Sep 17 00:00:00 2001 From: Else Someone Date: Thu, 19 Feb 2026 19:24:44 +0200 Subject: [PATCH 08/37] ch-runner: move more logic out of elb --- profiles/ch-runner.nix | 302 ++++++++++++++++++----------------------- 1 file changed, 131 insertions(+), 171 deletions(-) diff --git a/profiles/ch-runner.nix b/profiles/ch-runner.nix index 8b69117..dee946b 100644 --- a/profiles/ch-runner.nix +++ b/profiles/ch-runner.nix @@ -272,92 +272,85 @@ in # NOTE: Used to be an even uglier bash script, but, for now, execline makes for easier comparisons against spectrum uvms.cloud-hypervisor.runner = let - addProcess = getExe addProcess'; - addProcess' = pkgs.writers.writePython3Bin "add-process" { } '' + superviseVm = getExe superviseVm'; + superviseVm' = pkgs.writers.writePython3Bin "supervise-vm" { } '' import os - import select - import socket import subprocess - import sys from argparse import ArgumentParser from contextlib import contextmanager, ExitStack - from threading import Thread, Semaphore - parser = ArgumentParser() - parser.add_argument("events_path") - parser.add_argument("--then", action="append") + parser = ArgumentParser("supervise-vm") + parser.add_argument("--vm") + parser.add_argument("--prefix", default="$HOME/uvms/$VM") + parser.add_argument("--sock", default="$PREFIX/supervisor.sock") + parser.add_argument("--vm-config") MSG_SIZE = 16 - SHMEM = {} + ELB_DIR = "${lib.getBin pkgs.execline}/bin" # noqa: E501 + S6_DIR = "${lib.getBin pkgs.s6}/bin" # noqa: E501 + CH_DIR = "${lib.getBin package}/bin" # noqa: E501 + SOCKETBINDER_PATH = S6_DIR + "/s6-ipcserver-socketbinder" # noqa: E501 + CH_PATH = CH_DIR + "/cloud-hypervisor" + CHR_PATH = CH_DIR + "/ch-remote" + TAPS_PATH = "${lib.getExe uvmsPkgs.taps}" # noqa: E501 + + PASSTHRU_PATH = ":".join([ELB_DIR, S6_DIR, CH_DIR]) + PASSTHRU_ENV = { + **{ + k: v + for k, v in os.environ.items() + if k.startswith("RUST") + or k.startswith("WAYLAND") + or k in [ + "TAPS_SOCK", + ] + }, + "HOME": os.environ.get("HOME", os.getcwd()), + "PATH": PASSTHRU_PATH, + } - def send(sock, msg): - assert len(msg) <= MSG_SIZE, len(msg) - return sock.send(msg.ljust(MSG_SIZE)) + def configure_execline(prefix, vm, check=True, **defaults): + def execline(*args, check=check, **kwargs): + return subprocess.run( + ["execlineb", "-c", "\n".join(args)], + **defaults, + executable=ELB_DIR + "/execlineb", + env={ + **PASSTHRU_ENV, + "PATH": PASSTHRU_PATH, + "PREFIX": prefix, + "VM": vm, + }, + check=check, + cwd=prefix, + **kwargs) + return execline - def recv(sock): - msg = sock.recv(MSG_SIZE) - # assert len(msg) <= MSG_SIZE, len(msg) - assert len(msg) <= MSG_SIZE, len(msg) - return (msg.split() + [b""])[0] - - - def serve_impl(events_path, listener): - SHMEM["server"] = True - - cons = [] - state = "up" - while state == "up" or cons != []: - if state == "up": - rs, ws, es = select.select([listener, *cons], [], []) - else: - rs, ws, es = select.select(cons, cons, []) - events = [] - for r in rs: - if r is listener: - r, _ = r.accept() - cons.append(r) - else: - events.append(recv(r)) - if any(e == b"killall" for e in events): - state = "down" - if state == "down": - for w in ws: - with s_lock: - send(w, b"die") - w.close() - cons.remove(w) - for w in es: - w.close() - cons.remove(w) - - - def serve(events_path): - base_dir = os.path.dirname(events_path) - if base_dir: - os.makedirs(base_dir, exist_ok=True) - listener = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM, 0) - listener.setblocking(False) - - try: - listener.bind(events_path) - listener.listen() - return serve_impl(events_path, listener) - except OSError as e: - EADDRINUSE = 98 - if e.errno != EADDRINUSE: - raise - finally: - listener.close() - os.remove(events_path) - - - def register(events_path): - sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM, 0) - sock.connect(events_path) - return sock + def preprocess_args(args_mut): + keys = [ + k + for k, v + in args_mut._get_kwargs() + if isinstance(v, str)] + for k in keys: + v = getattr(args_mut, k) + if "$HOME" in v: + setattr( + args_mut, + k, + v.replace("$HOME", PASSTHRU_ENV["HOME"])) + for k in keys: + v = getattr(args_mut, k) + if "$VM" in v: + setattr(args_mut, k, v.replace("$VM", args.vm)) + for k in keys: + v = getattr(args_mut, k) + if "$PREFIX" in v: + setattr(args_mut, k, v.replace("$PREFIX", args.prefix)) + return args_mut @contextmanager @@ -368,110 +361,77 @@ in f() + @contextmanager + def run_ch(vm_prefix): + args = [ + SOCKETBINDER_PATH, + "-B", + vm_prefix + "/vmm.sock", + CH_PATH, + "--api-socket", + "fd=0", + ] + p = subprocess.Popen( + args, + shell=False) + try: + p.wait(1.0) + needs_cleanup = False + except subprocess.TimeoutExpired: + needs_cleanup = True + if not os.path.exists(vm_prefix + "/vmm.sock"): + raise RuntimeError(f"{vm_prefix}/vmm.sock should exist by now") + if p.returncode is not None: + raise RuntimeError("CH exited early") + try: + yield p + finally: + try: + p.poll() + except: # noqa: E722 + pass + if p.returncode is None: + p.terminate() # CH handles SIG{INT,TERM}? + p.wait() + unlink_paths = [ + vm_prefix + "/vmm.sock", + vm_prefix + "/vmm.sock.lock", + vm_prefix + "/vsock.sock", + ] if needs_cleanup else [] + for p in unlink_paths: + if os.path.exists(p): + os.remove(p) + + if __name__ == "__main__": args, args_next = parser.parse_known_args() + preprocess_args(args) + + os.makedirs(args.prefix, exist_ok=True) + execline = configure_execline( + prefix=args.prefix, + vm=args.vm) + + ch_remote = [ + "ch-remote", + "--api-socket", + args.prefix + "/vmm.sock", + ] with ExitStack() as cleanup: - if args_next: - p = subprocess.Popen( - args_next, - shell=False) - then_cmds = reversed(getattr(args, "then") or []) - if not args_next: - then_cmds = [] - try: - p.wait(0.5) - then_cmds = [] - except subprocess.TimeoutExpired: - pass - for f in then_cmds: - def run_f(): - subprocess.run(f) - cleanup.enter_context(defer(run_f)) - - maybe_server = Thread( - target=serve, - args=(args.events_path,), - daemon=True) - maybe_server.start() - maybe_server.join(0.5) - - assert ( - ("server" in SHMEM) == bool(maybe_server.is_alive) - ), (SHMEM, maybe_server) - - if args_next: - s = register(args.events_path) - s_lock = Semaphore() - - if args_next: - def watch_p(p, s): - p.wait() - with s_lock: - try: - send(s, b"killall") - except BrokenPipeError: - pass - - def watch_s(p, s): - while True: - if recv(s) == b"die": - p.terminate() - break - - s_watcher = Thread( - target=watch_s, - args=(p, s), - daemon=True) - s_watcher.start() - watch_p(p, s) - s_watcher.join() - s.close() - - if SHMEM.get("server", False): - maybe_server.join() - - exit_code = 0 - if args_next: - exit_code |= p.returncode - sys.exit(exit_code) + ch = cleanup.enter_context(run_ch(args.prefix)) + execline(*ch_remote, "create", args.vm_config) + execline( + TAPS_PATH, "pass", + *ch_remote, "add-net", + "id=wan,fd=3,mac=00:00:00:00:00:01") + execline(*ch_remote, "boot") + execline(*ch_remote, "info") + ch.wait() ''; - ch = getExe package; - chr = getExe' package "ch-remote"; in writeElb "run-${hostName}" '' - importas -i HOME HOME - importas -SsD "${chr} --api-socket=${vmmSock}" CHR - importas -SsD "${uvmPrefix}" PREFIX - define EVENTS ''${PREFIX}/events.sock - define -s ADD_PROC "${addProcess} ''${EVENTS}" - - cd $PREFIX - background { - $ADD_PROC --then ${getExe ( - writeElb "rm-vmmsock" '' - importas -i HOME HOME - rm -f ${vmmSock} - rm -f ${uvmPrefix}/vsock.sock - '' - )} ${getExe ( - writeElb "ch" '' - importas -Si 1 - importas -Si 2 - s6-ipcserver-socketbinder -B $1 - exec -a "uuvm/''${2} cloud-hypervisor" ${ch} --api-socket fd=0 - '' - )} ${vmmSock} ${hostName} - } - foreground { sleep 0.1 } - ifelse -n { test -S ${vmmSock} } { echo "Apparently ${vmmSock} does not exist" } - foreground { echo "Loading the configuration" } - if { $CHR create ${chSettingsFile} } - foreground { echo "Adding TAP" } - if { ${lib.getExe uvmsPkgs.taps} pass $CHR add-net "id=wan,fd=3,mac=00:00:00:00:00:01" } - foreground { echo "Booting" } - if { $CHR boot } - if { $CHR info } + ${superviseVm} --vm-config=${chSettingsFile} --vm=${hostName} ''; } (lib.mkIf cfg.enable { From 342a1dfe3a788ad524692f95be140aef9b0f7ff5 Mon Sep 17 00:00:00 2001 From: Else Someone Date: Fri, 20 Feb 2026 10:50:31 +0200 Subject: [PATCH 09/37] ch-runner: another iter of cleanup prior to moving things around --- profiles/ch-runner.nix | 42 +++++++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/profiles/ch-runner.nix b/profiles/ch-runner.nix index dee946b..b3d0385 100644 --- a/profiles/ch-runner.nix +++ b/profiles/ch-runner.nix @@ -311,10 +311,25 @@ in } - def configure_execline(prefix, vm, check=True, **defaults): - def execline(*args, check=check, **kwargs): + def configure_exec(prefix, vm, check=True, **defaults): + + def exec(*args, check=check, **kwargs): return subprocess.run( - ["execlineb", "-c", "\n".join(args)], + [*args], + **defaults, + env={ + **PASSTHRU_ENV, + "PATH": PASSTHRU_PATH, + "PREFIX": prefix, + "VM": vm, + }, + check=check, + cwd=prefix, + **kwargs) + + def execline(*args, check=check, **kwargs): + return exec( + "execlineb", "-c", "\n".join(args), **defaults, executable=ELB_DIR + "/execlineb", env={ @@ -326,7 +341,8 @@ in check=check, cwd=prefix, **kwargs) - return execline + + return exec, execline def preprocess_args(args_mut): @@ -373,7 +389,8 @@ in ] p = subprocess.Popen( args, - shell=False) + shell=False, + pass_fds=(2,)) try: p.wait(1.0) needs_cleanup = False @@ -408,7 +425,7 @@ in preprocess_args(args) os.makedirs(args.prefix, exist_ok=True) - execline = configure_execline( + exec, _ = configure_exec( prefix=args.prefix, vm=args.vm) @@ -420,14 +437,17 @@ in with ExitStack() as cleanup: ch = cleanup.enter_context(run_ch(args.prefix)) - execline(*ch_remote, "create", args.vm_config) - execline( + exec(*ch_remote, "create", args.vm_config) + exec( TAPS_PATH, "pass", *ch_remote, "add-net", "id=wan,fd=3,mac=00:00:00:00:00:01") - execline(*ch_remote, "boot") - execline(*ch_remote, "info") - ch.wait() + exec(*ch_remote, "boot") + exec(*ch_remote, "info") + try: + ch.wait() + except KeyboardInterrupt: + pass ''; in writeElb "run-${hostName}" '' From 7516eae7a6a83b1253441b56ceb3370248ac7fdd Mon Sep 17 00:00:00 2001 From: Else Someone Date: Sat, 21 Feb 2026 18:11:37 +0200 Subject: [PATCH 10/37] ch-runner: add minimal virtiofsd support Somehow it doesnt work if I open the socket in python and try to inherit it. Also didnt work with bwrap. And theres a bunch of warnings that virtiofsd keeps printing. Need investigation --- profiles/ch-runner.nix | 388 ++++++++++++++++++++++++++++++++--------- 1 file changed, 302 insertions(+), 86 deletions(-) diff --git a/profiles/ch-runner.nix b/profiles/ch-runner.nix index b3d0385..f156705 100644 --- a/profiles/ch-runner.nix +++ b/profiles/ch-runner.nix @@ -272,12 +272,22 @@ in # NOTE: Used to be an even uglier bash script, but, for now, execline makes for easier comparisons against spectrum uvms.cloud-hypervisor.runner = let + toolsClosure = pkgs.writeClosure [ + (lib.getBin pkgs.execline) + (lib.getBin pkgs.s6) + (lib.getBin package) + (lib.getBin pkgs.virtiofsd) + (lib.getBin pkgs.bubblewrap) + uvmsPkgs.taps + ]; + superviseVm = getExe superviseVm'; superviseVm' = pkgs.writers.writePython3Bin "supervise-vm" { } '' import os import subprocess + import socket from argparse import ArgumentParser - from contextlib import contextmanager, ExitStack + from contextlib import contextmanager, closing, ExitStack parser = ArgumentParser("supervise-vm") @@ -290,12 +300,21 @@ in ELB_DIR = "${lib.getBin pkgs.execline}/bin" # noqa: E501 S6_DIR = "${lib.getBin pkgs.s6}/bin" # noqa: E501 CH_DIR = "${lib.getBin package}/bin" # noqa: E501 + UTIL_LINUX_DIR = "${lib.getBin pkgs.util-linux}/bin" # noqa: E501 SOCKETBINDER_PATH = S6_DIR + "/s6-ipcserver-socketbinder" # noqa: E501 CH_PATH = CH_DIR + "/cloud-hypervisor" CHR_PATH = CH_DIR + "/ch-remote" TAPS_PATH = "${lib.getExe uvmsPkgs.taps}" # noqa: E501 + VIRTIOFSD_PATH = "${lib.getExe pkgs.virtiofsd}" # noqa: E501 + BWRAP_PATH = "${lib.getExe pkgs.bubblewrap}" # noqa: E501 - PASSTHRU_PATH = ":".join([ELB_DIR, S6_DIR, CH_DIR]) + with open("${toolsClosure}", mode="r") as f: # noqa: E501 + CLOSURE = [ + *(ln.rstrip() for ln in f.readlines()), + "${placeholder "out"}", # noqa: E501 + ] + + PASSTHRU_PATH = ":".join([ELB_DIR, S6_DIR, CH_DIR, UTIL_LINUX_DIR]) PASSTHRU_ENV = { **{ k: v @@ -311,40 +330,6 @@ in } - def configure_exec(prefix, vm, check=True, **defaults): - - def exec(*args, check=check, **kwargs): - return subprocess.run( - [*args], - **defaults, - env={ - **PASSTHRU_ENV, - "PATH": PASSTHRU_PATH, - "PREFIX": prefix, - "VM": vm, - }, - check=check, - cwd=prefix, - **kwargs) - - def execline(*args, check=check, **kwargs): - return exec( - "execlineb", "-c", "\n".join(args), - **defaults, - executable=ELB_DIR + "/execlineb", - env={ - **PASSTHRU_ENV, - "PATH": PASSTHRU_PATH, - "PREFIX": prefix, - "VM": vm, - }, - check=check, - cwd=prefix, - **kwargs) - - return exec, execline - - def preprocess_args(args_mut): keys = [ k @@ -369,6 +354,270 @@ in return args_mut + class Processes: + def __init__(self, prefix, vm, check=True, **defaults): + self.prefix = prefix + self.vm = vm + self.check = check + self.defaults = defaults + + def make_env(self): + return { + **PASSTHRU_ENV, + "PATH": PASSTHRU_PATH, + "PREFIX": self.prefix, + "VM": self.vm, + } + + def exec(self, *args, **kwargs): + kwargs["cwd"] = kwargs.get("cwd", self.prefix) + kwargs["check"] = kwargs.get("check", self.check) + kwargs["env"] = kwargs.get("env", self.make_env()) + return subprocess.run( + [*args], + **self.defaults, + **kwargs) + + def execline(self, *args, **kwargs): + return exec( + "execlineb", "-c", "\n".join(args), + **self.defaults, + executable=ELB_DIR + "/execlineb", + **{ + "env": self.make_env(), + "check": self.check, + "cwd": self.prefix, + **kwargs, + }, + ) + + def popen(self, *args, **kwargs): + kwargs["pass_fds"] = kwargs.get("pass_fds", ()) + kwargs["env"] = kwargs.get("env", self.make_env()) + kwargs["cwd"] = kwargs.get("cwd", self.prefix) + return subprocess.Popen( + args, + **kwargs, + ) + + @contextmanager + def bwrap( + self, + *bwrap_args, + + die_with_parent=True, + + # Based on the args from + # `host/rootfs/image/usr/bin/run-vmm` + unshare_all=True, + unshare_user=True, + unshare_ipc=None, + unshare_pid=None, + unshare_net=None, + unshare_uts=None, + unshare_cgroup_try=True, + bind=(), + dev_bind=("/dev/kvm", "/dev/vfio"), + dev="/dev", + proc="/proc", + ro_bind=( + "/etc", + "/sys", + "/proc/sys", + "/dev/null", + "/proc/kallsyms", + *CLOSURE), + ro_bind_extra=(), + remount_ro=("/proc/fs", "/proc/irq"), + tmpfs=("/dev/shm", "/tmp", "/var/tmp", "/proc/fs", "/proc/irq"), + tmpfs_extra=(), + + pass_fds=(2,), + **popen_kwargs): + + bwrap_args_sock, remote = socket.socketpair() + remote.set_inheritable(True) + bwrap_args_f = bwrap_args_sock.makefile("w") + with closing(bwrap_args_sock), closing(bwrap_args_f): + def print_arg(*args): + print(*args, file=bwrap_args_f, sep="\0", end="\0") + + if unshare_all: + print_arg("--unshare-all") + if unshare_user: + print_arg("--unshare-user") + if unshare_ipc: + print_arg("--unshare-ipc") + if unshare_pid: + print_arg("--unshare-pid") + if unshare_net: + print_arg("--unshare-net") + if unshare_uts: + print_arg("--unshare-uts") + if unshare_cgroup_try: + print_arg("--unshare-cgroup-try") + if die_with_parent: + print_arg("--die-with-parent") + + for p in bind: + p1, p2 = (p, p) if isinstance(p, str) else p + print_arg("--bind", p1, p2) + for p in (*ro_bind, *ro_bind_extra): + p1, p2 = (p, p) if isinstance(p, str) else p + print_arg("--ro-bind", p1, p2) + for p in dev_bind: + p1, p2 = (p, p) if isinstance(p, str) else p + print_arg("--dev-bind", p1, p2) + for p in (*tmpfs, *tmpfs_extra): + print_arg("--tmpfs", p) + # Hunch: order might matter... + for p in remount_ro: + print_arg("--remount-ro", p) + + bwrap_args_f.flush() + + with closing(remote): + proc = self.popen( + "bwrap", "--args", str(remote.fileno()), *bwrap_args, + **popen_kwargs, + executable=BWRAP_PATH, + pass_fds=(*pass_fds, remote.fileno()), + ) + + with proc as p: + try: + yield p + finally: + try: + p.poll() + except: # noqa: E722 + pass + if p.returncode is None: + p.terminate() + p.wait() + + @contextmanager + def run_ch(self): + args = [ + SOCKETBINDER_PATH, + "-B", + self.prefix + "/vmm.sock", + CH_PATH, + "--api-socket", + "fd=0", + ] + p = self.popen( + *args, + shell=False, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + pass_fds=(2,)) + try: + p.wait(0.125) + needs_cleanup = False + except subprocess.TimeoutExpired: + needs_cleanup = True + if not os.path.exists(self.prefix + "/vmm.sock"): + raise RuntimeError(f"{self.prefix}/vmm.sock should exist by now") + if p.returncode is not None: + raise RuntimeError("CH exited early") + try: + yield p + finally: + try: + p.poll() + except: # noqa: E722 + pass + if p.returncode is None: + p.terminate() # CH handles SIG{INT,TERM}? + p.wait() + unlink_paths = [ + self.prefix + "/vmm.sock", + self.prefix + "/vmm.sock.lock", + self.prefix + "/vsock.sock", + ] if needs_cleanup else [] + for p in unlink_paths: + if os.path.exists(p): + os.remove(p) + + @contextmanager + def add_virtiofsd( + self, + root_dir, + tag, + ro=False, + subdirs=None, + extra_flags=("--posix-acl",)): + + assert os.path.exists(root_dir) + + sock_path = self.prefix + f"/virtiofsd-{tag}.sock" + # s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + # NOTE: Nope. Virtiofsd actually expects a blocking socket + # s.setblocking(True) + + def rm_sock(): + if os.path.exists(sock_path): + os.remove(sock_path) + + with ExitStack() as cleanup: # noqa: F841 + # s.bind(sock_path.encode("utf8")) + # cleanup.enter_context(closing(s)) + cleanup.enter_context(defer(rm_sock)) + + args = [ + # If using bwrap(): + # "--argv0", "virtiofsd", + # "--uid", "1000", + # "--gid", "1000", + # "--", + "unshare", "-rUm", + "unshare", "--map-user", "1000", "--map-group", "1000", + VIRTIOFSD_PATH, + "--shared-dir", + root_dir, + "--tag", + tag, + + # "--fd", + # str(s.fileno()), + "--socket-path", + sock_path, + + # If relying on bwrap(): + # "--sandbox", + # "none", + ] + if ro: + args.append("--readonly") + kwargs = { + # If bwrap(): + # "bind": [], + # ("ro_bind_extra" if ro else "bind"): + # [*subdirs] + # if subdirs is not None + # else [root_dir], + + # "pass_fds": (2, s.fileno()), + } + proc_ctx = self.popen(*args, **kwargs) + with proc_ctx as p: + try: + try: + p.wait(0.125) + except subprocess.TimeoutExpired: + pass + if p.returncode is not None: + raise RuntimeError("virtiofsd exited too early") + yield p, sock_path + finally: + if p.returncode is None: + p.kill() + p.wait() + if os.path.exists(sock_path): + os.remove(sock_path) + + @contextmanager def defer(f): try: @@ -377,57 +626,15 @@ in f() - @contextmanager - def run_ch(vm_prefix): - args = [ - SOCKETBINDER_PATH, - "-B", - vm_prefix + "/vmm.sock", - CH_PATH, - "--api-socket", - "fd=0", - ] - p = subprocess.Popen( - args, - shell=False, - pass_fds=(2,)) - try: - p.wait(1.0) - needs_cleanup = False - except subprocess.TimeoutExpired: - needs_cleanup = True - if not os.path.exists(vm_prefix + "/vmm.sock"): - raise RuntimeError(f"{vm_prefix}/vmm.sock should exist by now") - if p.returncode is not None: - raise RuntimeError("CH exited early") - try: - yield p - finally: - try: - p.poll() - except: # noqa: E722 - pass - if p.returncode is None: - p.terminate() # CH handles SIG{INT,TERM}? - p.wait() - unlink_paths = [ - vm_prefix + "/vmm.sock", - vm_prefix + "/vmm.sock.lock", - vm_prefix + "/vsock.sock", - ] if needs_cleanup else [] - for p in unlink_paths: - if os.path.exists(p): - os.remove(p) - - if __name__ == "__main__": args, args_next = parser.parse_known_args() preprocess_args(args) os.makedirs(args.prefix, exist_ok=True) - exec, _ = configure_exec( + ps = Processes( prefix=args.prefix, - vm=args.vm) + vm=args.vm, + ) ch_remote = [ "ch-remote", @@ -436,14 +643,23 @@ in ] with ExitStack() as cleanup: - ch = cleanup.enter_context(run_ch(args.prefix)) - exec(*ch_remote, "create", args.vm_config) - exec( + ch = cleanup.enter_context(ps.run_ch()) + ps.exec(*ch_remote, "create", args.vm_config) + ps.exec( TAPS_PATH, "pass", *ch_remote, "add-net", "id=wan,fd=3,mac=00:00:00:00:00:01") - exec(*ch_remote, "boot") - exec(*ch_remote, "info") + + send_dir = PASSTHRU_ENV["HOME"] + f"/send/{args.vm}" + os.makedirs(send_dir, exist_ok=True) + vfsd, vfsd_path = cleanup.enter_context( + ps.add_virtiofsd( + send_dir, + tag="send", + )) + ps.exec(*ch_remote, "add-fs", f"tag=send,socket={vfsd_path},id=send") + ps.exec(*ch_remote, "boot") + ps.exec(*ch_remote, "info") try: ch.wait() except KeyboardInterrupt: From a3ed9cb4eaf6883e6973b3b4ab687f3982c5832d Mon Sep 17 00:00:00 2001 From: Else Someone Date: Sat, 21 Feb 2026 18:36:52 +0200 Subject: [PATCH 11/37] fixup! ch-runner: add minimal virtiofsd support --- profiles/ch-runner.nix | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/profiles/ch-runner.nix b/profiles/ch-runner.nix index f156705..b9e17cf 100644 --- a/profiles/ch-runner.nix +++ b/profiles/ch-runner.nix @@ -395,6 +395,9 @@ in kwargs["pass_fds"] = kwargs.get("pass_fds", ()) kwargs["env"] = kwargs.get("env", self.make_env()) kwargs["cwd"] = kwargs.get("cwd", self.prefix) + kwargs["stdin"] = kwargs.get("stdin", subprocess.DEVNULL) + kwargs["stdout"] = kwargs.get("stdout", subprocess.DEVNULL) + kwargs["stderr"] = kwargs.get("stderr", subprocess.DEVNULL) return subprocess.Popen( args, **kwargs, @@ -511,7 +514,7 @@ in shell=False, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, - pass_fds=(2,)) + pass_fds=()) try: p.wait(0.125) needs_cleanup = False @@ -529,6 +532,7 @@ in except: # noqa: E722 pass if p.returncode is None: + print("Terminating CH") p.terminate() # CH handles SIG{INT,TERM}? p.wait() unlink_paths = [ @@ -541,7 +545,7 @@ in os.remove(p) @contextmanager - def add_virtiofsd( + def start_virtiofsd( self, root_dir, tag, @@ -555,6 +559,7 @@ in # s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) # NOTE: Nope. Virtiofsd actually expects a blocking socket # s.setblocking(True) + # s.set_inheritable(True) def rm_sock(): if os.path.exists(sock_path): @@ -643,6 +648,15 @@ in ] with ExitStack() as cleanup: + + send_dir = PASSTHRU_ENV["HOME"] + f"/send/{args.vm}" + os.makedirs(send_dir, exist_ok=True) + vfsd, vfsd_path = cleanup.enter_context( + ps.start_virtiofsd( + send_dir, + tag="send", + )) + ch = cleanup.enter_context(ps.run_ch()) ps.exec(*ch_remote, "create", args.vm_config) ps.exec( @@ -650,13 +664,6 @@ in *ch_remote, "add-net", "id=wan,fd=3,mac=00:00:00:00:00:01") - send_dir = PASSTHRU_ENV["HOME"] + f"/send/{args.vm}" - os.makedirs(send_dir, exist_ok=True) - vfsd, vfsd_path = cleanup.enter_context( - ps.add_virtiofsd( - send_dir, - tag="send", - )) ps.exec(*ch_remote, "add-fs", f"tag=send,socket={vfsd_path},id=send") ps.exec(*ch_remote, "boot") ps.exec(*ch_remote, "info") From 04befb6328331a7002db66863bf3ce6304056b0b Mon Sep 17 00:00:00 2001 From: Else Someone Date: Mon, 23 Feb 2026 06:35:32 +0200 Subject: [PATCH 12/37] ch-runner: run CH in bwrap Albeit rather relaxed --- pkgs/taps/package.nix | 2 + profiles/ch-runner.nix | 104 ++++++++++++++++++++++++----------------- shell.nix | 33 +++++++++---- 3 files changed, 87 insertions(+), 52 deletions(-) diff --git a/pkgs/taps/package.nix b/pkgs/taps/package.nix index c666cd9..e396748 100644 --- a/pkgs/taps/package.nix +++ b/pkgs/taps/package.nix @@ -30,6 +30,8 @@ stdenv.mkDerivation { rustc ]; buildInputs = [ ch-proxy ]; + + meta.mainProgram = "taps"; } # { lib, rustPlatform }: # diff --git a/profiles/ch-runner.nix b/profiles/ch-runner.nix index b9e17cf..b82d700 100644 --- a/profiles/ch-runner.nix +++ b/profiles/ch-runner.nix @@ -278,6 +278,7 @@ in (lib.getBin package) (lib.getBin pkgs.virtiofsd) (lib.getBin pkgs.bubblewrap) + (lib.getBin pkgs.strace) uvmsPkgs.taps ]; @@ -403,7 +404,6 @@ in **kwargs, ) - @contextmanager def bwrap( self, *bwrap_args, @@ -420,20 +420,26 @@ in unshare_uts=None, unshare_cgroup_try=True, bind=(), - dev_bind=("/dev/kvm", "/dev/vfio"), + dev_bind=(), + dev_bind_implicit=("/dev/kvm", "/dev/vfio"), dev="/dev", proc="/proc", - ro_bind=( + ro_bind_implicit=( "/etc", "/sys", "/proc/sys", "/dev/null", "/proc/kallsyms", *CLOSURE), - ro_bind_extra=(), + ro_bind=(), remount_ro=("/proc/fs", "/proc/irq"), - tmpfs=("/dev/shm", "/tmp", "/var/tmp", "/proc/fs", "/proc/irq"), - tmpfs_extra=(), + tmpfs_implicit=( + "/dev/shm", + "/tmp", + "/var/tmp", + "/proc/fs", + "/proc/irq"), + tmpfs=(), pass_fds=(2,), **popen_kwargs): @@ -455,23 +461,29 @@ in print_arg("--unshare-pid") if unshare_net: print_arg("--unshare-net") + elif unshare_net is False: + print_arg("--share-net") if unshare_uts: print_arg("--unshare-uts") if unshare_cgroup_try: print_arg("--unshare-cgroup-try") if die_with_parent: print_arg("--die-with-parent") + if dev: + print_arg("--dev", dev) + if proc: + print_arg("--proc", proc) for p in bind: p1, p2 = (p, p) if isinstance(p, str) else p print_arg("--bind", p1, p2) - for p in (*ro_bind, *ro_bind_extra): + for p in (*ro_bind, *ro_bind_implicit): p1, p2 = (p, p) if isinstance(p, str) else p print_arg("--ro-bind", p1, p2) - for p in dev_bind: + for p in (*dev_bind, *dev_bind_implicit): p1, p2 = (p, p) if isinstance(p, str) else p print_arg("--dev-bind", p1, p2) - for p in (*tmpfs, *tmpfs_extra): + for p in (*tmpfs, *tmpfs_implicit): print_arg("--tmpfs", p) # Hunch: order might matter... for p in remount_ro: @@ -487,44 +499,48 @@ in pass_fds=(*pass_fds, remote.fileno()), ) - with proc as p: - try: - yield p - finally: - try: - p.poll() - except: # noqa: E722 - pass - if p.returncode is None: - p.terminate() - p.wait() + return proc @contextmanager def run_ch(self): - args = [ - SOCKETBINDER_PATH, - "-B", - self.prefix + "/vmm.sock", - CH_PATH, - "--api-socket", - "fd=0", - ] - p = self.popen( - *args, - shell=False, - stdin=subprocess.DEVNULL, - stdout=subprocess.DEVNULL, - pass_fds=()) - try: - p.wait(0.125) - needs_cleanup = False - except subprocess.TimeoutExpired: - needs_cleanup = True - if not os.path.exists(self.prefix + "/vmm.sock"): - raise RuntimeError(f"{self.prefix}/vmm.sock should exist by now") - if p.returncode is not None: - raise RuntimeError("CH exited early") try: + # s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM, 0) + # s.set_inheritable(True) + # s.setblocking(True) + # s.bind(self.prefix + "/vmm.sock") + args = [ + SOCKETBINDER_PATH, + "-B", + self.prefix + "/vmm.sock", + # "${lib.getExe pkgs.strace}", # noqa: E501 + # "-Z", + # "-ff", + CH_PATH, + "--api-socket", + "fd=0", + # f"fd={s.fileno()}" + ] + p = self.bwrap( + *args, + bind=[self.prefix], + # Probably just need the path to vmlinux + ro_bind=("/nix/store",), # A give up + unshare_net=False, + shell=False, + stderr=None, + # pass_fds=(s.fileno(),) + ) + # s.close() + try: + p.wait(0.125) + needs_cleanup = False + except subprocess.TimeoutExpired: + needs_cleanup = True + if not os.path.exists(self.prefix + "/vmm.sock"): + raise RuntimeError( + f"{self.prefix}/vmm.sock should exist by now") + if p.returncode is not None: + raise RuntimeError("CH exited early") yield p finally: try: @@ -598,7 +614,7 @@ in kwargs = { # If bwrap(): # "bind": [], - # ("ro_bind_extra" if ro else "bind"): + # ("ro_bind" if ro else "bind"): # [*subdirs] # if subdirs is not None # else [root_dir], diff --git a/shell.nix b/shell.nix index f8bb9a7..44fdd56 100644 --- a/shell.nix +++ b/shell.nix @@ -1,13 +1,30 @@ with import { }; +let + uvmPkgs = callPackage ./pkgs { }; +in mkShell.override { stdenv = stdenvNoCC; } { - packages = map lib.getBin [ - cloud-hypervisor - virtiofsd - crosvm # virtio-gpu - npins - ] ++ [ - man-pages - linux-manual + inputsFrom = with uvmPkgs; [ + ch-proxy + taps + writeErofsLayers + request-usb ]; + packages = + map lib.getBin [ + cloud-hypervisor + virtiofsd + crosvm # virtio-gpu + npins + strace + python3 + execline + s6 + wayland-proxy-virtwl + uvmPkgs.taps + ] + ++ [ + man-pages + linux-manual + ]; } From 8d1324da035cbba9b034df20422274f7449203bb Mon Sep 17 00:00:00 2001 From: Else Someone Date: Mon, 23 Feb 2026 22:32:32 +0200 Subject: [PATCH 13/37] ch-runner: move wait() out of run_ch --- profiles/ch-runner.nix | 101 +++++++++++++++++++++++------------------ 1 file changed, 56 insertions(+), 45 deletions(-) diff --git a/profiles/ch-runner.nix b/profiles/ch-runner.nix index b82d700..cd481fa 100644 --- a/profiles/ch-runner.nix +++ b/profiles/ch-runner.nix @@ -355,6 +355,16 @@ in return args_mut + def alive_after(proc, timeout): + if proc.returncode is not None: + return False + try: + proc.wait(timeout) + except subprocess.TimeoutExpired: + return True + return False + + class Processes: def __init__(self, prefix, vm, check=True, **defaults): self.prefix = prefix @@ -392,6 +402,7 @@ in }, ) + @contextmanager def popen(self, *args, **kwargs): kwargs["pass_fds"] = kwargs.get("pass_fds", ()) kwargs["env"] = kwargs.get("env", self.make_env()) @@ -399,11 +410,20 @@ in kwargs["stdin"] = kwargs.get("stdin", subprocess.DEVNULL) kwargs["stdout"] = kwargs.get("stdout", subprocess.DEVNULL) kwargs["stderr"] = kwargs.get("stderr", subprocess.DEVNULL) - return subprocess.Popen( - args, - **kwargs, - ) + try: + proc = subprocess.Popen( + args, + **kwargs, + ) + yield proc + finally: + try: + proc.wait(0.125) + except subprocess.TimeoutExpired: + proc.terminate() + proc.wait() + @contextmanager def bwrap( self, *bwrap_args, @@ -447,7 +467,10 @@ in bwrap_args_sock, remote = socket.socketpair() remote.set_inheritable(True) bwrap_args_f = bwrap_args_sock.makefile("w") - with closing(bwrap_args_sock), closing(bwrap_args_f): + with ExitStack() as cleanup: + # cleanup.enter_context(closing(bwrap_args_sock)) + # cleanup.enter_context(closing(bwrap_args_f)) + def print_arg(*args): print(*args, file=bwrap_args_f, sep="\0", end="\0") @@ -491,15 +514,17 @@ in bwrap_args_f.flush() - with closing(remote): - proc = self.popen( + with ExitStack() as es: + es.enter_context(closing(remote)) + es.enter_context(closing(bwrap_args_sock)) + es.enter_context(closing(bwrap_args_f)) + proc = cleanup.enter_context(self.popen( "bwrap", "--args", str(remote.fileno()), *bwrap_args, **popen_kwargs, executable=BWRAP_PATH, pass_fds=(*pass_fds, remote.fileno()), - ) - - return proc + )) + yield proc @contextmanager def run_ch(self): @@ -520,37 +545,27 @@ in "fd=0", # f"fd={s.fileno()}" ] - p = self.bwrap( - *args, - bind=[self.prefix], - # Probably just need the path to vmlinux - ro_bind=("/nix/store",), # A give up - unshare_net=False, - shell=False, - stderr=None, - # pass_fds=(s.fileno(),) - ) - # s.close() - try: - p.wait(0.125) - needs_cleanup = False - except subprocess.TimeoutExpired: + needs_cleanup = False + with self.bwrap( + *args, + bind=[self.prefix], + # Probably just need the path to vmlinux + ro_bind=("/nix/store",), # A give up + unshare_net=False, + shell=False, + stderr=None, + # pass_fds=(s.fileno(),) + ) as proc: + # s.close() + assert alive_after(proc, 0.125) + if not os.path.exists(self.prefix + "/vmm.sock"): + raise RuntimeError( + f"{self.prefix}/vmm.sock should exist by now") needs_cleanup = True - if not os.path.exists(self.prefix + "/vmm.sock"): - raise RuntimeError( - f"{self.prefix}/vmm.sock should exist by now") - if p.returncode is not None: - raise RuntimeError("CH exited early") - yield p + if proc.returncode is not None: + raise RuntimeError("CH exited early") + yield proc finally: - try: - p.poll() - except: # noqa: E722 - pass - if p.returncode is None: - print("Terminating CH") - p.terminate() # CH handles SIG{INT,TERM}? - p.wait() unlink_paths = [ self.prefix + "/vmm.sock", self.prefix + "/vmm.sock.lock", @@ -621,13 +636,9 @@ in # "pass_fds": (2, s.fileno()), } - proc_ctx = self.popen(*args, **kwargs) - with proc_ctx as p: + with self.popen(*args, **kwargs) as p: try: - try: - p.wait(0.125) - except subprocess.TimeoutExpired: - pass + assert alive_after(p, 0.125) if p.returncode is not None: raise RuntimeError("virtiofsd exited too early") yield p, sock_path From a772c5a727d82e65bdb9453d593272de5e60e657 Mon Sep 17 00:00:00 2001 From: Else Someone Date: Mon, 23 Feb 2026 22:32:51 +0200 Subject: [PATCH 14/37] shell: add deps of taps and ch-runner --- shell.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/shell.nix b/shell.nix index 44fdd56..a1f6174 100644 --- a/shell.nix +++ b/shell.nix @@ -17,6 +17,7 @@ mkShell.override { stdenv = stdenvNoCC; } { crosvm # virtio-gpu npins strace + bubblewrap python3 execline s6 From e077ad6858e8fc644d71566065e6d6d093edde3c Mon Sep 17 00:00:00 2001 From: Else Someone Date: Tue, 24 Feb 2026 01:21:21 +0200 Subject: [PATCH 15/37] ch-runner: move wait/cleanup out of start_virtiofsd --- profiles/ch-runner.nix | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/profiles/ch-runner.nix b/profiles/ch-runner.nix index cd481fa..1b46f22 100644 --- a/profiles/ch-runner.nix +++ b/profiles/ch-runner.nix @@ -415,11 +415,11 @@ in args, **kwargs, ) + if not alive_after(proc, 0.125): + raise RuntimeError("Failed to start", args) yield proc finally: - try: - proc.wait(0.125) - except subprocess.TimeoutExpired: + if alive_after(proc, 0.125): proc.terminate() proc.wait() @@ -636,18 +636,12 @@ in # "pass_fds": (2, s.fileno()), } - with self.popen(*args, **kwargs) as p: - try: - assert alive_after(p, 0.125) - if p.returncode is not None: - raise RuntimeError("virtiofsd exited too early") + try: + with self.popen(*args, **kwargs) as p: yield p, sock_path - finally: - if p.returncode is None: - p.kill() - p.wait() - if os.path.exists(sock_path): - os.remove(sock_path) + finally: + if os.path.exists(sock_path): + os.remove(sock_path) @contextmanager From 660bda3a4aae39d53f3d18c16fd063b1319dd9f2 Mon Sep 17 00:00:00 2001 From: Else Someone Date: Wed, 25 Feb 2026 17:37:06 +0200 Subject: [PATCH 16/37] ch-runner: add wayland-proxy --- examples/dummy.nix | 115 ++++++++++++++++++++++++++++++++++++++-- profiles/ch-runner.nix | 79 ++++++++++++++++++++++----- profiles/vmapp-demo.nix | 6 +-- shell.nix | 2 +- 4 files changed, 183 insertions(+), 19 deletions(-) diff --git a/examples/dummy.nix b/examples/dummy.nix index 7d8c813..02254c9 100644 --- a/examples/dummy.nix +++ b/examples/dummy.nix @@ -5,6 +5,28 @@ modulesPath, ... }: +let + uvmsPkgs = pkgs.callPackage ../pkgs { }; + waylandSock = "/run/user/1000/wayland-1"; + env = { + XDG_RUNTIME_DIR = "/run/user/1000"; + WAYLAND_DISPLAY = "wayland-1"; + + MESA_LOADER_DRIVER_OVERRIDE = "zink"; + + # WAYLAND_DEBUG = "1"; + # WAYLAND_DEBUG_PROXY = "1"; + + ELECTRON_OZONE_PLATFORM_HINT = "wayland"; + MOZ_ENABLE_WAYLAND = "1"; + QT_QPA_PLATFORM = "wayland"; # Qt Applications + GDK_BACKEND = "wayland"; # GTK Applications + XDG_SESSION_TYPE = "wayland"; # Electron Applications + SDL_VIDEODRIVER = "wayland"; + CLUTTER_BACKEND = "wayland"; + NIXOS_OZONE_WL = "1"; + }; +in { imports = [ ../profiles/all.nix @@ -18,30 +40,117 @@ vmapps.enable = true; _module.args.inputs = import ../npins; - # following microvm.nix: + # boot.isContainer = true; + # boot.initrd.enable = true; boot.loader.grub.enable = false; boot.initrd.systemd.enable = true; + services.logrotate.enable = false; + services.udisks2.enable = false; + system.tools.nixos-generate-config.enable = false; + # system.activationScripts.specialfs = lib.mkForce ""; + systemd.coredump.enable = false; + # networking.firewall.enable = false; + powerManagement.enable = false; + boot.kexec.enable = false; + # console.enable = false; + # system.switch.enable = false; + # services.udev.packages = lib.mkDefault [ ]; + services.resolved.enable = false; + systemd.services.generate-shutdown-ramfs.enable = lib.mkForce false; + systemd.services.systemd-remount-fs.enable = lib.mkForce false; + systemd.services.systemd-pstore.enable = lib.mkForce false; + systemd.services.lastlog2-import.enable = lib.mkForce false; + systemd.services.suid-sgid-wrappers.enable = lib.mkForce false; + fileSystems."/" = lib.mkDefault { device = "rootfs"; # how does this work? does this assign a label to the tmpfs? fsType = "tmpfs"; options = [ "size=20%,mode=0755" ]; neededForBoot = true; }; - boot.initrd.systemd.settings.Manager.DefaultTimeoutStartSec = 30; + boot.initrd.systemd.settings.Manager.DefaultTimeoutStartSec = 5; systemd.settings.Manager.DefaultTimeoutStopSec = 10; networking.useNetworkd = true; networking.nftables.enable = true; uvms.cloud-hypervisor.enable = true; + systemd.sysusers.enable = false; + services.userborn.enable = true; # nikstur it users.mutableUsers = false; + users.groups.user = { }; + users.users.user = { + isNormalUser = true; + password = "hacktheplanet!"; + extraGroups = [ + "video" + "render" + ]; + }; users.users.root.password = "hacktheplanet!"; - services.getty.autologinUser = "root"; systemd.services."suid-sgid-wrappers".serviceConfig = { StandardOutput = "journal+console"; StandardError = "journal+console"; }; + environment.variables = env; + systemd.globalEnvironment = env; + systemd.tmpfiles.settings."10-xdg" = { + ${env.XDG_RUNTIME_DIR}.d = { + user = "user"; + group = "user"; + mode = "0755"; + }; + }; + systemd.sockets."wayland-proxy" = { + listenStreams = [ + waylandSock + ]; + socketConfig = { + SocketUser = "user"; + SocketGroup = "user"; + FileDescriptorName = "wayland"; + }; + wantedBy = [ "sockets.target" ]; + partOf = [ "wayland-proxy.service" ]; + }; + systemd.services."wayland-proxy" = { + wantedBy = [ "default.target" ]; + serviceConfig = { + User = "user"; + Group = "user"; + ExecStart = "${lib.getExe pkgs.wayland-proxy-virtwl} --virtio-gpu"; + # ExecStart = "${lib.getExe uvmsPkgs.wl-cross-domain-proxy} --listen-fd --filter-global wp_presentation"; + ExecStartPre = [ + "+/run/current-system/sw/bin/chmod 0666 /dev/dri/card0 /dev/dri/renderD128" + ]; + StandardOutput = "journal+console"; + StandardError = "journal+console"; + Restart = "on-failure"; + RestartSec = 5; + }; + }; + fonts.enableDefaultPackages = true; + + systemd.services."terminal" = { + wantedBy = [ "multi-user.target" ]; + wants = [ "wayland-proxy.service" ]; + after = [ "wayland-proxy.service" ]; + environment = env; + serviceConfig = { + User = "user"; + WorkingDirectory = "/home/user"; + ExecStart = lib.getExe pkgs.alacritty; + StandardOutput = "journal+console"; + StandardError = "journal+console"; + }; + }; + boot.kernelModules = [ + "drm" + "virtio_gpu" + ]; + hardware.graphics.enable = true; + # TODO: cmdline, kernel, initrd, fileSystems } diff --git a/profiles/ch-runner.nix b/profiles/ch-runner.nix index 1b46f22..55be70a 100644 --- a/profiles/ch-runner.nix +++ b/profiles/ch-runner.nix @@ -22,13 +22,7 @@ let getBin ; - package = pkgs.cloud-hypervisor.overrideAttrs (oldAttrs: { - patches = oldAttrs.patches or [ ] ++ [ - # ../patches/ch.patch - ]; - buildType = "debug"; - dontStrip = true; - }); + package = uvmsPkgs.cloud-hypervisor-gpu; uvmsPkgs = pkgs.callPackage ../pkgs { }; chSettingsFile = (pkgs.formats.json { }).generate "vm.json" cfg.settings; @@ -279,11 +273,20 @@ in (lib.getBin pkgs.virtiofsd) (lib.getBin pkgs.bubblewrap) (lib.getBin pkgs.strace) + (lib.getBin pkgs.crosvm) uvmsPkgs.taps ]; superviseVm = getExe superviseVm'; superviseVm' = pkgs.writers.writePython3Bin "supervise-vm" { } '' + # NOTE: This would have been bash, + # and this was execlineb previously, + # but it was just easier to reason in terms of context managers + # and try-except-finally branches for the cleanup bit, + # than in terms of traps or such. + # Treat this as bash. + # Treat this as throwaway shitcode. + import os import subprocess import socket @@ -320,8 +323,10 @@ in **{ k: v for k, v in os.environ.items() - if k.startswith("RUST") + if k.startswith("RUST_") or k.startswith("WAYLAND") + or k.startswith("XDG_") + or k.startswith("DBUS_") or k in [ "TAPS_SOCK", ] @@ -356,6 +361,8 @@ in def alive_after(proc, timeout): + if proc is None: + return False if proc.returncode is not None: return False try: @@ -410,6 +417,7 @@ in kwargs["stdin"] = kwargs.get("stdin", subprocess.DEVNULL) kwargs["stdout"] = kwargs.get("stdout", subprocess.DEVNULL) kwargs["stderr"] = kwargs.get("stderr", subprocess.DEVNULL) + proc = None try: proc = subprocess.Popen( args, @@ -421,7 +429,8 @@ in finally: if alive_after(proc, 0.125): proc.terminate() - proc.wait() + if proc is not None: + proc.wait() @contextmanager def bwrap( @@ -550,7 +559,7 @@ in *args, bind=[self.prefix], # Probably just need the path to vmlinux - ro_bind=("/nix/store",), # A give up + ro_bind=["/nix/store"], # I give up unshare_net=False, shell=False, stderr=None, @@ -575,6 +584,35 @@ in if os.path.exists(p): os.remove(p) + @contextmanager + def start_gpu( + self, + ): + sock_path = self.prefix + "/gpu.sock" + args = [ + SOCKETBINDER_PATH, + "-b", "1", + sock_path, + "s6-ipcserverd", + "-1c1", + # "${lib.getExe pkgs.strace}", # noqa: E501 + # "-Z", + # "-ff", + "${lib.getExe pkgs.crosvm}", # noqa: E501 + "--no-syslog", + "device", "gpu", + "--fd", "0", + "--wayland-sock", + f'{PASSTHRU_ENV["XDG_RUNTIME_DIR"]}/{PASSTHRU_ENV["WAYLAND_DISPLAY"]}', # noqa: E501 + "--params", + "{ \"context-types\": \"cross-domain:virgl2:venus\" }", + ] + with self.popen( + *args, + stderr=None, + ) as proc, removing(sock_path): + yield proc, sock_path + @contextmanager def start_virtiofsd( self, @@ -652,11 +690,26 @@ in f() + @contextmanager + def removing(*paths): + try: + yield + finally: + for p in paths: + if os.path.exists(p): + os.remove(p) + + if __name__ == "__main__": args, args_next = parser.parse_known_args() preprocess_args(args) + send_dir = PASSTHRU_ENV["HOME"] + f"/send/{args.vm}" + + os.makedirs(send_dir, exist_ok=True) os.makedirs(args.prefix, exist_ok=True) + os.makedirs(args.prefix + "/pts", exist_ok=True) + ps = Processes( prefix=args.prefix, vm=args.vm, @@ -670,13 +723,14 @@ in with ExitStack() as cleanup: - send_dir = PASSTHRU_ENV["HOME"] + f"/send/{args.vm}" - os.makedirs(send_dir, exist_ok=True) vfsd, vfsd_path = cleanup.enter_context( ps.start_virtiofsd( send_dir, tag="send", )) + gpud, gpud_path = cleanup.enter_context( + ps.start_gpu() + ) ch = cleanup.enter_context(ps.run_ch()) ps.exec(*ch_remote, "create", args.vm_config) @@ -686,6 +740,7 @@ in "id=wan,fd=3,mac=00:00:00:00:00:01") ps.exec(*ch_remote, "add-fs", f"tag=send,socket={vfsd_path},id=send") + ps.exec(*ch_remote, "add-gpu", f"socket={gpud_path}") ps.exec(*ch_remote, "boot") ps.exec(*ch_remote, "info") try: diff --git a/profiles/vmapp-demo.nix b/profiles/vmapp-demo.nix index 2f960b2..3af62bc 100644 --- a/profiles/vmapp-demo.nix +++ b/profiles/vmapp-demo.nix @@ -212,9 +212,9 @@ in ''} %i"; }; - boot.initrd.systemd.settings.Manager.DefaultTimeoutStartSec = 30; - systemd.settings.Manager.DefaultTimeoutStopSec = 10; - systemd.services."user@".serviceConfig.TimeoutStopSec = 10; + boot.initrd.systemd.settings.Manager.DefaultTimeoutStartSec = lib.mkDefault 30; + systemd.settings.Manager.DefaultTimeoutStopSec = lib.mkDefault 10; + systemd.services."user@".serviceConfig.TimeoutStopSec = lib.mkDefault 10; services.openssh.enable = true; diff --git a/shell.nix b/shell.nix index a1f6174..83b3391 100644 --- a/shell.nix +++ b/shell.nix @@ -12,7 +12,7 @@ mkShell.override { stdenv = stdenvNoCC; } { ]; packages = map lib.getBin [ - cloud-hypervisor + uvmPkgs.cloud-hypervisor-gpu virtiofsd crosvm # virtio-gpu npins From f193787f4ea16146dcde01fcf583c0294d6e30c6 Mon Sep 17 00:00:00 2001 From: Else Someone Date: Wed, 25 Feb 2026 23:37:14 +0200 Subject: [PATCH 17/37] ch-runner: move the supervise/uvms script out --- pkgs/uvms/package.nix | 47 ++++ pkgs/uvms/uvms.py | 474 +++++++++++++++++++++++++++++++++++++++ profiles/ch-runner.nix | 491 +---------------------------------------- 3 files changed, 524 insertions(+), 488 deletions(-) create mode 100644 pkgs/uvms/package.nix create mode 100644 pkgs/uvms/uvms.py diff --git a/pkgs/uvms/package.nix b/pkgs/uvms/package.nix new file mode 100644 index 0000000..d5e84c3 --- /dev/null +++ b/pkgs/uvms/package.nix @@ -0,0 +1,47 @@ +{ + lib, + symlinkJoin, + writers, + writeClosure, + replaceVars, + bubblewrap, + cloud-hypervisor-gpu, + crosvm, + effective-cloud-hypervisor ? cloud-hypervisor-gpu, + execline, + s6, + strace, + taps, + util-linux, + virtiofsd, +}: + +let + tools = map lib.getBin [ + execline + s6 + effective-cloud-hypervisor + virtiofsd + bubblewrap + strace + crosvm + taps + util-linux + ]; + toolsFarm = symlinkJoin { + name = "tools"; + paths = tools; + }; + toolsClosure = writeClosure toolsFarm; +in +writers.writePython3Bin "uvms" { } ( + replaceVars ./uvms.py { + BWRAP = "${lib.getExe bubblewrap}"; + TOOLS = "${toolsFarm}/bin"; + TOOLS_CLOSURE = toolsClosure; + CROSVM = lib.getExe crosvm; + STRACE = lib.getExe strace; + TAPS = "${lib.getExe taps}"; + VIRTIOFSD = "${lib.getExe virtiofsd}"; + } +) diff --git a/pkgs/uvms/uvms.py b/pkgs/uvms/uvms.py new file mode 100644 index 0000000..ffdc28b --- /dev/null +++ b/pkgs/uvms/uvms.py @@ -0,0 +1,474 @@ +# NOTE: This would have been bash, +# and this was execlineb previously, +# but it was just easier to reason in terms of context managers +# and try-except-finally branches for the cleanup bit, +# than in terms of traps or such. +# Treat this as bash. +# Treat this as throwaway shitcode. + +import os +import subprocess +import socket +from argparse import ArgumentParser +from contextlib import contextmanager, closing, ExitStack + + +parser = ArgumentParser("supervise-vm") +parser.add_argument("--vm") +parser.add_argument("--prefix", default="$HOME/uvms/$VM") +parser.add_argument("--vm-config") + +TOOLS_DIR = "@TOOLS@" # noqa: E501 +SOCKETBINDER = TOOLS_DIR + "/s6-ipcserver-socketbinder" # noqa: E501 +CH = TOOLS_DIR + "/cloud-hypervisor" +CHR = TOOLS_DIR + "/ch-remote" +TAPS = "@TAPS@" # noqa: E501 +VIRTIOFSD = "@VIRTIOFSD@" # noqa: E501 +BWRAP = "@BWRAP@" # noqa: E501 + +with open("@TOOLS_CLOSURE@", mode="r") as f: # noqa: E501 + CLOSURE = [ + *(ln.rstrip() for ln in f.readlines()), + os.path.dirname(__file__), + ] + +PASSTHRU_PATH = ":".join([TOOLS_DIR]) +PASSTHRU_ENV = { + **{ + k: v + for k, v in os.environ.items() + if k.startswith("RUST_") + or k.startswith("WAYLAND") + or k.startswith("XDG_") + or k.startswith("DBUS_") + or k + in [ + "TAPS_SOCK", + ] + }, + "HOME": os.environ.get("HOME", os.getcwd()), + "PATH": PASSTHRU_PATH, +} + + +def preprocess_args(args_mut): + keys = [k for k, v in args_mut._get_kwargs() if isinstance(v, str)] + for k in keys: + v = getattr(args_mut, k) + if "$HOME" in v: + setattr(args_mut, k, v.replace("$HOME", PASSTHRU_ENV["HOME"])) + for k in keys: + v = getattr(args_mut, k) + if "$VM" in v: + setattr(args_mut, k, v.replace("$VM", args.vm)) + for k in keys: + v = getattr(args_mut, k) + if "$PREFIX" in v: + setattr(args_mut, k, v.replace("$PREFIX", args.prefix)) + return args_mut + + +def alive_after(proc, timeout): + if proc is None: + return False + if proc.returncode is not None: + return False + try: + proc.wait(timeout) + except subprocess.TimeoutExpired: + return True + return False + + +class Processes: + def __init__(self, prefix, vm, check=True, **defaults): + self.prefix = prefix + self.vm = vm + self.check = check + self.defaults = defaults + + def make_env(self): + return { + **PASSTHRU_ENV, + "PATH": PASSTHRU_PATH, + "PREFIX": self.prefix, + "VM": self.vm, + } + + def exec(self, *args, **kwargs): + kwargs["cwd"] = kwargs.get("cwd", self.prefix) + kwargs["check"] = kwargs.get("check", self.check) + kwargs["env"] = kwargs.get("env", self.make_env()) + return subprocess.run([*args], **self.defaults, **kwargs) + + def execline(self, *args, **kwargs): + return exec( + "execlineb", + "-c", + "\n".join(args), + **self.defaults, + executable=TOOLS_DIR + "/execlineb", + **{ + "env": self.make_env(), + "check": self.check, + "cwd": self.prefix, + **kwargs, + }, + ) + + @contextmanager + def popen(self, *args, **kwargs): + kwargs["pass_fds"] = kwargs.get("pass_fds", ()) + kwargs["env"] = kwargs.get("env", self.make_env()) + kwargs["cwd"] = kwargs.get("cwd", self.prefix) + kwargs["stdin"] = kwargs.get("stdin", subprocess.DEVNULL) + kwargs["stdout"] = kwargs.get("stdout", subprocess.DEVNULL) + kwargs["stderr"] = kwargs.get("stderr", subprocess.DEVNULL) + proc = None + try: + proc = subprocess.Popen( + args, + **kwargs, + ) + if not alive_after(proc, 0.125): + raise RuntimeError("Failed to start", args) + yield proc + finally: + if alive_after(proc, 0.125): + proc.terminate() + if proc is not None: + proc.wait() + + @contextmanager + def bwrap( + self, + *bwrap_args, + die_with_parent=True, + # Based on the args from + # `host/rootfs/image/usr/bin/run-vmm` + unshare_all=True, + unshare_user=True, + unshare_ipc=None, + unshare_pid=None, + unshare_net=None, + unshare_uts=None, + unshare_cgroup_try=True, + bind=(), + dev_bind=(), + dev_bind_implicit=("/dev/kvm", "/dev/vfio"), + dev="/dev", + proc="/proc", + ro_bind_implicit=( + "/etc", + "/sys", + "/proc/sys", + "/dev/null", + "/proc/kallsyms", + *CLOSURE, + ), + ro_bind=(), + remount_ro=("/proc/fs", "/proc/irq"), + tmpfs_implicit=( + "/dev/shm", + "/tmp", + "/var/tmp", + "/proc/fs", + "/proc/irq", + ), + tmpfs=(), + pass_fds=(2,), + **popen_kwargs, + ): + + bwrap_args_sock, remote = socket.socketpair() + remote.set_inheritable(True) + bwrap_args_f = bwrap_args_sock.makefile("w") + with ExitStack() as cleanup: + # cleanup.enter_context(closing(bwrap_args_sock)) + # cleanup.enter_context(closing(bwrap_args_f)) + + def print_arg(*args): + print(*args, file=bwrap_args_f, sep="\0", end="\0") + + if unshare_all: + print_arg("--unshare-all") + if unshare_user: + print_arg("--unshare-user") + if unshare_ipc: + print_arg("--unshare-ipc") + if unshare_pid: + print_arg("--unshare-pid") + if unshare_net: + print_arg("--unshare-net") + elif unshare_net is False: + print_arg("--share-net") + if unshare_uts: + print_arg("--unshare-uts") + if unshare_cgroup_try: + print_arg("--unshare-cgroup-try") + if die_with_parent: + print_arg("--die-with-parent") + if dev: + print_arg("--dev", dev) + if proc: + print_arg("--proc", proc) + + for p in bind: + p1, p2 = (p, p) if isinstance(p, str) else p + print_arg("--bind", p1, p2) + for p in (*ro_bind, *ro_bind_implicit): + p1, p2 = (p, p) if isinstance(p, str) else p + print_arg("--ro-bind", p1, p2) + for p in (*dev_bind, *dev_bind_implicit): + p1, p2 = (p, p) if isinstance(p, str) else p + print_arg("--dev-bind", p1, p2) + for p in (*tmpfs, *tmpfs_implicit): + print_arg("--tmpfs", p) + # Hunch: order might matter... + for p in remount_ro: + print_arg("--remount-ro", p) + + bwrap_args_f.flush() + + with ExitStack() as es: + es.enter_context(closing(remote)) + es.enter_context(closing(bwrap_args_sock)) + es.enter_context(closing(bwrap_args_f)) + proc = cleanup.enter_context( + self.popen( + "bwrap", + "--args", + str(remote.fileno()), + *bwrap_args, + **popen_kwargs, + executable=BWRAP, + pass_fds=(*pass_fds, remote.fileno()), + ) + ) + yield proc + + @contextmanager + def run_ch(self): + try: + # s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM, 0) + # s.set_inheritable(True) + # s.setblocking(True) + # s.bind(self.prefix + "/vmm.sock") + args = [ + SOCKETBINDER, + "-B", + self.prefix + "/vmm.sock", + # "@STRACE@", # noqa: E501 + # "-Z", + # "-ff", + CH, + "--api-socket", + "fd=0", + # f"fd={s.fileno()}" + ] + needs_cleanup = False + with self.bwrap( + *args, + bind=[self.prefix], + # Probably just need the path to vmlinux + ro_bind=["/nix/store"], # I give up + unshare_net=False, + shell=False, + stderr=None, + # pass_fds=(s.fileno(),) + ) as proc: + # s.close() + assert alive_after(proc, 0.125) + if not os.path.exists(self.prefix + "/vmm.sock"): + raise RuntimeError( + f"{self.prefix}/vmm.sock should exist by now", + ) + needs_cleanup = True + if proc.returncode is not None: + raise RuntimeError("CH exited early") + yield proc + finally: + unlink_paths = ( + [ + self.prefix + "/vmm.sock", + self.prefix + "/vmm.sock.lock", + self.prefix + "/vsock.sock", + ] + if needs_cleanup + else [] + ) + for p in unlink_paths: + if os.path.exists(p): + os.remove(p) + + @contextmanager + def start_gpu( + self, + ): + sock_path = self.prefix + "/gpu.sock" + args = [ + SOCKETBINDER, + "-b", + "1", + sock_path, + "s6-ipcserverd", + "-1c1", + # "@STRACE@", # noqa: E501 + # "-Z", + # "-ff", + "@CROSVM@", # noqa: E501 + "--no-syslog", + "device", + "gpu", + "--fd", + "0", + "--wayland-sock", + f'{PASSTHRU_ENV["XDG_RUNTIME_DIR"]}/{PASSTHRU_ENV["WAYLAND_DISPLAY"]}', # noqa: E501 + "--params", + '{ "context-types": "cross-domain:virgl2:venus" }', + ] + with self.popen( + *args, + stderr=None, + ) as proc, removing(sock_path): + yield proc, sock_path + + @contextmanager + def start_virtiofsd( + self, + root_dir, + tag, + ro=False, + subdirs=None, + extra_flags=("--posix-acl",), + ): + + assert os.path.exists(root_dir) + + sock_path = self.prefix + f"/virtiofsd-{tag}.sock" + # s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + # NOTE: Nope. Virtiofsd actually expects a blocking socket + # s.setblocking(True) + # s.set_inheritable(True) + + def rm_sock(): + if os.path.exists(sock_path): + os.remove(sock_path) + + with ExitStack() as cleanup: # noqa: F841 + # s.bind(sock_path.encode("utf8")) + # cleanup.enter_context(closing(s)) + cleanup.enter_context(defer(rm_sock)) + + args = [ + # If using bwrap(): + # "--argv0", "virtiofsd", + # "--uid", "1000", + # "--gid", "1000", + # "--", + "unshare", + "-rUm", + "unshare", + "--map-user", + "1000", + "--map-group", + "1000", + VIRTIOFSD, + "--shared-dir", + root_dir, + "--tag", + tag, + # "--fd", + # str(s.fileno()), + "--socket-path", + sock_path, + # If relying on bwrap(): + # "--sandbox", + # "none", + ] + if ro: + args.append("--readonly") + kwargs = { + # If bwrap(): + # "bind": [], + # ("ro_bind" if ro else "bind"): + # [*subdirs] + # if subdirs is not None + # else [root_dir], + # "pass_fds": (2, s.fileno()), + } + try: + with self.popen(*args, **kwargs) as p: + yield p, sock_path + finally: + if os.path.exists(sock_path): + os.remove(sock_path) + + +@contextmanager +def defer(f): + try: + yield + finally: + f() + + +@contextmanager +def removing(*paths): + try: + yield + finally: + for p in paths: + if os.path.exists(p): + os.remove(p) + + +if __name__ == "__main__": + args, args_next = parser.parse_known_args() + preprocess_args(args) + + send_dir = PASSTHRU_ENV["HOME"] + f"/send/{args.vm}" + + os.makedirs(send_dir, exist_ok=True) + os.makedirs(args.prefix, exist_ok=True) + os.makedirs(args.prefix + "/pts", exist_ok=True) + + ps = Processes( + prefix=args.prefix, + vm=args.vm, + ) + + ch_remote = [ + "ch-remote", + "--api-socket", + args.prefix + "/vmm.sock", + ] + + with ExitStack() as cleanup: + + vfsd, vfsd_path = cleanup.enter_context( + ps.start_virtiofsd( + send_dir, + tag="send", + ) + ) + gpud, gpud_path = cleanup.enter_context(ps.start_gpu()) + + ch = cleanup.enter_context(ps.run_ch()) + ps.exec(*ch_remote, "create", args.vm_config) + ps.exec( + TAPS, + "pass", + *ch_remote, + "add-net", + "id=wan,fd=3,mac=00:00:00:00:00:01", + ) + + ps.exec(*ch_remote, "add-fs", f"tag=send,socket={vfsd_path},id=send") + ps.exec(*ch_remote, "add-gpu", f"socket={gpud_path}") + ps.exec(*ch_remote, "boot") + ps.exec(*ch_remote, "info") + try: + ch.wait() + except KeyboardInterrupt: + pass diff --git a/profiles/ch-runner.nix b/profiles/ch-runner.nix index 55be70a..7a2e151 100644 --- a/profiles/ch-runner.nix +++ b/profiles/ch-runner.nix @@ -264,494 +264,9 @@ in ); # NOTE: Used to be an even uglier bash script, but, for now, execline makes for easier comparisons against spectrum - uvms.cloud-hypervisor.runner = - let - toolsClosure = pkgs.writeClosure [ - (lib.getBin pkgs.execline) - (lib.getBin pkgs.s6) - (lib.getBin package) - (lib.getBin pkgs.virtiofsd) - (lib.getBin pkgs.bubblewrap) - (lib.getBin pkgs.strace) - (lib.getBin pkgs.crosvm) - uvmsPkgs.taps - ]; - - superviseVm = getExe superviseVm'; - superviseVm' = pkgs.writers.writePython3Bin "supervise-vm" { } '' - # NOTE: This would have been bash, - # and this was execlineb previously, - # but it was just easier to reason in terms of context managers - # and try-except-finally branches for the cleanup bit, - # than in terms of traps or such. - # Treat this as bash. - # Treat this as throwaway shitcode. - - import os - import subprocess - import socket - from argparse import ArgumentParser - from contextlib import contextmanager, closing, ExitStack - - - parser = ArgumentParser("supervise-vm") - parser.add_argument("--vm") - parser.add_argument("--prefix", default="$HOME/uvms/$VM") - parser.add_argument("--sock", default="$PREFIX/supervisor.sock") - parser.add_argument("--vm-config") - - MSG_SIZE = 16 - ELB_DIR = "${lib.getBin pkgs.execline}/bin" # noqa: E501 - S6_DIR = "${lib.getBin pkgs.s6}/bin" # noqa: E501 - CH_DIR = "${lib.getBin package}/bin" # noqa: E501 - UTIL_LINUX_DIR = "${lib.getBin pkgs.util-linux}/bin" # noqa: E501 - SOCKETBINDER_PATH = S6_DIR + "/s6-ipcserver-socketbinder" # noqa: E501 - CH_PATH = CH_DIR + "/cloud-hypervisor" - CHR_PATH = CH_DIR + "/ch-remote" - TAPS_PATH = "${lib.getExe uvmsPkgs.taps}" # noqa: E501 - VIRTIOFSD_PATH = "${lib.getExe pkgs.virtiofsd}" # noqa: E501 - BWRAP_PATH = "${lib.getExe pkgs.bubblewrap}" # noqa: E501 - - with open("${toolsClosure}", mode="r") as f: # noqa: E501 - CLOSURE = [ - *(ln.rstrip() for ln in f.readlines()), - "${placeholder "out"}", # noqa: E501 - ] - - PASSTHRU_PATH = ":".join([ELB_DIR, S6_DIR, CH_DIR, UTIL_LINUX_DIR]) - PASSTHRU_ENV = { - **{ - k: v - for k, v in os.environ.items() - if k.startswith("RUST_") - or k.startswith("WAYLAND") - or k.startswith("XDG_") - or k.startswith("DBUS_") - or k in [ - "TAPS_SOCK", - ] - }, - "HOME": os.environ.get("HOME", os.getcwd()), - "PATH": PASSTHRU_PATH, - } - - - def preprocess_args(args_mut): - keys = [ - k - for k, v - in args_mut._get_kwargs() - if isinstance(v, str)] - for k in keys: - v = getattr(args_mut, k) - if "$HOME" in v: - setattr( - args_mut, - k, - v.replace("$HOME", PASSTHRU_ENV["HOME"])) - for k in keys: - v = getattr(args_mut, k) - if "$VM" in v: - setattr(args_mut, k, v.replace("$VM", args.vm)) - for k in keys: - v = getattr(args_mut, k) - if "$PREFIX" in v: - setattr(args_mut, k, v.replace("$PREFIX", args.prefix)) - return args_mut - - - def alive_after(proc, timeout): - if proc is None: - return False - if proc.returncode is not None: - return False - try: - proc.wait(timeout) - except subprocess.TimeoutExpired: - return True - return False - - - class Processes: - def __init__(self, prefix, vm, check=True, **defaults): - self.prefix = prefix - self.vm = vm - self.check = check - self.defaults = defaults - - def make_env(self): - return { - **PASSTHRU_ENV, - "PATH": PASSTHRU_PATH, - "PREFIX": self.prefix, - "VM": self.vm, - } - - def exec(self, *args, **kwargs): - kwargs["cwd"] = kwargs.get("cwd", self.prefix) - kwargs["check"] = kwargs.get("check", self.check) - kwargs["env"] = kwargs.get("env", self.make_env()) - return subprocess.run( - [*args], - **self.defaults, - **kwargs) - - def execline(self, *args, **kwargs): - return exec( - "execlineb", "-c", "\n".join(args), - **self.defaults, - executable=ELB_DIR + "/execlineb", - **{ - "env": self.make_env(), - "check": self.check, - "cwd": self.prefix, - **kwargs, - }, - ) - - @contextmanager - def popen(self, *args, **kwargs): - kwargs["pass_fds"] = kwargs.get("pass_fds", ()) - kwargs["env"] = kwargs.get("env", self.make_env()) - kwargs["cwd"] = kwargs.get("cwd", self.prefix) - kwargs["stdin"] = kwargs.get("stdin", subprocess.DEVNULL) - kwargs["stdout"] = kwargs.get("stdout", subprocess.DEVNULL) - kwargs["stderr"] = kwargs.get("stderr", subprocess.DEVNULL) - proc = None - try: - proc = subprocess.Popen( - args, - **kwargs, - ) - if not alive_after(proc, 0.125): - raise RuntimeError("Failed to start", args) - yield proc - finally: - if alive_after(proc, 0.125): - proc.terminate() - if proc is not None: - proc.wait() - - @contextmanager - def bwrap( - self, - *bwrap_args, - - die_with_parent=True, - - # Based on the args from - # `host/rootfs/image/usr/bin/run-vmm` - unshare_all=True, - unshare_user=True, - unshare_ipc=None, - unshare_pid=None, - unshare_net=None, - unshare_uts=None, - unshare_cgroup_try=True, - bind=(), - dev_bind=(), - dev_bind_implicit=("/dev/kvm", "/dev/vfio"), - dev="/dev", - proc="/proc", - ro_bind_implicit=( - "/etc", - "/sys", - "/proc/sys", - "/dev/null", - "/proc/kallsyms", - *CLOSURE), - ro_bind=(), - remount_ro=("/proc/fs", "/proc/irq"), - tmpfs_implicit=( - "/dev/shm", - "/tmp", - "/var/tmp", - "/proc/fs", - "/proc/irq"), - tmpfs=(), - - pass_fds=(2,), - **popen_kwargs): - - bwrap_args_sock, remote = socket.socketpair() - remote.set_inheritable(True) - bwrap_args_f = bwrap_args_sock.makefile("w") - with ExitStack() as cleanup: - # cleanup.enter_context(closing(bwrap_args_sock)) - # cleanup.enter_context(closing(bwrap_args_f)) - - def print_arg(*args): - print(*args, file=bwrap_args_f, sep="\0", end="\0") - - if unshare_all: - print_arg("--unshare-all") - if unshare_user: - print_arg("--unshare-user") - if unshare_ipc: - print_arg("--unshare-ipc") - if unshare_pid: - print_arg("--unshare-pid") - if unshare_net: - print_arg("--unshare-net") - elif unshare_net is False: - print_arg("--share-net") - if unshare_uts: - print_arg("--unshare-uts") - if unshare_cgroup_try: - print_arg("--unshare-cgroup-try") - if die_with_parent: - print_arg("--die-with-parent") - if dev: - print_arg("--dev", dev) - if proc: - print_arg("--proc", proc) - - for p in bind: - p1, p2 = (p, p) if isinstance(p, str) else p - print_arg("--bind", p1, p2) - for p in (*ro_bind, *ro_bind_implicit): - p1, p2 = (p, p) if isinstance(p, str) else p - print_arg("--ro-bind", p1, p2) - for p in (*dev_bind, *dev_bind_implicit): - p1, p2 = (p, p) if isinstance(p, str) else p - print_arg("--dev-bind", p1, p2) - for p in (*tmpfs, *tmpfs_implicit): - print_arg("--tmpfs", p) - # Hunch: order might matter... - for p in remount_ro: - print_arg("--remount-ro", p) - - bwrap_args_f.flush() - - with ExitStack() as es: - es.enter_context(closing(remote)) - es.enter_context(closing(bwrap_args_sock)) - es.enter_context(closing(bwrap_args_f)) - proc = cleanup.enter_context(self.popen( - "bwrap", "--args", str(remote.fileno()), *bwrap_args, - **popen_kwargs, - executable=BWRAP_PATH, - pass_fds=(*pass_fds, remote.fileno()), - )) - yield proc - - @contextmanager - def run_ch(self): - try: - # s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM, 0) - # s.set_inheritable(True) - # s.setblocking(True) - # s.bind(self.prefix + "/vmm.sock") - args = [ - SOCKETBINDER_PATH, - "-B", - self.prefix + "/vmm.sock", - # "${lib.getExe pkgs.strace}", # noqa: E501 - # "-Z", - # "-ff", - CH_PATH, - "--api-socket", - "fd=0", - # f"fd={s.fileno()}" - ] - needs_cleanup = False - with self.bwrap( - *args, - bind=[self.prefix], - # Probably just need the path to vmlinux - ro_bind=["/nix/store"], # I give up - unshare_net=False, - shell=False, - stderr=None, - # pass_fds=(s.fileno(),) - ) as proc: - # s.close() - assert alive_after(proc, 0.125) - if not os.path.exists(self.prefix + "/vmm.sock"): - raise RuntimeError( - f"{self.prefix}/vmm.sock should exist by now") - needs_cleanup = True - if proc.returncode is not None: - raise RuntimeError("CH exited early") - yield proc - finally: - unlink_paths = [ - self.prefix + "/vmm.sock", - self.prefix + "/vmm.sock.lock", - self.prefix + "/vsock.sock", - ] if needs_cleanup else [] - for p in unlink_paths: - if os.path.exists(p): - os.remove(p) - - @contextmanager - def start_gpu( - self, - ): - sock_path = self.prefix + "/gpu.sock" - args = [ - SOCKETBINDER_PATH, - "-b", "1", - sock_path, - "s6-ipcserverd", - "-1c1", - # "${lib.getExe pkgs.strace}", # noqa: E501 - # "-Z", - # "-ff", - "${lib.getExe pkgs.crosvm}", # noqa: E501 - "--no-syslog", - "device", "gpu", - "--fd", "0", - "--wayland-sock", - f'{PASSTHRU_ENV["XDG_RUNTIME_DIR"]}/{PASSTHRU_ENV["WAYLAND_DISPLAY"]}', # noqa: E501 - "--params", - "{ \"context-types\": \"cross-domain:virgl2:venus\" }", - ] - with self.popen( - *args, - stderr=None, - ) as proc, removing(sock_path): - yield proc, sock_path - - @contextmanager - def start_virtiofsd( - self, - root_dir, - tag, - ro=False, - subdirs=None, - extra_flags=("--posix-acl",)): - - assert os.path.exists(root_dir) - - sock_path = self.prefix + f"/virtiofsd-{tag}.sock" - # s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - # NOTE: Nope. Virtiofsd actually expects a blocking socket - # s.setblocking(True) - # s.set_inheritable(True) - - def rm_sock(): - if os.path.exists(sock_path): - os.remove(sock_path) - - with ExitStack() as cleanup: # noqa: F841 - # s.bind(sock_path.encode("utf8")) - # cleanup.enter_context(closing(s)) - cleanup.enter_context(defer(rm_sock)) - - args = [ - # If using bwrap(): - # "--argv0", "virtiofsd", - # "--uid", "1000", - # "--gid", "1000", - # "--", - "unshare", "-rUm", - "unshare", "--map-user", "1000", "--map-group", "1000", - VIRTIOFSD_PATH, - "--shared-dir", - root_dir, - "--tag", - tag, - - # "--fd", - # str(s.fileno()), - "--socket-path", - sock_path, - - # If relying on bwrap(): - # "--sandbox", - # "none", - ] - if ro: - args.append("--readonly") - kwargs = { - # If bwrap(): - # "bind": [], - # ("ro_bind" if ro else "bind"): - # [*subdirs] - # if subdirs is not None - # else [root_dir], - - # "pass_fds": (2, s.fileno()), - } - try: - with self.popen(*args, **kwargs) as p: - yield p, sock_path - finally: - if os.path.exists(sock_path): - os.remove(sock_path) - - - @contextmanager - def defer(f): - try: - yield - finally: - f() - - - @contextmanager - def removing(*paths): - try: - yield - finally: - for p in paths: - if os.path.exists(p): - os.remove(p) - - - if __name__ == "__main__": - args, args_next = parser.parse_known_args() - preprocess_args(args) - - send_dir = PASSTHRU_ENV["HOME"] + f"/send/{args.vm}" - - os.makedirs(send_dir, exist_ok=True) - os.makedirs(args.prefix, exist_ok=True) - os.makedirs(args.prefix + "/pts", exist_ok=True) - - ps = Processes( - prefix=args.prefix, - vm=args.vm, - ) - - ch_remote = [ - "ch-remote", - "--api-socket", - args.prefix + "/vmm.sock", - ] - - with ExitStack() as cleanup: - - vfsd, vfsd_path = cleanup.enter_context( - ps.start_virtiofsd( - send_dir, - tag="send", - )) - gpud, gpud_path = cleanup.enter_context( - ps.start_gpu() - ) - - ch = cleanup.enter_context(ps.run_ch()) - ps.exec(*ch_remote, "create", args.vm_config) - ps.exec( - TAPS_PATH, "pass", - *ch_remote, "add-net", - "id=wan,fd=3,mac=00:00:00:00:00:01") - - ps.exec(*ch_remote, "add-fs", f"tag=send,socket={vfsd_path},id=send") - ps.exec(*ch_remote, "add-gpu", f"socket={gpud_path}") - ps.exec(*ch_remote, "boot") - ps.exec(*ch_remote, "info") - try: - ch.wait() - except KeyboardInterrupt: - pass - ''; - in - writeElb "run-${hostName}" '' - ${superviseVm} --vm-config=${chSettingsFile} --vm=${hostName} - ''; + uvms.cloud-hypervisor.runner = writeElb "run-${hostName}" '' + ${lib.getExe uvmsPkgs.uvms} --vm-config=${chSettingsFile} --vm=${hostName} + ''; } (lib.mkIf cfg.enable { boot.initrd.availableKernelModules = [ From 22b613d157a9edc56815d0fa8e55625da0609981 Mon Sep 17 00:00:00 2001 From: Else Someone Date: Thu, 26 Feb 2026 02:36:23 +0200 Subject: [PATCH 18/37] linux-uvm: start adding overrides ...from spectrum and libkrunfw --- pkgs/linux-uvm.nix | 86 ++++++++++++++++++++++++++++++++++++++++++ profiles/ch-runner.nix | 1 + 2 files changed, 87 insertions(+) create mode 100644 pkgs/linux-uvm.nix diff --git a/pkgs/linux-uvm.nix b/pkgs/linux-uvm.nix new file mode 100644 index 0000000..02baab5 --- /dev/null +++ b/pkgs/linux-uvm.nix @@ -0,0 +1,86 @@ +{ + lib, + linux_latest, +}: + +let + inherit (lib.kernel) yes no unset; + inherit (lib) mkForce; +in +linux_latest.override { + structuredExtraConfig = { + BASE_SMALL = yes; + DRM_VIRTIO_GPU = yes; + EROFS_FS = yes; + # TSI = yes; + DAX = yes; + FS_DAX = yes; + FUSE_DAX = yes; + OVERLAY_FS = yes; + VIRTIO_BALLOON = yes; + VIRTIO_BLK = yes; + VIRTIO_CONSOLE = yes; + VIRTIO_PCI = yes; + VIRTIO_MMIO = yes; + VIRTIO = yes; + VSOCKETS = yes; + NO_HZ_IDLE = mkForce yes; + NO_HZ_FULL = mkForce unset; + HZ_1000 = unset; + HZ_250 = yes; # NixOS default: 1000 + + EXT4_FS = yes; + # EXT4_USE_FOR_EXT2 = yes; + XFS_FS = yes; + DEFAULT_SECURITY_APPARMOR = mkForce unset; + + XEN = mkForce unset; + XEN_BACKEND = mkForce unset; + XEN_BALLOON = mkForce unset; + XEN_BALLOON_MEMORY_HOTPLUG = mkForce unset; + XEN_DOM0 = mkForce unset; + XEN_HAVE_PVMMU = mkForce unset; + XEN_MCE_LOG = mkForce unset; + XEN_PVH = mkForce unset; + XEN_SAVE_RESTORE = mkForce unset; + XEN_SYS_HYPERVISOR = mkForce unset; + PCI_XEN = mkForce unset; + POWER_RESET_GPIO = mkForce unset; + POWER_RESET_GPIO_RESTART = mkForce unset; + RCU_LAZY = mkForce unset; + REISERFS_FS_POSIX_ACL = mkForce unset; + REISERFS_FS_SECURITY = mkForce unset; + REISERFS_FS_XATTR = mkForce unset; + SWIOTLB_XEN = mkForce unset; + SUSPEND = mkForce unset; + PM = mkForce unset; + HIBERNATION = mkForce unset; + ACPI = mkForce unset; + CPU_FREQ = mkForce unset; + CPU_FREQ_DT = mkForce unset; + INTEL_IDLE = mkForce unset; + ISA_DMA_API = mkForce unset; + IA32_EMULATION = mkForce unset; + COMPAT = mkForce unset; + COMPAT_32 = mkForce unset; + KVM = mkForce unset; + BLOCK_LEGACY_AUTOLOAD = mkForce unset; + SWAP = mkForce unset; + CMA = mkForce unset; + FB = mkForce unset; + FB_EFI = mkForce unset; + FB_VESA = mkForce unset; + SECURITY_APPARMOR = mkForce unset; + + VT = no; + DRM_FBDEV_EMULATION = lib.mkForce no; + FONTS = mkForce unset; + FONT_8x8 = mkForce unset; + FONT_TER16x32 = mkForce unset; + FRAMEBUFFER_CONSOLE = mkForce unset; + FRAMEBUFFER_CONSOLE_DEFERRED_TAKEOVER = mkForce unset; + FRAMEBUFFER_CONSOLE_DETECT_PRIMARY = mkForce unset; + FRAMEBUFFER_CONSOLE_ROTATION = mkForce unset; + RC_CORE = mkForce unset; + }; +} diff --git a/profiles/ch-runner.nix b/profiles/ch-runner.nix index 7a2e151..ad3685f 100644 --- a/profiles/ch-runner.nix +++ b/profiles/ch-runner.nix @@ -130,6 +130,7 @@ in }; config = lib.mkMerge [ { + # boot.kernelPackages = pkgs.linuxPackagesFor (uvmsPkgs.linux-uvm); uvms.cloud-hypervisor.settings = { payload = { cmdline = lib.concatStringsSep " " cfg.cmdline; From 384b45bdef025fbd4c139b20988f6cd5b4236e24 Mon Sep 17 00:00:00 2001 From: Else Someone Date: Fri, 27 Feb 2026 18:26:41 +0200 Subject: [PATCH 19/37] pkgs.uvms: init ...with some basic optional persistence and without having to rebuild images for every app nix run -f . pkgs.uvms -- --persist-home librewolf alacritty --run librewolf --run alacritty --- examples/dummy.nix | 107 +------- pkgs/baseImage.nix | 3 + pkgs/cloud-hypervisor-gpu.nix | 59 ++++ pkgs/default.nix | 12 +- pkgs/linux-uvm.nix | 153 ++++++----- pkgs/mkSystemdDropin.nix | 39 +++ pkgs/uvms-guest/guest.py | 76 +++++ pkgs/uvms-guest/package.nix | 5 + pkgs/uvms/package.nix | 11 +- pkgs/uvms/uvms.py | 503 +++++++++++++++++++++++----------- profiles/baseImage.nix | 407 +++++++++++++++++++++++++++ profiles/ch-runner.nix | 116 +------- profiles/debug-closure.nix | 8 +- profiles/minimal.nix | 36 +++ profiles/on-failure.nix | 72 +++++ 15 files changed, 1155 insertions(+), 452 deletions(-) create mode 100644 pkgs/baseImage.nix create mode 100644 pkgs/cloud-hypervisor-gpu.nix create mode 100644 pkgs/mkSystemdDropin.nix create mode 100644 pkgs/uvms-guest/guest.py create mode 100644 pkgs/uvms-guest/package.nix create mode 100644 profiles/baseImage.nix create mode 100644 profiles/minimal.nix create mode 100644 profiles/on-failure.nix diff --git a/examples/dummy.nix b/examples/dummy.nix index 02254c9..3e5ffd9 100644 --- a/examples/dummy.nix +++ b/examples/dummy.nix @@ -7,29 +7,11 @@ }: let uvmsPkgs = pkgs.callPackage ../pkgs { }; - waylandSock = "/run/user/1000/wayland-1"; - env = { - XDG_RUNTIME_DIR = "/run/user/1000"; - WAYLAND_DISPLAY = "wayland-1"; - - MESA_LOADER_DRIVER_OVERRIDE = "zink"; - - # WAYLAND_DEBUG = "1"; - # WAYLAND_DEBUG_PROXY = "1"; - - ELECTRON_OZONE_PLATFORM_HINT = "wayland"; - MOZ_ENABLE_WAYLAND = "1"; - QT_QPA_PLATFORM = "wayland"; # Qt Applications - GDK_BACKEND = "wayland"; # GTK Applications - XDG_SESSION_TYPE = "wayland"; # Electron Applications - SDL_VIDEODRIVER = "wayland"; - CLUTTER_BACKEND = "wayland"; - NIXOS_OZONE_WL = "1"; - }; in { imports = [ - ../profiles/all.nix + ../profiles/ch-runner.nix + ../profiles/baseImage.nix (modulesPath + "/profiles/minimal.nix") ]; @@ -41,103 +23,23 @@ in _module.args.inputs = import ../npins; # boot.isContainer = true; - # boot.initrd.enable = true; boot.loader.grub.enable = false; boot.initrd.systemd.enable = true; - services.logrotate.enable = false; - services.udisks2.enable = false; - system.tools.nixos-generate-config.enable = false; - # system.activationScripts.specialfs = lib.mkForce ""; - systemd.coredump.enable = false; - # networking.firewall.enable = false; - powerManagement.enable = false; - boot.kexec.enable = false; - # console.enable = false; - # system.switch.enable = false; - # services.udev.packages = lib.mkDefault [ ]; - services.resolved.enable = false; - systemd.services.generate-shutdown-ramfs.enable = lib.mkForce false; - systemd.services.systemd-remount-fs.enable = lib.mkForce false; - systemd.services.systemd-pstore.enable = lib.mkForce false; - systemd.services.lastlog2-import.enable = lib.mkForce false; - systemd.services.suid-sgid-wrappers.enable = lib.mkForce false; - fileSystems."/" = lib.mkDefault { - device = "rootfs"; # how does this work? does this assign a label to the tmpfs? - fsType = "tmpfs"; - options = [ "size=20%,mode=0755" ]; - neededForBoot = true; - }; boot.initrd.systemd.settings.Manager.DefaultTimeoutStartSec = 5; systemd.settings.Manager.DefaultTimeoutStopSec = 10; - networking.useNetworkd = true; - networking.nftables.enable = true; uvms.cloud-hypervisor.enable = true; - systemd.sysusers.enable = false; - services.userborn.enable = true; # nikstur it - users.mutableUsers = false; - users.groups.user = { }; - users.users.user = { - isNormalUser = true; - password = "hacktheplanet!"; - extraGroups = [ - "video" - "render" - ]; - }; - users.users.root.password = "hacktheplanet!"; - systemd.services."suid-sgid-wrappers".serviceConfig = { StandardOutput = "journal+console"; StandardError = "journal+console"; }; - environment.variables = env; - systemd.globalEnvironment = env; - systemd.tmpfiles.settings."10-xdg" = { - ${env.XDG_RUNTIME_DIR}.d = { - user = "user"; - group = "user"; - mode = "0755"; - }; - }; - systemd.sockets."wayland-proxy" = { - listenStreams = [ - waylandSock - ]; - socketConfig = { - SocketUser = "user"; - SocketGroup = "user"; - FileDescriptorName = "wayland"; - }; - wantedBy = [ "sockets.target" ]; - partOf = [ "wayland-proxy.service" ]; - }; - systemd.services."wayland-proxy" = { - wantedBy = [ "default.target" ]; - serviceConfig = { - User = "user"; - Group = "user"; - ExecStart = "${lib.getExe pkgs.wayland-proxy-virtwl} --virtio-gpu"; - # ExecStart = "${lib.getExe uvmsPkgs.wl-cross-domain-proxy} --listen-fd --filter-global wp_presentation"; - ExecStartPre = [ - "+/run/current-system/sw/bin/chmod 0666 /dev/dri/card0 /dev/dri/renderD128" - ]; - StandardOutput = "journal+console"; - StandardError = "journal+console"; - Restart = "on-failure"; - RestartSec = 5; - }; - }; - fonts.enableDefaultPackages = true; - systemd.services."terminal" = { wantedBy = [ "multi-user.target" ]; wants = [ "wayland-proxy.service" ]; after = [ "wayland-proxy.service" ]; - environment = env; serviceConfig = { User = "user"; WorkingDirectory = "/home/user"; @@ -146,11 +48,6 @@ in StandardError = "journal+console"; }; }; - boot.kernelModules = [ - "drm" - "virtio_gpu" - ]; - hardware.graphics.enable = true; # TODO: cmdline, kernel, initrd, fileSystems } diff --git a/pkgs/baseImage.nix b/pkgs/baseImage.nix new file mode 100644 index 0000000..3e67ba9 --- /dev/null +++ b/pkgs/baseImage.nix @@ -0,0 +1,3 @@ +{ nixos }: + +nixos ../profiles/baseImage.nix diff --git a/pkgs/cloud-hypervisor-gpu.nix b/pkgs/cloud-hypervisor-gpu.nix new file mode 100644 index 0000000..325b372 --- /dev/null +++ b/pkgs/cloud-hypervisor-gpu.nix @@ -0,0 +1,59 @@ +{ + lib, + cloud-hypervisor, + fetchFromGitHub, + rustPlatform, + enableDebug ? true, +}: + +let + spectrum = builtins.fetchTree { + url = "https://spectrum-os.org/git/spectrum"; + type = "git"; + rev = "0f3388f0191d9a03c7bf471c269a34a79f22018b"; + }; +in +cloud-hypervisor.overrideAttrs ( + finalAttrs: oldAttrs: + { + # Verbatim from spectrum + postUnpack = oldAttrs.postUnpack or "" + '' + unpackFile $vhost + chmod -R +w vhost + ''; + vhost = fetchFromGitHub { + name = "vhost"; + owner = "rust-vmm"; + repo = "vhost"; + rev = "vhost-user-backend-v0.20.0"; + hash = "sha256-KK1+mwYQr7YkyGT9+51v7TJael9D0lle2JXfRoTqYq8="; + }; + + patches = oldAttrs.patches or [ ] ++ [ + "${spectrum}/pkgs/cloud-hypervisor/0001-build-use-local-vhost.patch" + "${spectrum}/pkgs/cloud-hypervisor/0002-virtio-devices-add-a-GPU-device.patch" + ]; + vhostPatches = builtins.concatMap ( + name: + lib.optionals (lib.hasSuffix ".patch" name) [ "${spectrum}/pkgs/cloud-hypervisor/vhost/${name}" ] + ) (builtins.attrNames (builtins.readDir "${spectrum}/pkgs/cloud-hypervisor/vhost")); + # Verbatim copy from spectrum + postPatch = oldAttrs.postPatch or "" + '' + pushd ../vhost + for patch in $vhostPatches; do + echo applying patch $patch + patch -p1 < $patch + done + popd + ''; + cargoDeps = rustPlatform.fetchCargoVendor { + inherit (finalAttrs) patches; + inherit (oldAttrs) src; + hash = "sha256-wGtsyKDg1z1QK9mJ1Q43NSjoPbm3m81p++DoD8ipIUI="; + }; + } + // lib.optionalAttrs enableDebug { + buildType = "debug"; + dontStrip = true; + } +) diff --git a/pkgs/default.nix b/pkgs/default.nix index 6c710fa..ff3fa2e 100644 --- a/pkgs/default.nix +++ b/pkgs/default.nix @@ -4,6 +4,14 @@ let in lib.makeScope newScope ( self: + let + callPackage = + fun: overrides: + let + result = self.callPackage fun overrides; + in + result // { override = result.__originalOverride or result.override; }; + in dirToAttrs ./. [ ( @@ -14,9 +22,9 @@ lib.makeScope newScope ( ( name: fpath: typ: if typ == "regular" then - self.callPackage fpath { } + callPackage fpath { } else if typ == "directory" && builtins.pathExists (fpath + "/package.nix") then - self.callPackage (fpath + "/package.nix") { } + callPackage (fpath + "/package.nix") { } else null ) diff --git a/pkgs/linux-uvm.nix b/pkgs/linux-uvm.nix index 02baab5..bd11aef 100644 --- a/pkgs/linux-uvm.nix +++ b/pkgs/linux-uvm.nix @@ -6,81 +6,86 @@ let inherit (lib.kernel) yes no unset; inherit (lib) mkForce; -in -linux_latest.override { - structuredExtraConfig = { - BASE_SMALL = yes; - DRM_VIRTIO_GPU = yes; - EROFS_FS = yes; - # TSI = yes; - DAX = yes; - FS_DAX = yes; - FUSE_DAX = yes; - OVERLAY_FS = yes; - VIRTIO_BALLOON = yes; - VIRTIO_BLK = yes; - VIRTIO_CONSOLE = yes; - VIRTIO_PCI = yes; - VIRTIO_MMIO = yes; - VIRTIO = yes; - VSOCKETS = yes; - NO_HZ_IDLE = mkForce yes; - NO_HZ_FULL = mkForce unset; - HZ_1000 = unset; - HZ_250 = yes; # NixOS default: 1000 + result = linux_latest.override { + structuredExtraConfig = { + BASE_SMALL = yes; + DRM_VIRTIO_GPU = yes; + EROFS_FS = yes; + # TSI = yes; + DAX = yes; + FS_DAX = yes; + FUSE_DAX = yes; + OVERLAY_FS = yes; + VIRTIO_BALLOON = yes; + VIRTIO_BLK = yes; + VIRTIO_CONSOLE = yes; + VIRTIO_FS = yes; + VIRTIO_MMIO = yes; + VIRTIO_PCI = yes; + VIRTIO = yes; + FUSE_FS = yes; + VSOCKETS = yes; + NO_HZ_IDLE = mkForce yes; + NO_HZ_FULL = mkForce unset; + HZ_1000 = unset; + HZ_250 = yes; # NixOS default: 1000 - EXT4_FS = yes; - # EXT4_USE_FOR_EXT2 = yes; - XFS_FS = yes; - DEFAULT_SECURITY_APPARMOR = mkForce unset; + # LSM = "lockdown,yama,loadpin,safesetid,integrity,bpf"; - XEN = mkForce unset; - XEN_BACKEND = mkForce unset; - XEN_BALLOON = mkForce unset; - XEN_BALLOON_MEMORY_HOTPLUG = mkForce unset; - XEN_DOM0 = mkForce unset; - XEN_HAVE_PVMMU = mkForce unset; - XEN_MCE_LOG = mkForce unset; - XEN_PVH = mkForce unset; - XEN_SAVE_RESTORE = mkForce unset; - XEN_SYS_HYPERVISOR = mkForce unset; - PCI_XEN = mkForce unset; - POWER_RESET_GPIO = mkForce unset; - POWER_RESET_GPIO_RESTART = mkForce unset; - RCU_LAZY = mkForce unset; - REISERFS_FS_POSIX_ACL = mkForce unset; - REISERFS_FS_SECURITY = mkForce unset; - REISERFS_FS_XATTR = mkForce unset; - SWIOTLB_XEN = mkForce unset; - SUSPEND = mkForce unset; - PM = mkForce unset; - HIBERNATION = mkForce unset; - ACPI = mkForce unset; - CPU_FREQ = mkForce unset; - CPU_FREQ_DT = mkForce unset; - INTEL_IDLE = mkForce unset; - ISA_DMA_API = mkForce unset; - IA32_EMULATION = mkForce unset; - COMPAT = mkForce unset; - COMPAT_32 = mkForce unset; - KVM = mkForce unset; - BLOCK_LEGACY_AUTOLOAD = mkForce unset; - SWAP = mkForce unset; - CMA = mkForce unset; - FB = mkForce unset; - FB_EFI = mkForce unset; - FB_VESA = mkForce unset; - SECURITY_APPARMOR = mkForce unset; + EXT4_FS = yes; + # EXT4_USE_FOR_EXT2 = yes; + XFS_FS = yes; + DEFAULT_SECURITY_APPARMOR = mkForce unset; - VT = no; - DRM_FBDEV_EMULATION = lib.mkForce no; - FONTS = mkForce unset; - FONT_8x8 = mkForce unset; - FONT_TER16x32 = mkForce unset; - FRAMEBUFFER_CONSOLE = mkForce unset; - FRAMEBUFFER_CONSOLE_DEFERRED_TAKEOVER = mkForce unset; - FRAMEBUFFER_CONSOLE_DETECT_PRIMARY = mkForce unset; - FRAMEBUFFER_CONSOLE_ROTATION = mkForce unset; - RC_CORE = mkForce unset; + XEN = mkForce unset; + XEN_BACKEND = mkForce unset; + XEN_BALLOON = mkForce unset; + XEN_BALLOON_MEMORY_HOTPLUG = mkForce unset; + XEN_DOM0 = mkForce unset; + XEN_HAVE_PVMMU = mkForce unset; + XEN_MCE_LOG = mkForce unset; + XEN_PVH = mkForce unset; + XEN_SAVE_RESTORE = mkForce unset; + XEN_SYS_HYPERVISOR = mkForce unset; + PCI_XEN = mkForce unset; + POWER_RESET_GPIO = mkForce unset; + POWER_RESET_GPIO_RESTART = mkForce unset; + RCU_LAZY = mkForce unset; + REISERFS_FS_POSIX_ACL = mkForce unset; + REISERFS_FS_SECURITY = mkForce unset; + REISERFS_FS_XATTR = mkForce unset; + SWIOTLB_XEN = mkForce unset; + SUSPEND = mkForce unset; + PM = mkForce unset; + HIBERNATION = mkForce unset; + ACPI = mkForce unset; + CPU_FREQ = mkForce unset; + CPU_FREQ_DT = mkForce unset; + INTEL_IDLE = mkForce unset; + ISA_DMA_API = mkForce unset; + IA32_EMULATION = mkForce unset; + COMPAT = mkForce unset; + COMPAT_32 = mkForce unset; + KVM = mkForce unset; + BLOCK_LEGACY_AUTOLOAD = mkForce unset; + SWAP = mkForce unset; + CMA = mkForce unset; + FB = mkForce unset; + FB_EFI = mkForce unset; + FB_VESA = mkForce unset; + SECURITY_APPARMOR = mkForce unset; + + VT = no; + DRM_FBDEV_EMULATION = lib.mkForce no; + FONTS = mkForce unset; + FONT_8x8 = mkForce unset; + FONT_TER16x32 = mkForce unset; + FRAMEBUFFER_CONSOLE = mkForce unset; + FRAMEBUFFER_CONSOLE_DEFERRED_TAKEOVER = mkForce unset; + FRAMEBUFFER_CONSOLE_DETECT_PRIMARY = mkForce unset; + FRAMEBUFFER_CONSOLE_ROTATION = mkForce unset; + RC_CORE = mkForce unset; + }; }; -} +in +result // { __originalOverride = result.override; } diff --git a/pkgs/mkSystemdDropin.nix b/pkgs/mkSystemdDropin.nix new file mode 100644 index 0000000..56407bc --- /dev/null +++ b/pkgs/mkSystemdDropin.nix @@ -0,0 +1,39 @@ +{ + lib, + runCommand, + writeShellScriptBin, +}: +{ + name, + prefix ? "10-all-", + dirs ? [ + "service" + "mount" + "socket" + "timer" + "target" + ], + + dropinText ? null, + extraCommands ? "", + ... +}@args: + +runCommand "${name}-dropin" + ( + lib.removeAttrs args [ + "name" + ] + // { + inherit dirs dropinText extraCommands; + } + ) + '' + set -euo pipefail + root=$out/lib/systemd/system + for dir in $dirs ; do + mkdir -p "$root/$dir".d + printf "%s" "$dropinText" > "$root/$dir.d/${prefix}${name}.conf" + done + runHook extraCommands + '' diff --git a/pkgs/uvms-guest/guest.py b/pkgs/uvms-guest/guest.py new file mode 100644 index 0000000..87b3d1a --- /dev/null +++ b/pkgs/uvms-guest/guest.py @@ -0,0 +1,76 @@ +import json +import os +import select +import socket +import subprocess + + +def handle_run(run: dict) -> dict: + res = {} + text = run.get("text", False) + env = { + **os.environ, + "PATH": ":".join( + os.environ.get("PATH", "").split(":") + run.get("EXTRA_PATH", []) + ), + } + proc = None + try: + proc = subprocess.Popen( + req["run"]["argv"], + text=text, + env=env, + cwd="/home/user", + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + res["status"] = "exec succeeded" + except Exception as e: + res["status"] = "exec failed" + res["exception"] = repr(e) + res["pid"] = getattr(proc, "pid", None) + try: + if proc is not None: + proc.wait(0.125) + res["long_running"] = False + res["returncode"] = getattr(proc, "returncode", None) + except subprocess.TimeoutExpired: + res["long_running"] = True + return res, proc + + +if __name__ == "__main__": + serv = socket.fromfd(3, socket.AF_VSOCK, socket.SOCK_STREAM) + + procs = [] + conns = [serv] + + while True: + rr, rw, xs = select.select(conns, [], []) + + for con in rr: + if con is serv: + con, (cid, port) = serv.accept() + assert cid == 2, cid + conns.append(con) + continue + req = con.recv(8192) + # IDK why but I keep getting empty messages + if req == b"": + continue + try: + req = json.loads(req) + print(f"Received {req=}") + except json.JSONDecodeError as e: + print(f"Couldn't interpret {req=}: {e}") + continue + if "run" in req: + res, proc = handle_run(req["run"]) + procs.append(proc) + else: + res = {"status": "unknown command"} + _, rw, _ = select.select([], [con], []) + assert rw, rw + res = json.dumps(res).encode("utf8") + print(f"Responding with {res=}") + con.send(res) diff --git a/pkgs/uvms-guest/package.nix b/pkgs/uvms-guest/package.nix new file mode 100644 index 0000000..66cfa2d --- /dev/null +++ b/pkgs/uvms-guest/package.nix @@ -0,0 +1,5 @@ +{ + lib, + writers, +}: +writers.writePython3Bin "uvms-guest" { } ./guest.py diff --git a/pkgs/uvms/package.nix b/pkgs/uvms/package.nix index d5e84c3..109235e 100644 --- a/pkgs/uvms/package.nix +++ b/pkgs/uvms/package.nix @@ -11,9 +11,11 @@ execline, s6, strace, - taps, util-linux, virtiofsd, + + taps, + baseImage, }: let @@ -43,5 +45,12 @@ writers.writePython3Bin "uvms" { } ( STRACE = lib.getExe strace; TAPS = "${lib.getExe taps}"; VIRTIOFSD = "${lib.getExe virtiofsd}"; + + BASE_CONFIG = baseImage.config.system.build.ch; + SYSTEM = baseImage.config.system.build.toplevel; + SYSTEM_CLOSURE = writeClosure [ + baseImage.config.system.build.toplevel + baseImage.config.system.build.ch + ]; } ) diff --git a/pkgs/uvms/uvms.py b/pkgs/uvms/uvms.py index ffdc28b..ab71c6e 100644 --- a/pkgs/uvms/uvms.py +++ b/pkgs/uvms/uvms.py @@ -9,14 +9,18 @@ import os import subprocess import socket +import json from argparse import ArgumentParser from contextlib import contextmanager, closing, ExitStack parser = ArgumentParser("supervise-vm") -parser.add_argument("--vm") +parser.add_argument("--vm", default=None) parser.add_argument("--prefix", default="$HOME/uvms/$VM") -parser.add_argument("--vm-config") +parser.add_argument("--vm-config", default="@BASE_CONFIG@") # noqa: E501 +parser.add_argument("--persist-home", action="store_true") +parser.add_argument("--run", action="append") +parser.add_argument("app", nargs="*", default=()) TOOLS_DIR = "@TOOLS@" # noqa: E501 SOCKETBINDER = TOOLS_DIR + "/s6-ipcserver-socketbinder" # noqa: E501 @@ -27,12 +31,18 @@ VIRTIOFSD = "@VIRTIOFSD@" # noqa: E501 BWRAP = "@BWRAP@" # noqa: E501 with open("@TOOLS_CLOSURE@", mode="r") as f: # noqa: E501 - CLOSURE = [ + TOOLS_CLOSURE = [ *(ln.rstrip() for ln in f.readlines()), os.path.dirname(__file__), ] -PASSTHRU_PATH = ":".join([TOOLS_DIR]) +BASE_SYSTEM = "@SYSTEM@" # noqa: E501 +with open("@SYSTEM_CLOSURE@", mode="r") as f: # noqa: E501 + BASE_SYSTEM_CLOSURE = [ + *(ln.rstrip() for ln in f.readlines()), + ] + +PASSTHRU_PATH = ":".join([TOOLS_DIR, *os.environ.get("PATH", "").split(":")]) PASSTHRU_ENV = { **{ k: v @@ -41,6 +51,7 @@ PASSTHRU_ENV = { or k.startswith("WAYLAND") or k.startswith("XDG_") or k.startswith("DBUS_") + or k.startswith("NIX_") or k in [ "TAPS_SOCK", @@ -52,6 +63,10 @@ PASSTHRU_ENV = { def preprocess_args(args_mut): + if not args_mut.app and args_mut.run: + args_mut.app = [*args_mut.run] + if not args_mut.vm: + args_mut.vm = args_mut.run[0] keys = [k for k, v in args_mut._get_kwargs() if isinstance(v, str)] for k in keys: v = getattr(args_mut, k) @@ -86,6 +101,7 @@ class Processes: self.vm = vm self.check = check self.defaults = defaults + self.processes = [] def make_env(self): return { @@ -121,6 +137,7 @@ class Processes: kwargs["pass_fds"] = kwargs.get("pass_fds", ()) kwargs["env"] = kwargs.get("env", self.make_env()) kwargs["cwd"] = kwargs.get("cwd", self.prefix) + kwargs["text"] = kwargs.get("text", True) kwargs["stdin"] = kwargs.get("stdin", subprocess.DEVNULL) kwargs["stdout"] = kwargs.get("stdout", subprocess.DEVNULL) kwargs["stderr"] = kwargs.get("stderr", subprocess.DEVNULL) @@ -132,12 +149,19 @@ class Processes: ) if not alive_after(proc, 0.125): raise RuntimeError("Failed to start", args) + print(f"Started {args}") + self.processes.append(proc) yield proc + print(f"Releasing {args}") finally: - if alive_after(proc, 0.125): - proc.terminate() - if proc is not None: - proc.wait() + if subprocess.PIPE in (kwargs["stderr"], kwargs["stdout"]): + print(proc.communicate()) + while alive_after(proc, 0.125): + try: + proc.terminate() + proc.wait() + except Exception as e: + print(f"Cleanup failing: {e}") @contextmanager def bwrap( @@ -147,6 +171,8 @@ class Processes: # Based on the args from # `host/rootfs/image/usr/bin/run-vmm` unshare_all=True, + uid=1000, + gid=100, unshare_user=True, unshare_ipc=None, unshare_pid=None, @@ -164,7 +190,7 @@ class Processes: "/proc/sys", "/dev/null", "/proc/kallsyms", - *CLOSURE, + *sorted(set([*TOOLS_CLOSURE, *BASE_SYSTEM_CLOSURE])), ), ro_bind=(), remount_ro=("/proc/fs", "/proc/irq"), @@ -183,123 +209,128 @@ class Processes: bwrap_args_sock, remote = socket.socketpair() remote.set_inheritable(True) bwrap_args_f = bwrap_args_sock.makefile("w") - with ExitStack() as cleanup: - # cleanup.enter_context(closing(bwrap_args_sock)) - # cleanup.enter_context(closing(bwrap_args_f)) - def print_arg(*args): - print(*args, file=bwrap_args_f, sep="\0", end="\0") + def print_arg(*args): + print(*args, file=bwrap_args_f, sep="\0", end="\0") - if unshare_all: - print_arg("--unshare-all") - if unshare_user: - print_arg("--unshare-user") - if unshare_ipc: - print_arg("--unshare-ipc") - if unshare_pid: - print_arg("--unshare-pid") - if unshare_net: - print_arg("--unshare-net") - elif unshare_net is False: - print_arg("--share-net") - if unshare_uts: - print_arg("--unshare-uts") - if unshare_cgroup_try: - print_arg("--unshare-cgroup-try") - if die_with_parent: - print_arg("--die-with-parent") - if dev: - print_arg("--dev", dev) - if proc: - print_arg("--proc", proc) + if unshare_all: + print_arg("--unshare-all") + if unshare_user: + print_arg("--unshare-user") + if uid is not None: + assert unshare_user + print_arg("--uid", uid) + if gid is not None: + assert unshare_user + print_arg("--gid", gid) + if unshare_ipc: + print_arg("--unshare-ipc") + if unshare_pid: + print_arg("--unshare-pid") + if unshare_net: + print_arg("--unshare-net") + elif unshare_net is False: + print_arg("--share-net") + if unshare_uts: + print_arg("--unshare-uts") + if unshare_cgroup_try: + print_arg("--unshare-cgroup-try") + if die_with_parent: + print_arg("--die-with-parent") + if dev: + print_arg("--dev", dev) + if proc: + print_arg("--proc", proc) - for p in bind: - p1, p2 = (p, p) if isinstance(p, str) else p - print_arg("--bind", p1, p2) - for p in (*ro_bind, *ro_bind_implicit): - p1, p2 = (p, p) if isinstance(p, str) else p - print_arg("--ro-bind", p1, p2) - for p in (*dev_bind, *dev_bind_implicit): - p1, p2 = (p, p) if isinstance(p, str) else p - print_arg("--dev-bind", p1, p2) - for p in (*tmpfs, *tmpfs_implicit): - print_arg("--tmpfs", p) - # Hunch: order might matter... - for p in remount_ro: - print_arg("--remount-ro", p) + for p in bind: + assert isinstance(p, (str, tuple)), p + p1, p2 = (p, p) if isinstance(p, str) else p + print_arg("--bind", p1, p2) + for p in (*ro_bind, *ro_bind_implicit): + assert isinstance(p, (str, tuple)), p + p1, p2 = (p, p) if isinstance(p, str) else p + print_arg("--ro-bind", p1, p2) + for p in (*dev_bind, *dev_bind_implicit): + assert isinstance(p, (str, tuple)), p + p1, p2 = (p, p) if isinstance(p, str) else p + print_arg("--dev-bind", p1, p2) + for p in (*tmpfs, *tmpfs_implicit): + print_arg("--tmpfs", p) + # Hunch: order might matter... + for p in remount_ro: + print_arg("--remount-ro", p) - bwrap_args_f.flush() + bwrap_args_f.flush() - with ExitStack() as es: - es.enter_context(closing(remote)) - es.enter_context(closing(bwrap_args_sock)) - es.enter_context(closing(bwrap_args_f)) - proc = cleanup.enter_context( - self.popen( - "bwrap", - "--args", - str(remote.fileno()), - *bwrap_args, - **popen_kwargs, - executable=BWRAP, - pass_fds=(*pass_fds, remote.fileno()), + try: + with ExitStack() as proc_es: + with ExitStack() as es: + es.enter_context(closing(remote)) + es.enter_context(closing(bwrap_args_sock)) + es.enter_context(closing(bwrap_args_f)) + proc = proc_es.enter_context( + self.popen( + "bwrap", + "--args", + str(remote.fileno()), + *bwrap_args, + **popen_kwargs, + executable=BWRAP, + pass_fds=(*pass_fds, remote.fileno()), + ) ) - ) - yield proc + yield proc + finally: + assert proc.returncode is not None, proc @contextmanager def run_ch(self): - try: - # s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM, 0) - # s.set_inheritable(True) - # s.setblocking(True) - # s.bind(self.prefix + "/vmm.sock") - args = [ - SOCKETBINDER, - "-B", - self.prefix + "/vmm.sock", - # "@STRACE@", # noqa: E501 - # "-Z", - # "-ff", - CH, - "--api-socket", - "fd=0", - # f"fd={s.fileno()}" - ] - needs_cleanup = False - with self.bwrap( - *args, - bind=[self.prefix], - # Probably just need the path to vmlinux - ro_bind=["/nix/store"], # I give up - unshare_net=False, - shell=False, - stderr=None, - # pass_fds=(s.fileno(),) - ) as proc: - # s.close() - assert alive_after(proc, 0.125) - if not os.path.exists(self.prefix + "/vmm.sock"): - raise RuntimeError( - f"{self.prefix}/vmm.sock should exist by now", - ) - needs_cleanup = True - if proc.returncode is not None: - raise RuntimeError("CH exited early") - yield proc - finally: - unlink_paths = ( - [ - self.prefix + "/vmm.sock", - self.prefix + "/vmm.sock.lock", - self.prefix + "/vsock.sock", - ] - if needs_cleanup - else [] + # s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM, 0) + # s.set_inheritable(True) + # s.setblocking(True) + # s.bind(self.prefix + "/vmm.sock") + args = [ + SOCKETBINDER, + "-B", + self.prefix + "/vmm.sock", + # "@STRACE@", # noqa: E501 + # "-Z", + # "-ff", + CH, + "--api-socket", + "fd=0", + # f"fd={s.fileno()}" + ] + cleanup_paths = [ + self.prefix + "/vmm.sock", + self.prefix + "/vmm.sock.lock", + self.prefix + "/vsock.sock", + ] + new_paths = [p for p in cleanup_paths if not os.path.exists(p)] + old_paths = [p for p in cleanup_paths if p not in new_paths] + with ExitStack() as cleanup: + cleanup.enter_context(removing(*new_paths)) + proc = cleanup.enter_context( + self.bwrap( + *args, + bind=[self.prefix], + # Probably just need the path to vmlinux + # ro_bind=["/nix/store"], # I give up + unshare_net=False, + shell=False, + # pass_fds=(s.fileno(),) + ) ) - for p in unlink_paths: - if os.path.exists(p): - os.remove(p) + # s.close() + cleanup.enter_context(removing(*old_paths)) + assert alive_after(proc, 1.0), proc + if not os.path.exists(self.prefix + "/vmm.sock"): + raise RuntimeError( + f"{self.prefix}/vmm.sock should exist by now", + ) + if proc.returncode is not None: + raise RuntimeError("CH exited early") + yield proc @contextmanager def start_gpu( @@ -330,7 +361,7 @@ class Processes: with self.popen( *args, stderr=None, - ) as proc, removing(sock_path): + ) as proc, removing(sock_path, sock_path + ".lock"): yield proc, sock_path @contextmanager @@ -338,9 +369,9 @@ class Processes: self, root_dir, tag, - ro=False, + ro=True, subdirs=None, - extra_flags=("--posix-acl",), + extra_flags=("--posix-acl", "--xattr"), ): assert os.path.exists(root_dir) @@ -351,20 +382,16 @@ class Processes: # s.setblocking(True) # s.set_inheritable(True) - def rm_sock(): - if os.path.exists(sock_path): - os.remove(sock_path) - with ExitStack() as cleanup: # noqa: F841 # s.bind(sock_path.encode("utf8")) # cleanup.enter_context(closing(s)) - cleanup.enter_context(defer(rm_sock)) + cleanup.enter_context(removing(sock_path, sock_path + ".pid")) args = [ # If using bwrap(): # "--argv0", "virtiofsd", # "--uid", "1000", - # "--gid", "1000", + # "--gid", "100", # "--", "unshare", "-rUm", @@ -372,7 +399,7 @@ class Processes: "--map-user", "1000", "--map-group", - "1000", + "100", VIRTIOFSD, "--shared-dir", root_dir, @@ -396,6 +423,8 @@ class Processes: # if subdirs is not None # else [root_dir], # "pass_fds": (2, s.fileno()), + "stdout": subprocess.PIPE, + "stderr": subprocess.PIPE, } try: with self.popen(*args, **kwargs) as p: @@ -423,20 +452,43 @@ def removing(*paths): os.remove(p) -if __name__ == "__main__": - args, args_next = parser.parse_known_args() - preprocess_args(args) +def connect_ch_vsock( + vsock_sock_path, + port: int, + type=socket.SOCK_STREAM, + blocking=True, +) -> socket.socket: + s = socket.socket(socket.AF_UNIX, type, 0) + s.setblocking(blocking) + s.connect(vsock_sock_path) + s.send(b"CONNECT %d\n" % port) + return s + + +@contextmanager +def listen_ch_vsock( + vsock_sock_path, + port: int, + type=socket.SOCK_STREAM, + blocking=True, +) -> socket.socket: + listen_path = vsock_sock_path + "_%d" % port + s = socket.socket(socket.AF_UNIX, type, 0) + s.setblocking(blocking) + s.bind(listen_path) + s.listen() + try: + yield s + finally: + os.remove(listen_path) + + +def main(args, args_next, cleanup, ps): send_dir = PASSTHRU_ENV["HOME"] + f"/send/{args.vm}" os.makedirs(send_dir, exist_ok=True) os.makedirs(args.prefix, exist_ok=True) - os.makedirs(args.prefix + "/pts", exist_ok=True) - - ps = Processes( - prefix=args.prefix, - vm=args.vm, - ) ch_remote = [ "ch-remote", @@ -444,31 +496,170 @@ if __name__ == "__main__": args.prefix + "/vmm.sock", ] - with ExitStack() as cleanup: + with open(args.vm_config) as f: + config = json.load(f) - vfsd, vfsd_path = cleanup.enter_context( + app_paths = [] + for a in args.app: + out_path = ps.exec( + "nix-build", + "", + "-A", + a, + "--no-out-link", + capture_output=True, + text=True, + ).stdout.strip() + assert out_path.startswith("/nix/store/") + app_paths.append(out_path) + apps_closure = ps.exec( # noqa: F841 + "nix-store", + "-qR", + *app_paths, + capture_output=True, + text=True, + ).stdout.split() + + ready_sock = cleanup.enter_context( + listen_ch_vsock(ps.prefix + "/vsock.sock", 8888), + ) + + virtiofs_socks = [] + _, sock_path = cleanup.enter_context( + ps.start_virtiofsd( + send_dir, + tag="send", + ro=False, + ) + ) + virtiofs_socks.append(("send", sock_path)) + _, sock_path = cleanup.enter_context( + ps.start_virtiofsd( + "/nix/store", + subdirs=apps_closure, + tag="apps", + ) + ) + virtiofs_socks.append(("apps", sock_path)) + _, sock_path = cleanup.enter_context( + ps.start_virtiofsd( + "/nix/store", + subdirs=BASE_SYSTEM_CLOSURE, + tag="system", + ) + ) + virtiofs_socks.append(("system", sock_path)) + + if args.persist_home: + os.makedirs(args.prefix + "/home", exist_ok=True) + _, sock_path = cleanup.enter_context( ps.start_virtiofsd( - send_dir, - tag="send", + args.prefix + "/home", + subdirs=BASE_SYSTEM_CLOSURE, + tag="home", + ro=False, ) ) - gpud, gpud_path = cleanup.enter_context(ps.start_gpu()) + virtiofs_socks.append(("home", sock_path)) + config["payload"]["cmdline"] += " uvms.persist-home=1" - ch = cleanup.enter_context(ps.run_ch()) - ps.exec(*ch_remote, "create", args.vm_config) - ps.exec( - TAPS, - "pass", - *ch_remote, - "add-net", - "id=wan,fd=3,mac=00:00:00:00:00:01", - ) + gpud, gpud_path = cleanup.enter_context(ps.start_gpu()) - ps.exec(*ch_remote, "add-fs", f"tag=send,socket={vfsd_path},id=send") - ps.exec(*ch_remote, "add-gpu", f"socket={gpud_path}") - ps.exec(*ch_remote, "boot") - ps.exec(*ch_remote, "info") + ch = cleanup.enter_context(ps.run_ch()) + + ps.exec( + *ch_remote, + "create", + input=json.dumps(config), + text=True, + ) + ps.exec( + TAPS, + "pass", + *ch_remote, + "add-net", + "id=wan,fd=3,mac=00:00:00:00:00:01", + ) + + # TODO: add-fs apps closure separately + for tag, sock_path in virtiofs_socks: + ps.exec(*ch_remote, "add-fs", f"tag={tag},socket={sock_path},id={tag}") + ps.exec(*ch_remote, "add-gpu", f"socket={gpud_path}") + ps.exec(*ch_remote, "boot") + ps.exec(*ch_remote, "info") + + with ready_sock: + ready_sock.settimeout(16.0) try: - ch.wait() - except KeyboardInterrupt: - pass + con, _ = ready_sock.accept() + except: # noqa: E722 + print( + "CH didn't try connecting to the readiness notification socket" + ) # noqa: E501 + else: + with con: + msg = con.recv(128) + assert msg.startswith(b"READY=1"), msg + + with connect_ch_vsock(ps.prefix + "/vsock.sock", 24601) as guest: + for r in args.run: + try: + guest.send( + json.dumps( + { + "run": { + "argv": [r], + "EXTRA_PATH": [ + f"{a}/bin" for a in app_paths + ], # noqa: E501 + } + } + ).encode("utf8") + ) + res = guest.recv(8192) + try: + res = json.loads(guest.recv(8192)) + except json.JSONDecodeError as e: + print(f"Couldn't interpret --run {r} response: {e} {res}") + continue + adverb = ( + "Successfully" + if res["status"] == "exec succeeded" + else "Failed to" # noqa: E501 + ) + print(f"{adverb} --run {r}: {res}") + except Exception as e: + print(f"Couldn't --run {r}: {repr(e)}") + try: + ch.wait() + except KeyboardInterrupt: + pass + + +if __name__ == "__main__": + args, args_next = parser.parse_known_args() + preprocess_args(args) + ps = Processes( + prefix=args.prefix, + vm=args.vm, + ) + + try: + with ExitStack() as cleanup: + main(args, args_next, cleanup, ps) + finally: + for p in ps.processes: + if p.returncode is not None: + continue + try: + print(f"Cleanup failed. Re-trying the killing of {p}") + p.terminate() + except: # noqa: E722 + pass + for p in ps.processes: + if p.returncode is not None: + continue + try: + p.wait() + except: # noqa: E722 + pass diff --git a/profiles/baseImage.nix b/profiles/baseImage.nix new file mode 100644 index 0000000..87f8df5 --- /dev/null +++ b/profiles/baseImage.nix @@ -0,0 +1,407 @@ +{ + lib, + config, + modulesPath, + pkgs, + ... +}: +let + inherit (lib) mkOption types concatStringsSep; + jsonType = (pkgs.formats.json { }).type; + + inherit (config.system.build) initialRamdisk; + inherit (config.system.boot.loader) initrdFile; + inherit (config.boot.kernelPackages) kernel; + kernelTarget = pkgs.stdenv.hostPlatform.linux-kernel.target; + uvmsPkgs = pkgs.callPackage ../pkgs { }; + waylandSock = "/run/user/1000/wayland-1"; + env = { + XDG_RUNTIME_DIR = "/run/user/1000"; + WAYLAND_DISPLAY = "wayland-1"; + + # MESA_LOADER_DRIVER_OVERRIDE = "zink"; + + ELECTRON_OZONE_PLATFORM_HINT = "wayland"; + MOZ_ENABLE_WAYLAND = "1"; + QT_QPA_PLATFORM = "wayland"; # Qt Applications + GDK_BACKEND = "wayland"; # GTK Applications + XDG_SESSION_TYPE = "wayland"; # Electron Applications + SDL_VIDEODRIVER = "wayland"; + CLUTTER_BACKEND = "wayland"; + NIXOS_OZONE_WL = "1"; + }; +in +{ + imports = [ + (modulesPath + "/profiles/minimal.nix") + ./debug-closure.nix + ./minimal.nix + ./on-failure.nix + ]; + config = { + some.failure-handler.enable = true; + hardware.graphics.enable = true; + # boot.kernelPackages = pkgs.linuxPackagesFor uvmsPkgs.linux-uvm; + # boot.isContainer = true; + boot.initrd.kernelModules = [ + "drm" + "virtio_blk" + "virtiofs" + "virtio_gpu" + "virtio_mmio" + "virtio_pci" + "overlay" + ]; + boot.kernelModules = [ + "drm" + "erofs" + "overlay" + "virtio_blk" + "virtiofs" + "virtio_gpu" + "virtio_mmio" + "virtio_pci" + ]; + boot.initrd.systemd.initrdBin = [ + pkgs.fuse + pkgs.fuse3 + ]; + fileSystems = { + "/" = lib.mkDefault { + device = "rootfs"; # how does this work? does this assign a label to the tmpfs? + fsType = "tmpfs"; + options = [ "size=20%,mode=0755" ]; + neededForBoot = true; + }; + "/nix/store" = { + fsType = "overlay"; + overlay.lowerdir = [ + "/nix/.ro-stores/system" + "/nix/.ro-stores/apps" + ]; + neededForBoot = true; + }; + "/nix/.ro-stores/system" = { + device = "system"; + fsType = "virtiofs"; + options = [ + "defaults" + "ro" + "x-systemd.requires=systemd-modules-load.service" + ]; + neededForBoot = true; + }; + "/nix/.ro-stores/apps" = { + device = "apps"; + fsType = "virtiofs"; + options = [ + "defaults" + "ro" + "x-systemd.requires=systemd-modules-load.service" + ]; + neededForBoot = true; + }; + }; + + systemd.mounts = [ + { + type = "virtiofs"; + where = "/home/user"; + what = "home"; + after = [ "systemd-modules-load.service" ]; + wantedBy = [ "local-fs.target" ]; + before = [ "local-fs.target" ]; + requires = [ "systemd-modules-load.service" ]; + options = lib.concatStringsSep "," [ + "defaults" + "rw" + "X-mount.owner=1000" + "X-mount.group=100" + ]; + unitConfig = { + ConditionKernelCommandLine = "uvms.persist-home=1"; + }; + } + { + type = "virtiofs"; + where = "/home/user/send"; + what = "send"; + wants = [ + "home-user.mount" + "-.mount" + ]; + after = [ + "systemd-modules-load.service" + "home-user.mount" + "-.mount" + ]; + wantedBy = [ "local-fs.target" ]; + before = [ "local-fs.target" ]; + options = lib.concatStringsSep "," [ + "defaults" + "rw" + "X-mount.owner=1000" + "X-mount.group=100" + ]; + unitConfig = { + DefaultDependencies = false; + }; + } + ]; + # systemd.services."mount-home-user-send" = { + # wants = [ "home-user.mount" ]; + # after = [ + # "systemd-modules-load.service" + # "home-user.mount" + # "-.mount" + # ]; + # wantedBy = [ "local-fs.target" ]; + # before = [ "local-fs.target" ]; + # unitConfig = { + # DefaultDependencies = false; + # }; + # environment.PATH = lib.mkForce ( + # lib.makeBinPath [ + # pkgs.fuse + # pkgs.fuse3 + # pkgs.coreutils + # ] + # ); + # serviceConfig = { + # Type = "oneshot"; + # RemainsAfterExit = true; + # ExecStart = [ + # "/run/current-system/sw/bin/mkdir -p /home/user/send" + # "/run/current-system/sw/bin/chown user /home/user/send" + # "/run/current-system/sw/sbin/mount -t virtiofs -o defaults,rw send /home/user/send" + # ]; + # StandardOutput = "journal+console"; + # StandardError = "journal+console"; + # }; + # }; + + systemd.network.enable = true; + networking.useNetworkd = true; + networking.nftables.enable = true; + networking.useDHCP = true; + networking.nameservers = [ "1.1.1.1" ]; + services.resolved.enable = lib.mkForce true; + + system.activationScripts.specialfs = lib.mkForce ""; + # networking.firewall.enable = false; + console.enable = false; + services.udev.packages = lib.mkDefault [ ]; + systemd.services."systemd-oomd".enable = false; + + users.mutableUsers = false; + users.users.root.password = "hacktheplanet!"; + users.groups.users = { }; + users.users.user = { + uid = 1000; + isNormalUser = true; + password = "hacktheplanet!"; + extraGroups = [ + "video" + "render" + "users" + "wheel" + ]; + }; + + environment.variables = env; + systemd.globalEnvironment = env; + + systemd.tmpfiles.settings."10-xdg" = { + ${env.XDG_RUNTIME_DIR}.d = { + user = "user"; + group = "users"; + mode = "0755"; + }; + }; + + systemd.sockets."wayland-proxy" = { + listenStreams = [ + waylandSock + ]; + socketConfig = { + SocketUser = "user"; + SocketGroup = "users"; + FileDescriptorName = "wayland"; + }; + wantedBy = [ "sockets.target" ]; + partOf = [ "wayland-proxy.service" ]; + }; + systemd.services."wayland-proxy" = { + wantedBy = [ "default.target" ]; + serviceConfig = { + User = "user"; + Group = "users"; + ExecStart = "${lib.getExe pkgs.wayland-proxy-virtwl} --virtio-gpu"; + # ExecStart = "${lib.getExe uvmsPkgs.wl-cross-domain-proxy} --listen-fd --filter-global wp_presentation"; + ExecStartPre = [ + "+/run/current-system/sw/bin/chmod 0666 /dev/dri/card0 /dev/dri/renderD128" + ]; + StandardOutput = "journal+console"; + StandardError = "journal+console"; + Restart = "on-failure"; + RestartSec = 5; + }; + }; + + systemd.sockets."uvms-guest" = { + wantedBy = [ "default.target" ]; + listenStreams = [ + "vsock::24601" + ]; + partOf = [ "uvms-guest.service" ]; + }; + systemd.services."uvms-guest" = { + serviceConfig = { + User = "user"; + Group = "users"; + ExecStart = "${lib.getExe uvmsPkgs.uvms-guest}"; + StandardOutput = "journal+console"; + StandardError = "journal+console"; + Restart = "on-failure"; + RestartSec = 5; + }; + }; + + fonts.enableDefaultPackages = true; + + boot.kernelParams = [ + "earlyprintk=ttyS0" + "console=ttyS0" + "reboot=t" + "panic=-1" + "io.systemd.credential:vmm.notify_socket=vsock-stream:2:8888" + # "rootfstype=virtiofs" + # "root=rootstore" + ]; + }; + + options = { + system.build.ch = mkOption { + type = types.package; + default = (pkgs.formats.json { }).generate "vm.json" config.uvms.ch.settings; + }; + uvms.ch.settings = mkOption { + default = { }; + type = types.submodule { + freeformType = jsonType; + options = { + payload = { + cmdline = mkOption { + type = types.str; + default = concatStringsSep " " ( + config.boot.kernelParams + ++ [ + # "init=${lib.removePrefix "/nix/store" "${config.system.build.toplevel}"}/init" + "init=${config.system.build.toplevel}/init" + ] + ); + defaultText = ''concatStringsSep " " ${config.boot.kernelParams}''; + }; + kernel = mkOption { + type = types.str; + default = "${kernel}/${kernelTarget}"; + }; + initramfs = mkOption { + type = types.nullOr types.str; + default = "${initialRamdisk}/${initrdFile}"; + }; + }; + vsock = { + cid = mkOption { + type = types.int; + default = 4; + }; + socket = mkOption { + type = types.str; + default = "vsock.sock"; + }; + }; + "api-socket" = mkOption { + type = types.str; + default = "vmm.sock"; + }; + "serial".mode = mkOption { + type = types.str; + default = "File"; + }; + "serial".file = mkOption { + type = types.nullOr types.str; + default = "serial"; + }; + "console".mode = mkOption { + type = types.str; + default = "Pty"; + }; + "console".file = mkOption { + type = types.nullOr types.str; + default = null; + }; + # "watchdog" = true; + # "seccomp" = true; + disks = mkOption { + default = [ ]; + type = types.listOf ( + types.submodule { + freeformType = jsonType; + options = { + path = mkOption { + type = types.oneOf [ + types.path + types.str + ]; + }; + readonly = mkOption { + type = types.bool; + default = true; + }; + id = mkOption { type = types.str; }; + }; + } + ); + }; + memory = mkOption { + default = { }; + type = types.submodule { + freeformType = jsonType; + options = { + size = mkOption { + type = types.int; + default = 1536 * 1048576; + }; + shared = mkOption { + type = types.bool; + default = true; + }; + mergeable = mkOption { + type = types.bool; + default = true; + }; + }; + }; + }; + cpus = mkOption { + default = { }; + type = types.submodule { + freeformType = jsonType; + options = { + boot_vcpus = mkOption { + type = types.int; + default = 4; + }; + max_vcpus = mkOption { + type = types.int; + default = 4; + }; + }; + }; + }; + }; + }; + }; + }; +} diff --git a/profiles/ch-runner.nix b/profiles/ch-runner.nix index ad3685f..ef32247 100644 --- a/profiles/ch-runner.nix +++ b/profiles/ch-runner.nix @@ -9,7 +9,7 @@ # but we shall begin by reproducing at least some of their work. let - cfg = config.uvms.cloud-hypervisor; + cfg = config.uvms.ch; inherit (config.networking) hostName; inherit (config.debug.closure.erofs) layers; @@ -48,69 +48,21 @@ let in { options = { - uvms.cloud-hypervisor.enable = lib.mkEnableOption "Configure guest (e.g. fileSystems)"; - uvms.cloud-hypervisor.runner = mkOption { + uvms.ch.enable = lib.mkEnableOption "Configure guest (e.g. fileSystems)"; + uvms.ch.runner = mkOption { type = types.package; description = "A naive script for running this system in cloud-hypervisor"; }; - uvms.cloud-hypervisor.debugger = mkOption { + uvms.ch.debugger = mkOption { type = types.lazyAttrsOf types.anything; description = "Same but you can debug the kernel"; }; - uvms.cloud-hypervisor.settingsFile = mkOption { + uvms.ch.settingsFile = mkOption { type = types.package; default = chSettingsFile; defaultText = "..."; readOnly = true; }; - uvms.cloud-hypervisor.settings = mkOption { - default = { }; - type = types.submodule { - freeformType = (pkgs.formats.json { }).type; - options = { - payload = { - cmdline = mkOption { type = types.str; }; - kernel = mkOption { type = types.str; }; - initramfs = mkOption { - type = types.str; - default = "${config.system.build.initialRamdisk}/${config.system.boot.loader.initrdFile}"; - }; - }; - vsock = { - cid = mkOption { - type = types.int; - default = 4; - }; - socket = mkOption { - type = types.str; - default = "vsock.sock"; - }; - }; - "api-socket" = mkOption { - type = types.str; - default = "vmm.sock"; - }; - "serial".mode = mkOption { - type = types.str; - default = "File"; - }; - "serial".file = mkOption { - type = types.nullOr types.str; - default = "serial"; - }; - "console".mode = mkOption { - type = types.str; - default = "Pty"; - }; - "console".file = mkOption { - type = types.nullOr types.str; - default = null; - }; - # "watchdog" = true; - # "seccomp" = true; - }; - }; - }; uvms.cloud-hypervisor.extraCmdline = lib.mkOption { type = lib.types.listOf lib.types.str; default = [ ]; @@ -118,44 +70,24 @@ in uvms.cloud-hypervisor.cmdline = lib.mkOption { type = lib.types.listOf lib.types.str; default = [ - "earlyprintk=ttyS0" - "console=ttyS0" - "reboot=t" - "panic=-1" - "init=${config.system.build.toplevel}/init" ] ++ config.boot.kernelParams ++ config.uvms.cloud-hypervisor.extraCmdline; }; }; + imports = [ ./baseImage.nix ]; config = lib.mkMerge [ { # boot.kernelPackages = pkgs.linuxPackagesFor (uvmsPkgs.linux-uvm); - uvms.cloud-hypervisor.settings = { - payload = { - cmdline = lib.concatStringsSep " " cfg.cmdline; - kernel = "${config.boot.kernelPackages.kernel}/${pkgs.stdenv.hostPlatform.linux-kernel.target}"; - }; - disks = map (img: { - path = img; - readonly = true; - id = toString img.label; - }) layers; + uvms.ch.settings = { memory = { - size = 1536 * 1048576; - shared = true; - mergeable = true; # hotplugged_size = 512 * 1048576; # hotplugd_size = 1536 * 1048576; # hotplug_method = "virtio-mem" }; - cpus = { - boot_vcpus = 4; - max_vcpus = 4; - }; }; - uvms.cloud-hypervisor.debugger = pkgs.testers.runNixOSTest ( + uvms.ch.debugger = pkgs.testers.runNixOSTest ( { config, ... }: { name = "test-run-${hostName}"; @@ -265,39 +197,9 @@ in ); # NOTE: Used to be an even uglier bash script, but, for now, execline makes for easier comparisons against spectrum - uvms.cloud-hypervisor.runner = writeElb "run-${hostName}" '' + uvms.ch.runner = writeElb "run-${hostName}" '' ${lib.getExe uvmsPkgs.uvms} --vm-config=${chSettingsFile} --vm=${hostName} ''; } - (lib.mkIf cfg.enable { - boot.initrd.availableKernelModules = [ - "erofs" - "overlay" - "virtio_mmio" - "virtio_pci" - "virtio_blk" - # "9pnet_virtio" - # "9p" - "virtiofs" - ]; - boot.initrd.systemd.enable = lib.mkDefault true; - fileSystems = { - "/nix/store" = { - fsType = "overlay"; - overlay.lowerdir = map (img: "/nix/.ro-stores/${toString img.seq}") layers; - neededForBoot = true; - }; - } - // lib.listToAttrs ( - map ( - img: - lib.nameValuePair "/nix/.ro-stores/${toString img.seq}" { - device = "/dev/disk/by-label/${img.label}"; - neededForBoot = true; - options = [ "x-systemd.device-timeout=5" ]; - } - ) layers - ); - }) ]; } diff --git a/profiles/debug-closure.nix b/profiles/debug-closure.nix index d1772da..86137c0 100644 --- a/profiles/debug-closure.nix +++ b/profiles/debug-closure.nix @@ -15,15 +15,9 @@ let inherit (ps) writeErofsLayers; emptySystem = import (pkgs.path + "/nixos/lib/eval-config.nix") { modules = [ - (modulesPath + "/profiles/minimal.nix") + ./minimal.nix { system.stateVersion = config.system.stateVersion; - fileSystems."/".fsType = "tmpfs"; - boot.loader.grub.enable = false; - networking.hostName = "base"; - networking.nftables.enable = true; - networking.useNetworkd = true; - systemd.network.enable = true; } ]; }; diff --git a/profiles/minimal.nix b/profiles/minimal.nix new file mode 100644 index 0000000..1ac85c8 --- /dev/null +++ b/profiles/minimal.nix @@ -0,0 +1,36 @@ +{ + lib, + config, + modulesPath, + ... +}: +{ + imports = [ + (modulesPath + "/profiles/minimal.nix") + ]; + boot.loader.grub.enable = false; + boot.initrd.systemd.enable = true; + networking.useNetworkd = true; + networking.nftables.enable = config.networking.firewall.enable || config.networking.nat.enable; + fileSystems."/".fsType = lib.mkDefault "tmpfs"; + networking.hostName = lib.mkDefault "base"; + + systemd.sysusers.enable = false; + services.userborn.enable = true; # nikstur it + + nix.enable = false; + services.logrotate.enable = false; + services.udisks2.enable = false; + system.tools.nixos-generate-config.enable = false; + systemd.coredump.enable = false; + powerManagement.enable = false; + boot.kexec.enable = false; + system.switch.enable = false; + services.resolved.enable = false; + + systemd.services.generate-shutdown-ramfs.enable = lib.mkForce false; + systemd.services.systemd-remount-fs.enable = lib.mkForce false; + systemd.services.systemd-pstore.enable = lib.mkForce false; + systemd.services.lastlog2-import.enable = lib.mkForce false; + # systemd.services.suid-sgid-wrappers.enable = lib.mkForce false; +} diff --git a/profiles/on-failure.nix b/profiles/on-failure.nix new file mode 100644 index 0000000..c5c256d --- /dev/null +++ b/profiles/on-failure.nix @@ -0,0 +1,72 @@ +{ + lib, + config, + pkgs, + ... +}: +let + cfg = config.some.failure-handler; + jobScript = pkgs.writeShellScriptBin "show-status" '' + set -euo pipefail + + export PATH=${lib.getBin config.boot.initrd.systemd.package}/bin''${PATH:+:}$PATH + export PATH=${lib.getBin pkgs.util-linux}/bin''${PATH:+:}$PATH + export PATH=${lib.getBin pkgs.gnugrep}/bin''${PATH:+:}$PATH + + unit="$1" + shift + + systemctl status "$unit" >&2 || true + patterns=$unit$'\n'error + dmesg | grep -Fi "$patterns" || true + ''; + mkSystemdDropin = pkgs.callPackage ../pkgs/mkSystemdDropin.nix { }; +in +{ + options.some.failure-handler = { + enable = lib.mkEnableOption "Set up show-status@.service as a default OnFailure dependency"; + stage-1.enable = + lib.mkEnableOption "Set up show-status@.service as a default OnFailure dependency in initramfs/initrd" + // { + default = cfg.enable; + }; + package = lib.mkOption { + type = lib.types.package; + readOnly = true; + description = "The internal package with the drop-ins"; + }; + }; + config = { + some.failure-handler.package = mkSystemdDropin { + name = "status-on-failure"; + inherit jobScript; + dropinText = '' + [Unit] + OnFailure=status@%n.service + ''; + serviceText = '' + [Unit] + DefaultDependencies=no + Description=Show status for %i + + [Service] + Type=oneshot + StandardOutput=journal+console + StandardError=journal+console + ExecStart=${lib.getExe jobScript} "%i" + JoinsNamespaceOf= + DelegateNamespaces= + ''; + extraCommands = '' + printf "%s" "$serviceText" > "$root/status@.service" + ''; + }; + boot.initrd.systemd.packages = lib.optionals cfg.stage-1.enable [ cfg.package ]; + boot.initrd.systemd.storePaths = lib.optionals cfg.stage-1.enable [ + jobScript + pkgs.util-linux + pkgs.gnugrep + ]; + systemd.packages = lib.optionals cfg.enable [ cfg.package ]; + }; +} From b84ef62e8f84b94126c6dc4b92672e1c66d0e98f Mon Sep 17 00:00:00 2001 From: Else Someone Date: Fri, 27 Feb 2026 19:05:45 +0200 Subject: [PATCH 20/37] cloud-hypervisor-gpu: make it easier to ignore patches from other ovrlays --- pkgs/cloud-hypervisor-gpu.nix | 80 +++++++++++++++++++++++------------ 1 file changed, 52 insertions(+), 28 deletions(-) diff --git a/pkgs/cloud-hypervisor-gpu.nix b/pkgs/cloud-hypervisor-gpu.nix index 325b372..27cc554 100644 --- a/pkgs/cloud-hypervisor-gpu.nix +++ b/pkgs/cloud-hypervisor-gpu.nix @@ -7,6 +7,7 @@ }: let + inherit (lib) optionals optionalString; spectrum = builtins.fetchTree { url = "https://spectrum-os.org/git/spectrum"; type = "git"; @@ -15,37 +16,60 @@ let in cloud-hypervisor.overrideAttrs ( finalAttrs: oldAttrs: - { - # Verbatim from spectrum - postUnpack = oldAttrs.postUnpack or "" + '' - unpackFile $vhost - chmod -R +w vhost - ''; - vhost = fetchFromGitHub { - name = "vhost"; - owner = "rust-vmm"; - repo = "vhost"; - rev = "vhost-user-backend-v0.20.0"; - hash = "sha256-KK1+mwYQr7YkyGT9+51v7TJael9D0lle2JXfRoTqYq8="; + let + patchesFromDir = + root: + builtins.concatMap (name: lib.optionals (lib.hasSuffix ".patch" name) [ (root + "/${name}") ]) ( + builtins.attrNames (builtins.readDir root) + ); + spectrumPatches = { + version = "2025-12-20"; + vhost = fetchFromGitHub { + name = "vhost"; + owner = "rust-vmm"; + repo = "vhost"; + rev = "vhost-user-backend-v0.20.0"; + hash = "sha256-KK1+mwYQr7YkyGT9+51v7TJael9D0lle2JXfRoTqYq8="; + }; + patches = patchesFromDir (spectrum + "/pkgs/cloud-hypervisor"); + vhostPatches = patchesFromDir (spectrum + "/pkgs/cloud-hypervisor/vhost"); }; + previouslyPatched = oldAttrs ? spectrumPatches; + patchPhases = !previouslyPatched; + overridePatches = + lib.versionOlder oldAttrs.spectrumPatches.version or "2000-00-00" + spectrumPatches.version; + oldPatches = oldAttrs.spectrumPatches.patches or [ ]; + removeAll = removeElts: lst: builtins.filter (x: !(builtins.elem x removeElts)) lst; + in + { + passthru = + oldAttrs.passthru or { } + // lib.optionalAttrs overridePatches { + inherit spectrumPatches; + }; + # Verbatim from spectrum + postUnpack = + oldAttrs.postUnpack or "" + + optionalString patchPhases '' + unpackFile $vhost + chmod -R +w vhost + ''; + vhost = if overridePatches then spectrumPatches.vhost else oldAttrs.vhost or null; - patches = oldAttrs.patches or [ ] ++ [ - "${spectrum}/pkgs/cloud-hypervisor/0001-build-use-local-vhost.patch" - "${spectrum}/pkgs/cloud-hypervisor/0002-virtio-devices-add-a-GPU-device.patch" - ]; - vhostPatches = builtins.concatMap ( - name: - lib.optionals (lib.hasSuffix ".patch" name) [ "${spectrum}/pkgs/cloud-hypervisor/vhost/${name}" ] - ) (builtins.attrNames (builtins.readDir "${spectrum}/pkgs/cloud-hypervisor/vhost")); + patches = + removeAll oldPatches (oldAttrs.patches or [ ]) ++ optionals overridePatches spectrumPatches.patches; # Verbatim copy from spectrum - postPatch = oldAttrs.postPatch or "" + '' - pushd ../vhost - for patch in $vhostPatches; do - echo applying patch $patch - patch -p1 < $patch - done - popd - ''; + postPatch = + oldAttrs.postPatch or "" + + optionalString (!previouslyPatched) '' + pushd ../vhost + for patch in $vhostPatches; do + echo applying patch $patch + patch -p1 < $patch + done + popd + ''; cargoDeps = rustPlatform.fetchCargoVendor { inherit (finalAttrs) patches; inherit (oldAttrs) src; From dd6609ba3f6a18b9d9dd9b3611fb9f7868a62ff1 Mon Sep 17 00:00:00 2001 From: Else Someone Date: Fri, 27 Feb 2026 19:38:26 +0200 Subject: [PATCH 21/37] fixup! cloud-hypervisor-gpu: make it easier to ignore patches from other ovrlays --- pkgs/cloud-hypervisor-gpu.nix | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/pkgs/cloud-hypervisor-gpu.nix b/pkgs/cloud-hypervisor-gpu.nix index 27cc554..a99ce6f 100644 --- a/pkgs/cloud-hypervisor-gpu.nix +++ b/pkgs/cloud-hypervisor-gpu.nix @@ -7,7 +7,7 @@ }: let - inherit (lib) optionals optionalString; + inherit (lib) optionals optionalAttrs optionalString; spectrum = builtins.fetchTree { url = "https://spectrum-os.org/git/spectrum"; type = "git"; @@ -36,18 +36,14 @@ cloud-hypervisor.overrideAttrs ( }; previouslyPatched = oldAttrs ? spectrumPatches; patchPhases = !previouslyPatched; - overridePatches = - lib.versionOlder oldAttrs.spectrumPatches.version or "2000-00-00" - spectrumPatches.version; + isNewer = lib.versionOlder oldAttrs.spectrumPatches.version or "2000-00-00" spectrumPatches.version; oldPatches = oldAttrs.spectrumPatches.patches or [ ]; removeAll = removeElts: lst: builtins.filter (x: !(builtins.elem x removeElts)) lst; in - { - passthru = - oldAttrs.passthru or { } - // lib.optionalAttrs overridePatches { - inherit spectrumPatches; - }; + optionalAttrs isNewer { + passthru = oldAttrs.passthru or { } // { + inherit spectrumPatches; + }; # Verbatim from spectrum postUnpack = oldAttrs.postUnpack or "" @@ -55,14 +51,14 @@ cloud-hypervisor.overrideAttrs ( unpackFile $vhost chmod -R +w vhost ''; - vhost = if overridePatches then spectrumPatches.vhost else oldAttrs.vhost or null; + inherit (spectrumPatches) vhost; - patches = - removeAll oldPatches (oldAttrs.patches or [ ]) ++ optionals overridePatches spectrumPatches.patches; + patches = removeAll oldPatches (oldAttrs.patches or [ ]) ++ spectrumPatches.patches; + inherit (spectrumPatches) vhostPatches; # Verbatim copy from spectrum postPatch = oldAttrs.postPatch or "" - + optionalString (!previouslyPatched) '' + + optionalString patchPhases '' pushd ../vhost for patch in $vhostPatches; do echo applying patch $patch From 00ae88e8adb5ce96d91d0225992b5dc482ab3984 Mon Sep 17 00:00:00 2001 From: Else Someone Date: Mon, 2 Mar 2026 05:07:52 +0200 Subject: [PATCH 22/37] uvms-guest: select() to detect dead subprocesses --- pkgs/uvms-guest/guest.py | 161 +++++++++++++++++++++++++-------------- pkgs/uvms/uvms.py | 14 ++-- profiles/baseImage.nix | 22 ++++-- 3 files changed, 127 insertions(+), 70 deletions(-) diff --git a/pkgs/uvms-guest/guest.py b/pkgs/uvms-guest/guest.py index 87b3d1a..575245e 100644 --- a/pkgs/uvms-guest/guest.py +++ b/pkgs/uvms-guest/guest.py @@ -3,74 +3,117 @@ import os import select import socket import subprocess +import sys -def handle_run(run: dict) -> dict: - res = {} - text = run.get("text", False) - env = { - **os.environ, - "PATH": ":".join( - os.environ.get("PATH", "").split(":") + run.get("EXTRA_PATH", []) - ), - } - proc = None - try: - proc = subprocess.Popen( - req["run"]["argv"], - text=text, - env=env, - cwd="/home/user", - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - ) - res["status"] = "exec succeeded" - except Exception as e: - res["status"] = "exec failed" - res["exception"] = repr(e) - res["pid"] = getattr(proc, "pid", None) - try: - if proc is not None: - proc.wait(0.125) - res["long_running"] = False - res["returncode"] = getattr(proc, "returncode", None) - except subprocess.TimeoutExpired: - res["long_running"] = True - return res, proc +class Processes: + def __init__(self): + self.processes = [] + self.sources = [] + self.liveness_fds = dict() + self.client_fds = set() + + def popen(self, *args, **kwargs): + a, b = socket.socketpair() + pass_fds = [*kwargs.get("pass_fds", ()), b.fileno()] + proc = subprocess.Popen(*args, **kwargs, pass_fds=pass_fds) + self.processes.append(proc) + self.sources.append(a) + assert a.fileno() not in self.liveness_fds + self.liveness_fds[a.fileno()] = proc + b.close() + return proc + + def handle_run(self, run: dict) -> dict: + res = {} + text = run.get("text", False) + env = { + **os.environ, + "PATH": ":".join( + [ + *os.environ.get("PATH", "").split(":"), + *run.get( + "EXTRA_PATH", + [], + ), + ], + ), + } + proc = None + try: + proc = self.popen( + req["run"]["argv"], + text=text, + env=env, + cwd="/home/user", + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + res["status"] = "exec succeeded" + except Exception as e: + print(e) + res["status"] = "exec failed" + res["exception"] = repr(e) + res["pid"] = getattr(proc, "pid", None) + try: + if proc is not None: + proc.wait(0.125) + res["long_running"] = False + res["returncode"] = getattr(proc, "returncode", None) + except subprocess.TimeoutExpired: + res["long_running"] = True + return res, proc + + def accept_vsock(self, s): + con, (cid, port) = serv.accept() + assert cid == 2, cid + self.sources.append(con) + self.client_fds.insert(con.fileno()) + return con, (cid, port) if __name__ == "__main__": + ps = Processes() serv = socket.fromfd(3, socket.AF_VSOCK, socket.SOCK_STREAM) - - procs = [] - conns = [serv] + ps.sources.append(serv) while True: - rr, rw, xs = select.select(conns, [], []) + rr, rw, xs = select.select(ps.sources, [], ps.sources) + for con in (*rr, *xs): + if con.fileno() in ps.liveness_fds: + assert con.recv(128) == b"" + proc = ps.liveness_fds[con.fileno()] + proc.wait() + assert proc.returncode is not None, proc + print(f"{proc} has terminated, shutting down") + sys.exit(proc.returncode) for con in rr: if con is serv: - con, (cid, port) = serv.accept() - assert cid == 2, cid - conns.append(con) - continue - req = con.recv(8192) - # IDK why but I keep getting empty messages - if req == b"": - continue - try: - req = json.loads(req) - print(f"Received {req=}") - except json.JSONDecodeError as e: - print(f"Couldn't interpret {req=}: {e}") - continue - if "run" in req: - res, proc = handle_run(req["run"]) - procs.append(proc) + con, _ = ps.accept_vsock(serv) + print(f"Open [{con.fileno()}]") + if con.fileno() in ps.liveness_fds: + assert False, "Must already be processed" + elif con.fileno() in ps.client_fds: + req = con.recv(8192) + # IDK why but I keep getting empty messages + if req == b"": + print(f"Lost [{con.fileno()}]") + continue + try: + req = json.loads(req) + print(f"Received {req=}") + except json.JSONDecodeError as e: + print(f"Couldn't interpret {req=}: {e}") + continue + if "run" in req: + res, proc = ps.handle_run(req["run"]) + else: + res = {"status": "unknown command"} + _, rw, _ = select.select([], [con], []) + assert rw, rw + res = json.dumps(res).encode("utf8") + print(f"Responding with {res=}") + con.send(res) else: - res = {"status": "unknown command"} - _, rw, _ = select.select([], [con], []) - assert rw, rw - res = json.dumps(res).encode("utf8") - print(f"Responding with {res=}") - con.send(res) + assert False, con.fileno() diff --git a/pkgs/uvms/uvms.py b/pkgs/uvms/uvms.py index ab71c6e..c473f11 100644 --- a/pkgs/uvms/uvms.py +++ b/pkgs/uvms/uvms.py @@ -452,18 +452,21 @@ def removing(*paths): os.remove(p) +@contextmanager def connect_ch_vsock( vsock_sock_path, port: int, type=socket.SOCK_STREAM, blocking=True, ) -> socket.socket: + os.makedirs(os.path.dirname(vsock_sock_path), exist_ok=True) s = socket.socket(socket.AF_UNIX, type, 0) s.setblocking(blocking) s.connect(vsock_sock_path) - s.send(b"CONNECT %d\n" % port) - return s + with removing(vsock_sock_path): + s.send(b"CONNECT %d\n" % port) + yield s @contextmanager @@ -473,15 +476,14 @@ def listen_ch_vsock( type=socket.SOCK_STREAM, blocking=True, ) -> socket.socket: + os.makedirs(os.path.dirname(vsock_sock_path), exist_ok=True) listen_path = vsock_sock_path + "_%d" % port s = socket.socket(socket.AF_UNIX, type, 0) s.setblocking(blocking) s.bind(listen_path) s.listen() - try: + with removing(listen_path): yield s - finally: - os.remove(listen_path) def main(args, args_next, cleanup, ps): @@ -589,7 +591,7 @@ def main(args, args_next, cleanup, ps): ps.exec(*ch_remote, "info") with ready_sock: - ready_sock.settimeout(16.0) + ready_sock.settimeout(20.0) try: con, _ = ready_sock.accept() except: # noqa: E722 diff --git a/profiles/baseImage.nix b/profiles/baseImage.nix index 87f8df5..8ba9767 100644 --- a/profiles/baseImage.nix +++ b/profiles/baseImage.nix @@ -39,9 +39,9 @@ in ./on-failure.nix ]; config = { - some.failure-handler.enable = true; + # some.failure-handler.enable = true; hardware.graphics.enable = true; - # boot.kernelPackages = pkgs.linuxPackagesFor uvmsPkgs.linux-uvm; + boot.kernelPackages = pkgs.linuxPackagesFor uvmsPkgs.linux-uvm; # boot.isContainer = true; boot.initrd.kernelModules = [ "drm" @@ -256,14 +256,26 @@ in partOf = [ "uvms-guest.service" ]; }; systemd.services."uvms-guest" = { + requiredBy = [ "multi-user.target" ]; + onFailure = [ "shutdown.service" ]; serviceConfig = { User = "user"; Group = "users"; ExecStart = "${lib.getExe uvmsPkgs.uvms-guest}"; + ExecStop = [ + "/run/current-system/sw/bin/echo GUEST DOWN" + "/run/current-system/sw/bin/systemctl poweroff" + ]; + StandardOutput = "journal+console"; + StandardError = "journal+console"; + Restart = "no"; + }; + }; + systemd.services."shutdown" = { + serviceConfig = { + ExecStart = [ "/run/current-system/sw/bin/systemctl poweroff" ]; StandardOutput = "journal+console"; StandardError = "journal+console"; - Restart = "on-failure"; - RestartSec = 5; }; }; @@ -371,7 +383,7 @@ in options = { size = mkOption { type = types.int; - default = 1536 * 1048576; + default = 3 * 1024 * 1048576; }; shared = mkOption { type = types.bool; From 18d08e727b1f31ed88244afe2ab0a9cfc8d6f714 Mon Sep 17 00:00:00 2001 From: Else Someone Date: Mon, 2 Mar 2026 05:08:26 +0200 Subject: [PATCH 23/37] linux-uvm: stage cached kernel, add future as comments --- pkgs/linux-uvm.nix | 369 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 302 insertions(+), 67 deletions(-) diff --git a/pkgs/linux-uvm.nix b/pkgs/linux-uvm.nix index bd11aef..ea077dd 100644 --- a/pkgs/linux-uvm.nix +++ b/pkgs/linux-uvm.nix @@ -8,73 +8,7 @@ let inherit (lib) mkForce; result = linux_latest.override { structuredExtraConfig = { - BASE_SMALL = yes; - DRM_VIRTIO_GPU = yes; - EROFS_FS = yes; - # TSI = yes; - DAX = yes; - FS_DAX = yes; - FUSE_DAX = yes; - OVERLAY_FS = yes; - VIRTIO_BALLOON = yes; - VIRTIO_BLK = yes; - VIRTIO_CONSOLE = yes; - VIRTIO_FS = yes; - VIRTIO_MMIO = yes; - VIRTIO_PCI = yes; - VIRTIO = yes; - FUSE_FS = yes; - VSOCKETS = yes; - NO_HZ_IDLE = mkForce yes; - NO_HZ_FULL = mkForce unset; - HZ_1000 = unset; - HZ_250 = yes; # NixOS default: 1000 - - # LSM = "lockdown,yama,loadpin,safesetid,integrity,bpf"; - - EXT4_FS = yes; - # EXT4_USE_FOR_EXT2 = yes; - XFS_FS = yes; - DEFAULT_SECURITY_APPARMOR = mkForce unset; - - XEN = mkForce unset; - XEN_BACKEND = mkForce unset; - XEN_BALLOON = mkForce unset; - XEN_BALLOON_MEMORY_HOTPLUG = mkForce unset; - XEN_DOM0 = mkForce unset; - XEN_HAVE_PVMMU = mkForce unset; - XEN_MCE_LOG = mkForce unset; - XEN_PVH = mkForce unset; - XEN_SAVE_RESTORE = mkForce unset; - XEN_SYS_HYPERVISOR = mkForce unset; - PCI_XEN = mkForce unset; - POWER_RESET_GPIO = mkForce unset; - POWER_RESET_GPIO_RESTART = mkForce unset; - RCU_LAZY = mkForce unset; - REISERFS_FS_POSIX_ACL = mkForce unset; - REISERFS_FS_SECURITY = mkForce unset; - REISERFS_FS_XATTR = mkForce unset; - SWIOTLB_XEN = mkForce unset; - SUSPEND = mkForce unset; - PM = mkForce unset; - HIBERNATION = mkForce unset; - ACPI = mkForce unset; - CPU_FREQ = mkForce unset; - CPU_FREQ_DT = mkForce unset; - INTEL_IDLE = mkForce unset; - ISA_DMA_API = mkForce unset; - IA32_EMULATION = mkForce unset; - COMPAT = mkForce unset; - COMPAT_32 = mkForce unset; - KVM = mkForce unset; - BLOCK_LEGACY_AUTOLOAD = mkForce unset; - SWAP = mkForce unset; - CMA = mkForce unset; - FB = mkForce unset; - FB_EFI = mkForce unset; - FB_VESA = mkForce unset; - SECURITY_APPARMOR = mkForce unset; - + # From spectrum VT = no; DRM_FBDEV_EMULATION = lib.mkForce no; FONTS = mkForce unset; @@ -85,6 +19,307 @@ let FRAMEBUFFER_CONSOLE_DETECT_PRIMARY = mkForce unset; FRAMEBUFFER_CONSOLE_ROTATION = mkForce unset; RC_CORE = mkForce unset; + + # Manually specified: + EROFS_FS = yes; + # TSI = yes; # Needs external patches + DAX = yes; + FS_DAX = yes; + FUSE_DAX = yes; + OVERLAY_FS = yes; + # OVERLAY_FS_REDIRECT_ALWAYS_FOLLOW = yes; + VIRTIO_BALLOON = yes; + VIRTIO_BLK = yes; + VIRTIO_CONSOLE = yes; + # VIRTIO_FS = yes; + VIRTIO_MMIO = yes; + VIRTIO_PCI = yes; + VIRTIO = yes; + # FUSE_FS = yes; + VSOCKETS = yes; + + # # libkurnfw a70c65d97eda1d53a55602bdef984aba82383097 + # DRM = yes; + # DRM_KMS_HELPER = yes; + # DRM_GEM_SHMEM_HELPER = yes; + # # Can't select??? + # VIRTIO_GPU = yes; + # # VIRTIO_GPU_KMS = yes; + + DRM_VIRTIO_GPU = yes; # Based on compiled diff? + + BASE_SMALL = yes; # libkurnfw 33a72344da4938c41616d200372542b3f7eb4412 + + # # Based on compiled config diff with libkurnfw + # NO_HZ_COMMON = mkForce unset; + NO_HZ_IDLE = mkForce yes; + NO_HZ_FULL = mkForce unset; + HZ_1000 = unset; + HZ_250 = yes; # NixOS default: 1000 + + # # LSM = "lockdown,yama,loadpin,safesetid,integrity,bpf"; + + EXT4_FS = yes; + # # EXT4_USE_FOR_EXT2 = yes; + XFS_FS = yes; + DEFAULT_SECURITY_APPARMOR = mkForce unset; + + # HW_RANDOM = yes; + # HW_RANDOM_VIRTIO = yes; + + # # libkurnfw + # POSIX_MQUEUE = mkForce no; + + # # libkurnfw 0ad58f60dc061e61f088a8b9b2758bea4a3bd41d + # HAVE_RELIABLE_STACKTRACE = mkForce unset; + # STACKDEPOT = mkForce unset; + # ARCH_WANT_FRAME_POINTERS = mkForce unset; + # FRAME_POINTER = mkForce unset; + # STACK_VALIDATION = mkForce unset; + # SLUB_DEBUG = mkForce unset; + # UNWINDER_FRAME_POINTER = mkForce unset; + # # UNWINDER_GUESS = mkForce yes; + # UNWINDER_ORC = mkForce unset; + + # # libkurnfw a5094ce1633889250482b812265c15d9a252f6a7 + # PROFILING = mkForce no; + # KEXEC_CORE = mkForce unset; # Still selected? + # KEXEC_FILE = mkForce unset; # Still selected? + # DEBUG_MISC = mkForce no; + # MAGIC_SYSRQ = mkForce no; + # DEBUG_FS = mkForce unset; # mkForce no; + # DEBUG_INFO_NONE = mkForce yes; + + # KEXEC_JUMP = mkForce unset; + # DEBUG_INFO = mkForce unset; + # DEBUG_INFO_REDUCED = mkForce unset; + # DEBUG_LIST = mkForce unset; + # DEBUG_INFO_BTF = mkForce unset; + # DYNAMIC_DEBUG = mkForce unset; + # SUNRPC_DEBUG = mkForce unset; + # MAC80211_DEBUGFS = mkForce unset; + # CFG80211_DEBUGFS = mkForce unset; + # ACPI_DEBUG = mkForce unset; + # DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT = mkForce unset; + # MODULE_ALLOW_BTF_MISMATCH = mkForce unset; + # CRC32_SELFTEST = mkForce unset; + # BLK_DEBUG_FS = mkForce unset; + + # # Conflicts: + # MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT = mkForce unset; + # KEXEC_HANDOVER = mkForce unset; + # MEM_ALLOC_PROFILING = mkForce unset; + + # # libkurnfw d28d2632b704335c4065648afa490b63cfe49393 + # X86_MSR = mkForce no; + # PERF_EVENTS_AMD_UNCORE = mkForce unset; + # ARCH_MEMORY_PROBE = mkForce unset; + # X86_CHECK_BIOS_CORRUPTION = mkForce unset; + # X86_REROUTE_FOR_BROKEN_BOOT_IRQS = mkForce unset; + # MTRR = mkForce no; + # MTRR_SANITIZER = mkForce unset; + # X86_PAT = mkForce unset; + # X86_SGX = mkForce unset; + # MODIFY_LDT_SYSCALL = mkForce unset; + # ARCH_HAS_PKEYS = mkForce unset; + # ARCH_USES_HIGH_VMA_FLAGS = mkForce unset; + + # X86_INTEL_PSTATE = mkForce unset; + # X86_AMD_PSTATE = mkForce unset; + # X86_SGX_KVM = mkForce unset; + # UCLAMP_TASK = mkForce unset; + + # # libkurnfw 8043cecb92f7384f648bfb2fde9e19653fe877e8 + # PM = mkForce unset; # want "no" but can't select? + # CPU_FREQ = mkForce unset; + # CPU_IDLE_GOV_LADDER = mkForce unset; + INTEL_IDLE = mkForce unset; + # SCHED_MC_PRIO = mkForce unset; + # BLK_PM = mkForce unset; + # CPU_IDLE = yes; + # CPU_IDLE_GOV_HALTPOLL = yes; + # HALTPOLL_CPUIDLE = yes; + + # # Conflicts: + # PM_DEBUG = mkForce unset; + # PM_ADVANCED_DEBUG = mkForce unset; + # PM_WAKELOCKS = mkForce unset; + # TPMI = mkForce unset; + # INTEL_TPMI = mkForce unset; + + # # libkurnfw 63c4d29cd28ab73ea1c3a85c9df0c71a6101dd41 + # IA32_EMULATION = mkForce no; + # COMPAT_32BIT_TIME = mkForce no; + + # # libkurnfw b085fa0f8958ea37f0e3c16dfec6cd7cc58c6b41 + # HIBERNATION = mkForce unset; + # SWAP = mkForce no; + # ZSWAP = mkForce unset; + # ZSWAP_COMPRESSOR_DEFAULT_ZSTD = mkForce unset; + # ZONE_DMA = mkForce no; + # VM_EVENT_COUNTERS = mkForce unset; # Still selected as a dep? + # PERCPU_STATS = mkForce no; + + # # libkurnfw 123090b6960cd0beb7fcb19e2fc383d24c6b74e9 + # XFRM_USER = mkForce unset; + # SYN_COOKIES = mkForce unset; + # TCP_CONG_ADVANCED = mkForce no; + # TCP_MD5SIG = mkForce no; + # NETLABEL = mkForce no; + # ETHTOOL_NETLINK = mkForce no; + + # # libkurnfw b31a4ab84f13bde66497fd21b277503f7ad4b541 + # FW_LOADER = mkForce unset; + + # SOUND = mkForce unset; + # USB_SUPPORT = mkForce unset; + # MEMSTICK = mkForce unset; + # NEW_LEDS = mkForce unset; + # ACCESSIBILITY = mkForce unset; + # INFINIBAND = mkForce unset; + # INFINIBAND_ADDR_TRANS = mkForce unset; + + # UIO = mkForce unset; + # VFIO = mkForce unset; + # VIRTIO_PMEM = mkForce unset; + # VDPA = mkForce unset; + # HYPERV = mkForce unset; + # FB_HYPERV = mkForce unset; + # DRM_HYPERV = mkForce unset; + # KVM_HYPERV = mkForce unset; + # COMMON_CLK = mkForce unset; + # EXT2_FS = mkForce unset; + # EXT3_FS = mkForce unset; + + # EXT3_FS_POSIX_ACL = mkForce unset; + # EXT3_FS_SECURITY = mkForce unset; + + # # libkrunfw e2fd98beb10f2fd9b827e188cc55ec0f90d44932 + # PAGE_SIZE_LESS_THAN_64KB = yes; + # PAGE_SIZE_LESS_THAN_256KB = yes; + # CHROME_PLATFORMS = mkForce unset; + # X86_PLATFORM_DEVICES = mkForce unset; + # SURFACE_PLATFORMS = mkForce unset; + # INTEL_UNCORE_FREQ_CONTROL = mkForce unset; + # INTEL_TURBO_MAX_3 = mkForce unset; + + # CHROMEOS_PSTORE = mkForce unset; + # CHROMEOS_LAPTOP = mkForce unset; + # CHROMEOS_TBMC = mkForce unset; + + # CROS_EC = mkForce unset; + # CRYPTO_TEST = mkForce unset; + # NFS_SWAP = mkForce unset; + # CROS_EC_I2C = mkForce unset; + # CROS_EC_ISHTP = mkForce unset; + # CROS_EC_LPC = mkForce unset; + # CROS_EC_SPI = mkForce unset; + + # NET_SCH_BPF = mkForce unset; + + # UCLAMP_TASK_GROUP = mkForce unset; + # SCHED_CLASS_EXT = mkForce unset; + # DRM_NOUVEAU_SVM = mkForce unset; + # CROS_KBD_LED_BACKLIGHT = mkForce unset; + + # # Based on compiled config diff with libkurnfw + XEN = mkForce unset; + # XEN_EFI = mkForce unset; + # HVC_XEN = mkForce unset; + # HVC_XEN_FRONTEND = mkForce unset; + XEN_BACKEND = mkForce unset; + XEN_BALLOON = mkForce unset; + XEN_BALLOON_MEMORY_HOTPLUG = mkForce unset; + XEN_DOM0 = mkForce unset; + XEN_HAVE_PVMMU = mkForce unset; + XEN_MCE_LOG = mkForce unset; + XEN_PVH = mkForce unset; + # XEN_PVHVM = mkForce unset; + XEN_SAVE_RESTORE = mkForce unset; + XEN_SYS_HYPERVISOR = mkForce unset; + PCI_XEN = mkForce unset; + POWER_RESET_GPIO = mkForce unset; + POWER_RESET_GPIO_RESTART = mkForce unset; + RCU_LAZY = mkForce unset; + REISERFS_FS_POSIX_ACL = mkForce unset; + REISERFS_FS_SECURITY = mkForce unset; + REISERFS_FS_XATTR = mkForce unset; + SWIOTLB_XEN = mkForce unset; + # SUSPEND = mkForce unset; + # ACPI = mkForce unset; + # ISA_DMA_API = mkForce unset; + # COMPAT = mkForce unset; + # COMPAT_32 = mkForce unset; + # KVM = mkForce unset; + # BLOCK_LEGACY_AUTOLOAD = mkForce unset; + # CMA = mkForce unset; + FB = mkForce unset; + FB_EFI = mkForce unset; + FB_VESA = mkForce unset; + SECURITY_APPARMOR = mkForce unset; + # SERIAL_8250 = mkForce unset; + # ISA_BUS = unset; + # X86_X32_ABI = unset; + # SCHED_AUTOGROUP = mkForce unset; # libkurnfw + # RAID6_PQ = mkForce unset; # libkurnfw: yes + # ARCH_HAS_FAST_MULTIPLIER = yes; + # GENERIC_NET_UTILS = yes; + # BITREVERSE = yes; + + # # Conflicts + # IIO = mkForce unset; + # IIO_TRIGGERED_EVENT = mkForce unset; + # IIO_CONFIGFS = mkForce unset; + # IIO_KFIFO_BUF = mkForce unset; + # IIO_TRIGGER = mkForce unset; + # STAGING = mkForce unset; + # STAGING_MEDIA = mkForce unset; + + # LEGACY_PTYS = mkForce unset; + # NULL_TTY = mkForce unset; + # N_GSM = mkForce unset; + # SERIAL_NONSTANDARD = mkForce unset; + # TTY_PRINTK = mkForce unset; + # IPMI_HANDLER = mkForce unset; + # TELCLOCK = mkForce unset; + # TCG_TPM = mkForce unset; + # SERIAL_UARTLITE = mkForce unset; + # SERIAL_LANTIQ = mkForce unset; + # MWAVE = mkForce unset; + # SERIAL_CORE = mkForce unset; + # SERIAL_MCTRL_GPIO = mkForce unset; + # I2C = mkForce unset; + + # # Interactively solving conflicts... + + # COMEDI = mkForce unset; + + # NLS = mkForce yes; + # NLS_CODEPAGE_437 = mkForce yes; + # NLS_CODEPAGE_737 = mkForce unset; + # NLS_CODEPAGE_775 = mkForce unset; + # NLS_CODEPAGE_850 = mkForce unset; + # NLS_CODEPAGE_852 = mkForce unset; + # NLS_CODEPAGE_855 = mkForce unset; + # NLS_CODEPAGE_857 = mkForce unset; + # NLS_CODEPAGE_860 = mkForce unset; + # NLS_CODEPAGE_861 = mkForce unset; + # NLS_CODEPAGE_862 = mkForce unset; + # NLS_CODEPAGE_863 = mkForce unset; + # NLS_CODEPAGE_864 = mkForce unset; + # NLS_CODEPAGE_865 = mkForce unset; + # NLS_CODEPAGE_866 = mkForce unset; + # NLS_CODEPAGE_869 = mkForce unset; + # NLS_CODEPAGE_936 = mkForce unset; + # NLS_CODEPAGE_950 = mkForce unset; + # NLS_CODEPAGE_932 = mkForce unset; + # NLS_CODEPAGE_949 = mkForce unset; + # NLS_CODEPAGE_874 = mkForce unset; + # NLS_ISO8859_8 = mkForce unset; + # NLS_CODEPAGE_1250 = mkForce unset; + # NLS_CODEPAGE_1251 = mkForce unset; + + # ZSMALLOC = mkForce unset; }; }; in From a40925c3419627cbd08cf2d616deeb08dcf2c683 Mon Sep 17 00:00:00 2001 From: Else Someone Date: Mon, 2 Mar 2026 10:49:32 +0200 Subject: [PATCH 24/37] baseImage: overridable uvmsPkgs/inherit from scope --- pkgs/baseImage.nix | 10 ++++++++-- profiles/baseImage.nix | 3 ++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/pkgs/baseImage.nix b/pkgs/baseImage.nix index 3e67ba9..8f59460 100644 --- a/pkgs/baseImage.nix +++ b/pkgs/baseImage.nix @@ -1,3 +1,9 @@ -{ nixos }: +{ nixos, overrideScope }: -nixos ../profiles/baseImage.nix +nixos { + imports = [ + ../profiles/baseImage.nix + ]; + _module.args.uvmsPkgs = (overrideScope (fi: _: { uvmsPkgs = fi; })).uvmsPkgs; + _file = __curPos.file; +} diff --git a/profiles/baseImage.nix b/profiles/baseImage.nix index 8ba9767..9d9b5ae 100644 --- a/profiles/baseImage.nix +++ b/profiles/baseImage.nix @@ -3,6 +3,7 @@ config, modulesPath, pkgs, + uvmsPkgs, ... }: let @@ -13,7 +14,6 @@ let inherit (config.system.boot.loader) initrdFile; inherit (config.boot.kernelPackages) kernel; kernelTarget = pkgs.stdenv.hostPlatform.linux-kernel.target; - uvmsPkgs = pkgs.callPackage ../pkgs { }; waylandSock = "/run/user/1000/wayland-1"; env = { XDG_RUNTIME_DIR = "/run/user/1000"; @@ -39,6 +39,7 @@ in ./on-failure.nix ]; config = { + _module.args.uvmsPkgs = lib.mkDefault (pkgs.callPackage ../pkgs { }); # some.failure-handler.enable = true; hardware.graphics.enable = true; boot.kernelPackages = pkgs.linuxPackagesFor uvmsPkgs.linux-uvm; From 20b7a83181ff54674b9b8338abbcbe8ba686c24e Mon Sep 17 00:00:00 2001 From: Else Someone Date: Mon, 2 Mar 2026 10:54:51 +0200 Subject: [PATCH 25/37] fixup! baseImage: overridable uvmsPkgs/inherit from scope --- pkgs/baseImage.nix | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/pkgs/baseImage.nix b/pkgs/baseImage.nix index 8f59460..63eed63 100644 --- a/pkgs/baseImage.nix +++ b/pkgs/baseImage.nix @@ -1,9 +1,11 @@ { nixos, overrideScope }: nixos { - imports = [ - ../profiles/baseImage.nix - ]; - _module.args.uvmsPkgs = (overrideScope (fi: _: { uvmsPkgs = fi; })).uvmsPkgs; - _file = __curPos.file; + imports = [ + ../profiles/baseImage.nix + ]; + _module.args.uvmsPkgs = builtins.removeAttrs (overrideScope (fi: _: { _self = fi; }))._self [ + "_self" + ]; + _file = __curPos.file; } From 95c0a1d72b7d35cfb12bc0c85e9f709fb46acc32 Mon Sep 17 00:00:00 2001 From: Else Someone Date: Mon, 2 Mar 2026 15:28:31 +0200 Subject: [PATCH 26/37] fixup! fixup! baseImage: overridable uvmsPkgs/inherit from scope --- pkgs/uvms-guest/guest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/uvms-guest/guest.py b/pkgs/uvms-guest/guest.py index 575245e..04914a9 100644 --- a/pkgs/uvms-guest/guest.py +++ b/pkgs/uvms-guest/guest.py @@ -68,7 +68,7 @@ class Processes: con, (cid, port) = serv.accept() assert cid == 2, cid self.sources.append(con) - self.client_fds.insert(con.fileno()) + self.client_fds.add(con.fileno()) return con, (cid, port) From db4e7809d142e757719b5f357505ade09ccfa56b Mon Sep 17 00:00:00 2001 From: Else Someone Date: Wed, 4 Mar 2026 01:41:33 +0200 Subject: [PATCH 27/37] uvms: start actually using the recompiled kernel ...still with an initrd, and very much not useful - bot it boots! --- pkgs/linux-uvm.nix | 406 +++++++++++++++++++-------------------- pkgs/uvms-guest/guest.py | 19 +- profiles/baseImage.nix | 5 +- 3 files changed, 224 insertions(+), 206 deletions(-) diff --git a/pkgs/linux-uvm.nix b/pkgs/linux-uvm.nix index ea077dd..adc05ff 100644 --- a/pkgs/linux-uvm.nix +++ b/pkgs/linux-uvm.nix @@ -27,206 +27,206 @@ let FS_DAX = yes; FUSE_DAX = yes; OVERLAY_FS = yes; - # OVERLAY_FS_REDIRECT_ALWAYS_FOLLOW = yes; + OVERLAY_FS_REDIRECT_ALWAYS_FOLLOW = yes; VIRTIO_BALLOON = yes; VIRTIO_BLK = yes; VIRTIO_CONSOLE = yes; - # VIRTIO_FS = yes; + VIRTIO_FS = yes; VIRTIO_MMIO = yes; VIRTIO_PCI = yes; VIRTIO = yes; - # FUSE_FS = yes; + FUSE_FS = yes; VSOCKETS = yes; # # libkurnfw a70c65d97eda1d53a55602bdef984aba82383097 - # DRM = yes; - # DRM_KMS_HELPER = yes; - # DRM_GEM_SHMEM_HELPER = yes; - # # Can't select??? + DRM = yes; + DRM_KMS_HELPER = yes; + DRM_GEM_SHMEM_HELPER = yes; + # Can't select??? # VIRTIO_GPU = yes; - # # VIRTIO_GPU_KMS = yes; + # VIRTIO_GPU_KMS = yes; DRM_VIRTIO_GPU = yes; # Based on compiled diff? BASE_SMALL = yes; # libkurnfw 33a72344da4938c41616d200372542b3f7eb4412 # # Based on compiled config diff with libkurnfw - # NO_HZ_COMMON = mkForce unset; + NO_HZ_COMMON = mkForce unset; NO_HZ_IDLE = mkForce yes; NO_HZ_FULL = mkForce unset; HZ_1000 = unset; HZ_250 = yes; # NixOS default: 1000 - # # LSM = "lockdown,yama,loadpin,safesetid,integrity,bpf"; + LSM = lib.kernel.freeform "lockdown,yama,loadpin,safesetid,integrity,bpf"; EXT4_FS = yes; # # EXT4_USE_FOR_EXT2 = yes; XFS_FS = yes; DEFAULT_SECURITY_APPARMOR = mkForce unset; - # HW_RANDOM = yes; - # HW_RANDOM_VIRTIO = yes; + HW_RANDOM = yes; + HW_RANDOM_VIRTIO = yes; # # libkurnfw - # POSIX_MQUEUE = mkForce no; + POSIX_MQUEUE = mkForce unset; # # libkurnfw 0ad58f60dc061e61f088a8b9b2758bea4a3bd41d - # HAVE_RELIABLE_STACKTRACE = mkForce unset; - # STACKDEPOT = mkForce unset; - # ARCH_WANT_FRAME_POINTERS = mkForce unset; - # FRAME_POINTER = mkForce unset; - # STACK_VALIDATION = mkForce unset; - # SLUB_DEBUG = mkForce unset; - # UNWINDER_FRAME_POINTER = mkForce unset; - # # UNWINDER_GUESS = mkForce yes; - # UNWINDER_ORC = mkForce unset; + HAVE_RELIABLE_STACKTRACE = mkForce unset; + STACKDEPOT = mkForce unset; + ARCH_WANT_FRAME_POINTERS = mkForce unset; + FRAME_POINTER = mkForce unset; + STACK_VALIDATION = mkForce unset; + SLUB_DEBUG = mkForce unset; + UNWINDER_FRAME_POINTER = mkForce unset; + # UNWINDER_GUESS = mkForce yes; + UNWINDER_ORC = mkForce unset; # # libkurnfw a5094ce1633889250482b812265c15d9a252f6a7 - # PROFILING = mkForce no; - # KEXEC_CORE = mkForce unset; # Still selected? - # KEXEC_FILE = mkForce unset; # Still selected? - # DEBUG_MISC = mkForce no; - # MAGIC_SYSRQ = mkForce no; - # DEBUG_FS = mkForce unset; # mkForce no; - # DEBUG_INFO_NONE = mkForce yes; + PROFILING = mkForce no; + KEXEC_CORE = mkForce unset; # Still selected? + KEXEC_FILE = mkForce unset; # Still selected? + DEBUG_MISC = mkForce no; + MAGIC_SYSRQ = mkForce no; + DEBUG_FS = mkForce unset; # mkForce no; + DEBUG_INFO_NONE = mkForce yes; - # KEXEC_JUMP = mkForce unset; - # DEBUG_INFO = mkForce unset; - # DEBUG_INFO_REDUCED = mkForce unset; - # DEBUG_LIST = mkForce unset; - # DEBUG_INFO_BTF = mkForce unset; - # DYNAMIC_DEBUG = mkForce unset; - # SUNRPC_DEBUG = mkForce unset; - # MAC80211_DEBUGFS = mkForce unset; - # CFG80211_DEBUGFS = mkForce unset; - # ACPI_DEBUG = mkForce unset; - # DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT = mkForce unset; - # MODULE_ALLOW_BTF_MISMATCH = mkForce unset; - # CRC32_SELFTEST = mkForce unset; - # BLK_DEBUG_FS = mkForce unset; + KEXEC_JUMP = mkForce unset; + DEBUG_INFO = mkForce unset; + DEBUG_INFO_REDUCED = mkForce unset; + DEBUG_LIST = mkForce unset; + DEBUG_INFO_BTF = mkForce unset; + DYNAMIC_DEBUG = mkForce unset; + SUNRPC_DEBUG = mkForce unset; + MAC80211_DEBUGFS = mkForce unset; + CFG80211_DEBUGFS = mkForce unset; + ACPI_DEBUG = mkForce unset; + DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT = mkForce unset; + MODULE_ALLOW_BTF_MISMATCH = mkForce unset; + CRC32_SELFTEST = mkForce unset; + BLK_DEBUG_FS = mkForce unset; - # # Conflicts: - # MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT = mkForce unset; - # KEXEC_HANDOVER = mkForce unset; - # MEM_ALLOC_PROFILING = mkForce unset; + # Conflicts: + MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT = mkForce unset; + KEXEC_HANDOVER = mkForce unset; + MEM_ALLOC_PROFILING = mkForce unset; # # libkurnfw d28d2632b704335c4065648afa490b63cfe49393 - # X86_MSR = mkForce no; - # PERF_EVENTS_AMD_UNCORE = mkForce unset; - # ARCH_MEMORY_PROBE = mkForce unset; - # X86_CHECK_BIOS_CORRUPTION = mkForce unset; - # X86_REROUTE_FOR_BROKEN_BOOT_IRQS = mkForce unset; - # MTRR = mkForce no; - # MTRR_SANITIZER = mkForce unset; - # X86_PAT = mkForce unset; - # X86_SGX = mkForce unset; - # MODIFY_LDT_SYSCALL = mkForce unset; - # ARCH_HAS_PKEYS = mkForce unset; - # ARCH_USES_HIGH_VMA_FLAGS = mkForce unset; + X86_MSR = mkForce no; + PERF_EVENTS_AMD_UNCORE = mkForce unset; + ARCH_MEMORY_PROBE = mkForce unset; + X86_CHECK_BIOS_CORRUPTION = mkForce unset; + X86_REROUTE_FOR_BROKEN_BOOT_IRQS = mkForce unset; + MTRR = mkForce no; + MTRR_SANITIZER = mkForce unset; + X86_PAT = mkForce unset; + X86_SGX = mkForce unset; + MODIFY_LDT_SYSCALL = mkForce unset; + ARCH_HAS_PKEYS = mkForce unset; + ARCH_USES_HIGH_VMA_FLAGS = mkForce unset; - # X86_INTEL_PSTATE = mkForce unset; - # X86_AMD_PSTATE = mkForce unset; - # X86_SGX_KVM = mkForce unset; - # UCLAMP_TASK = mkForce unset; + X86_INTEL_PSTATE = mkForce unset; + X86_AMD_PSTATE = mkForce unset; + X86_SGX_KVM = mkForce unset; + UCLAMP_TASK = mkForce unset; # # libkurnfw 8043cecb92f7384f648bfb2fde9e19653fe877e8 - # PM = mkForce unset; # want "no" but can't select? - # CPU_FREQ = mkForce unset; - # CPU_IDLE_GOV_LADDER = mkForce unset; + PM = mkForce unset; # want "no" but can't select? + CPU_FREQ = mkForce unset; + CPU_IDLE_GOV_LADDER = mkForce unset; INTEL_IDLE = mkForce unset; - # SCHED_MC_PRIO = mkForce unset; - # BLK_PM = mkForce unset; - # CPU_IDLE = yes; - # CPU_IDLE_GOV_HALTPOLL = yes; - # HALTPOLL_CPUIDLE = yes; + SCHED_MC_PRIO = mkForce unset; + BLK_PM = mkForce unset; + CPU_IDLE = yes; + CPU_IDLE_GOV_HALTPOLL = yes; + HALTPOLL_CPUIDLE = yes; # # Conflicts: - # PM_DEBUG = mkForce unset; - # PM_ADVANCED_DEBUG = mkForce unset; - # PM_WAKELOCKS = mkForce unset; - # TPMI = mkForce unset; - # INTEL_TPMI = mkForce unset; + PM_DEBUG = mkForce unset; + PM_ADVANCED_DEBUG = mkForce unset; + PM_WAKELOCKS = mkForce unset; + TPMI = mkForce unset; + INTEL_TPMI = mkForce unset; # # libkurnfw 63c4d29cd28ab73ea1c3a85c9df0c71a6101dd41 - # IA32_EMULATION = mkForce no; - # COMPAT_32BIT_TIME = mkForce no; + IA32_EMULATION = mkForce no; + COMPAT_32BIT_TIME = mkForce no; # # libkurnfw b085fa0f8958ea37f0e3c16dfec6cd7cc58c6b41 - # HIBERNATION = mkForce unset; - # SWAP = mkForce no; - # ZSWAP = mkForce unset; - # ZSWAP_COMPRESSOR_DEFAULT_ZSTD = mkForce unset; - # ZONE_DMA = mkForce no; - # VM_EVENT_COUNTERS = mkForce unset; # Still selected as a dep? - # PERCPU_STATS = mkForce no; + HIBERNATION = mkForce unset; + SWAP = mkForce no; + ZSWAP = mkForce unset; + ZSWAP_COMPRESSOR_DEFAULT_ZSTD = mkForce unset; + ZONE_DMA = mkForce no; + VM_EVENT_COUNTERS = mkForce unset; # Still selected as a dep? + PERCPU_STATS = mkForce no; # # libkurnfw 123090b6960cd0beb7fcb19e2fc383d24c6b74e9 - # XFRM_USER = mkForce unset; - # SYN_COOKIES = mkForce unset; - # TCP_CONG_ADVANCED = mkForce no; - # TCP_MD5SIG = mkForce no; - # NETLABEL = mkForce no; - # ETHTOOL_NETLINK = mkForce no; + XFRM_USER = mkForce unset; + SYN_COOKIES = mkForce unset; + TCP_CONG_ADVANCED = mkForce no; + TCP_MD5SIG = mkForce no; + NETLABEL = mkForce no; + ETHTOOL_NETLINK = mkForce no; # # libkurnfw b31a4ab84f13bde66497fd21b277503f7ad4b541 - # FW_LOADER = mkForce unset; + FW_LOADER = mkForce unset; # SOUND = mkForce unset; - # USB_SUPPORT = mkForce unset; - # MEMSTICK = mkForce unset; - # NEW_LEDS = mkForce unset; - # ACCESSIBILITY = mkForce unset; - # INFINIBAND = mkForce unset; - # INFINIBAND_ADDR_TRANS = mkForce unset; + USB_SUPPORT = mkForce unset; + MEMSTICK = mkForce unset; + NEW_LEDS = mkForce unset; + ACCESSIBILITY = mkForce unset; + INFINIBAND = mkForce unset; + INFINIBAND_ADDR_TRANS = mkForce unset; # UIO = mkForce unset; - # VFIO = mkForce unset; - # VIRTIO_PMEM = mkForce unset; - # VDPA = mkForce unset; - # HYPERV = mkForce unset; - # FB_HYPERV = mkForce unset; - # DRM_HYPERV = mkForce unset; - # KVM_HYPERV = mkForce unset; - # COMMON_CLK = mkForce unset; - # EXT2_FS = mkForce unset; - # EXT3_FS = mkForce unset; + VFIO = mkForce unset; + VIRTIO_PMEM = mkForce unset; + VDPA = mkForce unset; + HYPERV = mkForce unset; + FB_HYPERV = mkForce unset; + DRM_HYPERV = mkForce unset; + KVM_HYPERV = mkForce unset; + COMMON_CLK = mkForce unset; + EXT2_FS = mkForce unset; + EXT3_FS = mkForce unset; - # EXT3_FS_POSIX_ACL = mkForce unset; - # EXT3_FS_SECURITY = mkForce unset; + EXT3_FS_POSIX_ACL = mkForce unset; + EXT3_FS_SECURITY = mkForce unset; - # # libkrunfw e2fd98beb10f2fd9b827e188cc55ec0f90d44932 - # PAGE_SIZE_LESS_THAN_64KB = yes; - # PAGE_SIZE_LESS_THAN_256KB = yes; - # CHROME_PLATFORMS = mkForce unset; - # X86_PLATFORM_DEVICES = mkForce unset; - # SURFACE_PLATFORMS = mkForce unset; - # INTEL_UNCORE_FREQ_CONTROL = mkForce unset; - # INTEL_TURBO_MAX_3 = mkForce unset; + # libkrunfw e2fd98beb10f2fd9b827e188cc55ec0f90d44932 + PAGE_SIZE_LESS_THAN_64KB = yes; + PAGE_SIZE_LESS_THAN_256KB = yes; + CHROME_PLATFORMS = mkForce unset; + X86_PLATFORM_DEVICES = mkForce unset; + SURFACE_PLATFORMS = mkForce unset; + INTEL_UNCORE_FREQ_CONTROL = mkForce unset; + INTEL_TURBO_MAX_3 = mkForce unset; - # CHROMEOS_PSTORE = mkForce unset; - # CHROMEOS_LAPTOP = mkForce unset; - # CHROMEOS_TBMC = mkForce unset; + CHROMEOS_PSTORE = mkForce unset; + CHROMEOS_LAPTOP = mkForce unset; + CHROMEOS_TBMC = mkForce unset; - # CROS_EC = mkForce unset; - # CRYPTO_TEST = mkForce unset; - # NFS_SWAP = mkForce unset; - # CROS_EC_I2C = mkForce unset; - # CROS_EC_ISHTP = mkForce unset; - # CROS_EC_LPC = mkForce unset; - # CROS_EC_SPI = mkForce unset; + CROS_EC = mkForce unset; + CRYPTO_TEST = mkForce unset; + NFS_SWAP = mkForce unset; + CROS_EC_I2C = mkForce unset; + CROS_EC_ISHTP = mkForce unset; + CROS_EC_LPC = mkForce unset; + CROS_EC_SPI = mkForce unset; - # NET_SCH_BPF = mkForce unset; + NET_SCH_BPF = mkForce unset; - # UCLAMP_TASK_GROUP = mkForce unset; - # SCHED_CLASS_EXT = mkForce unset; - # DRM_NOUVEAU_SVM = mkForce unset; - # CROS_KBD_LED_BACKLIGHT = mkForce unset; + UCLAMP_TASK_GROUP = mkForce unset; + SCHED_CLASS_EXT = mkForce unset; + DRM_NOUVEAU_SVM = mkForce unset; + CROS_KBD_LED_BACKLIGHT = mkForce unset; - # # Based on compiled config diff with libkurnfw + # Based on compiled config diff with libkurnfw XEN = mkForce unset; - # XEN_EFI = mkForce unset; - # HVC_XEN = mkForce unset; - # HVC_XEN_FRONTEND = mkForce unset; + XEN_EFI = mkForce unset; + HVC_XEN = mkForce unset; + HVC_XEN_FRONTEND = mkForce unset; XEN_BACKEND = mkForce unset; XEN_BALLOON = mkForce unset; XEN_BALLOON_MEMORY_HOTPLUG = mkForce unset; @@ -234,7 +234,7 @@ let XEN_HAVE_PVMMU = mkForce unset; XEN_MCE_LOG = mkForce unset; XEN_PVH = mkForce unset; - # XEN_PVHVM = mkForce unset; + XEN_PVHVM = mkForce unset; XEN_SAVE_RESTORE = mkForce unset; XEN_SYS_HYPERVISOR = mkForce unset; PCI_XEN = mkForce unset; @@ -245,81 +245,81 @@ let REISERFS_FS_SECURITY = mkForce unset; REISERFS_FS_XATTR = mkForce unset; SWIOTLB_XEN = mkForce unset; - # SUSPEND = mkForce unset; - # ACPI = mkForce unset; - # ISA_DMA_API = mkForce unset; - # COMPAT = mkForce unset; - # COMPAT_32 = mkForce unset; - # KVM = mkForce unset; - # BLOCK_LEGACY_AUTOLOAD = mkForce unset; - # CMA = mkForce unset; + SUSPEND = mkForce unset; + ACPI = mkForce unset; + ISA_DMA_API = mkForce unset; + COMPAT = mkForce unset; + COMPAT_32 = mkForce unset; + KVM = mkForce unset; + BLOCK_LEGACY_AUTOLOAD = mkForce unset; + CMA = mkForce unset; FB = mkForce unset; FB_EFI = mkForce unset; FB_VESA = mkForce unset; SECURITY_APPARMOR = mkForce unset; - # SERIAL_8250 = mkForce unset; - # ISA_BUS = unset; - # X86_X32_ABI = unset; - # SCHED_AUTOGROUP = mkForce unset; # libkurnfw - # RAID6_PQ = mkForce unset; # libkurnfw: yes - # ARCH_HAS_FAST_MULTIPLIER = yes; - # GENERIC_NET_UTILS = yes; - # BITREVERSE = yes; + SERIAL_8250 = mkForce unset; + ISA_BUS = unset; + X86_X32_ABI = unset; + SCHED_AUTOGROUP = mkForce unset; # libkurnfw + RAID6_PQ = mkForce unset; # libkurnfw: yes + ARCH_HAS_FAST_MULTIPLIER = yes; + GENERIC_NET_UTILS = yes; + BITREVERSE = yes; - # # Conflicts - # IIO = mkForce unset; - # IIO_TRIGGERED_EVENT = mkForce unset; - # IIO_CONFIGFS = mkForce unset; - # IIO_KFIFO_BUF = mkForce unset; - # IIO_TRIGGER = mkForce unset; - # STAGING = mkForce unset; - # STAGING_MEDIA = mkForce unset; + # Conflicts + IIO = mkForce unset; + IIO_TRIGGERED_EVENT = mkForce unset; + IIO_CONFIGFS = mkForce unset; + IIO_KFIFO_BUF = mkForce unset; + IIO_TRIGGER = mkForce unset; + STAGING = mkForce unset; + STAGING_MEDIA = mkForce unset; - # LEGACY_PTYS = mkForce unset; - # NULL_TTY = mkForce unset; - # N_GSM = mkForce unset; - # SERIAL_NONSTANDARD = mkForce unset; - # TTY_PRINTK = mkForce unset; - # IPMI_HANDLER = mkForce unset; - # TELCLOCK = mkForce unset; - # TCG_TPM = mkForce unset; - # SERIAL_UARTLITE = mkForce unset; - # SERIAL_LANTIQ = mkForce unset; - # MWAVE = mkForce unset; - # SERIAL_CORE = mkForce unset; - # SERIAL_MCTRL_GPIO = mkForce unset; - # I2C = mkForce unset; + LEGACY_PTYS = mkForce unset; + NULL_TTY = mkForce unset; + N_GSM = mkForce unset; + SERIAL_NONSTANDARD = mkForce unset; + TTY_PRINTK = mkForce unset; + IPMI_HANDLER = mkForce unset; + TELCLOCK = mkForce unset; + TCG_TPM = mkForce unset; + SERIAL_UARTLITE = mkForce unset; + SERIAL_LANTIQ = mkForce unset; + MWAVE = mkForce unset; + SERIAL_CORE = mkForce unset; + SERIAL_MCTRL_GPIO = mkForce unset; + I2C = mkForce unset; - # # Interactively solving conflicts... + # Interactively solving conflicts... - # COMEDI = mkForce unset; + COMEDI = mkForce unset; - # NLS = mkForce yes; - # NLS_CODEPAGE_437 = mkForce yes; - # NLS_CODEPAGE_737 = mkForce unset; - # NLS_CODEPAGE_775 = mkForce unset; - # NLS_CODEPAGE_850 = mkForce unset; - # NLS_CODEPAGE_852 = mkForce unset; - # NLS_CODEPAGE_855 = mkForce unset; - # NLS_CODEPAGE_857 = mkForce unset; - # NLS_CODEPAGE_860 = mkForce unset; - # NLS_CODEPAGE_861 = mkForce unset; - # NLS_CODEPAGE_862 = mkForce unset; - # NLS_CODEPAGE_863 = mkForce unset; - # NLS_CODEPAGE_864 = mkForce unset; - # NLS_CODEPAGE_865 = mkForce unset; - # NLS_CODEPAGE_866 = mkForce unset; - # NLS_CODEPAGE_869 = mkForce unset; - # NLS_CODEPAGE_936 = mkForce unset; - # NLS_CODEPAGE_950 = mkForce unset; - # NLS_CODEPAGE_932 = mkForce unset; - # NLS_CODEPAGE_949 = mkForce unset; - # NLS_CODEPAGE_874 = mkForce unset; - # NLS_ISO8859_8 = mkForce unset; - # NLS_CODEPAGE_1250 = mkForce unset; - # NLS_CODEPAGE_1251 = mkForce unset; + NLS = mkForce yes; + NLS_CODEPAGE_437 = mkForce yes; + NLS_CODEPAGE_737 = mkForce unset; + NLS_CODEPAGE_775 = mkForce unset; + NLS_CODEPAGE_850 = mkForce unset; + NLS_CODEPAGE_852 = mkForce unset; + NLS_CODEPAGE_855 = mkForce unset; + NLS_CODEPAGE_857 = mkForce unset; + NLS_CODEPAGE_860 = mkForce unset; + NLS_CODEPAGE_861 = mkForce unset; + NLS_CODEPAGE_862 = mkForce unset; + NLS_CODEPAGE_863 = mkForce unset; + NLS_CODEPAGE_864 = mkForce unset; + NLS_CODEPAGE_865 = mkForce unset; + NLS_CODEPAGE_866 = mkForce unset; + NLS_CODEPAGE_869 = mkForce unset; + NLS_CODEPAGE_936 = mkForce unset; + NLS_CODEPAGE_950 = mkForce unset; + NLS_CODEPAGE_932 = mkForce unset; + NLS_CODEPAGE_949 = mkForce unset; + NLS_CODEPAGE_874 = mkForce unset; + NLS_ISO8859_8 = mkForce unset; + NLS_CODEPAGE_1250 = mkForce unset; + NLS_CODEPAGE_1251 = mkForce unset; - # ZSMALLOC = mkForce unset; + ZSMALLOC = mkForce unset; }; }; in diff --git a/pkgs/uvms-guest/guest.py b/pkgs/uvms-guest/guest.py index 04914a9..9fd9397 100644 --- a/pkgs/uvms-guest/guest.py +++ b/pkgs/uvms-guest/guest.py @@ -28,7 +28,22 @@ class Processes: res = {} text = run.get("text", False) env = { - **os.environ, + **{ + k: v + for k, v in os.environ.items() + # if any( + # k.startswith(prefix) + # for prefix in ( + # "XDG_", + # "NIX_", + # "RUST_", + # "WAYLAND_", + # "DBUS_", + # "HOME", + # "PS", + # ) + # ) + }, "PATH": ":".join( [ *os.environ.get("PATH", "").split(":"), @@ -36,6 +51,8 @@ class Processes: "EXTRA_PATH", [], ), + "/run/wrappers/bin", + "/run/current-system/sw/bin", ], ), } diff --git a/profiles/baseImage.nix b/profiles/baseImage.nix index 9d9b5ae..2f9029a 100644 --- a/profiles/baseImage.nix +++ b/profiles/baseImage.nix @@ -209,6 +209,7 @@ in ]; }; + environment.profileRelativeSessionVariables.PATH = lib.mkForce [ "/bin\${PATH:+:}$PATH" ]; environment.variables = env; systemd.globalEnvironment = env; @@ -264,8 +265,8 @@ in Group = "users"; ExecStart = "${lib.getExe uvmsPkgs.uvms-guest}"; ExecStop = [ - "/run/current-system/sw/bin/echo GUEST DOWN" - "/run/current-system/sw/bin/systemctl poweroff" + "/run/current-system/sw/bin/echo GUEST DOWN" + "/run/current-system/sw/bin/systemctl poweroff" ]; StandardOutput = "journal+console"; StandardError = "journal+console"; From cf95fd33b07bd9cb1d9c98378d29d1e371c93b9b Mon Sep 17 00:00:00 2001 From: Else Someone Date: Sat, 7 Mar 2026 21:16:53 +0200 Subject: [PATCH 28/37] uvms: add --mem Yeah, yeah, I know, it's stupid, adding add_argument() statements manually, et c. I'll throw the whole Python thing away whenever I might have the time --- pkgs/uvms/uvms.py | 20 ++++ profiles/baseImage.nix | 221 +++++++++++++++++++++++------------------ 2 files changed, 143 insertions(+), 98 deletions(-) diff --git a/pkgs/uvms/uvms.py b/pkgs/uvms/uvms.py index c473f11..534a7b9 100644 --- a/pkgs/uvms/uvms.py +++ b/pkgs/uvms/uvms.py @@ -10,6 +10,7 @@ import os import subprocess import socket import json +import re from argparse import ArgumentParser from contextlib import contextmanager, closing, ExitStack @@ -20,6 +21,7 @@ parser.add_argument("--prefix", default="$HOME/uvms/$VM") parser.add_argument("--vm-config", default="@BASE_CONFIG@") # noqa: E501 parser.add_argument("--persist-home", action="store_true") parser.add_argument("--run", action="append") +parser.add_argument("--mem", default=None) parser.add_argument("app", nargs="*", default=()) TOOLS_DIR = "@TOOLS@" # noqa: E501 @@ -469,6 +471,21 @@ def connect_ch_vsock( yield s +BYTES_PATTERN = re.compile(r"^([0-9]+)([MmGgKk]?)$") +BYTES_UNITS = { + "k": 1024, + "m": 1048576, + "g": 1024 * 1048576, +} + + +def parse_bytes(s): + m = BYTES_PATTERN.match(s) + assert m, s + size, unit = m.groups() + return int(size) * BYTES_UNITS.get(unit.lower(), 1) + + @contextmanager def listen_ch_vsock( vsock_sock_path, @@ -564,6 +581,9 @@ def main(args, args_next, cleanup, ps): ) virtiofs_socks.append(("home", sock_path)) config["payload"]["cmdline"] += " uvms.persist-home=1" + if args.mem is not None: + config["memory"]["size"] = parse_bytes(args.mem) + config["memory"]["hotplug_size"] = parse_bytes(args.mem) gpud, gpud_path = cleanup.enter_context(ps.start_gpu()) diff --git a/profiles/baseImage.nix b/profiles/baseImage.nix index 2f9029a..289f903 100644 --- a/profiles/baseImage.nix +++ b/profiles/baseImage.nix @@ -301,121 +301,146 @@ in }; uvms.ch.settings = mkOption { default = { }; - type = types.submodule { - freeformType = jsonType; - options = { - payload = { - cmdline = mkOption { - type = types.str; - default = concatStringsSep " " ( - config.boot.kernelParams - ++ [ - # "init=${lib.removePrefix "/nix/store" "${config.system.build.toplevel}"}/init" - "init=${config.system.build.toplevel}/init" - ] - ); - defaultText = ''concatStringsSep " " ${config.boot.kernelParams}''; + type = types.submodule ( + let + osConfig = config; + in + { config, ... }: + { + freeformType = jsonType; + options = { + payload = { + cmdline = mkOption { + type = types.str; + default = concatStringsSep " " ( + osConfig.boot.kernelParams + ++ [ + # "init=${lib.removePrefix "/nix/store" "${osConfig.system.build.toplevel}"}/init" + "init=${osConfig.system.build.toplevel}/init" + ] + ); + defaultText = ''concatStringsSep " " ${osConfig.boot.kernelParams}''; + }; + kernel = mkOption { + type = types.str; + default = "${kernel}/${kernelTarget}"; + }; + initramfs = mkOption { + type = types.nullOr types.str; + default = "${initialRamdisk}/${initrdFile}"; + }; }; - kernel = mkOption { - type = types.str; - default = "${kernel}/${kernelTarget}"; + vsock = { + cid = mkOption { + type = types.int; + default = 4; + }; + socket = mkOption { + type = types.str; + default = "vsock.sock"; + }; }; - initramfs = mkOption { + "api-socket" = mkOption { + type = types.str; + default = "vmm.sock"; + }; + "serial".mode = mkOption { + type = types.str; + default = "File"; + }; + "serial".file = mkOption { type = types.nullOr types.str; - default = "${initialRamdisk}/${initrdFile}"; + default = "serial"; }; - }; - vsock = { - cid = mkOption { - type = types.int; - default = 4; - }; - socket = mkOption { + "console".mode = mkOption { type = types.str; - default = "vsock.sock"; + default = "Pty"; }; - }; - "api-socket" = mkOption { - type = types.str; - default = "vmm.sock"; - }; - "serial".mode = mkOption { - type = types.str; - default = "File"; - }; - "serial".file = mkOption { - type = types.nullOr types.str; - default = "serial"; - }; - "console".mode = mkOption { - type = types.str; - default = "Pty"; - }; - "console".file = mkOption { - type = types.nullOr types.str; - default = null; - }; - # "watchdog" = true; - # "seccomp" = true; - disks = mkOption { - default = [ ]; - type = types.listOf ( - types.submodule { + "console".file = mkOption { + type = types.nullOr types.str; + default = null; + }; + # "watchdog" = true; + # "seccomp" = true; + disks = mkOption { + default = [ ]; + type = types.listOf ( + types.submodule { + freeformType = jsonType; + options = { + path = mkOption { + type = types.oneOf [ + types.path + types.str + ]; + }; + readonly = mkOption { + type = types.bool; + default = true; + }; + id = mkOption { type = types.str; }; + }; + } + ); + }; + memory = mkOption { + default = { }; + type = types.submodule { freeformType = jsonType; options = { - path = mkOption { - type = types.oneOf [ - types.path - types.str - ]; + size = mkOption { + type = types.int; + default = 2 * 1024 * 1048576; }; - readonly = mkOption { + shared = mkOption { type = types.bool; default = true; }; - id = mkOption { type = types.str; }; + mergeable = mkOption { + type = types.bool; + default = true; + }; + hotplug_method = mkOption { + default = "VirtioMem"; + type = types.enum [ + "Acpi" + "VirtioMem" + ]; + }; + hotplugged_size = mkOption { + type = types.int; + default = 512 * 1048576; + }; + hotplug_size = mkOption { + type = types.int; + default = config.memory.size; + }; + # hugepages = mkOption { + # type = types.bool; + # default = true; + # }; }; - } - ); - }; - memory = mkOption { - default = { }; - type = types.submodule { - freeformType = jsonType; - options = { - size = mkOption { - type = types.int; - default = 3 * 1024 * 1048576; - }; - shared = mkOption { - type = types.bool; - default = true; - }; - mergeable = mkOption { - type = types.bool; - default = true; + }; + }; + cpus = mkOption { + default = { }; + type = types.submodule { + freeformType = jsonType; + options = { + boot_vcpus = mkOption { + type = types.int; + default = 4; + }; + max_vcpus = mkOption { + type = types.int; + default = 4; + }; }; }; }; }; - cpus = mkOption { - default = { }; - type = types.submodule { - freeformType = jsonType; - options = { - boot_vcpus = mkOption { - type = types.int; - default = 4; - }; - max_vcpus = mkOption { - type = types.int; - default = 4; - }; - }; - }; - }; - }; - }; + } + ); }; }; } From add1c4f6bd15f4ccc626edd3eb71b3e7a2959067 Mon Sep 17 00:00:00 2001 From: Else Someone Date: Tue, 10 Mar 2026 02:01:54 +0200 Subject: [PATCH 29/37] uvms: churn --- pkgs/uvms-guest/package.nix | 5 - pkgs/uvms/package.nix | 40 +++--- pkgs/uvms/uvms.py | 125 +++++++++++------- pkgs/uvmslib/package.nix | 21 +++ pkgs/uvmslib/pyproject.toml | 10 ++ .../guest.py => uvmslib/uvmslib.py} | 10 +- profiles/baseImage.nix | 12 +- 7 files changed, 143 insertions(+), 80 deletions(-) delete mode 100644 pkgs/uvms-guest/package.nix create mode 100644 pkgs/uvmslib/package.nix create mode 100644 pkgs/uvmslib/pyproject.toml rename pkgs/{uvms-guest/guest.py => uvmslib/uvmslib.py} (97%) diff --git a/pkgs/uvms-guest/package.nix b/pkgs/uvms-guest/package.nix deleted file mode 100644 index 66cfa2d..0000000 --- a/pkgs/uvms-guest/package.nix +++ /dev/null @@ -1,5 +0,0 @@ -{ - lib, - writers, -}: -writers.writePython3Bin "uvms-guest" { } ./guest.py diff --git a/pkgs/uvms/package.nix b/pkgs/uvms/package.nix index 109235e..5ebf056 100644 --- a/pkgs/uvms/package.nix +++ b/pkgs/uvms/package.nix @@ -13,6 +13,8 @@ strace, util-linux, virtiofsd, + python3Packages, + uvmslib, taps, baseImage, @@ -36,21 +38,25 @@ let }; toolsClosure = writeClosure toolsFarm; in -writers.writePython3Bin "uvms" { } ( - replaceVars ./uvms.py { - BWRAP = "${lib.getExe bubblewrap}"; - TOOLS = "${toolsFarm}/bin"; - TOOLS_CLOSURE = toolsClosure; - CROSVM = lib.getExe crosvm; - STRACE = lib.getExe strace; - TAPS = "${lib.getExe taps}"; - VIRTIOFSD = "${lib.getExe virtiofsd}"; - - BASE_CONFIG = baseImage.config.system.build.ch; - SYSTEM = baseImage.config.system.build.toplevel; - SYSTEM_CLOSURE = writeClosure [ - baseImage.config.system.build.toplevel - baseImage.config.system.build.ch - ]; +writers.writePython3Bin "uvms" + { + libraries = [ uvmslib ]; } -) + ( + replaceVars ./uvms.py { + BWRAP = "${lib.getExe bubblewrap}"; + TOOLS = "${toolsFarm}/bin"; + TOOLS_CLOSURE = toolsClosure; + CROSVM = lib.getExe crosvm; + STRACE = lib.getExe strace; + TAPS = "${lib.getExe taps}"; + VIRTIOFSD = "${lib.getExe virtiofsd}"; + + BASE_CONFIG = baseImage.config.system.build.ch; + SYSTEM = baseImage.config.system.build.toplevel; + SYSTEM_CLOSURE = writeClosure [ + baseImage.config.system.build.toplevel + baseImage.config.system.build.ch + ]; + } + ) diff --git a/pkgs/uvms/uvms.py b/pkgs/uvms/uvms.py index 534a7b9..ec4a139 100644 --- a/pkgs/uvms/uvms.py +++ b/pkgs/uvms/uvms.py @@ -299,6 +299,7 @@ class Processes: # "-Z", # "-ff", CH, + # "-v", "--api-socket", "fd=0", # f"fd={s.fileno()}" @@ -320,6 +321,8 @@ class Processes: # ro_bind=["/nix/store"], # I give up unshare_net=False, shell=False, + stdout=None, + stderr=None, # pass_fds=(s.fileno(),) ) ) @@ -340,29 +343,23 @@ class Processes: ): sock_path = self.prefix + "/gpu.sock" args = [ - SOCKETBINDER, - "-b", - "1", - sock_path, - "s6-ipcserverd", - "-1c1", - # "@STRACE@", # noqa: E501 - # "-Z", - # "-ff", "@CROSVM@", # noqa: E501 "--no-syslog", + "--log-level", + "debug", "device", "gpu", - "--fd", - "0", + "--socket-path", + sock_path, "--wayland-sock", f'{PASSTHRU_ENV["XDG_RUNTIME_DIR"]}/{PASSTHRU_ENV["WAYLAND_DISPLAY"]}', # noqa: E501 "--params", - '{ "context-types": "cross-domain:virgl2:venus" }', + '{ "context-types": "cross-domain" }', ] with self.popen( *args, stderr=None, + stdout=None, ) as proc, removing(sock_path, sock_path + ".lock"): yield proc, sock_path @@ -585,6 +582,13 @@ def main(args, args_next, cleanup, ps): config["memory"]["size"] = parse_bytes(args.mem) config["memory"]["hotplug_size"] = parse_bytes(args.mem) + if "platform" not in config: + config["platform"] = {} + config["platform"]["oem_strings"] = [ + "io.systemd.credential:vmm.notify_socket=vsock-stream:2:8888", + *config["platform"].get("oem_strings", []), + ] + gpud, gpud_path = cleanup.enter_context(ps.start_gpu()) ch = cleanup.enter_context(ps.run_ch()) @@ -610,48 +614,71 @@ def main(args, args_next, cleanup, ps): ps.exec(*ch_remote, "boot") ps.exec(*ch_remote, "info") + ready = False with ready_sock: - ready_sock.settimeout(20.0) - try: - con, _ = ready_sock.accept() - except: # noqa: E722 - print( - "CH didn't try connecting to the readiness notification socket" - ) # noqa: E501 - else: - with con: - msg = con.recv(128) - assert msg.startswith(b"READY=1"), msg + ready_sock.settimeout(8.0) + for _ in range(1048576): + if ready: + break + try: + con, _ = ready_sock.accept() + except: # noqa: E722 + print( + "WARNING: CH didn't try connecting to the readiness notification socket" # noqa: E501 + ) + ready = True + break + else: + with con: + msg = con.recv(1024) + for ln in msg.split(b"\n"): + ln = ln.strip() + print(ln) + # if ln.startswith(b"X_SYSTEMD_UNIT_ACTIVE=uvms-guest.service"): # noqa: E501 + if ln.startswith(b"READY=1"): # noqa: E501 + ready = True + break + + assert ready with connect_ch_vsock(ps.prefix + "/vsock.sock", 24601) as guest: for r in args.run: - try: - guest.send( - json.dumps( - { - "run": { - "argv": [r], - "EXTRA_PATH": [ - f"{a}/bin" for a in app_paths - ], # noqa: E501 - } - } - ).encode("utf8") - ) - res = guest.recv(8192) + res = {} + for _ in range(1): + if "status" in res: + break try: - res = json.loads(guest.recv(8192)) - except json.JSONDecodeError as e: - print(f"Couldn't interpret --run {r} response: {e} {res}") - continue - adverb = ( - "Successfully" - if res["status"] == "exec succeeded" - else "Failed to" # noqa: E501 - ) - print(f"{adverb} --run {r}: {res}") - except Exception as e: - print(f"Couldn't --run {r}: {repr(e)}") + guest.send( + json.dumps( + { + "run": { + "argv": [r], + "EXTRA_PATH": [ + f"{a}/bin" for a in app_paths + ], # noqa: E501 + } + } + ).encode("utf8") + ) + res = guest.recv(8192) + try: + res = json.loads(guest.recv(8192)) + except json.JSONDecodeError as e: + print( + f"Couldn't interpret --run {r} response: {e} {res}" + ) # noqa: E501 + res = {} + # res = {"status": "failed"} + except Exception as e: + print(f"Couldn't --run {r}: {repr(e)}") + if "status" not in res: + res["status"] = "fail" + adverb = ( + "Successfully" + if res["status"] == "exec succeeded" + else "Failed to" # noqa: E501 + ) + print(f"{adverb} --run {r}: {res}") try: ch.wait() except KeyboardInterrupt: diff --git a/pkgs/uvmslib/package.nix b/pkgs/uvmslib/package.nix new file mode 100644 index 0000000..1bd51d1 --- /dev/null +++ b/pkgs/uvmslib/package.nix @@ -0,0 +1,21 @@ +{ + lib, + python3Packages, +}: +python3Packages.buildPythonPackage { + pname = "uvmslib"; + version = "0.0.0"; + pyproject = true; + src = + let + fs = lib.fileset; + in + fs.toSource { + root = ./.; + fileset = fs.unions [ + ./pyproject.toml + ./uvmslib.py + ]; + }; + build-system = [ python3Packages.setuptools ]; +} diff --git a/pkgs/uvmslib/pyproject.toml b/pkgs/uvmslib/pyproject.toml new file mode 100644 index 0000000..5995470 --- /dev/null +++ b/pkgs/uvmslib/pyproject.toml @@ -0,0 +1,10 @@ +[build-system] +build-backend = "setuptools.build_meta" +requires = [ "setuptools" ] + +[project] +name = "uvms" +version = "0.0.0" + +[project.scripts] +"uvms-guest" = "uvmslib:guest_main" diff --git a/pkgs/uvms-guest/guest.py b/pkgs/uvmslib/uvmslib.py similarity index 97% rename from pkgs/uvms-guest/guest.py rename to pkgs/uvmslib/uvmslib.py index 9fd9397..d6279ae 100644 --- a/pkgs/uvms-guest/guest.py +++ b/pkgs/uvmslib/uvmslib.py @@ -59,7 +59,7 @@ class Processes: proc = None try: proc = self.popen( - req["run"]["argv"], + run["argv"], text=text, env=env, cwd="/home/user", @@ -82,14 +82,14 @@ class Processes: return res, proc def accept_vsock(self, s): - con, (cid, port) = serv.accept() + con, (cid, port) = s.accept() assert cid == 2, cid self.sources.append(con) self.client_fds.add(con.fileno()) return con, (cid, port) -if __name__ == "__main__": +def guest_main(): ps = Processes() serv = socket.fromfd(3, socket.AF_VSOCK, socket.SOCK_STREAM) ps.sources.append(serv) @@ -134,3 +134,7 @@ if __name__ == "__main__": con.send(res) else: assert False, con.fileno() + + +if __name__ == "__main__": + guest_main() diff --git a/profiles/baseImage.nix b/profiles/baseImage.nix index 289f903..683f061 100644 --- a/profiles/baseImage.nix +++ b/profiles/baseImage.nix @@ -42,7 +42,7 @@ in _module.args.uvmsPkgs = lib.mkDefault (pkgs.callPackage ../pkgs { }); # some.failure-handler.enable = true; hardware.graphics.enable = true; - boot.kernelPackages = pkgs.linuxPackagesFor uvmsPkgs.linux-uvm; + # boot.kernelPackages = pkgs.linuxPackagesFor uvmsPkgs.linux-uvm; # boot.isContainer = true; boot.initrd.kernelModules = [ "drm" @@ -251,19 +251,20 @@ in }; systemd.sockets."uvms-guest" = { - wantedBy = [ "default.target" ]; + requiredBy = [ "multi-user.target" ]; + before = [ "multi-user.target" ]; listenStreams = [ "vsock::24601" ]; partOf = [ "uvms-guest.service" ]; }; systemd.services."uvms-guest" = { - requiredBy = [ "multi-user.target" ]; + before = [ "multi-user.target" ]; onFailure = [ "shutdown.service" ]; serviceConfig = { User = "user"; Group = "users"; - ExecStart = "${lib.getExe uvmsPkgs.uvms-guest}"; + ExecStart = "${lib.getExe' uvmsPkgs.uvmslib "uvms-guest"}"; ExecStop = [ "/run/current-system/sw/bin/echo GUEST DOWN" "/run/current-system/sw/bin/systemctl poweroff" @@ -288,7 +289,6 @@ in "console=ttyS0" "reboot=t" "panic=-1" - "io.systemd.credential:vmm.notify_socket=vsock-stream:2:8888" # "rootfstype=virtiofs" # "root=rootstore" ]; @@ -390,7 +390,7 @@ in options = { size = mkOption { type = types.int; - default = 2 * 1024 * 1048576; + default = 4 * 1024 * 1048576; }; shared = mkOption { type = types.bool; From f8cc57f146f0bf84ebf3797763bcd9d6a80eda99 Mon Sep 17 00:00:00 2001 From: Else Someone Date: Wed, 11 Mar 2026 17:38:49 +0200 Subject: [PATCH 30/37] fixup! uvms: churn --- pkgs/uvms/uvms.py | 2 +- profiles/baseImage.nix | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/pkgs/uvms/uvms.py b/pkgs/uvms/uvms.py index ec4a139..d58fb28 100644 --- a/pkgs/uvms/uvms.py +++ b/pkgs/uvms/uvms.py @@ -354,7 +354,7 @@ class Processes: "--wayland-sock", f'{PASSTHRU_ENV["XDG_RUNTIME_DIR"]}/{PASSTHRU_ENV["WAYLAND_DISPLAY"]}', # noqa: E501 "--params", - '{ "context-types": "cross-domain" }', + '{ "context-types": "cross-domain:virgl2" }', ] with self.popen( *args, diff --git a/profiles/baseImage.nix b/profiles/baseImage.nix index 683f061..294c58e 100644 --- a/profiles/baseImage.nix +++ b/profiles/baseImage.nix @@ -16,6 +16,7 @@ let kernelTarget = pkgs.stdenv.hostPlatform.linux-kernel.target; waylandSock = "/run/user/1000/wayland-1"; env = { + DBUS_SESSION_BUS_ADDRESS = "unix:path=/run/user/1000/bus"; XDG_RUNTIME_DIR = "/run/user/1000"; WAYLAND_DISPLAY = "wayland-1"; @@ -104,6 +105,7 @@ in }; }; + programs.dconf.enable = true; systemd.mounts = [ { type = "virtiofs"; @@ -210,6 +212,7 @@ in }; environment.profileRelativeSessionVariables.PATH = lib.mkForce [ "/bin\${PATH:+:}$PATH" ]; + environment.sessionVariables = env; environment.variables = env; systemd.globalEnvironment = env; @@ -302,9 +305,9 @@ in uvms.ch.settings = mkOption { default = { }; type = types.submodule ( - let - osConfig = config; - in + let + osConfig = config; + in { config, ... }: { freeformType = jsonType; From 42fca624741704a2841e525bbeb3c2435f5f70a7 Mon Sep 17 00:00:00 2001 From: Else Someone Date: Wed, 11 Mar 2026 17:39:00 +0200 Subject: [PATCH 31/37] uvms: init app urls, kinda --- pkgs/uvms/uvms.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/pkgs/uvms/uvms.py b/pkgs/uvms/uvms.py index d58fb28..d89a6ff 100644 --- a/pkgs/uvms/uvms.py +++ b/pkgs/uvms/uvms.py @@ -13,6 +13,7 @@ import json import re from argparse import ArgumentParser from contextlib import contextmanager, closing, ExitStack +from urllib import urlparse parser = ArgumentParser("supervise-vm") @@ -517,11 +518,24 @@ def main(args, args_next, cleanup, ps): app_paths = [] for a in args.app: + a = urlparse(a) + nix_file = None + attr = None + if a.scheme == "": + nix_file = "" + attr = a.path + elif a.scheme == "getexe": + nix_file = a.netloc or "./." + attr = a.path.lstrip("/") + else: + raise RuntimeError("Unknown app url", a) + assert nix_file is not None, a + assert attr is not None, a out_path = ps.exec( "nix-build", - "", + nix_file, "-A", - a, + attr, "--no-out-link", capture_output=True, text=True, From a13e9c9393421e3df6d1daacb1e8c3baa18e84ce Mon Sep 17 00:00:00 2001 From: Else Someone Date: Sun, 15 Mar 2026 01:30:18 +0200 Subject: [PATCH 32/37] uvms: support args in app urls --- pkgs/uvms/uvms.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/pkgs/uvms/uvms.py b/pkgs/uvms/uvms.py index d89a6ff..9a8582c 100644 --- a/pkgs/uvms/uvms.py +++ b/pkgs/uvms/uvms.py @@ -13,7 +13,7 @@ import json import re from argparse import ArgumentParser from contextlib import contextmanager, closing, ExitStack -from urllib import urlparse +from urllib.parse import urlparse, parse_qs parser = ArgumentParser("supervise-vm") @@ -519,16 +519,21 @@ def main(args, args_next, cleanup, ps): app_paths = [] for a in args.app: a = urlparse(a) - nix_file = None + nix_file = "./." attr = None if a.scheme == "": nix_file = "" attr = a.path - elif a.scheme == "getexe": - nix_file = a.netloc or "./." - attr = a.path.lstrip("/") else: - raise RuntimeError("Unknown app url", a) + assert a.fragment, a + attr = a.fragment + nix_file = a.path or "./." + arglist = [] + for k, v in parse_qs(a.query).items(): + arglist.append("--arg") + arglist.append(k) + arglist.append(v) + assert nix_file is not None, a assert attr is not None, a out_path = ps.exec( @@ -536,7 +541,9 @@ def main(args, args_next, cleanup, ps): nix_file, "-A", attr, + *arglist, "--no-out-link", + cwd=os.getcwd(), capture_output=True, text=True, ).stdout.strip() From 0692a20ae93be8801d39a04c5ba280b4edf339db Mon Sep 17 00:00:00 2001 From: Else Someone Date: Sun, 15 Mar 2026 01:30:27 +0200 Subject: [PATCH 33/37] uvms: do not hang on communicate() --- pkgs/uvms/uvms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/uvms/uvms.py b/pkgs/uvms/uvms.py index 9a8582c..4833752 100644 --- a/pkgs/uvms/uvms.py +++ b/pkgs/uvms/uvms.py @@ -158,7 +158,7 @@ class Processes: print(f"Releasing {args}") finally: if subprocess.PIPE in (kwargs["stderr"], kwargs["stdout"]): - print(proc.communicate()) + print(proc.communicate(timeout=0.125)) while alive_after(proc, 0.125): try: proc.terminate() From 90614bdf749d48df8292cab215e4c25f42ab04b2 Mon Sep 17 00:00:00 2001 From: Else Someone Date: Sun, 15 Mar 2026 01:32:28 +0200 Subject: [PATCH 34/37] uvms: do not rm vsock.sock until vmm is down --- pkgs/uvms/uvms.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkgs/uvms/uvms.py b/pkgs/uvms/uvms.py index 4833752..b7443b4 100644 --- a/pkgs/uvms/uvms.py +++ b/pkgs/uvms/uvms.py @@ -464,7 +464,7 @@ def connect_ch_vsock( s.setblocking(blocking) s.connect(vsock_sock_path) - with removing(vsock_sock_path): + with closing(s): s.send(b"CONNECT %d\n" % port) yield s @@ -662,6 +662,7 @@ def main(args, args_next, cleanup, ps): assert ready + cleanup.enter_context(removing(ps.prefix + "/vsock.sock")) with connect_ch_vsock(ps.prefix + "/vsock.sock", 24601) as guest: for r in args.run: res = {} From 3540b5aba24fe933403d8afb3855af9a188df896 Mon Sep 17 00:00:00 2001 From: Else Someone Date: Sun, 15 Mar 2026 02:15:58 +0200 Subject: [PATCH 35/37] uvms: support .desktop applications --- pkgs/uvms/uvms.py | 3 +++ pkgs/uvmslib/uvmslib.py | 11 +++++++++++ profiles/baseImage.nix | 1 + 3 files changed, 15 insertions(+) diff --git a/pkgs/uvms/uvms.py b/pkgs/uvms/uvms.py index b7443b4..72d941c 100644 --- a/pkgs/uvms/uvms.py +++ b/pkgs/uvms/uvms.py @@ -678,6 +678,9 @@ def main(args, args_next, cleanup, ps): "EXTRA_PATH": [ f"{a}/bin" for a in app_paths ], # noqa: E501 + "EXTRA_XDG_DATA_DIRS": [ + f"{a}/share" for a in app_paths + ], # noqa: E501 } } ).encode("utf8") diff --git a/pkgs/uvmslib/uvmslib.py b/pkgs/uvmslib/uvmslib.py index d6279ae..9428edb 100644 --- a/pkgs/uvmslib/uvmslib.py +++ b/pkgs/uvmslib/uvmslib.py @@ -55,6 +55,16 @@ class Processes: "/run/current-system/sw/bin", ], ), + "XDG_DATA_DIRS": ":".join( + [ + *os.environ.get("XDG_DATA_DIRS", "").split(":"), + *run.get( + "EXTRA_XDG_DATA_DIRS", + [], + ), + "/run/current-system/sw/share", + ], + ), } proc = None try: @@ -116,6 +126,7 @@ def guest_main(): # IDK why but I keep getting empty messages if req == b"": print(f"Lost [{con.fileno()}]") + ps.sources = [s for s in ps.sources if s.fileno() != con.fileno()] continue try: req = json.loads(req) diff --git a/profiles/baseImage.nix b/profiles/baseImage.nix index 294c58e..66e95b8 100644 --- a/profiles/baseImage.nix +++ b/profiles/baseImage.nix @@ -212,6 +212,7 @@ in }; environment.profileRelativeSessionVariables.PATH = lib.mkForce [ "/bin\${PATH:+:}$PATH" ]; + environment.profileRelativeSessionVariables.XDG_DATA_DIRS = lib.mkForce [ "/run/current-system/sw/share/\${XDG_DATA_DIRS:+:}$XDG_DATA_DIRS" ]; environment.sessionVariables = env; environment.variables = env; systemd.globalEnvironment = env; From 42180096aff11da18e2fdac0c8695f1e36dc2098 Mon Sep 17 00:00:00 2001 From: Else Someone Date: Sun, 22 Mar 2026 16:19:11 +0200 Subject: [PATCH 36/37] baseImage: tinkering with overcommit --- pkgs/uvmslib/uvmslib.py | 4 ++-- profiles/baseImage.nix | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/pkgs/uvmslib/uvmslib.py b/pkgs/uvmslib/uvmslib.py index 9428edb..f7319bc 100644 --- a/pkgs/uvmslib/uvmslib.py +++ b/pkgs/uvmslib/uvmslib.py @@ -73,8 +73,8 @@ class Processes: text=text, env=env, cwd="/home/user", - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, + stdin=None, + stdout=None, ) res["status"] = "exec succeeded" except Exception as e: diff --git a/profiles/baseImage.nix b/profiles/baseImage.nix index 66e95b8..60a8cc5 100644 --- a/profiles/baseImage.nix +++ b/profiles/baseImage.nix @@ -45,6 +45,11 @@ in hardware.graphics.enable = true; # boot.kernelPackages = pkgs.linuxPackagesFor uvmsPkgs.linux-uvm; # boot.isContainer = true; + boot.kernel.sysctl = { + "vm.overcommit_memory" = 1; # "always" + # "vm.overcommit_memory" = 2; # "never" + "vm.panic_on_oom" = 1; + }; boot.initrd.kernelModules = [ "drm" "virtio_blk" From f79adcdcd6915c228ddee5cdaa4a2abcdacea669 Mon Sep 17 00:00:00 2001 From: Else Someone Date: Sun, 22 Mar 2026 21:25:48 +0200 Subject: [PATCH 37/37] cloud-hypervisor: clean up the short-circuiting logic --- pkgs/cloud-hypervisor-gpu.nix | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkgs/cloud-hypervisor-gpu.nix b/pkgs/cloud-hypervisor-gpu.nix index a99ce6f..0b5ddbd 100644 --- a/pkgs/cloud-hypervisor-gpu.nix +++ b/pkgs/cloud-hypervisor-gpu.nix @@ -34,10 +34,10 @@ cloud-hypervisor.overrideAttrs ( patches = patchesFromDir (spectrum + "/pkgs/cloud-hypervisor"); vhostPatches = patchesFromDir (spectrum + "/pkgs/cloud-hypervisor/vhost"); }; - previouslyPatched = oldAttrs ? spectrumPatches; - patchPhases = !previouslyPatched; - isNewer = lib.versionOlder oldAttrs.spectrumPatches.version or "2000-00-00" spectrumPatches.version; - oldPatches = oldAttrs.spectrumPatches.patches or [ ]; + oldPatchesStruct = oldAttrs.passthru.spectrumPatches or { }; + missingPatchPhases = oldPatchesStruct != { }; + isNewer = lib.versionOlder oldPatchesStruct.version or "2000-00-00" spectrumPatches.version; + oldPatches = oldPatchesStruct.patches or [ ]; removeAll = removeElts: lst: builtins.filter (x: !(builtins.elem x removeElts)) lst; in optionalAttrs isNewer { @@ -47,7 +47,7 @@ cloud-hypervisor.overrideAttrs ( # Verbatim from spectrum postUnpack = oldAttrs.postUnpack or "" - + optionalString patchPhases '' + + optionalString missingPatchPhases '' unpackFile $vhost chmod -R +w vhost ''; @@ -58,7 +58,7 @@ cloud-hypervisor.overrideAttrs ( # Verbatim copy from spectrum postPatch = oldAttrs.postPatch or "" - + optionalString patchPhases '' + + optionalString missingPatchPhases '' pushd ../vhost for patch in $vhostPatches; do echo applying patch $patch