else/uvms
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

pkgs.uvms: init

Browse source 
...with some basic optional persistence and without having to rebuild
images for every app

nix run -f . pkgs.uvms -- --persist-home librewolf alacritty --run librewolf --run alacritty
This commit is contained in:
Else Someone 2026-02-27 18:26:41 +02:00
parent 22b613d157
commit 384b45bdef
15 changed files with 1155 additions and 452 deletions
Download patch file Download diff file Expand all files Collapse all files

3
pkgs/baseImage.nix Normal file
View file

@ -0,0 +1,3 @@
{ nixos }:
nixos ../profiles/baseImage.nix

59
pkgs/cloud-hypervisor-gpu.nix Normal file
View file

@ -0,0 +1,59 @@
{
lib,
cloud-hypervisor,
fetchFromGitHub,
rustPlatform,
enableDebug ? true,
}:
let
spectrum = builtins.fetchTree {
url = "https://spectrum-os.org/git/spectrum";
type = "git";
rev = "0f3388f0191d9a03c7bf471c269a34a79f22018b";
};
in
cloud-hypervisor.overrideAttrs (
finalAttrs: oldAttrs:
{
# Verbatim from spectrum
postUnpack = oldAttrs.postUnpack or "" + ''
unpackFile $vhost
chmod -R +w vhost
'';
vhost = fetchFromGitHub {
name = "vhost";
owner = "rust-vmm";
repo = "vhost";
rev = "vhost-user-backend-v0.20.0";
hash = "sha256-KK1+mwYQr7YkyGT9+51v7TJael9D0lle2JXfRoTqYq8=";
};
patches = oldAttrs.patches or [ ] ++ [
"${spectrum}/pkgs/cloud-hypervisor/0001-build-use-local-vhost.patch"
"${spectrum}/pkgs/cloud-hypervisor/0002-virtio-devices-add-a-GPU-device.patch"
];
vhostPatches = builtins.concatMap (
name:
lib.optionals (lib.hasSuffix ".patch" name) [ "${spectrum}/pkgs/cloud-hypervisor/vhost/${name}" ]
) (builtins.attrNames (builtins.readDir "${spectrum}/pkgs/cloud-hypervisor/vhost"));
# Verbatim copy from spectrum
postPatch = oldAttrs.postPatch or "" + ''
pushd ../vhost
for patch in $vhostPatches; do
echo applying patch $patch
patch -p1 < $patch
done
popd
'';
cargoDeps = rustPlatform.fetchCargoVendor {
inherit (finalAttrs) patches;
inherit (oldAttrs) src;
hash = "sha256-wGtsyKDg1z1QK9mJ1Q43NSjoPbm3m81p++DoD8ipIUI=";
};
}
// lib.optionalAttrs enableDebug {
buildType = "debug";
dontStrip = true;
}
)

12
pkgs/default.nix
View file

@ -4,6 +4,14 @@ let
in
lib.makeScope newScope (
self:
let
callPackage =
fun: overrides:
let
result = self.callPackage fun overrides;
in
result // { override = result.__originalOverride or result.override; };
in
dirToAttrs ./.
[
(
@ -14,9 +22,9 @@ lib.makeScope newScope (
(
name: fpath: typ:
if typ == "regular" then
self.callPackage fpath { }
callPackage fpath { }
else if typ == "directory" && builtins.pathExists (fpath + "/package.nix") then
self.callPackage (fpath + "/package.nix") { }
callPackage (fpath + "/package.nix") { }
else
null
)

153
pkgs/linux-uvm.nix
View file

@ -6,81 +6,86 @@
let
inherit (lib.kernel) yes no unset;
inherit (lib) mkForce;
in
linux_latest.override {
structuredExtraConfig = {
BASE_SMALL = yes;
DRM_VIRTIO_GPU = yes;
EROFS_FS = yes;
# TSI = yes;
DAX = yes;
FS_DAX = yes;
FUSE_DAX = yes;
OVERLAY_FS = yes;
VIRTIO_BALLOON = yes;
VIRTIO_BLK = yes;
VIRTIO_CONSOLE = yes;
VIRTIO_PCI = yes;
VIRTIO_MMIO = yes;
VIRTIO = yes;
VSOCKETS = yes;
NO_HZ_IDLE = mkForce yes;
NO_HZ_FULL = mkForce unset;
HZ_1000 = unset;
HZ_250 = yes; # NixOS default: 1000
result = linux_latest.override {
structuredExtraConfig = {
BASE_SMALL = yes;
DRM_VIRTIO_GPU = yes;
EROFS_FS = yes;
# TSI = yes;
DAX = yes;
FS_DAX = yes;
FUSE_DAX = yes;
OVERLAY_FS = yes;
VIRTIO_BALLOON = yes;
VIRTIO_BLK = yes;
VIRTIO_CONSOLE = yes;
VIRTIO_FS = yes;
VIRTIO_MMIO = yes;
VIRTIO_PCI = yes;
VIRTIO = yes;
FUSE_FS = yes;
VSOCKETS = yes;
NO_HZ_IDLE = mkForce yes;
NO_HZ_FULL = mkForce unset;
HZ_1000 = unset;
HZ_250 = yes; # NixOS default: 1000
EXT4_FS = yes;
# EXT4_USE_FOR_EXT2 = yes;
XFS_FS = yes;
DEFAULT_SECURITY_APPARMOR = mkForce unset;
# LSM = "lockdown,yama,loadpin,safesetid,integrity,bpf";
XEN = mkForce unset;
XEN_BACKEND = mkForce unset;
XEN_BALLOON = mkForce unset;
XEN_BALLOON_MEMORY_HOTPLUG = mkForce unset;
XEN_DOM0 = mkForce unset;
XEN_HAVE_PVMMU = mkForce unset;
XEN_MCE_LOG = mkForce unset;
XEN_PVH = mkForce unset;
XEN_SAVE_RESTORE = mkForce unset;
XEN_SYS_HYPERVISOR = mkForce unset;
PCI_XEN = mkForce unset;
POWER_RESET_GPIO = mkForce unset;
POWER_RESET_GPIO_RESTART = mkForce unset;
RCU_LAZY = mkForce unset;
REISERFS_FS_POSIX_ACL = mkForce unset;
REISERFS_FS_SECURITY = mkForce unset;
REISERFS_FS_XATTR = mkForce unset;
SWIOTLB_XEN = mkForce unset;
SUSPEND = mkForce unset;
PM = mkForce unset;
HIBERNATION = mkForce unset;
ACPI = mkForce unset;
CPU_FREQ = mkForce unset;
CPU_FREQ_DT = mkForce unset;
INTEL_IDLE = mkForce unset;
ISA_DMA_API = mkForce unset;
IA32_EMULATION = mkForce unset;
COMPAT = mkForce unset;
COMPAT_32 = mkForce unset;
KVM = mkForce unset;
BLOCK_LEGACY_AUTOLOAD = mkForce unset;
SWAP = mkForce unset;
CMA = mkForce unset;
FB = mkForce unset;
FB_EFI = mkForce unset;
FB_VESA = mkForce unset;
SECURITY_APPARMOR = mkForce unset;
EXT4_FS = yes;
# EXT4_USE_FOR_EXT2 = yes;
XFS_FS = yes;
DEFAULT_SECURITY_APPARMOR = mkForce unset;
VT = no;
DRM_FBDEV_EMULATION = lib.mkForce no;
FONTS = mkForce unset;
FONT_8x8 = mkForce unset;
FONT_TER16x32 = mkForce unset;
FRAMEBUFFER_CONSOLE = mkForce unset;
FRAMEBUFFER_CONSOLE_DEFERRED_TAKEOVER = mkForce unset;
FRAMEBUFFER_CONSOLE_DETECT_PRIMARY = mkForce unset;
FRAMEBUFFER_CONSOLE_ROTATION = mkForce unset;
RC_CORE = mkForce unset;
XEN = mkForce unset;
XEN_BACKEND = mkForce unset;
XEN_BALLOON = mkForce unset;
XEN_BALLOON_MEMORY_HOTPLUG = mkForce unset;
XEN_DOM0 = mkForce unset;
XEN_HAVE_PVMMU = mkForce unset;
XEN_MCE_LOG = mkForce unset;
XEN_PVH = mkForce unset;
XEN_SAVE_RESTORE = mkForce unset;
XEN_SYS_HYPERVISOR = mkForce unset;
PCI_XEN = mkForce unset;
POWER_RESET_GPIO = mkForce unset;
POWER_RESET_GPIO_RESTART = mkForce unset;
RCU_LAZY = mkForce unset;
REISERFS_FS_POSIX_ACL = mkForce unset;
REISERFS_FS_SECURITY = mkForce unset;
REISERFS_FS_XATTR = mkForce unset;
SWIOTLB_XEN = mkForce unset;
SUSPEND = mkForce unset;
PM = mkForce unset;
HIBERNATION = mkForce unset;
ACPI = mkForce unset;
CPU_FREQ = mkForce unset;
CPU_FREQ_DT = mkForce unset;
INTEL_IDLE = mkForce unset;
ISA_DMA_API = mkForce unset;
IA32_EMULATION = mkForce unset;
COMPAT = mkForce unset;
COMPAT_32 = mkForce unset;
KVM = mkForce unset;
BLOCK_LEGACY_AUTOLOAD = mkForce unset;
SWAP = mkForce unset;
CMA = mkForce unset;
FB = mkForce unset;
FB_EFI = mkForce unset;
FB_VESA = mkForce unset;
SECURITY_APPARMOR = mkForce unset;
VT = no;
DRM_FBDEV_EMULATION = lib.mkForce no;
FONTS = mkForce unset;
FONT_8x8 = mkForce unset;
FONT_TER16x32 = mkForce unset;
FRAMEBUFFER_CONSOLE = mkForce unset;
FRAMEBUFFER_CONSOLE_DEFERRED_TAKEOVER = mkForce unset;
FRAMEBUFFER_CONSOLE_DETECT_PRIMARY = mkForce unset;
FRAMEBUFFER_CONSOLE_ROTATION = mkForce unset;
RC_CORE = mkForce unset;
};
};
}
in
result // { __originalOverride = result.override; }

39
pkgs/mkSystemdDropin.nix Normal file
View file

@ -0,0 +1,39 @@
{
lib,
runCommand,
writeShellScriptBin,
}:
{
name,
prefix ? "10-all-",
dirs ? [
"service"
"mount"
"socket"
"timer"
"target"
],
dropinText ? null,
extraCommands ? "",
...
}@args:
runCommand "${name}-dropin"
(
lib.removeAttrs args [
"name"
]
// {
inherit dirs dropinText extraCommands;
}
)
''
set -euo pipefail
root=$out/lib/systemd/system
for dir in $dirs ; do
mkdir -p "$root/$dir".d
printf "%s" "$dropinText" > "$root/$dir.d/${prefix}${name}.conf"
done
runHook extraCommands
''

76
pkgs/uvms-guest/guest.py Normal file
View file

@ -0,0 +1,76 @@
import json
import os
import select
import socket
import subprocess
def handle_run(run: dict) -> dict:
res = {}
text = run.get("text", False)
env = {
**os.environ,
"PATH": ":".join(
os.environ.get("PATH", "").split(":") + run.get("EXTRA_PATH", [])
),
}
proc = None
try:
proc = subprocess.Popen(
req["run"]["argv"],
text=text,
env=env,
cwd="/home/user",
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
)
res["status"] = "exec succeeded"
except Exception as e:
res["status"] = "exec failed"
res["exception"] = repr(e)
res["pid"] = getattr(proc, "pid", None)
try:
if proc is not None:
proc.wait(0.125)
res["long_running"] = False
res["returncode"] = getattr(proc, "returncode", None)
except subprocess.TimeoutExpired:
res["long_running"] = True
return res, proc
if __name__ == "__main__":
serv = socket.fromfd(3, socket.AF_VSOCK, socket.SOCK_STREAM)
procs = []
conns = [serv]
while True:
rr, rw, xs = select.select(conns, [], [])
for con in rr:
if con is serv:
con, (cid, port) = serv.accept()
assert cid == 2, cid
conns.append(con)
continue
req = con.recv(8192)
# IDK why but I keep getting empty messages
if req == b"":
continue
try:
req = json.loads(req)
print(f"Received {req=}")
except json.JSONDecodeError as e:
print(f"Couldn't interpret {req=}: {e}")
continue
if "run" in req:
res, proc = handle_run(req["run"])
procs.append(proc)
else:
res = {"status": "unknown command"}
_, rw, _ = select.select([], [con], [])
assert rw, rw
res = json.dumps(res).encode("utf8")
print(f"Responding with {res=}")
con.send(res)

5
pkgs/uvms-guest/package.nix Normal file
View file

@ -0,0 +1,5 @@
{
lib,
writers,
}:
writers.writePython3Bin "uvms-guest" { } ./guest.py

11
pkgs/uvms/package.nix
View file

@ -11,9 +11,11 @@
execline,
s6,
strace,
taps,
util-linux,
virtiofsd,
taps,
baseImage,
}:
let
@ -43,5 +45,12 @@ writers.writePython3Bin "uvms" { } (
STRACE = lib.getExe strace;
TAPS = "${lib.getExe taps}";
VIRTIOFSD = "${lib.getExe virtiofsd}";
BASE_CONFIG = baseImage.config.system.build.ch;
SYSTEM = baseImage.config.system.build.toplevel;
SYSTEM_CLOSURE = writeClosure [
baseImage.config.system.build.toplevel
baseImage.config.system.build.ch
];
}
)

503
pkgs/uvms/uvms.py
View file

@ -9,14 +9,18 @@
import os
import subprocess
import socket
import json
from argparse import ArgumentParser
from contextlib import contextmanager, closing, ExitStack
parser = ArgumentParser("supervise-vm")
parser.add_argument("--vm")
parser.add_argument("--vm", default=None)
parser.add_argument("--prefix", default="$HOME/uvms/$VM")
parser.add_argument("--vm-config")
parser.add_argument("--vm-config", default="@BASE_CONFIG@") # noqa: E501
parser.add_argument("--persist-home", action="store_true")
parser.add_argument("--run", action="append")
parser.add_argument("app", nargs="*", default=())
TOOLS_DIR = "@TOOLS@" # noqa: E501
SOCKETBINDER = TOOLS_DIR + "/s6-ipcserver-socketbinder" # noqa: E501
@ -27,12 +31,18 @@ VIRTIOFSD = "@VIRTIOFSD@" # noqa: E501
BWRAP = "@BWRAP@" # noqa: E501
with open("@TOOLS_CLOSURE@", mode="r") as f: # noqa: E501
CLOSURE = [
TOOLS_CLOSURE = [
*(ln.rstrip() for ln in f.readlines()),
os.path.dirname(__file__),
]
PASSTHRU_PATH = ":".join([TOOLS_DIR])
BASE_SYSTEM = "@SYSTEM@" # noqa: E501
with open("@SYSTEM_CLOSURE@", mode="r") as f: # noqa: E501
BASE_SYSTEM_CLOSURE = [
*(ln.rstrip() for ln in f.readlines()),
]
PASSTHRU_PATH = ":".join([TOOLS_DIR, *os.environ.get("PATH", "").split(":")])
PASSTHRU_ENV = {
**{
k: v
@ -41,6 +51,7 @@ PASSTHRU_ENV = {
or k.startswith("WAYLAND")
or k.startswith("XDG_")
or k.startswith("DBUS_")
or k.startswith("NIX_")
or k
in [
"TAPS_SOCK",
@ -52,6 +63,10 @@ PASSTHRU_ENV = {
def preprocess_args(args_mut):
if not args_mut.app and args_mut.run:
args_mut.app = [*args_mut.run]
if not args_mut.vm:
args_mut.vm = args_mut.run[0]
keys = [k for k, v in args_mut._get_kwargs() if isinstance(v, str)]
for k in keys:
v = getattr(args_mut, k)
@ -86,6 +101,7 @@ class Processes:
self.vm = vm
self.check = check
self.defaults = defaults
self.processes = []
def make_env(self):
return {
@ -121,6 +137,7 @@ class Processes:
kwargs["pass_fds"] = kwargs.get("pass_fds", ())
kwargs["env"] = kwargs.get("env", self.make_env())
kwargs["cwd"] = kwargs.get("cwd", self.prefix)
kwargs["text"] = kwargs.get("text", True)
kwargs["stdin"] = kwargs.get("stdin", subprocess.DEVNULL)
kwargs["stdout"] = kwargs.get("stdout", subprocess.DEVNULL)
kwargs["stderr"] = kwargs.get("stderr", subprocess.DEVNULL)
@ -132,12 +149,19 @@ class Processes:
)
if not alive_after(proc, 0.125):
raise RuntimeError("Failed to start", args)
print(f"Started {args}")
self.processes.append(proc)
yield proc
print(f"Releasing {args}")
finally:
if alive_after(proc, 0.125):
proc.terminate()
if proc is not None:
proc.wait()
if subprocess.PIPE in (kwargs["stderr"], kwargs["stdout"]):
print(proc.communicate())
while alive_after(proc, 0.125):
try:
proc.terminate()
proc.wait()
except Exception as e:
print(f"Cleanup failing: {e}")
@contextmanager
def bwrap(
@ -147,6 +171,8 @@ class Processes:
# Based on the args from
# `host/rootfs/image/usr/bin/run-vmm`
unshare_all=True,
uid=1000,
gid=100,
unshare_user=True,
unshare_ipc=None,
unshare_pid=None,
@ -164,7 +190,7 @@ class Processes:
"/proc/sys",
"/dev/null",
"/proc/kallsyms",
*CLOSURE,
*sorted(set([*TOOLS_CLOSURE, *BASE_SYSTEM_CLOSURE])),
),
ro_bind=(),
remount_ro=("/proc/fs", "/proc/irq"),
@ -183,123 +209,128 @@ class Processes:
bwrap_args_sock, remote = socket.socketpair()
remote.set_inheritable(True)
bwrap_args_f = bwrap_args_sock.makefile("w")
with ExitStack() as cleanup:
# cleanup.enter_context(closing(bwrap_args_sock))
# cleanup.enter_context(closing(bwrap_args_f))
def print_arg(*args):
print(*args, file=bwrap_args_f, sep="\0", end="\0")
def print_arg(*args):
print(*args, file=bwrap_args_f, sep="\0", end="\0")
if unshare_all:
print_arg("--unshare-all")
if unshare_user:
print_arg("--unshare-user")
if unshare_ipc:
print_arg("--unshare-ipc")
if unshare_pid:
print_arg("--unshare-pid")
if unshare_net:
print_arg("--unshare-net")
elif unshare_net is False:
print_arg("--share-net")
if unshare_uts:
print_arg("--unshare-uts")
if unshare_cgroup_try:
print_arg("--unshare-cgroup-try")
if die_with_parent:
print_arg("--die-with-parent")
if dev:
print_arg("--dev", dev)
if proc:
print_arg("--proc", proc)
if unshare_all:
print_arg("--unshare-all")
if unshare_user:
print_arg("--unshare-user")
if uid is not None:
assert unshare_user
print_arg("--uid", uid)
if gid is not None:
assert unshare_user
print_arg("--gid", gid)
if unshare_ipc:
print_arg("--unshare-ipc")
if unshare_pid:
print_arg("--unshare-pid")
if unshare_net:
print_arg("--unshare-net")
elif unshare_net is False:
print_arg("--share-net")
if unshare_uts:
print_arg("--unshare-uts")
if unshare_cgroup_try:
print_arg("--unshare-cgroup-try")
if die_with_parent:
print_arg("--die-with-parent")
if dev:
print_arg("--dev", dev)
if proc:
print_arg("--proc", proc)
for p in bind:
p1, p2 = (p, p) if isinstance(p, str) else p
print_arg("--bind", p1, p2)
for p in (*ro_bind, *ro_bind_implicit):
p1, p2 = (p, p) if isinstance(p, str) else p
print_arg("--ro-bind", p1, p2)
for p in (*dev_bind, *dev_bind_implicit):
p1, p2 = (p, p) if isinstance(p, str) else p
print_arg("--dev-bind", p1, p2)
for p in (*tmpfs, *tmpfs_implicit):
print_arg("--tmpfs", p)
# Hunch: order might matter...
for p in remount_ro:
print_arg("--remount-ro", p)
for p in bind:
assert isinstance(p, (str, tuple)), p
p1, p2 = (p, p) if isinstance(p, str) else p
print_arg("--bind", p1, p2)
for p in (*ro_bind, *ro_bind_implicit):
assert isinstance(p, (str, tuple)), p
p1, p2 = (p, p) if isinstance(p, str) else p
print_arg("--ro-bind", p1, p2)
for p in (*dev_bind, *dev_bind_implicit):
assert isinstance(p, (str, tuple)), p
p1, p2 = (p, p) if isinstance(p, str) else p
print_arg("--dev-bind", p1, p2)
for p in (*tmpfs, *tmpfs_implicit):
print_arg("--tmpfs", p)
# Hunch: order might matter...
for p in remount_ro:
print_arg("--remount-ro", p)
bwrap_args_f.flush()
bwrap_args_f.flush()
with ExitStack() as es:
es.enter_context(closing(remote))
es.enter_context(closing(bwrap_args_sock))
es.enter_context(closing(bwrap_args_f))
proc = cleanup.enter_context(
self.popen(
"bwrap",
"--args",
str(remote.fileno()),
*bwrap_args,
**popen_kwargs,
executable=BWRAP,
pass_fds=(*pass_fds, remote.fileno()),
try:
with ExitStack() as proc_es:
with ExitStack() as es:
es.enter_context(closing(remote))
es.enter_context(closing(bwrap_args_sock))
es.enter_context(closing(bwrap_args_f))
proc = proc_es.enter_context(
self.popen(
"bwrap",
"--args",
str(remote.fileno()),
*bwrap_args,
**popen_kwargs,
executable=BWRAP,
pass_fds=(*pass_fds, remote.fileno()),
)
)
)
yield proc
yield proc
finally:
assert proc.returncode is not None, proc
@contextmanager
def run_ch(self):
try:
# s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM, 0)
# s.set_inheritable(True)
# s.setblocking(True)
# s.bind(self.prefix + "/vmm.sock")
args = [
SOCKETBINDER,
"-B",
self.prefix + "/vmm.sock",
# "@STRACE@", # noqa: E501
# "-Z",
# "-ff",
CH,
"--api-socket",
"fd=0",
# f"fd={s.fileno()}"
]
needs_cleanup = False
with self.bwrap(
*args,
bind=[self.prefix],
# Probably just need the path to vmlinux
ro_bind=["/nix/store"], # I give up
unshare_net=False,
shell=False,
stderr=None,
# pass_fds=(s.fileno(),)
) as proc:
# s.close()
assert alive_after(proc, 0.125)
if not os.path.exists(self.prefix + "/vmm.sock"):
raise RuntimeError(
f"{self.prefix}/vmm.sock should exist by now",
)
needs_cleanup = True
if proc.returncode is not None:
raise RuntimeError("CH exited early")
yield proc
finally:
unlink_paths = (
[
self.prefix + "/vmm.sock",
self.prefix + "/vmm.sock.lock",
self.prefix + "/vsock.sock",
]
if needs_cleanup
else []
# s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM, 0)
# s.set_inheritable(True)
# s.setblocking(True)
# s.bind(self.prefix + "/vmm.sock")
args = [
SOCKETBINDER,
"-B",
self.prefix + "/vmm.sock",
# "@STRACE@", # noqa: E501
# "-Z",
# "-ff",
CH,
"--api-socket",
"fd=0",
# f"fd={s.fileno()}"
]
cleanup_paths = [
self.prefix + "/vmm.sock",
self.prefix + "/vmm.sock.lock",
self.prefix + "/vsock.sock",
]
new_paths = [p for p in cleanup_paths if not os.path.exists(p)]
old_paths = [p for p in cleanup_paths if p not in new_paths]
with ExitStack() as cleanup:
cleanup.enter_context(removing(*new_paths))
proc = cleanup.enter_context(
self.bwrap(
*args,
bind=[self.prefix],
# Probably just need the path to vmlinux
# ro_bind=["/nix/store"], # I give up
unshare_net=False,
shell=False,
# pass_fds=(s.fileno(),)
)
)
for p in unlink_paths:
if os.path.exists(p):
os.remove(p)
# s.close()
cleanup.enter_context(removing(*old_paths))
assert alive_after(proc, 1.0), proc
if not os.path.exists(self.prefix + "/vmm.sock"):
raise RuntimeError(
f"{self.prefix}/vmm.sock should exist by now",
)
if proc.returncode is not None:
raise RuntimeError("CH exited early")
yield proc
@contextmanager
def start_gpu(
@ -330,7 +361,7 @@ class Processes:
with self.popen(
*args,
stderr=None,
) as proc, removing(sock_path):
) as proc, removing(sock_path, sock_path + ".lock"):
yield proc, sock_path
@contextmanager
@ -338,9 +369,9 @@ class Processes:
self,
root_dir,
tag,
ro=False,
ro=True,
subdirs=None,
extra_flags=("--posix-acl",),
extra_flags=("--posix-acl", "--xattr"),
):
assert os.path.exists(root_dir)
@ -351,20 +382,16 @@ class Processes:
# s.setblocking(True)
# s.set_inheritable(True)
def rm_sock():
if os.path.exists(sock_path):
os.remove(sock_path)
with ExitStack() as cleanup: # noqa: F841
# s.bind(sock_path.encode("utf8"))
# cleanup.enter_context(closing(s))
cleanup.enter_context(defer(rm_sock))
cleanup.enter_context(removing(sock_path, sock_path + ".pid"))
args = [
# If using bwrap():
# "--argv0", "virtiofsd",
# "--uid", "1000",
# "--gid", "1000",
# "--gid", "100",
# "--",
"unshare",
"-rUm",
@ -372,7 +399,7 @@ class Processes:
"--map-user",
"1000",
"--map-group",
"1000",
"100",
VIRTIOFSD,
"--shared-dir",
root_dir,
@ -396,6 +423,8 @@ class Processes:
# if subdirs is not None
# else [root_dir],
# "pass_fds": (2, s.fileno()),
"stdout": subprocess.PIPE,
"stderr": subprocess.PIPE,
}
try:
with self.popen(*args, **kwargs) as p:
@ -423,20 +452,43 @@ def removing(*paths):
os.remove(p)
if __name__ == "__main__":
args, args_next = parser.parse_known_args()
preprocess_args(args)
def connect_ch_vsock(
vsock_sock_path,
port: int,
type=socket.SOCK_STREAM,
blocking=True,
) -> socket.socket:
s = socket.socket(socket.AF_UNIX, type, 0)
s.setblocking(blocking)
s.connect(vsock_sock_path)
s.send(b"CONNECT %d\n" % port)
return s
@contextmanager
def listen_ch_vsock(
vsock_sock_path,
port: int,
type=socket.SOCK_STREAM,
blocking=True,
) -> socket.socket:
listen_path = vsock_sock_path + "_%d" % port
s = socket.socket(socket.AF_UNIX, type, 0)
s.setblocking(blocking)
s.bind(listen_path)
s.listen()
try:
yield s
finally:
os.remove(listen_path)
def main(args, args_next, cleanup, ps):
send_dir = PASSTHRU_ENV["HOME"] + f"/send/{args.vm}"
os.makedirs(send_dir, exist_ok=True)
os.makedirs(args.prefix, exist_ok=True)
os.makedirs(args.prefix + "/pts", exist_ok=True)
ps = Processes(
prefix=args.prefix,
vm=args.vm,
)
ch_remote = [
"ch-remote",
@ -444,31 +496,170 @@ if __name__ == "__main__":
args.prefix + "/vmm.sock",
]
with ExitStack() as cleanup:
with open(args.vm_config) as f:
config = json.load(f)
vfsd, vfsd_path = cleanup.enter_context(
app_paths = []
for a in args.app:
out_path = ps.exec(
"nix-build",
"<nixpkgs>",
"-A",
a,
"--no-out-link",
capture_output=True,
text=True,
).stdout.strip()
assert out_path.startswith("/nix/store/")
app_paths.append(out_path)
apps_closure = ps.exec( # noqa: F841
"nix-store",
"-qR",
*app_paths,
capture_output=True,
text=True,
).stdout.split()
ready_sock = cleanup.enter_context(
listen_ch_vsock(ps.prefix + "/vsock.sock", 8888),
)
virtiofs_socks = []
_, sock_path = cleanup.enter_context(
ps.start_virtiofsd(
send_dir,
tag="send",
ro=False,
)
)
virtiofs_socks.append(("send", sock_path))
_, sock_path = cleanup.enter_context(
ps.start_virtiofsd(
"/nix/store",
subdirs=apps_closure,
tag="apps",
)
)
virtiofs_socks.append(("apps", sock_path))
_, sock_path = cleanup.enter_context(
ps.start_virtiofsd(
"/nix/store",
subdirs=BASE_SYSTEM_CLOSURE,
tag="system",
)
)
virtiofs_socks.append(("system", sock_path))
if args.persist_home:
os.makedirs(args.prefix + "/home", exist_ok=True)
_, sock_path = cleanup.enter_context(
ps.start_virtiofsd(
send_dir,
tag="send",
args.prefix + "/home",
subdirs=BASE_SYSTEM_CLOSURE,
tag="home",
ro=False,
)
)
gpud, gpud_path = cleanup.enter_context(ps.start_gpu())
virtiofs_socks.append(("home", sock_path))
config["payload"]["cmdline"] += " uvms.persist-home=1"
ch = cleanup.enter_context(ps.run_ch())
ps.exec(*ch_remote, "create", args.vm_config)
ps.exec(
TAPS,
"pass",
*ch_remote,
"add-net",
"id=wan,fd=3,mac=00:00:00:00:00:01",
)
gpud, gpud_path = cleanup.enter_context(ps.start_gpu())
ps.exec(*ch_remote, "add-fs", f"tag=send,socket={vfsd_path},id=send")
ps.exec(*ch_remote, "add-gpu", f"socket={gpud_path}")
ps.exec(*ch_remote, "boot")
ps.exec(*ch_remote, "info")
ch = cleanup.enter_context(ps.run_ch())
ps.exec(
*ch_remote,
"create",
input=json.dumps(config),
text=True,
)
ps.exec(
TAPS,
"pass",
*ch_remote,
"add-net",
"id=wan,fd=3,mac=00:00:00:00:00:01",
)
# TODO: add-fs apps closure separately
for tag, sock_path in virtiofs_socks:
ps.exec(*ch_remote, "add-fs", f"tag={tag},socket={sock_path},id={tag}")
ps.exec(*ch_remote, "add-gpu", f"socket={gpud_path}")
ps.exec(*ch_remote, "boot")
ps.exec(*ch_remote, "info")
with ready_sock:
ready_sock.settimeout(16.0)
try:
ch.wait()
except KeyboardInterrupt:
pass
con, _ = ready_sock.accept()
except: # noqa: E722
print(
"CH didn't try connecting to the readiness notification socket"
) # noqa: E501
else:
with con:
msg = con.recv(128)
assert msg.startswith(b"READY=1"), msg
with connect_ch_vsock(ps.prefix + "/vsock.sock", 24601) as guest:
for r in args.run:
try:
guest.send(
json.dumps(
{
"run": {
"argv": [r],
"EXTRA_PATH": [
f"{a}/bin" for a in app_paths
], # noqa: E501
}
}
).encode("utf8")
)
res = guest.recv(8192)
try:
res = json.loads(guest.recv(8192))
except json.JSONDecodeError as e:
print(f"Couldn't interpret --run {r} response: {e} {res}")
continue
adverb = (
"Successfully"
if res["status"] == "exec succeeded"
else "Failed to" # noqa: E501
)
print(f"{adverb} --run {r}: {res}")
except Exception as e:
print(f"Couldn't --run {r}: {repr(e)}")
try:
ch.wait()
except KeyboardInterrupt:
pass
if __name__ == "__main__":
args, args_next = parser.parse_known_args()
preprocess_args(args)
ps = Processes(
prefix=args.prefix,
vm=args.vm,
)
try:
with ExitStack() as cleanup:
main(args, args_next, cleanup, ps)
finally:
for p in ps.processes:
if p.returncode is not None:
continue
try:
print(f"Cleanup failed. Re-trying the killing of {p}")
p.terminate()
except: # noqa: E722
pass
for p in ps.processes:
if p.returncode is not None:
continue
try:
p.wait()
except: # noqa: E722
pass