Compare commits

...

2 commits

Author SHA1 Message Date
Else Someone
18d08e727b linux-uvm: stage cached kernel, add future as comments 2026-03-02 05:08:26 +02:00
Else Someone
00ae88e8ad uvms-guest: select() to detect dead subprocesses 2026-03-02 05:07:52 +02:00
4 changed files with 429 additions and 137 deletions

View file

@ -8,73 +8,7 @@ let
inherit (lib) mkForce;
result = linux_latest.override {
structuredExtraConfig = {
BASE_SMALL = yes;
DRM_VIRTIO_GPU = yes;
EROFS_FS = yes;
# TSI = yes;
DAX = yes;
FS_DAX = yes;
FUSE_DAX = yes;
OVERLAY_FS = yes;
VIRTIO_BALLOON = yes;
VIRTIO_BLK = yes;
VIRTIO_CONSOLE = yes;
VIRTIO_FS = yes;
VIRTIO_MMIO = yes;
VIRTIO_PCI = yes;
VIRTIO = yes;
FUSE_FS = yes;
VSOCKETS = yes;
NO_HZ_IDLE = mkForce yes;
NO_HZ_FULL = mkForce unset;
HZ_1000 = unset;
HZ_250 = yes; # NixOS default: 1000
# LSM = "lockdown,yama,loadpin,safesetid,integrity,bpf";
EXT4_FS = yes;
# EXT4_USE_FOR_EXT2 = yes;
XFS_FS = yes;
DEFAULT_SECURITY_APPARMOR = mkForce unset;
XEN = mkForce unset;
XEN_BACKEND = mkForce unset;
XEN_BALLOON = mkForce unset;
XEN_BALLOON_MEMORY_HOTPLUG = mkForce unset;
XEN_DOM0 = mkForce unset;
XEN_HAVE_PVMMU = mkForce unset;
XEN_MCE_LOG = mkForce unset;
XEN_PVH = mkForce unset;
XEN_SAVE_RESTORE = mkForce unset;
XEN_SYS_HYPERVISOR = mkForce unset;
PCI_XEN = mkForce unset;
POWER_RESET_GPIO = mkForce unset;
POWER_RESET_GPIO_RESTART = mkForce unset;
RCU_LAZY = mkForce unset;
REISERFS_FS_POSIX_ACL = mkForce unset;
REISERFS_FS_SECURITY = mkForce unset;
REISERFS_FS_XATTR = mkForce unset;
SWIOTLB_XEN = mkForce unset;
SUSPEND = mkForce unset;
PM = mkForce unset;
HIBERNATION = mkForce unset;
ACPI = mkForce unset;
CPU_FREQ = mkForce unset;
CPU_FREQ_DT = mkForce unset;
INTEL_IDLE = mkForce unset;
ISA_DMA_API = mkForce unset;
IA32_EMULATION = mkForce unset;
COMPAT = mkForce unset;
COMPAT_32 = mkForce unset;
KVM = mkForce unset;
BLOCK_LEGACY_AUTOLOAD = mkForce unset;
SWAP = mkForce unset;
CMA = mkForce unset;
FB = mkForce unset;
FB_EFI = mkForce unset;
FB_VESA = mkForce unset;
SECURITY_APPARMOR = mkForce unset;
# From spectrum
VT = no;
DRM_FBDEV_EMULATION = lib.mkForce no;
FONTS = mkForce unset;
@ -85,6 +19,307 @@ let
FRAMEBUFFER_CONSOLE_DETECT_PRIMARY = mkForce unset;
FRAMEBUFFER_CONSOLE_ROTATION = mkForce unset;
RC_CORE = mkForce unset;
# Manually specified:
EROFS_FS = yes;
# TSI = yes; # Needs external patches
DAX = yes;
FS_DAX = yes;
FUSE_DAX = yes;
OVERLAY_FS = yes;
# OVERLAY_FS_REDIRECT_ALWAYS_FOLLOW = yes;
VIRTIO_BALLOON = yes;
VIRTIO_BLK = yes;
VIRTIO_CONSOLE = yes;
# VIRTIO_FS = yes;
VIRTIO_MMIO = yes;
VIRTIO_PCI = yes;
VIRTIO = yes;
# FUSE_FS = yes;
VSOCKETS = yes;
# # libkurnfw a70c65d97eda1d53a55602bdef984aba82383097
# DRM = yes;
# DRM_KMS_HELPER = yes;
# DRM_GEM_SHMEM_HELPER = yes;
# # Can't select???
# VIRTIO_GPU = yes;
# # VIRTIO_GPU_KMS = yes;
DRM_VIRTIO_GPU = yes; # Based on compiled diff?
BASE_SMALL = yes; # libkurnfw 33a72344da4938c41616d200372542b3f7eb4412
# # Based on compiled config diff with libkurnfw
# NO_HZ_COMMON = mkForce unset;
NO_HZ_IDLE = mkForce yes;
NO_HZ_FULL = mkForce unset;
HZ_1000 = unset;
HZ_250 = yes; # NixOS default: 1000
# # LSM = "lockdown,yama,loadpin,safesetid,integrity,bpf";
EXT4_FS = yes;
# # EXT4_USE_FOR_EXT2 = yes;
XFS_FS = yes;
DEFAULT_SECURITY_APPARMOR = mkForce unset;
# HW_RANDOM = yes;
# HW_RANDOM_VIRTIO = yes;
# # libkurnfw
# POSIX_MQUEUE = mkForce no;
# # libkurnfw 0ad58f60dc061e61f088a8b9b2758bea4a3bd41d
# HAVE_RELIABLE_STACKTRACE = mkForce unset;
# STACKDEPOT = mkForce unset;
# ARCH_WANT_FRAME_POINTERS = mkForce unset;
# FRAME_POINTER = mkForce unset;
# STACK_VALIDATION = mkForce unset;
# SLUB_DEBUG = mkForce unset;
# UNWINDER_FRAME_POINTER = mkForce unset;
# # UNWINDER_GUESS = mkForce yes;
# UNWINDER_ORC = mkForce unset;
# # libkurnfw a5094ce1633889250482b812265c15d9a252f6a7
# PROFILING = mkForce no;
# KEXEC_CORE = mkForce unset; # Still selected?
# KEXEC_FILE = mkForce unset; # Still selected?
# DEBUG_MISC = mkForce no;
# MAGIC_SYSRQ = mkForce no;
# DEBUG_FS = mkForce unset; # mkForce no;
# DEBUG_INFO_NONE = mkForce yes;
# KEXEC_JUMP = mkForce unset;
# DEBUG_INFO = mkForce unset;
# DEBUG_INFO_REDUCED = mkForce unset;
# DEBUG_LIST = mkForce unset;
# DEBUG_INFO_BTF = mkForce unset;
# DYNAMIC_DEBUG = mkForce unset;
# SUNRPC_DEBUG = mkForce unset;
# MAC80211_DEBUGFS = mkForce unset;
# CFG80211_DEBUGFS = mkForce unset;
# ACPI_DEBUG = mkForce unset;
# DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT = mkForce unset;
# MODULE_ALLOW_BTF_MISMATCH = mkForce unset;
# CRC32_SELFTEST = mkForce unset;
# BLK_DEBUG_FS = mkForce unset;
# # Conflicts:
# MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT = mkForce unset;
# KEXEC_HANDOVER = mkForce unset;
# MEM_ALLOC_PROFILING = mkForce unset;
# # libkurnfw d28d2632b704335c4065648afa490b63cfe49393
# X86_MSR = mkForce no;
# PERF_EVENTS_AMD_UNCORE = mkForce unset;
# ARCH_MEMORY_PROBE = mkForce unset;
# X86_CHECK_BIOS_CORRUPTION = mkForce unset;
# X86_REROUTE_FOR_BROKEN_BOOT_IRQS = mkForce unset;
# MTRR = mkForce no;
# MTRR_SANITIZER = mkForce unset;
# X86_PAT = mkForce unset;
# X86_SGX = mkForce unset;
# MODIFY_LDT_SYSCALL = mkForce unset;
# ARCH_HAS_PKEYS = mkForce unset;
# ARCH_USES_HIGH_VMA_FLAGS = mkForce unset;
# X86_INTEL_PSTATE = mkForce unset;
# X86_AMD_PSTATE = mkForce unset;
# X86_SGX_KVM = mkForce unset;
# UCLAMP_TASK = mkForce unset;
# # libkurnfw 8043cecb92f7384f648bfb2fde9e19653fe877e8
# PM = mkForce unset; # want "no" but can't select?
# CPU_FREQ = mkForce unset;
# CPU_IDLE_GOV_LADDER = mkForce unset;
INTEL_IDLE = mkForce unset;
# SCHED_MC_PRIO = mkForce unset;
# BLK_PM = mkForce unset;
# CPU_IDLE = yes;
# CPU_IDLE_GOV_HALTPOLL = yes;
# HALTPOLL_CPUIDLE = yes;
# # Conflicts:
# PM_DEBUG = mkForce unset;
# PM_ADVANCED_DEBUG = mkForce unset;
# PM_WAKELOCKS = mkForce unset;
# TPMI = mkForce unset;
# INTEL_TPMI = mkForce unset;
# # libkurnfw 63c4d29cd28ab73ea1c3a85c9df0c71a6101dd41
# IA32_EMULATION = mkForce no;
# COMPAT_32BIT_TIME = mkForce no;
# # libkurnfw b085fa0f8958ea37f0e3c16dfec6cd7cc58c6b41
# HIBERNATION = mkForce unset;
# SWAP = mkForce no;
# ZSWAP = mkForce unset;
# ZSWAP_COMPRESSOR_DEFAULT_ZSTD = mkForce unset;
# ZONE_DMA = mkForce no;
# VM_EVENT_COUNTERS = mkForce unset; # Still selected as a dep?
# PERCPU_STATS = mkForce no;
# # libkurnfw 123090b6960cd0beb7fcb19e2fc383d24c6b74e9
# XFRM_USER = mkForce unset;
# SYN_COOKIES = mkForce unset;
# TCP_CONG_ADVANCED = mkForce no;
# TCP_MD5SIG = mkForce no;
# NETLABEL = mkForce no;
# ETHTOOL_NETLINK = mkForce no;
# # libkurnfw b31a4ab84f13bde66497fd21b277503f7ad4b541
# FW_LOADER = mkForce unset;
# SOUND = mkForce unset;
# USB_SUPPORT = mkForce unset;
# MEMSTICK = mkForce unset;
# NEW_LEDS = mkForce unset;
# ACCESSIBILITY = mkForce unset;
# INFINIBAND = mkForce unset;
# INFINIBAND_ADDR_TRANS = mkForce unset;
# UIO = mkForce unset;
# VFIO = mkForce unset;
# VIRTIO_PMEM = mkForce unset;
# VDPA = mkForce unset;
# HYPERV = mkForce unset;
# FB_HYPERV = mkForce unset;
# DRM_HYPERV = mkForce unset;
# KVM_HYPERV = mkForce unset;
# COMMON_CLK = mkForce unset;
# EXT2_FS = mkForce unset;
# EXT3_FS = mkForce unset;
# EXT3_FS_POSIX_ACL = mkForce unset;
# EXT3_FS_SECURITY = mkForce unset;
# # libkrunfw e2fd98beb10f2fd9b827e188cc55ec0f90d44932
# PAGE_SIZE_LESS_THAN_64KB = yes;
# PAGE_SIZE_LESS_THAN_256KB = yes;
# CHROME_PLATFORMS = mkForce unset;
# X86_PLATFORM_DEVICES = mkForce unset;
# SURFACE_PLATFORMS = mkForce unset;
# INTEL_UNCORE_FREQ_CONTROL = mkForce unset;
# INTEL_TURBO_MAX_3 = mkForce unset;
# CHROMEOS_PSTORE = mkForce unset;
# CHROMEOS_LAPTOP = mkForce unset;
# CHROMEOS_TBMC = mkForce unset;
# CROS_EC = mkForce unset;
# CRYPTO_TEST = mkForce unset;
# NFS_SWAP = mkForce unset;
# CROS_EC_I2C = mkForce unset;
# CROS_EC_ISHTP = mkForce unset;
# CROS_EC_LPC = mkForce unset;
# CROS_EC_SPI = mkForce unset;
# NET_SCH_BPF = mkForce unset;
# UCLAMP_TASK_GROUP = mkForce unset;
# SCHED_CLASS_EXT = mkForce unset;
# DRM_NOUVEAU_SVM = mkForce unset;
# CROS_KBD_LED_BACKLIGHT = mkForce unset;
# # Based on compiled config diff with libkurnfw
XEN = mkForce unset;
# XEN_EFI = mkForce unset;
# HVC_XEN = mkForce unset;
# HVC_XEN_FRONTEND = mkForce unset;
XEN_BACKEND = mkForce unset;
XEN_BALLOON = mkForce unset;
XEN_BALLOON_MEMORY_HOTPLUG = mkForce unset;
XEN_DOM0 = mkForce unset;
XEN_HAVE_PVMMU = mkForce unset;
XEN_MCE_LOG = mkForce unset;
XEN_PVH = mkForce unset;
# XEN_PVHVM = mkForce unset;
XEN_SAVE_RESTORE = mkForce unset;
XEN_SYS_HYPERVISOR = mkForce unset;
PCI_XEN = mkForce unset;
POWER_RESET_GPIO = mkForce unset;
POWER_RESET_GPIO_RESTART = mkForce unset;
RCU_LAZY = mkForce unset;
REISERFS_FS_POSIX_ACL = mkForce unset;
REISERFS_FS_SECURITY = mkForce unset;
REISERFS_FS_XATTR = mkForce unset;
SWIOTLB_XEN = mkForce unset;
# SUSPEND = mkForce unset;
# ACPI = mkForce unset;
# ISA_DMA_API = mkForce unset;
# COMPAT = mkForce unset;
# COMPAT_32 = mkForce unset;
# KVM = mkForce unset;
# BLOCK_LEGACY_AUTOLOAD = mkForce unset;
# CMA = mkForce unset;
FB = mkForce unset;
FB_EFI = mkForce unset;
FB_VESA = mkForce unset;
SECURITY_APPARMOR = mkForce unset;
# SERIAL_8250 = mkForce unset;
# ISA_BUS = unset;
# X86_X32_ABI = unset;
# SCHED_AUTOGROUP = mkForce unset; # libkurnfw
# RAID6_PQ = mkForce unset; # libkurnfw: yes
# ARCH_HAS_FAST_MULTIPLIER = yes;
# GENERIC_NET_UTILS = yes;
# BITREVERSE = yes;
# # Conflicts
# IIO = mkForce unset;
# IIO_TRIGGERED_EVENT = mkForce unset;
# IIO_CONFIGFS = mkForce unset;
# IIO_KFIFO_BUF = mkForce unset;
# IIO_TRIGGER = mkForce unset;
# STAGING = mkForce unset;
# STAGING_MEDIA = mkForce unset;
# LEGACY_PTYS = mkForce unset;
# NULL_TTY = mkForce unset;
# N_GSM = mkForce unset;
# SERIAL_NONSTANDARD = mkForce unset;
# TTY_PRINTK = mkForce unset;
# IPMI_HANDLER = mkForce unset;
# TELCLOCK = mkForce unset;
# TCG_TPM = mkForce unset;
# SERIAL_UARTLITE = mkForce unset;
# SERIAL_LANTIQ = mkForce unset;
# MWAVE = mkForce unset;
# SERIAL_CORE = mkForce unset;
# SERIAL_MCTRL_GPIO = mkForce unset;
# I2C = mkForce unset;
# # Interactively solving conflicts...
# COMEDI = mkForce unset;
# NLS = mkForce yes;
# NLS_CODEPAGE_437 = mkForce yes;
# NLS_CODEPAGE_737 = mkForce unset;
# NLS_CODEPAGE_775 = mkForce unset;
# NLS_CODEPAGE_850 = mkForce unset;
# NLS_CODEPAGE_852 = mkForce unset;
# NLS_CODEPAGE_855 = mkForce unset;
# NLS_CODEPAGE_857 = mkForce unset;
# NLS_CODEPAGE_860 = mkForce unset;
# NLS_CODEPAGE_861 = mkForce unset;
# NLS_CODEPAGE_862 = mkForce unset;
# NLS_CODEPAGE_863 = mkForce unset;
# NLS_CODEPAGE_864 = mkForce unset;
# NLS_CODEPAGE_865 = mkForce unset;
# NLS_CODEPAGE_866 = mkForce unset;
# NLS_CODEPAGE_869 = mkForce unset;
# NLS_CODEPAGE_936 = mkForce unset;
# NLS_CODEPAGE_950 = mkForce unset;
# NLS_CODEPAGE_932 = mkForce unset;
# NLS_CODEPAGE_949 = mkForce unset;
# NLS_CODEPAGE_874 = mkForce unset;
# NLS_ISO8859_8 = mkForce unset;
# NLS_CODEPAGE_1250 = mkForce unset;
# NLS_CODEPAGE_1251 = mkForce unset;
# ZSMALLOC = mkForce unset;
};
};
in

View file

@ -3,74 +3,117 @@ import os
import select
import socket
import subprocess
import sys
def handle_run(run: dict) -> dict:
res = {}
text = run.get("text", False)
env = {
**os.environ,
"PATH": ":".join(
os.environ.get("PATH", "").split(":") + run.get("EXTRA_PATH", [])
),
}
proc = None
try:
proc = subprocess.Popen(
req["run"]["argv"],
text=text,
env=env,
cwd="/home/user",
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
)
res["status"] = "exec succeeded"
except Exception as e:
res["status"] = "exec failed"
res["exception"] = repr(e)
res["pid"] = getattr(proc, "pid", None)
try:
if proc is not None:
proc.wait(0.125)
res["long_running"] = False
res["returncode"] = getattr(proc, "returncode", None)
except subprocess.TimeoutExpired:
res["long_running"] = True
return res, proc
class Processes:
def __init__(self):
self.processes = []
self.sources = []
self.liveness_fds = dict()
self.client_fds = set()
def popen(self, *args, **kwargs):
a, b = socket.socketpair()
pass_fds = [*kwargs.get("pass_fds", ()), b.fileno()]
proc = subprocess.Popen(*args, **kwargs, pass_fds=pass_fds)
self.processes.append(proc)
self.sources.append(a)
assert a.fileno() not in self.liveness_fds
self.liveness_fds[a.fileno()] = proc
b.close()
return proc
def handle_run(self, run: dict) -> dict:
res = {}
text = run.get("text", False)
env = {
**os.environ,
"PATH": ":".join(
[
*os.environ.get("PATH", "").split(":"),
*run.get(
"EXTRA_PATH",
[],
),
],
),
}
proc = None
try:
proc = self.popen(
req["run"]["argv"],
text=text,
env=env,
cwd="/home/user",
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
)
res["status"] = "exec succeeded"
except Exception as e:
print(e)
res["status"] = "exec failed"
res["exception"] = repr(e)
res["pid"] = getattr(proc, "pid", None)
try:
if proc is not None:
proc.wait(0.125)
res["long_running"] = False
res["returncode"] = getattr(proc, "returncode", None)
except subprocess.TimeoutExpired:
res["long_running"] = True
return res, proc
def accept_vsock(self, s):
con, (cid, port) = serv.accept()
assert cid == 2, cid
self.sources.append(con)
self.client_fds.insert(con.fileno())
return con, (cid, port)
if __name__ == "__main__":
ps = Processes()
serv = socket.fromfd(3, socket.AF_VSOCK, socket.SOCK_STREAM)
procs = []
conns = [serv]
ps.sources.append(serv)
while True:
rr, rw, xs = select.select(conns, [], [])
rr, rw, xs = select.select(ps.sources, [], ps.sources)
for con in (*rr, *xs):
if con.fileno() in ps.liveness_fds:
assert con.recv(128) == b""
proc = ps.liveness_fds[con.fileno()]
proc.wait()
assert proc.returncode is not None, proc
print(f"{proc} has terminated, shutting down")
sys.exit(proc.returncode)
for con in rr:
if con is serv:
con, (cid, port) = serv.accept()
assert cid == 2, cid
conns.append(con)
continue
req = con.recv(8192)
# IDK why but I keep getting empty messages
if req == b"":
continue
try:
req = json.loads(req)
print(f"Received {req=}")
except json.JSONDecodeError as e:
print(f"Couldn't interpret {req=}: {e}")
continue
if "run" in req:
res, proc = handle_run(req["run"])
procs.append(proc)
con, _ = ps.accept_vsock(serv)
print(f"Open [{con.fileno()}]")
if con.fileno() in ps.liveness_fds:
assert False, "Must already be processed"
elif con.fileno() in ps.client_fds:
req = con.recv(8192)
# IDK why but I keep getting empty messages
if req == b"":
print(f"Lost [{con.fileno()}]")
continue
try:
req = json.loads(req)
print(f"Received {req=}")
except json.JSONDecodeError as e:
print(f"Couldn't interpret {req=}: {e}")
continue
if "run" in req:
res, proc = ps.handle_run(req["run"])
else:
res = {"status": "unknown command"}
_, rw, _ = select.select([], [con], [])
assert rw, rw
res = json.dumps(res).encode("utf8")
print(f"Responding with {res=}")
con.send(res)
else:
res = {"status": "unknown command"}
_, rw, _ = select.select([], [con], [])
assert rw, rw
res = json.dumps(res).encode("utf8")
print(f"Responding with {res=}")
con.send(res)
assert False, con.fileno()

View file

@ -452,18 +452,21 @@ def removing(*paths):
os.remove(p)
@contextmanager
def connect_ch_vsock(
vsock_sock_path,
port: int,
type=socket.SOCK_STREAM,
blocking=True,
) -> socket.socket:
os.makedirs(os.path.dirname(vsock_sock_path), exist_ok=True)
s = socket.socket(socket.AF_UNIX, type, 0)
s.setblocking(blocking)
s.connect(vsock_sock_path)
s.send(b"CONNECT %d\n" % port)
return s
with removing(vsock_sock_path):
s.send(b"CONNECT %d\n" % port)
yield s
@contextmanager
@ -473,15 +476,14 @@ def listen_ch_vsock(
type=socket.SOCK_STREAM,
blocking=True,
) -> socket.socket:
os.makedirs(os.path.dirname(vsock_sock_path), exist_ok=True)
listen_path = vsock_sock_path + "_%d" % port
s = socket.socket(socket.AF_UNIX, type, 0)
s.setblocking(blocking)
s.bind(listen_path)
s.listen()
try:
with removing(listen_path):
yield s
finally:
os.remove(listen_path)
def main(args, args_next, cleanup, ps):
@ -589,7 +591,7 @@ def main(args, args_next, cleanup, ps):
ps.exec(*ch_remote, "info")
with ready_sock:
ready_sock.settimeout(16.0)
ready_sock.settimeout(20.0)
try:
con, _ = ready_sock.accept()
except: # noqa: E722

View file

@ -39,9 +39,9 @@ in
./on-failure.nix
];
config = {
some.failure-handler.enable = true;
# some.failure-handler.enable = true;
hardware.graphics.enable = true;
# boot.kernelPackages = pkgs.linuxPackagesFor uvmsPkgs.linux-uvm;
boot.kernelPackages = pkgs.linuxPackagesFor uvmsPkgs.linux-uvm;
# boot.isContainer = true;
boot.initrd.kernelModules = [
"drm"
@ -256,14 +256,26 @@ in
partOf = [ "uvms-guest.service" ];
};
systemd.services."uvms-guest" = {
requiredBy = [ "multi-user.target" ];
onFailure = [ "shutdown.service" ];
serviceConfig = {
User = "user";
Group = "users";
ExecStart = "${lib.getExe uvmsPkgs.uvms-guest}";
ExecStop = [
"/run/current-system/sw/bin/echo GUEST DOWN"
"/run/current-system/sw/bin/systemctl poweroff"
];
StandardOutput = "journal+console";
StandardError = "journal+console";
Restart = "no";
};
};
systemd.services."shutdown" = {
serviceConfig = {
ExecStart = [ "/run/current-system/sw/bin/systemctl poweroff" ];
StandardOutput = "journal+console";
StandardError = "journal+console";
Restart = "on-failure";
RestartSec = 5;
};
};
@ -371,7 +383,7 @@ in
options = {
size = mkOption {
type = types.int;
default = 1536 * 1048576;
default = 3 * 1024 * 1048576;
};
shared = mkOption {
type = types.bool;