pkgs.uvms: init

...with some basic optional persistence and without having to rebuild
images for every app

nix run -f . pkgs.uvms -- --persist-home librewolf alacritty --run librewolf --run alacritty
This commit is contained in:
Else Someone 2026-02-27 18:26:41 +02:00
parent 22b613d157
commit 384b45bdef
15 changed files with 1155 additions and 452 deletions

407
profiles/baseImage.nix Normal file
View file

@ -0,0 +1,407 @@
{
lib,
config,
modulesPath,
pkgs,
...
}:
let
inherit (lib) mkOption types concatStringsSep;
jsonType = (pkgs.formats.json { }).type;
inherit (config.system.build) initialRamdisk;
inherit (config.system.boot.loader) initrdFile;
inherit (config.boot.kernelPackages) kernel;
kernelTarget = pkgs.stdenv.hostPlatform.linux-kernel.target;
uvmsPkgs = pkgs.callPackage ../pkgs { };
waylandSock = "/run/user/1000/wayland-1";
env = {
XDG_RUNTIME_DIR = "/run/user/1000";
WAYLAND_DISPLAY = "wayland-1";
# MESA_LOADER_DRIVER_OVERRIDE = "zink";
ELECTRON_OZONE_PLATFORM_HINT = "wayland";
MOZ_ENABLE_WAYLAND = "1";
QT_QPA_PLATFORM = "wayland"; # Qt Applications
GDK_BACKEND = "wayland"; # GTK Applications
XDG_SESSION_TYPE = "wayland"; # Electron Applications
SDL_VIDEODRIVER = "wayland";
CLUTTER_BACKEND = "wayland";
NIXOS_OZONE_WL = "1";
};
in
{
imports = [
(modulesPath + "/profiles/minimal.nix")
./debug-closure.nix
./minimal.nix
./on-failure.nix
];
config = {
some.failure-handler.enable = true;
hardware.graphics.enable = true;
# boot.kernelPackages = pkgs.linuxPackagesFor uvmsPkgs.linux-uvm;
# boot.isContainer = true;
boot.initrd.kernelModules = [
"drm"
"virtio_blk"
"virtiofs"
"virtio_gpu"
"virtio_mmio"
"virtio_pci"
"overlay"
];
boot.kernelModules = [
"drm"
"erofs"
"overlay"
"virtio_blk"
"virtiofs"
"virtio_gpu"
"virtio_mmio"
"virtio_pci"
];
boot.initrd.systemd.initrdBin = [
pkgs.fuse
pkgs.fuse3
];
fileSystems = {
"/" = lib.mkDefault {
device = "rootfs"; # how does this work? does this assign a label to the tmpfs?
fsType = "tmpfs";
options = [ "size=20%,mode=0755" ];
neededForBoot = true;
};
"/nix/store" = {
fsType = "overlay";
overlay.lowerdir = [
"/nix/.ro-stores/system"
"/nix/.ro-stores/apps"
];
neededForBoot = true;
};
"/nix/.ro-stores/system" = {
device = "system";
fsType = "virtiofs";
options = [
"defaults"
"ro"
"x-systemd.requires=systemd-modules-load.service"
];
neededForBoot = true;
};
"/nix/.ro-stores/apps" = {
device = "apps";
fsType = "virtiofs";
options = [
"defaults"
"ro"
"x-systemd.requires=systemd-modules-load.service"
];
neededForBoot = true;
};
};
systemd.mounts = [
{
type = "virtiofs";
where = "/home/user";
what = "home";
after = [ "systemd-modules-load.service" ];
wantedBy = [ "local-fs.target" ];
before = [ "local-fs.target" ];
requires = [ "systemd-modules-load.service" ];
options = lib.concatStringsSep "," [
"defaults"
"rw"
"X-mount.owner=1000"
"X-mount.group=100"
];
unitConfig = {
ConditionKernelCommandLine = "uvms.persist-home=1";
};
}
{
type = "virtiofs";
where = "/home/user/send";
what = "send";
wants = [
"home-user.mount"
"-.mount"
];
after = [
"systemd-modules-load.service"
"home-user.mount"
"-.mount"
];
wantedBy = [ "local-fs.target" ];
before = [ "local-fs.target" ];
options = lib.concatStringsSep "," [
"defaults"
"rw"
"X-mount.owner=1000"
"X-mount.group=100"
];
unitConfig = {
DefaultDependencies = false;
};
}
];
# systemd.services."mount-home-user-send" = {
# wants = [ "home-user.mount" ];
# after = [
# "systemd-modules-load.service"
# "home-user.mount"
# "-.mount"
# ];
# wantedBy = [ "local-fs.target" ];
# before = [ "local-fs.target" ];
# unitConfig = {
# DefaultDependencies = false;
# };
# environment.PATH = lib.mkForce (
# lib.makeBinPath [
# pkgs.fuse
# pkgs.fuse3
# pkgs.coreutils
# ]
# );
# serviceConfig = {
# Type = "oneshot";
# RemainsAfterExit = true;
# ExecStart = [
# "/run/current-system/sw/bin/mkdir -p /home/user/send"
# "/run/current-system/sw/bin/chown user /home/user/send"
# "/run/current-system/sw/sbin/mount -t virtiofs -o defaults,rw send /home/user/send"
# ];
# StandardOutput = "journal+console";
# StandardError = "journal+console";
# };
# };
systemd.network.enable = true;
networking.useNetworkd = true;
networking.nftables.enable = true;
networking.useDHCP = true;
networking.nameservers = [ "1.1.1.1" ];
services.resolved.enable = lib.mkForce true;
system.activationScripts.specialfs = lib.mkForce "";
# networking.firewall.enable = false;
console.enable = false;
services.udev.packages = lib.mkDefault [ ];
systemd.services."systemd-oomd".enable = false;
users.mutableUsers = false;
users.users.root.password = "hacktheplanet!";
users.groups.users = { };
users.users.user = {
uid = 1000;
isNormalUser = true;
password = "hacktheplanet!";
extraGroups = [
"video"
"render"
"users"
"wheel"
];
};
environment.variables = env;
systemd.globalEnvironment = env;
systemd.tmpfiles.settings."10-xdg" = {
${env.XDG_RUNTIME_DIR}.d = {
user = "user";
group = "users";
mode = "0755";
};
};
systemd.sockets."wayland-proxy" = {
listenStreams = [
waylandSock
];
socketConfig = {
SocketUser = "user";
SocketGroup = "users";
FileDescriptorName = "wayland";
};
wantedBy = [ "sockets.target" ];
partOf = [ "wayland-proxy.service" ];
};
systemd.services."wayland-proxy" = {
wantedBy = [ "default.target" ];
serviceConfig = {
User = "user";
Group = "users";
ExecStart = "${lib.getExe pkgs.wayland-proxy-virtwl} --virtio-gpu";
# ExecStart = "${lib.getExe uvmsPkgs.wl-cross-domain-proxy} --listen-fd --filter-global wp_presentation";
ExecStartPre = [
"+/run/current-system/sw/bin/chmod 0666 /dev/dri/card0 /dev/dri/renderD128"
];
StandardOutput = "journal+console";
StandardError = "journal+console";
Restart = "on-failure";
RestartSec = 5;
};
};
systemd.sockets."uvms-guest" = {
wantedBy = [ "default.target" ];
listenStreams = [
"vsock::24601"
];
partOf = [ "uvms-guest.service" ];
};
systemd.services."uvms-guest" = {
serviceConfig = {
User = "user";
Group = "users";
ExecStart = "${lib.getExe uvmsPkgs.uvms-guest}";
StandardOutput = "journal+console";
StandardError = "journal+console";
Restart = "on-failure";
RestartSec = 5;
};
};
fonts.enableDefaultPackages = true;
boot.kernelParams = [
"earlyprintk=ttyS0"
"console=ttyS0"
"reboot=t"
"panic=-1"
"io.systemd.credential:vmm.notify_socket=vsock-stream:2:8888"
# "rootfstype=virtiofs"
# "root=rootstore"
];
};
options = {
system.build.ch = mkOption {
type = types.package;
default = (pkgs.formats.json { }).generate "vm.json" config.uvms.ch.settings;
};
uvms.ch.settings = mkOption {
default = { };
type = types.submodule {
freeformType = jsonType;
options = {
payload = {
cmdline = mkOption {
type = types.str;
default = concatStringsSep " " (
config.boot.kernelParams
++ [
# "init=${lib.removePrefix "/nix/store" "${config.system.build.toplevel}"}/init"
"init=${config.system.build.toplevel}/init"
]
);
defaultText = ''concatStringsSep " " ${config.boot.kernelParams}'';
};
kernel = mkOption {
type = types.str;
default = "${kernel}/${kernelTarget}";
};
initramfs = mkOption {
type = types.nullOr types.str;
default = "${initialRamdisk}/${initrdFile}";
};
};
vsock = {
cid = mkOption {
type = types.int;
default = 4;
};
socket = mkOption {
type = types.str;
default = "vsock.sock";
};
};
"api-socket" = mkOption {
type = types.str;
default = "vmm.sock";
};
"serial".mode = mkOption {
type = types.str;
default = "File";
};
"serial".file = mkOption {
type = types.nullOr types.str;
default = "serial";
};
"console".mode = mkOption {
type = types.str;
default = "Pty";
};
"console".file = mkOption {
type = types.nullOr types.str;
default = null;
};
# "watchdog" = true;
# "seccomp" = true;
disks = mkOption {
default = [ ];
type = types.listOf (
types.submodule {
freeformType = jsonType;
options = {
path = mkOption {
type = types.oneOf [
types.path
types.str
];
};
readonly = mkOption {
type = types.bool;
default = true;
};
id = mkOption { type = types.str; };
};
}
);
};
memory = mkOption {
default = { };
type = types.submodule {
freeformType = jsonType;
options = {
size = mkOption {
type = types.int;
default = 1536 * 1048576;
};
shared = mkOption {
type = types.bool;
default = true;
};
mergeable = mkOption {
type = types.bool;
default = true;
};
};
};
};
cpus = mkOption {
default = { };
type = types.submodule {
freeformType = jsonType;
options = {
boot_vcpus = mkOption {
type = types.int;
default = 4;
};
max_vcpus = mkOption {
type = types.int;
default = 4;
};
};
};
};
};
};
};
};
}

View file

@ -9,7 +9,7 @@
# but we shall begin by reproducing at least some of their work.
let
cfg = config.uvms.cloud-hypervisor;
cfg = config.uvms.ch;
inherit (config.networking) hostName;
inherit (config.debug.closure.erofs) layers;
@ -48,69 +48,21 @@ let
in
{
options = {
uvms.cloud-hypervisor.enable = lib.mkEnableOption "Configure guest (e.g. fileSystems)";
uvms.cloud-hypervisor.runner = mkOption {
uvms.ch.enable = lib.mkEnableOption "Configure guest (e.g. fileSystems)";
uvms.ch.runner = mkOption {
type = types.package;
description = "A naive script for running this system in cloud-hypervisor";
};
uvms.cloud-hypervisor.debugger = mkOption {
uvms.ch.debugger = mkOption {
type = types.lazyAttrsOf types.anything;
description = "Same but you can debug the kernel";
};
uvms.cloud-hypervisor.settingsFile = mkOption {
uvms.ch.settingsFile = mkOption {
type = types.package;
default = chSettingsFile;
defaultText = "...";
readOnly = true;
};
uvms.cloud-hypervisor.settings = mkOption {
default = { };
type = types.submodule {
freeformType = (pkgs.formats.json { }).type;
options = {
payload = {
cmdline = mkOption { type = types.str; };
kernel = mkOption { type = types.str; };
initramfs = mkOption {
type = types.str;
default = "${config.system.build.initialRamdisk}/${config.system.boot.loader.initrdFile}";
};
};
vsock = {
cid = mkOption {
type = types.int;
default = 4;
};
socket = mkOption {
type = types.str;
default = "vsock.sock";
};
};
"api-socket" = mkOption {
type = types.str;
default = "vmm.sock";
};
"serial".mode = mkOption {
type = types.str;
default = "File";
};
"serial".file = mkOption {
type = types.nullOr types.str;
default = "serial";
};
"console".mode = mkOption {
type = types.str;
default = "Pty";
};
"console".file = mkOption {
type = types.nullOr types.str;
default = null;
};
# "watchdog" = true;
# "seccomp" = true;
};
};
};
uvms.cloud-hypervisor.extraCmdline = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ ];
@ -118,44 +70,24 @@ in
uvms.cloud-hypervisor.cmdline = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [
"earlyprintk=ttyS0"
"console=ttyS0"
"reboot=t"
"panic=-1"
"init=${config.system.build.toplevel}/init"
]
++ config.boot.kernelParams
++ config.uvms.cloud-hypervisor.extraCmdline;
};
};
imports = [ ./baseImage.nix ];
config = lib.mkMerge [
{
# boot.kernelPackages = pkgs.linuxPackagesFor (uvmsPkgs.linux-uvm);
uvms.cloud-hypervisor.settings = {
payload = {
cmdline = lib.concatStringsSep " " cfg.cmdline;
kernel = "${config.boot.kernelPackages.kernel}/${pkgs.stdenv.hostPlatform.linux-kernel.target}";
};
disks = map (img: {
path = img;
readonly = true;
id = toString img.label;
}) layers;
uvms.ch.settings = {
memory = {
size = 1536 * 1048576;
shared = true;
mergeable = true;
# hotplugged_size = 512 * 1048576;
# hotplugd_size = 1536 * 1048576;
# hotplug_method = "virtio-mem"
};
cpus = {
boot_vcpus = 4;
max_vcpus = 4;
};
};
uvms.cloud-hypervisor.debugger = pkgs.testers.runNixOSTest (
uvms.ch.debugger = pkgs.testers.runNixOSTest (
{ config, ... }:
{
name = "test-run-${hostName}";
@ -265,39 +197,9 @@ in
);
# NOTE: Used to be an even uglier bash script, but, for now, execline makes for easier comparisons against spectrum
uvms.cloud-hypervisor.runner = writeElb "run-${hostName}" ''
uvms.ch.runner = writeElb "run-${hostName}" ''
${lib.getExe uvmsPkgs.uvms} --vm-config=${chSettingsFile} --vm=${hostName}
'';
}
(lib.mkIf cfg.enable {
boot.initrd.availableKernelModules = [
"erofs"
"overlay"
"virtio_mmio"
"virtio_pci"
"virtio_blk"
# "9pnet_virtio"
# "9p"
"virtiofs"
];
boot.initrd.systemd.enable = lib.mkDefault true;
fileSystems = {
"/nix/store" = {
fsType = "overlay";
overlay.lowerdir = map (img: "/nix/.ro-stores/${toString img.seq}") layers;
neededForBoot = true;
};
}
// lib.listToAttrs (
map (
img:
lib.nameValuePair "/nix/.ro-stores/${toString img.seq}" {
device = "/dev/disk/by-label/${img.label}";
neededForBoot = true;
options = [ "x-systemd.device-timeout=5" ];
}
) layers
);
})
];
}

View file

@ -15,15 +15,9 @@ let
inherit (ps) writeErofsLayers;
emptySystem = import (pkgs.path + "/nixos/lib/eval-config.nix") {
modules = [
(modulesPath + "/profiles/minimal.nix")
./minimal.nix
{
system.stateVersion = config.system.stateVersion;
fileSystems."/".fsType = "tmpfs";
boot.loader.grub.enable = false;
networking.hostName = "base";
networking.nftables.enable = true;
networking.useNetworkd = true;
systemd.network.enable = true;
}
];
};

36
profiles/minimal.nix Normal file
View file

@ -0,0 +1,36 @@
{
lib,
config,
modulesPath,
...
}:
{
imports = [
(modulesPath + "/profiles/minimal.nix")
];
boot.loader.grub.enable = false;
boot.initrd.systemd.enable = true;
networking.useNetworkd = true;
networking.nftables.enable = config.networking.firewall.enable || config.networking.nat.enable;
fileSystems."/".fsType = lib.mkDefault "tmpfs";
networking.hostName = lib.mkDefault "base";
systemd.sysusers.enable = false;
services.userborn.enable = true; # nikstur it
nix.enable = false;
services.logrotate.enable = false;
services.udisks2.enable = false;
system.tools.nixos-generate-config.enable = false;
systemd.coredump.enable = false;
powerManagement.enable = false;
boot.kexec.enable = false;
system.switch.enable = false;
services.resolved.enable = false;
systemd.services.generate-shutdown-ramfs.enable = lib.mkForce false;
systemd.services.systemd-remount-fs.enable = lib.mkForce false;
systemd.services.systemd-pstore.enable = lib.mkForce false;
systemd.services.lastlog2-import.enable = lib.mkForce false;
# systemd.services.suid-sgid-wrappers.enable = lib.mkForce false;
}

72
profiles/on-failure.nix Normal file
View file

@ -0,0 +1,72 @@
{
lib,
config,
pkgs,
...
}:
let
cfg = config.some.failure-handler;
jobScript = pkgs.writeShellScriptBin "show-status" ''
set -euo pipefail
export PATH=${lib.getBin config.boot.initrd.systemd.package}/bin''${PATH:+:}$PATH
export PATH=${lib.getBin pkgs.util-linux}/bin''${PATH:+:}$PATH
export PATH=${lib.getBin pkgs.gnugrep}/bin''${PATH:+:}$PATH
unit="$1"
shift
systemctl status "$unit" >&2 || true
patterns=$unit$'\n'error
dmesg | grep -Fi "$patterns" || true
'';
mkSystemdDropin = pkgs.callPackage ../pkgs/mkSystemdDropin.nix { };
in
{
options.some.failure-handler = {
enable = lib.mkEnableOption "Set up show-status@.service as a default OnFailure dependency";
stage-1.enable =
lib.mkEnableOption "Set up show-status@.service as a default OnFailure dependency in initramfs/initrd"
// {
default = cfg.enable;
};
package = lib.mkOption {
type = lib.types.package;
readOnly = true;
description = "The internal package with the drop-ins";
};
};
config = {
some.failure-handler.package = mkSystemdDropin {
name = "status-on-failure";
inherit jobScript;
dropinText = ''
[Unit]
OnFailure=status@%n.service
'';
serviceText = ''
[Unit]
DefaultDependencies=no
Description=Show status for %i
[Service]
Type=oneshot
StandardOutput=journal+console
StandardError=journal+console
ExecStart=${lib.getExe jobScript} "%i"
JoinsNamespaceOf=
DelegateNamespaces=
'';
extraCommands = ''
printf "%s" "$serviceText" > "$root/status@.service"
'';
};
boot.initrd.systemd.packages = lib.optionals cfg.stage-1.enable [ cfg.package ];
boot.initrd.systemd.storePaths = lib.optionals cfg.stage-1.enable [
jobScript
pkgs.util-linux
pkgs.gnugrep
];
systemd.packages = lib.optionals cfg.enable [ cfg.package ];
};
}