{ config, lib, pkgs, ... }: # It is not the intent to stick to the microvm.nix-like static interface, # but we shall begin by reproducing at least some of their work. let cfg = config.uvms.cloud-hypervisor; inherit (config.networking) hostName; inherit (config.debug.closure.erofs) layers; inherit (lib) mkOption types; package = pkgs.cloud-hypervisor.overrideAttrs (oldAttrs: { patches = oldAttrs.patches or [ ] ++ [ # ../patches/ch.patch ]; buildType = "debug"; dontStrip = true; }); uvmsPkgs = pkgs.callPackage ../pkgs { }; chSettingsFile = (pkgs.formats.json { }).generate "vm.json" cfg.settings; uvmPrefix = "\${HOME}/uvms/${hostName}"; vmmSock = "${uvmPrefix}/vmm.sock"; pastaSock = "${uvmPrefix}/pasta.sock"; pastaPidPath = "${uvmPrefix}/pasta.pid"; elbPrefix = "${lib.getBin pkgs.execline}/bin"; s6Prefix = "${lib.getBin pkgs.s6}/bin"; writeElb = name: text: writeElb' name "-W" text; writeElb' = name: elArgs: text: pkgs.writeTextFile { inherit name; destination = "/bin/${name}"; executable = true; text = '' #!${lib.getExe' pkgs.execline "execlineb"}${lib.optionalString (elArgs != null) " "}${elArgs} importas OLDPATH PATH export PATH "${elbPrefix}:${s6Prefix}:''${OLDPATH}" ${text} ''; }; in { options = { uvms.cloud-hypervisor.enable = lib.mkEnableOption "Configure guest (e.g. fileSystems)"; uvms.cloud-hypervisor.runner = mkOption { type = types.package; description = "A naive script for running this system in cloud-hypervisor"; }; uvms.cloud-hypervisor.debugger = mkOption { type = types.lazyAttrsOf types.anything; description = "Same but you can debug the kernel"; }; uvms.cloud-hypervisor.settingsFile = mkOption { type = types.package; default = chSettingsFile; defaultText = "..."; readOnly = true; }; uvms.cloud-hypervisor.settings = mkOption { default = { }; type = types.submodule { freeformType = (pkgs.formats.json { }).type; options = { payload = { cmdline = mkOption { type = types.str; }; kernel = mkOption { type = types.str; }; initramfs = mkOption { type = types.str; default = "${config.system.build.initialRamdisk}/${config.system.boot.loader.initrdFile}"; }; }; vsock = { cid = mkOption { type = types.int; default = 4; }; socket = mkOption { type = types.str; default = "vsock.sock"; }; }; "api-socket" = mkOption { type = types.str; default = "vmm.sock"; }; "serial".mode = mkOption { type = types.str; default = "File"; }; "serial".file = mkOption { type = types.nullOr types.str; default = "serial"; }; "console".mode = mkOption { type = types.str; default = "Pty"; }; "console".file = mkOption { type = types.nullOr types.str; default = null; }; # "watchdog" = true; # "seccomp" = true; }; }; }; uvms.cloud-hypervisor.extraCmdline = lib.mkOption { type = lib.types.listOf lib.types.str; default = [ ]; }; uvms.cloud-hypervisor.cmdline = lib.mkOption { type = lib.types.listOf lib.types.str; default = [ "earlyprintk=ttyS0" "console=ttyS0" "reboot=t" "panic=-1" "init=${config.system.build.toplevel}/init" ] ++ config.boot.kernelParams ++ config.uvms.cloud-hypervisor.extraCmdline; }; }; config = lib.mkMerge [ { uvms.cloud-hypervisor.settings = { payload = { cmdline = lib.concatStringsSep " " cfg.cmdline; kernel = "${config.boot.kernelPackages.kernel}/${pkgs.stdenv.hostPlatform.linux-kernel.target}"; }; disks = map (img: { path = img; readonly = true; id = toString img.label; }) layers; memory = { size = 1536 * 1048576; shared = true; mergeable = true; # hotplugged_size = 512 * 1048576; # hotplugd_size = 1536 * 1048576; # hotplug_method = "virtio-mem" }; cpus = { boot_vcpus = 4; max_vcpus = 4; }; }; uvms.cloud-hypervisor.debugger = pkgs.testers.runNixOSTest ( { config, ... }: { name = "test-run-${hostName}"; passthru = rec { inherit (config.nodes.machine.system.build) gdbScript; inherit (config.nodes.machine.boot.kernelPackages) kernel; kernelSrc = pkgs.srcOnly kernel; }; nodes.machine = { config, ... }: let kernel = config.boot.kernelPackages.kernel; kernelSrc = pkgs.srcOnly kernel; gdbScript = writeElb "attach-gdb" '' if { rm -rf /tmp/gdb } if { mkdir -p /tmp/gdb/kos } cd /tmp/gdb if { elglob -0 files ${kernelSrc}/* forx -E f { $files } ln -s $f ./ } if { mkdir -p build } cd build if { forx -E pattern { ${kernel.modules}/lib/modules/*/kernel/drivers/net/tun* ${kernel.modules}/lib/modules/*/kernel/drivers/net/tap* } elglob -0 files $pattern forx -E f { $files } if { cp $f . } backtick -E COMPRESSED { basename $f } xz -d $COMPRESSED } elglob -0 GDB_SCRIPT_DIR ${lib.getDev kernel}/lib/modules/*/build/scripts/gdb if { if { cp -r --no-preserve=all $GDB_SCRIPT_DIR gdb_scripts } mv gdb_scripts/linux/constants.py.in gdb_scripts/linux/constants.py } ${lib.getExe pkgs.gdb} -ex "python import sys; sys.path.insert(0, \"''${GDB_SCRIPT_DIR}\")" -ex "target remote :1234" -ex "source ''${GDB_SCRIPT_DIR}/vmlinux-gdb.py" -ex "lx-symbols" ${kernel.dev}/vmlinux ''; in { boot.kernelPackages = pkgs.linuxPackagesFor ( (pkgs.linux.override (oldArgs: { # extraMakeFlags = oldArgs.extraMakeFlags or [ ] ++ [ # "scripts_gdb" # ]; kernelPatches = oldArgs.kernelPatches or [ ] ++ [ { name = "debug"; patch = null; structuredExtraConfig = { GDB_SCRIPTS = lib.kernel.yes; DEBUG_INFO = lib.kernel.yes; DEBUG_INFO_REDUCED = lib.kernel.no; # FRAME_POINTER = lib.kernel.yes; # "unused option"??? KALLSYMS = lib.kernel.yes; KGDB = lib.kernel.yes; }; } ]; })).overrideAttrs (oldAttrs: { dontStrip = true; postInstall = oldAttrs.postInstall or "" + '' cp "$buildRoot/scripts/gdb/linux/constants.py" $dev/lib/modules/*/build/scripts/gdb/linux/ || echo "$buildRoot/scripts/gdb/linux/constants.py doesn't exist" ''; }) ); boot.kernelParams = [ "nokaslr" ]; networking.useNetworkd = true; virtualisation.qemu.options = [ "-s" ]; environment.systemPackages = [ pkgs.gdb package # CH cfg.runner uvmsPkgs.taps ]; system.build.gdbScript = gdbScript; systemd.services.taps = { wantedBy = [ "multi-user.target" ]; environment.TAPS_SOCK = "/run/taps/taps.sock"; serviceConfig = { UMask = "0007"; ExecStart = "${lib.getExe uvmsPkgs.taps} serve"; RuntimeDirectory = "taps"; DynamicUser = true; AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" "CAP_NET_ADMIN" ]; NoNewPrivileges = true; }; }; }; testScript = '' machine.succeed("${lib.getExe cfg.runner}") ''; } ); # NOTE: Used to be an even uglier bash script, but, for now, execline makes for easier comparisons against spectrum uvms.cloud-hypervisor.runner = writeElb "run-${hostName}" '' importas -i HOME HOME importas -SsD ${lib.getExe' pkgs.passt "passt"} PASST importas -SsD ${lib.getExe package} CH importas -SsD "${lib.getExe' package "ch-remote"} --api-socket=${vmmSock}" CHR foreground { mkdir -p "${uvmPrefix}" "${uvmPrefix}" } cd "${uvmPrefix}" background { s6-ipcserver-socketbinder -B ${vmmSock} trap { default { foreground { s6-ipcclient -q ${uvmPrefix}/cleanup.sock true } exit } } fdmove -c 3 0 redirfd -r 0 /dev/null exec -a "uuvm/${hostName} cloud-hypervisor" $CH --api-socket fd=3 } importas CH_PID ! background { s6-ipcserver-socketbinder -B ${pastaSock} s6-ipcserverd trap { default { foreground { s6-ipcclient -q ${uvmPrefix}/cleanup.sock true } exit } } exec -a "uuvm/${hostName} passt" $PASST --vhost-user -fF 0 } importas PASST_PID ! background { getpid -E CLEANUP_PID s6-ipcserver-socketbinder -B "${uvmPrefix}/cleanup.sock" s6-ipcserverd foreground { kill $CH_PID $PASST_PID } foreground { rm -f ${vmmSock} ${uvmPrefix}/vsock.sock } kill $CLEANUP_PID } trap { default { foreground { s6-ipcclient -q ${uvmPrefix}/cleanup.sock true } exit } } if { $CHR create ${chSettingsFile} } # if { $CHR add-net "vhost_user=on,socket=${pastaSock}" } if { ${lib.getExe uvmsPkgs.taps} pass $CHR add-net "id=wan,fd=3,mac=00:00:00:00:00:01" } if { $CHR boot } if { $CHR info } ''; } (lib.mkIf cfg.enable { boot.initrd.availableKernelModules = [ "erofs" "overlay" "virtio_mmio" "virtio_pci" "virtio_blk" # "9pnet_virtio" # "9p" "virtiofs" ]; boot.initrd.systemd.enable = lib.mkDefault true; fileSystems = { "/nix/store" = { fsType = "overlay"; overlay.lowerdir = map (img: "/nix/.ro-stores/${toString img.seq}") layers; neededForBoot = true; }; } // lib.listToAttrs ( map ( img: lib.nameValuePair "/nix/.ro-stores/${toString img.seq}" { device = "/dev/disk/by-label/${img.label}"; neededForBoot = true; options = [ "x-systemd.device-timeout=5" ]; } ) layers ); }) ]; }