diff --git a/examples/dummy.nix b/examples/dummy.nix index 4689af1..7d8c813 100644 --- a/examples/dummy.nix +++ b/examples/dummy.nix @@ -13,6 +13,8 @@ system.stateVersion = "25.11"; + networking.hostName = "dummy"; + vmapps.enable = true; _module.args.inputs = import ../npins; diff --git a/pkgs/taps/main.c b/pkgs/taps/main.c index 05edb64..3b276df 100644 --- a/pkgs/taps/main.c +++ b/pkgs/taps/main.c @@ -1,5 +1,6 @@ #define _GNU_SOURCE +#include #include /* secure_getenv */ #include #include @@ -15,10 +16,12 @@ #include #include #include +#include #define __UAPI_DEF_IF_IFNAMSIZ 1 #include #include +#include #include "sendfd.h" @@ -108,6 +111,11 @@ int tuntap_alloc(char *dev, short openFlags, short ifrFlags, int *out_fd) { strncpy(dev, ifr.ifr_name, IFNAMSIZ); *out_fd = fd; + + { + int sz = sizeof(struct virtio_net_hdr_v1); + DO_OR_DIE(ioctl(fd, TUNSETVNETHDRSZ, &sz)); + } return 0; } @@ -257,6 +265,13 @@ void cleanup(int signo, siginfo_t *info, void *_context) { errx(EXIT_FAILURE, "Exiting with signal %d", signo); } +/* skarlibs under ISC */ +int uncoe (int fd) +{ + int flags = fcntl(fd, F_GETFD, 0) ; + return flags < 0 ? flags : flags & FD_CLOEXEC ? fcntl(fd, F_SETFD, flags & ~FD_CLOEXEC) : 0 ; +} + int main(int argc, char **argv) { struct sigaction act = { 0 }; act.sa_flags = SA_SIGINFO; @@ -266,7 +281,7 @@ int main(int argc, char **argv) { bool cmdServe = false; bool cmdPass = false; - char *ifname = "vt%d"; + char *ifname = "vt-%d"; char **rest = argv + 1; char **end = argv + argc; @@ -303,7 +318,7 @@ int main(int argc, char **argv) { const char *servePath = secure_getenv("TAPS_SOCK"); if (servePath == NULL) { - servePath = "taps.sock"; + servePath = "/run/taps/taps.sock"; } if (cmdServe) { @@ -317,6 +332,7 @@ int main(int argc, char **argv) { close(fd); fd = 3; } + uncoe(fd); DO_OR_DIE(execvp(nextArgv[0], nextArgv)); } else { error(EINVAL, EINVAL, "subcommand args"); diff --git a/profiles/ch-runner.nix b/profiles/ch-runner.nix index fbb09a4..f680354 100644 --- a/profiles/ch-runner.nix +++ b/profiles/ch-runner.nix @@ -10,21 +10,107 @@ let cfg = config.uvms.cloud-hypervisor; + + inherit (config.networking) hostName; inherit (config.debug.closure.erofs) layers; + inherit (lib) mkOption types; + + package = pkgs.cloud-hypervisor.overrideAttrs (oldAttrs: { + patches = oldAttrs.patches or [ ] ++ [ + # ../patches/ch.patch + ]; + buildType = "debug"; + dontStrip = true; + }); + uvmsPkgs = pkgs.callPackage ../pkgs { }; + + chSettingsFile = (pkgs.formats.json { }).generate "vm.json" cfg.settings; + + uvmPrefix = "\${HOME}/uvms/${hostName}"; + vmmSock = "${uvmPrefix}/vmm.sock"; + pastaSock = "${uvmPrefix}/pasta.sock"; + pastaPidPath = "${uvmPrefix}/pasta.pid"; + elbPrefix = "${lib.getBin pkgs.execline}/bin"; + s6Prefix = "${lib.getBin pkgs.s6}/bin"; + writeElb = name: text: writeElb' name "-W" text; + writeElb' = + name: elArgs: text: + pkgs.writeTextFile { + inherit name; + destination = "/bin/${name}"; + executable = true; + text = '' + #!${lib.getExe' pkgs.execline "execlineb"}${lib.optionalString (elArgs != null) " "}${elArgs} + importas OLDPATH PATH + export PATH "${elbPrefix}:${s6Prefix}:''${OLDPATH}" + ${text} + ''; + }; in { options = { uvms.cloud-hypervisor.enable = lib.mkEnableOption "Configure guest (e.g. fileSystems)"; - uvms.cloud-hypervisor.runner = lib.mkOption { - type = lib.types.package; + uvms.cloud-hypervisor.runner = mkOption { + type = types.package; description = "A naive script for running this system in cloud-hypervisor"; }; - uvms.cloud-hypervisor.extraArgv = lib.mkOption { - type = lib.types.listOf lib.types.str; - default = [ ]; + uvms.cloud-hypervisor.debugger = mkOption { + type = types.lazyAttrsOf types.anything; + description = "Same but you can debug the kernel"; }; - uvms.cloud-hypervisor.argv = lib.mkOption { - type = lib.types.listOf lib.types.str; + uvms.cloud-hypervisor.settingsFile = mkOption { + type = types.package; + default = chSettingsFile; + defaultText = "..."; + readOnly = true; + }; + uvms.cloud-hypervisor.settings = mkOption { + default = { }; + type = types.submodule { + freeformType = (pkgs.formats.json { }).type; + options = { + payload = { + cmdline = mkOption { type = types.str; }; + kernel = mkOption { type = types.str; }; + initramfs = mkOption { + type = types.str; + default = "${config.system.build.initialRamdisk}/${config.system.boot.loader.initrdFile}"; + }; + }; + vsock = { + cid = mkOption { + type = types.int; + default = 4; + }; + socket = mkOption { + type = types.str; + default = "vsock.sock"; + }; + }; + "api-socket" = mkOption { + type = types.str; + default = "vmm.sock"; + }; + "serial".mode = mkOption { + type = types.str; + default = "File"; + }; + "serial".file = mkOption { + type = types.nullOr types.str; + default = "serial"; + }; + "console".mode = mkOption { + type = types.str; + default = "Pty"; + }; + "console".file = mkOption { + type = types.nullOr types.str; + default = null; + }; + # "watchdog" = true; + # "seccomp" = true; + }; + }; }; uvms.cloud-hypervisor.extraCmdline = lib.mkOption { type = lib.types.listOf lib.types.str; @@ -45,33 +131,178 @@ in }; config = lib.mkMerge [ { - uvms.cloud-hypervisor.argv = lib.mkBefore ( - [ - (lib.getExe pkgs.cloud-hypervisor) - "--cmdline=${lib.concatStringsSep " " cfg.cmdline}" - "--kernel=${config.boot.kernelPackages.kernel}/${pkgs.stdenv.hostPlatform.linux-kernel.target}" - "--initramfs=${config.system.build.initialRamdisk}/${config.system.boot.loader.initrdFile}" - "--vsock=cid=4,socket=vsock.sock" - "--api-socket=vmm.sock" - "--serial=tty" - "--console=null" - "--watchdog" - "--seccomp=true" - ] - ++ cfg.extraArgv - ); - uvms.cloud-hypervisor.runner = pkgs.writeShellScriptBin "run-${config.networking.hostName}" '' - set -euo pipefail - GUESTNAME=${config.networking.hostName} - args=( - ${lib.concatMapStringsSep "\n" lib.escapeShellArg cfg.argv} - ) - mkdir -p "$HOME/uvms/$GUESTNAME" - cd "$HOME/uvms/$GUESTNAME" - cleanup() { - rm "$HOME/uvms/$GUESTNAME"/{vmm,vsock}.sock + uvms.cloud-hypervisor.settings = { + payload = { + cmdline = lib.concatStringsSep " " cfg.cmdline; + kernel = "${config.boot.kernelPackages.kernel}/${pkgs.stdenv.hostPlatform.linux-kernel.target}"; + }; + disks = map (img: { + path = img; + readonly = true; + id = toString img.label; + }) layers; + memory = { + size = 1536 * 1048576; + shared = true; + mergeable = true; + # hotplugged_size = 512 * 1048576; + # hotplugd_size = 1536 * 1048576; + # hotplug_method = "virtio-mem" + }; + cpus = { + boot_vcpus = 4; + max_vcpus = 4; + }; + }; + + uvms.cloud-hypervisor.debugger = pkgs.testers.runNixOSTest ( + { config, ... }: + { + name = "test-run-${hostName}"; + passthru = rec { + inherit (config.nodes.machine.system.build) gdbScript; + inherit (config.nodes.machine.boot.kernelPackages) kernel; + kernelSrc = pkgs.srcOnly kernel; + }; + nodes.machine = + { config, ... }: + let + kernel = config.boot.kernelPackages.kernel; + kernelSrc = pkgs.srcOnly kernel; + gdbScript = writeElb "attach-gdb" '' + if { rm -rf /tmp/gdb } + if { mkdir -p /tmp/gdb/kos } + cd /tmp/gdb + if { + elglob -0 files ${kernelSrc}/* + forx -E f { $files } + ln -s $f ./ + } + if { mkdir -p build } + cd build + if { + forx -E pattern { + ${kernel.modules}/lib/modules/*/kernel/drivers/net/tun* + ${kernel.modules}/lib/modules/*/kernel/drivers/net/tap* + } + elglob -0 files $pattern + forx -E f { $files } + if { cp $f . } + backtick -E COMPRESSED { basename $f } + xz -d $COMPRESSED + } + elglob -0 GDB_SCRIPT_DIR ${lib.getDev kernel}/lib/modules/*/build/scripts/gdb + if { + if { cp -r --no-preserve=all $GDB_SCRIPT_DIR gdb_scripts } + mv gdb_scripts/linux/constants.py.in gdb_scripts/linux/constants.py + } + ${lib.getExe pkgs.gdb} + -ex "python import sys; sys.path.insert(0, \"''${GDB_SCRIPT_DIR}\")" + -ex "target remote :1234" + -ex "source ''${GDB_SCRIPT_DIR}/vmlinux-gdb.py" + -ex "lx-symbols" + ${kernel.dev}/vmlinux + ''; + in + { + boot.kernelPackages = pkgs.linuxPackagesFor ( + (pkgs.linux.override (oldArgs: { + # extraMakeFlags = oldArgs.extraMakeFlags or [ ] ++ [ + # "scripts_gdb" + # ]; + kernelPatches = oldArgs.kernelPatches or [ ] ++ [ + { + name = "debug"; + patch = null; + structuredExtraConfig = { + GDB_SCRIPTS = lib.kernel.yes; + DEBUG_INFO = lib.kernel.yes; + DEBUG_INFO_REDUCED = lib.kernel.no; + # FRAME_POINTER = lib.kernel.yes; # "unused option"??? + KALLSYMS = lib.kernel.yes; + KGDB = lib.kernel.yes; + }; + } + ]; + })).overrideAttrs + (oldAttrs: { + dontStrip = true; + postInstall = oldAttrs.postInstall or "" + '' + cp "$buildRoot/scripts/gdb/linux/constants.py" $dev/lib/modules/*/build/scripts/gdb/linux/ || echo "$buildRoot/scripts/gdb/linux/constants.py doesn't exist" + ''; + }) + ); + boot.kernelParams = [ "nokaslr" ]; + networking.useNetworkd = true; + virtualisation.qemu.options = [ "-s" ]; + environment.systemPackages = [ + pkgs.gdb + package # CH + cfg.runner + uvmsPkgs.taps + ]; + system.build.gdbScript = gdbScript; + systemd.services.taps = { + wantedBy = [ "multi-user.target" ]; + environment.TAPS_SOCK = "/run/taps/taps.sock"; + serviceConfig = { + UMask = "0007"; + ExecStart = "${lib.getExe uvmsPkgs.taps} serve"; + RuntimeDirectory = "taps"; + DynamicUser = true; + AmbientCapabilities = [ + "CAP_NET_BIND_SERVICE" + "CAP_NET_ADMIN" + ]; + NoNewPrivileges = true; + }; + }; + }; + testScript = '' + machine.succeed("${lib.getExe cfg.runner}") + ''; } - exec -a "uuvm/$GUESTNAME" "''${args[@]}" + ); + + # NOTE: Used to be an even uglier bash script, but, for now, execline makes for easier comparisons against spectrum + uvms.cloud-hypervisor.runner = writeElb "run-${hostName}" '' + importas -i HOME HOME + importas -SsD ${lib.getExe' pkgs.passt "passt"} PASST + importas -SsD ${lib.getExe package} CH + importas -SsD "${lib.getExe' package "ch-remote"} --api-socket=${vmmSock}" CHR + foreground { mkdir -p "${uvmPrefix}" "${uvmPrefix}" } + cd "${uvmPrefix}" + background { + s6-ipcserver-socketbinder -B ${vmmSock} + trap { default { foreground { s6-ipcclient -q ${uvmPrefix}/cleanup.sock true } exit } } + fdmove -c 3 0 + redirfd -r 0 /dev/null + exec -a "uuvm/${hostName} cloud-hypervisor" $CH --api-socket fd=3 + } + importas CH_PID ! + background { + s6-ipcserver-socketbinder -B ${pastaSock} + s6-ipcserverd + trap { default { foreground { s6-ipcclient -q ${uvmPrefix}/cleanup.sock true } exit } } + exec -a "uuvm/${hostName} passt" $PASST --vhost-user -fF 0 + } + importas PASST_PID ! + background { + getpid -E CLEANUP_PID + s6-ipcserver-socketbinder -B "${uvmPrefix}/cleanup.sock" + s6-ipcserverd + foreground { kill $CH_PID $PASST_PID } + foreground { + rm -f ${vmmSock} ${uvmPrefix}/vsock.sock + } + kill $CLEANUP_PID + } + trap { default { foreground { s6-ipcclient -q ${uvmPrefix}/cleanup.sock true } exit } } + if { $CHR create ${chSettingsFile} } + # if { $CHR add-net "vhost_user=on,socket=${pastaSock}" } + if { ${lib.getExe uvmsPkgs.taps} pass $CHR add-net "id=wan,fd=3,mac=00:00:00:00:00:01" } + if { $CHR boot } + if { $CHR info } ''; } (lib.mkIf cfg.enable { @@ -103,12 +334,6 @@ in } ) layers ); - uvms.cloud-hypervisor.argv = [ - "--memory=size=1536M,hotplug_size=1536M,hotplugged_size=512M,hotplug_method=virtio-mem,mergeable=on,shared=on" - "--cpus=boot=4" - "--disk" - ] - ++ map (img: "path=${img},readonly=true,id=${toString img.label}") layers; }) ]; }