uvms/profiles/ch-runner.nix

{
  config,
  lib,
  pkgs,
  ...
}:

# It is not the intent to stick to the microvm.nix-like static interface,
# but we shall begin by reproducing at least some of their work.

let
  cfg = config.uvms.ch;

  inherit (config.networking) hostName;
  inherit (config.debug.closure.erofs) layers;
  inherit (lib)
    mkOption
    types
    concatMapStringsSep
    getExe
    getExe'
    getBin
    ;

  package = uvmsPkgs.cloud-hypervisor-gpu;
  uvmsPkgs = pkgs.callPackage ../pkgs { };

  chSettingsFile = (pkgs.formats.json { }).generate "vm.json" cfg.settings;

  uvmPrefix = "\${HOME}/uvms/${hostName}";
  vmmSock = "${uvmPrefix}/vmm.sock";
  elbPrefix = "${lib.getBin pkgs.execline}/bin";
  s6Prefix = "${lib.getBin pkgs.s6}/bin";
  writeElb = name: text: writeElb' name "-W" text;
  writeElb' =
    name: elArgs: text:
    pkgs.writeTextFile {
      inherit name;
      destination = "/bin/${name}";
      executable = true;
      text = ''
        #!${getExe' pkgs.execline "execlineb"}${lib.optionalString (elArgs != null) " "}${elArgs}
        importas OLDPATH PATH
        export PATH "${elbPrefix}:${s6Prefix}:''${OLDPATH}"
        ${text}
      '';
    };
in
{
  options = {
    uvms.ch.enable = lib.mkEnableOption "Configure guest (e.g. fileSystems)";
    uvms.ch.runner = mkOption {
      type = types.package;
      description = "A naive script for running this system in cloud-hypervisor";
    };
    uvms.ch.debugger = mkOption {
      type = types.lazyAttrsOf types.anything;
      description = "Same but you can debug the kernel";
    };
    uvms.ch.settingsFile = mkOption {
      type = types.package;
      default = chSettingsFile;
      defaultText = "...";
      readOnly = true;
    };
    uvms.cloud-hypervisor.extraCmdline = lib.mkOption {
      type = lib.types.listOf lib.types.str;
      default = [ ];
    };
    uvms.cloud-hypervisor.cmdline = lib.mkOption {
      type = lib.types.listOf lib.types.str;
      default = [
      ]
      ++ config.boot.kernelParams
      ++ config.uvms.cloud-hypervisor.extraCmdline;
    };
  };
  imports = [ ./baseImage.nix ];
  config = lib.mkMerge [
    {
      # boot.kernelPackages = pkgs.linuxPackagesFor (uvmsPkgs.linux-uvm);
      uvms.ch.settings = {
        memory = {
          # hotplugged_size = 512 * 1048576;
          # hotplugd_size = 1536 * 1048576;
          # hotplug_method = "virtio-mem"
        };
      };

      uvms.ch.debugger = pkgs.testers.runNixOSTest (
        { config, ... }:
        {
          name = "test-run-${hostName}";
          passthru = rec {
            inherit (config.nodes.machine.system.build) gdbScript;
            inherit (config.nodes.machine.boot.kernelPackages) kernel;
            kernelSrc = pkgs.srcOnly kernel;
          };
          nodes.machine =
            { config, ... }:
            let
              kernel = config.boot.kernelPackages.kernel;
              kernelSrc = pkgs.srcOnly kernel;
              gdbScript = writeElb "attach-gdb" ''
                if { rm -rf /tmp/gdb }
                if { mkdir -p /tmp/gdb/kos }
                cd /tmp/gdb
                if {
                    elglob -0 files ${kernelSrc}/*
                    forx -E f { $files }
                    ln -s $f ./
                }
                if { mkdir -p build }
                cd build
                if {
                    forx -E pattern { 
                        ${kernel.modules}/lib/modules/*/kernel/drivers/net/tun*
                        ${kernel.modules}/lib/modules/*/kernel/drivers/net/tap*
                    }
                    elglob -0 files $pattern
                    forx -E f { $files }
                    if { cp $f . }
                    backtick -E COMPRESSED { basename $f }
                    xz -d $COMPRESSED
                }
                elglob -0 GDB_SCRIPT_DIR ${lib.getDev kernel}/lib/modules/*/build/scripts/gdb
                if {
                    if { cp -r --no-preserve=all $GDB_SCRIPT_DIR gdb_scripts }
                    mv gdb_scripts/linux/constants.py.in gdb_scripts/linux/constants.py
                }
                ${getExe pkgs.gdb}
                -ex "python import sys; sys.path.insert(0, \"''${GDB_SCRIPT_DIR}\")"
                -ex "target remote :1234"
                -ex "source ''${GDB_SCRIPT_DIR}/vmlinux-gdb.py"
                -ex "lx-symbols"
                ${kernel.dev}/vmlinux
              '';
            in
            {
              boot.kernelPackages = pkgs.linuxPackagesFor (
                (pkgs.linux.override (oldArgs: {
                  # extraMakeFlags = oldArgs.extraMakeFlags or [ ] ++ [
                  #   "scripts_gdb"
                  # ];
                  kernelPatches = oldArgs.kernelPatches or [ ] ++ [
                    {
                      name = "debug";
                      patch = null;
                      structuredExtraConfig = {
                        GDB_SCRIPTS = lib.kernel.yes;
                        DEBUG_INFO = lib.kernel.yes;
                        DEBUG_INFO_REDUCED = lib.kernel.no;
                        # FRAME_POINTER = lib.kernel.yes; # "unused option"???
                        KALLSYMS = lib.kernel.yes;
                        KGDB = lib.kernel.yes;
                      };
                    }
                  ];
                })).overrideAttrs
                  (oldAttrs: {
                    dontStrip = true;
                    postInstall = oldAttrs.postInstall or "" + ''
                      cp "$buildRoot/scripts/gdb/linux/constants.py" $dev/lib/modules/*/build/scripts/gdb/linux/ || echo "$buildRoot/scripts/gdb/linux/constants.py doesn't exist"
                    '';
                  })
              );
              boot.kernelParams = [ "nokaslr" ];
              networking.useNetworkd = true;
              virtualisation.qemu.options = [ "-s" ];
              environment.systemPackages = [
                pkgs.gdb
                package # CH
                cfg.runner
                uvmsPkgs.taps
              ];
              system.build.gdbScript = gdbScript;
              systemd.services.taps = {
                wantedBy = [ "multi-user.target" ];
                environment.TAPS_SOCK = "/run/taps/taps.sock";
                serviceConfig = {
                  UMask = "0007";
                  ExecStart = "${getExe uvmsPkgs.taps} serve";
                  RuntimeDirectory = "taps";
                  DynamicUser = true;
                  AmbientCapabilities = [
                    "CAP_NET_BIND_SERVICE"
                    "CAP_NET_ADMIN"
                  ];
                  NoNewPrivileges = true;
                };
              };
            };
          testScript = ''
            machine.succeed("${getExe cfg.runner}")
          '';
        }
      );

      # NOTE: Used to be an even uglier bash script, but, for now, execline makes for easier comparisons against spectrum
      uvms.ch.runner = writeElb "run-${hostName}" ''
        ${lib.getExe uvmsPkgs.uvms} --vm-config=${chSettingsFile} --vm=${hostName}
      '';
    }
  ];
}