clan-munix/nixosModules/default.nix

{
  self,
  virtwl,
  sidebus,
}:
{
  pkgs,
  lib,
  utils,
  config,
  ...
}:
let
  useTTY = {
    TTYPath = "/dev/hvc0";
    StandardOutput = "tty";
    StandardInput = "tty";
    StandardError = "tty";
  };
  runtimeDir = "/run/vm-user";
  system = pkgs.stdenv.hostPlatform.system;
in
{
  options.virtualisation.munix.defaultCommand = lib.mkOption {
    type = lib.types.str;
    default = "bash";
    description = "Default command to run when starting the VM without arguments.";
  };

  config = {
    boot.isContainer = true;
    fileSystems."/".device = lib.mkDefault "/dev/sda"; # dummy

    # Disable unused things
    environment.defaultPackages = lib.mkDefault [ ];
    documentation = {
      enable = lib.mkDefault false;
      doc.enable = lib.mkDefault false;
      info.enable = lib.mkDefault false;
      man.enable = lib.mkDefault false;
      nixos.enable = lib.mkDefault false;
    };
    services.logrotate.enable = false;
    services.udisks2.enable = false;
    system.tools.nixos-generate-config.enable = false;
    system.activationScripts.specialfs = lib.mkForce "";
    systemd.coredump.enable = false;
    networking.firewall.enable = false;
    powerManagement.enable = false;
    boot.kexec.enable = false;
    console.enable = false;

    # Configure activation / systemd
    boot.initrd.systemd.enable = true; # for etc.overlay, but we don't have initrd
    system.etc.overlay.enable = true; # erofs
    system.etc.overlay.mutable = false;
    system.switch.enable = false;
    services.udev.enable = lib.mkDefault true;
    services.udev.packages = lib.mkDefault [ ];
    services.resolved.enable = false;
    environment.etc."resolv.conf".source = "/run/resolv.conf";
    environment.etc."machine-id".source = "/run/machine-id";
    environment.etc."systemd/system".source = lib.mkForce (
      utils.systemdUtils.lib.generateUnits {
        type = "system";
        units = config.systemd.units;
        upstreamUnits = [
          "sysinit.target"
          "local-fs.target"
          "nss-user-lookup.target"
          "umount.target"
          "sockets.target"
          "shutdown.target"
          "reboot.target"
          "exit.target"
          "final.target"
          "systemd-exit.service"
          "systemd-journald.socket"
          "systemd-journald-audit.socket"
          "systemd-journald-dev-log.socket"
          "systemd-journald.service"
          "systemd-udevd-kernel.socket"
          "systemd-udevd-control.socket"
          "user.slice"
        ];
        upstreamWants = [ "multi-user.target.wants" ];
      }
    );

    # systemd.package = pkgs.systemdMinimal; # no analyze
    systemd.defaultUnit = "microvm.target";
    systemd.targets.microvm = {
      description = "Minimal microVM system";
      wants = [
        "systemd-journald.socket"
        "systemd-udevd.service"
        "dbus.socket"
      ];
      unitConfig.AllowIsolate = "yes";
    };
    systemd.services.generate-shutdown-ramfs.enable = lib.mkForce false;
    systemd.services.systemd-remount-fs.enable = lib.mkForce false;
    systemd.services.systemd-pstore.enable = lib.mkForce false;
    systemd.services.lastlog2-import.enable = lib.mkForce false;
    systemd.services.suid-sgid-wrappers.enable = lib.mkForce false;
    systemd.services.systemd-udevd = {
      # Redefine to remove the Before deps and get out of the critical chain
      enable = true;
      description = "Rule-based Manager for Device Events and Files";
      unitConfig.DefaultDependencies = "no";
      serviceConfig = {
        CapabilityBoundingSet = "~CAP_SYS_TIME CAP_WAKE_ALARM";
        Delegate = "";
        DelegateSubgroup = "udev";
        Type = "notify-reload";
        OOMScoreAdjust = "-1000";
        Sockets = "systemd-udevd-control.socket systemd-udevd-kernel.socket systemd-udevd-varlink.socket";
        Restart = "always";
        RestartSec = "0";
        ExecStart = "${pkgs.systemd}/lib/systemd/systemd-udevd";
        FileDescriptorStoreMax = "512";
        FileDescriptorStorePreserve = "yes";
        KillMode = "mixed";
        TasksMax = "infinity";
        PrivateMounts = "yes";
        ProtectHostname = "yes";
        MemoryDenyWriteExecute = "yes";
        RestrictAddressFamilies = "AF_UNIX AF_NETLINK AF_INET AF_INET6";
        RestrictRealtime = "yes";
        RestrictSUIDSGID = "yes";
        SystemCallFilter = [
          "@system-service @module @raw-io bpf"
          "~@clock"
        ];
        SystemCallErrorNumber = "EPERM";
        SystemCallArchitectures = "native";
        LockPersonality = "yes";
        IPAddressDeny = "any";
        WatchdogSec = "3min";
      };
    };

    # Configure user accounts
    systemd.sysusers.enable = false;
    services.userborn.enable = true;
    services.userborn.static = true;
    users.mutableUsers = false;
    users.users.appvm = {
      uid = 1337;
      group = "appvm";
      isNormalUser = false;
      isSystemUser = true;
      home = "/home/appvm";
      description = "microVM User";
      shell = pkgs.bash; # not nologin, despite being a "system" user
      extraGroups = [
        "wheel"
        "video"
        "input"
        "systemd-journal"
      ];
    };
    users.groups.appvm.gid = 1337;
    users.allowNoPasswordLogin = true;

    # Configure services

    systemd.settings.Manager.DefaultEnvironment = "XDG_RUNTIME_DIR=${runtimeDir}";

    systemd.services.muvm-remote = {
      enable = true;
      description = "microVM Application runner";
      onFailure = [ "exit.target" ];
      onSuccess = [ "exit.target" ];
      wants = [ "sockets.target" ];
      after = [ "sockets.target" ];
      wantedBy = [ "microvm.target" ];
      serviceConfig = {
        Type = "exec";
        PassEnvironment = [
          "TERM"
          "MESA_LOADER_DRIVER_OVERRIDE"
          "MUVM_REMOTE_CONFIG"
        ]; # "KRUN_CONFIG"];
        Environment = [
          "WAYLAND_DISPLAY=wayland-1"
          "DBUS_SESSION_BUS_ADDRESS=unix:path=${runtimeDir}/dbus.sock"
          "PATH=/run/current-system/sw/bin"
        ];
        User = "appvm";
        Group = "appvm";
        ExecStartPre = "+/run/current-system/sw/bin/chown appvm:appvm ${runtimeDir}";
        ExecStart = "/opt/bin/muvm-remote";
        ExecStopPost = ''+${pkgs.python3}/bin/python -c "import os,fcntl,struct;print(os.getenv('EXIT_STATUS', '1'));fcntl.ioctl(os.open('/', os.O_RDONLY), 0x7602, int(os.getenv('EXIT_STATUS', '1')))"'';
      }
      // useTTY;
    };

    systemd.services.muvm-configure-network = {
      enable = true;
      description = "microVM Network configuration";
      wantedBy = [ "microvm.target" ];
      serviceConfig.Type = "oneshot";
      serviceConfig.ExecStart = "/opt/bin/muvm-configure-network";
    };

    systemd.sockets.muvm-pwbridge = {
      enable = true;
      description = "PipeWire cross-domain proxy socket";
      wantedBy = [ "microvm.target" ];
      partOf = [ "muvm-pwbridge.service" ];
      listenStreams = [ "${runtimeDir}/pipewire-0" ];
      socketConfig = {
        SocketUser = "appvm";
        SocketGroup = "appvm";
      };
    };
    systemd.services.muvm-pwbridge = {
      enable = true;
      description = "PipeWire cross-domain proxy";
      requires = [ "muvm-pwbridge.socket" ];
      serviceConfig.Type = "exec";
      serviceConfig.ExecStart = "/opt/bin/muvm-pwbridge";
    };

    systemd.sockets.wayland-proxy-virtwl = {
      enable = true;
      description = "Wayland cross-domain proxy socket";
      wantedBy = [ "microvm.target" ];
      partOf = [ "wayland-proxy-virtwl.service" ];
      listenStreams = [ "${runtimeDir}/wayland-1" ];
      socketConfig = {
        SocketUser = "appvm";
        SocketGroup = "appvm";
        FileDescriptorName = "wayland";
      };
    };
    systemd.services.wayland-proxy-virtwl = {
      enable = true;
      description = "Wayland cross-domain proxy";
      requires = [ "wayland-proxy-virtwl.socket" ];
      serviceConfig = {
        ExecStartPre = "+/run/current-system/sw/bin/chmod 0666 /dev/dri/card0 /dev/dri/renderD128";
        ExecStart = "${virtwl.packages.${system}.proxy}/bin/wayland-proxy-virtwl --virtio-gpu";
        User = "appvm";
        Group = "appvm";
      };
    };

    systemd.sockets.session-bus = {
      enable = true;
      description = "D-Bus session bus socket";
      wantedBy = [ "microvm.target" ];
      partOf = [ "session-bus.service" ];
      listenStreams = [ "${runtimeDir}/dbus.sock" ];
      socketConfig = {
        SocketUser = "appvm";
        SocketGroup = "appvm";
      };
    };
    systemd.services.session-bus = {
      enable = true;
      description = "D-Bus session bus";
      requires = [ "session-bus.socket" ];
      serviceConfig = {
        ExecStart = "${pkgs.dbus}/bin/dbus-daemon --session --address=systemd: --nofork --nopidfile --syslog-only"; # no systemd activation, we don't run a *session* systemd
        User = "appvm";
        Group = "appvm";
      };
    };
    systemd.services.sidebus-agent = {
      enable = true;
      description = "D-Bus session bus";
      wantedBy = ["microvm.target"];
      requires = ["session-bus.socket" "session-bus.service"];
      after = ["session-bus.service"];
      serviceConfig = {
        ImportCredential = "sidebus.port";
        Environment = ["DBUS_SESSION_BUS_ADDRESS=unix:path=${runtimeDir}/dbus.sock"];
        ExecStart = "${sidebus.packages.${system}.sidebus-agent}/bin/sidebus-agent";
        User = "appvm";
        Group = "appvm";
      };
    };

    hardware.graphics.enable = true;
    hardware.graphics.package = self.packages.${system}.mesa;

    system.build.munix = pkgs.symlinkJoin {
      name = "munix";
      paths = [ self.packages.${system}.munix ];
      buildInputs = [ pkgs.makeWrapper ];
      postBuild = ''
        wrapProgram $out/bin/munix \
          --add-flags ${config.system.build.toplevel} \
          --set MICROVM_DEFAULT_COMMAND ${lib.escapeShellArg config.virtualisation.munix.defaultCommand}
      '';
    };
  };
}