taps: MVE
EIO crash in CH was caused by CH trying to writev before the interface has been assigned any addresses; adding an address prior to passing the FD (via 80-vm-vt.network) solves the issue. It is still unclear what causes the interrupt=None in the vhost_user/passt branch
This commit is contained in:
Else Someone
parent
1c5e2b7e89
commit
17bde34c96
3 changed files with 284 additions and 41 deletions
|
|
@ -10,21 +10,107 @@
|
|||
|
||||
let
|
||||
cfg = config.uvms.cloud-hypervisor;
|
||||
|
||||
inherit (config.networking) hostName;
|
||||
inherit (config.debug.closure.erofs) layers;
|
||||
inherit (lib) mkOption types;
|
||||
|
||||
package = pkgs.cloud-hypervisor.overrideAttrs (oldAttrs: {
|
||||
patches = oldAttrs.patches or [ ] ++ [
|
||||
# ../patches/ch.patch
|
||||
];
|
||||
buildType = "debug";
|
||||
dontStrip = true;
|
||||
});
|
||||
uvmsPkgs = pkgs.callPackage ../pkgs { };
|
||||
|
||||
chSettingsFile = (pkgs.formats.json { }).generate "vm.json" cfg.settings;
|
||||
|
||||
uvmPrefix = "\${HOME}/uvms/${hostName}";
|
||||
vmmSock = "${uvmPrefix}/vmm.sock";
|
||||
pastaSock = "${uvmPrefix}/pasta.sock";
|
||||
pastaPidPath = "${uvmPrefix}/pasta.pid";
|
||||
elbPrefix = "${lib.getBin pkgs.execline}/bin";
|
||||
s6Prefix = "${lib.getBin pkgs.s6}/bin";
|
||||
writeElb = name: text: writeElb' name "-W" text;
|
||||
writeElb' =
|
||||
name: elArgs: text:
|
||||
pkgs.writeTextFile {
|
||||
inherit name;
|
||||
destination = "/bin/${name}";
|
||||
executable = true;
|
||||
text = ''
|
||||
#!${lib.getExe' pkgs.execline "execlineb"}${lib.optionalString (elArgs != null) " "}${elArgs}
|
||||
importas OLDPATH PATH
|
||||
export PATH "${elbPrefix}:${s6Prefix}:''${OLDPATH}"
|
||||
${text}
|
||||
'';
|
||||
};
|
||||
in
|
||||
{
|
||||
options = {
|
||||
uvms.cloud-hypervisor.enable = lib.mkEnableOption "Configure guest (e.g. fileSystems)";
|
||||
uvms.cloud-hypervisor.runner = lib.mkOption {
|
||||
type = lib.types.package;
|
||||
uvms.cloud-hypervisor.runner = mkOption {
|
||||
type = types.package;
|
||||
description = "A naive script for running this system in cloud-hypervisor";
|
||||
};
|
||||
uvms.cloud-hypervisor.extraArgv = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
default = [ ];
|
||||
uvms.cloud-hypervisor.debugger = mkOption {
|
||||
type = types.lazyAttrsOf types.anything;
|
||||
description = "Same but you can debug the kernel";
|
||||
};
|
||||
uvms.cloud-hypervisor.argv = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
uvms.cloud-hypervisor.settingsFile = mkOption {
|
||||
type = types.package;
|
||||
default = chSettingsFile;
|
||||
defaultText = "...";
|
||||
readOnly = true;
|
||||
};
|
||||
uvms.cloud-hypervisor.settings = mkOption {
|
||||
default = { };
|
||||
type = types.submodule {
|
||||
freeformType = (pkgs.formats.json { }).type;
|
||||
options = {
|
||||
payload = {
|
||||
cmdline = mkOption { type = types.str; };
|
||||
kernel = mkOption { type = types.str; };
|
||||
initramfs = mkOption {
|
||||
type = types.str;
|
||||
default = "${config.system.build.initialRamdisk}/${config.system.boot.loader.initrdFile}";
|
||||
};
|
||||
};
|
||||
vsock = {
|
||||
cid = mkOption {
|
||||
type = types.int;
|
||||
default = 4;
|
||||
};
|
||||
socket = mkOption {
|
||||
type = types.str;
|
||||
default = "vsock.sock";
|
||||
};
|
||||
};
|
||||
"api-socket" = mkOption {
|
||||
type = types.str;
|
||||
default = "vmm.sock";
|
||||
};
|
||||
"serial".mode = mkOption {
|
||||
type = types.str;
|
||||
default = "File";
|
||||
};
|
||||
"serial".file = mkOption {
|
||||
type = types.nullOr types.str;
|
||||
default = "serial";
|
||||
};
|
||||
"console".mode = mkOption {
|
||||
type = types.str;
|
||||
default = "Pty";
|
||||
};
|
||||
"console".file = mkOption {
|
||||
type = types.nullOr types.str;
|
||||
default = null;
|
||||
};
|
||||
# "watchdog" = true;
|
||||
# "seccomp" = true;
|
||||
};
|
||||
};
|
||||
};
|
||||
uvms.cloud-hypervisor.extraCmdline = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
|
|
@ -45,33 +131,178 @@ in
|
|||
};
|
||||
config = lib.mkMerge [
|
||||
{
|
||||
uvms.cloud-hypervisor.argv = lib.mkBefore (
|
||||
[
|
||||
(lib.getExe pkgs.cloud-hypervisor)
|
||||
"--cmdline=${lib.concatStringsSep " " cfg.cmdline}"
|
||||
"--kernel=${config.boot.kernelPackages.kernel}/${pkgs.stdenv.hostPlatform.linux-kernel.target}"
|
||||
"--initramfs=${config.system.build.initialRamdisk}/${config.system.boot.loader.initrdFile}"
|
||||
"--vsock=cid=4,socket=vsock.sock"
|
||||
"--api-socket=vmm.sock"
|
||||
"--serial=tty"
|
||||
"--console=null"
|
||||
"--watchdog"
|
||||
"--seccomp=true"
|
||||
]
|
||||
++ cfg.extraArgv
|
||||
);
|
||||
uvms.cloud-hypervisor.runner = pkgs.writeShellScriptBin "run-${config.networking.hostName}" ''
|
||||
set -euo pipefail
|
||||
GUESTNAME=${config.networking.hostName}
|
||||
args=(
|
||||
${lib.concatMapStringsSep "\n" lib.escapeShellArg cfg.argv}
|
||||
)
|
||||
mkdir -p "$HOME/uvms/$GUESTNAME"
|
||||
cd "$HOME/uvms/$GUESTNAME"
|
||||
cleanup() {
|
||||
rm "$HOME/uvms/$GUESTNAME"/{vmm,vsock}.sock
|
||||
uvms.cloud-hypervisor.settings = {
|
||||
payload = {
|
||||
cmdline = lib.concatStringsSep " " cfg.cmdline;
|
||||
kernel = "${config.boot.kernelPackages.kernel}/${pkgs.stdenv.hostPlatform.linux-kernel.target}";
|
||||
};
|
||||
disks = map (img: {
|
||||
path = img;
|
||||
readonly = true;
|
||||
id = toString img.label;
|
||||
}) layers;
|
||||
memory = {
|
||||
size = 1536 * 1048576;
|
||||
shared = true;
|
||||
mergeable = true;
|
||||
# hotplugged_size = 512 * 1048576;
|
||||
# hotplugd_size = 1536 * 1048576;
|
||||
# hotplug_method = "virtio-mem"
|
||||
};
|
||||
cpus = {
|
||||
boot_vcpus = 4;
|
||||
max_vcpus = 4;
|
||||
};
|
||||
};
|
||||
|
||||
uvms.cloud-hypervisor.debugger = pkgs.testers.runNixOSTest (
|
||||
{ config, ... }:
|
||||
{
|
||||
name = "test-run-${hostName}";
|
||||
passthru = rec {
|
||||
inherit (config.nodes.machine.system.build) gdbScript;
|
||||
inherit (config.nodes.machine.boot.kernelPackages) kernel;
|
||||
kernelSrc = pkgs.srcOnly kernel;
|
||||
};
|
||||
nodes.machine =
|
||||
{ config, ... }:
|
||||
let
|
||||
kernel = config.boot.kernelPackages.kernel;
|
||||
kernelSrc = pkgs.srcOnly kernel;
|
||||
gdbScript = writeElb "attach-gdb" ''
|
||||
if { rm -rf /tmp/gdb }
|
||||
if { mkdir -p /tmp/gdb/kos }
|
||||
cd /tmp/gdb
|
||||
if {
|
||||
elglob -0 files ${kernelSrc}/*
|
||||
forx -E f { $files }
|
||||
ln -s $f ./
|
||||
}
|
||||
if { mkdir -p build }
|
||||
cd build
|
||||
if {
|
||||
forx -E pattern {
|
||||
${kernel.modules}/lib/modules/*/kernel/drivers/net/tun*
|
||||
${kernel.modules}/lib/modules/*/kernel/drivers/net/tap*
|
||||
}
|
||||
elglob -0 files $pattern
|
||||
forx -E f { $files }
|
||||
if { cp $f . }
|
||||
backtick -E COMPRESSED { basename $f }
|
||||
xz -d $COMPRESSED
|
||||
}
|
||||
elglob -0 GDB_SCRIPT_DIR ${lib.getDev kernel}/lib/modules/*/build/scripts/gdb
|
||||
if {
|
||||
if { cp -r --no-preserve=all $GDB_SCRIPT_DIR gdb_scripts }
|
||||
mv gdb_scripts/linux/constants.py.in gdb_scripts/linux/constants.py
|
||||
}
|
||||
${lib.getExe pkgs.gdb}
|
||||
-ex "python import sys; sys.path.insert(0, \"''${GDB_SCRIPT_DIR}\")"
|
||||
-ex "target remote :1234"
|
||||
-ex "source ''${GDB_SCRIPT_DIR}/vmlinux-gdb.py"
|
||||
-ex "lx-symbols"
|
||||
${kernel.dev}/vmlinux
|
||||
'';
|
||||
in
|
||||
{
|
||||
boot.kernelPackages = pkgs.linuxPackagesFor (
|
||||
(pkgs.linux.override (oldArgs: {
|
||||
# extraMakeFlags = oldArgs.extraMakeFlags or [ ] ++ [
|
||||
# "scripts_gdb"
|
||||
# ];
|
||||
kernelPatches = oldArgs.kernelPatches or [ ] ++ [
|
||||
{
|
||||
name = "debug";
|
||||
patch = null;
|
||||
structuredExtraConfig = {
|
||||
GDB_SCRIPTS = lib.kernel.yes;
|
||||
DEBUG_INFO = lib.kernel.yes;
|
||||
DEBUG_INFO_REDUCED = lib.kernel.no;
|
||||
# FRAME_POINTER = lib.kernel.yes; # "unused option"???
|
||||
KALLSYMS = lib.kernel.yes;
|
||||
KGDB = lib.kernel.yes;
|
||||
};
|
||||
}
|
||||
];
|
||||
})).overrideAttrs
|
||||
(oldAttrs: {
|
||||
dontStrip = true;
|
||||
postInstall = oldAttrs.postInstall or "" + ''
|
||||
cp "$buildRoot/scripts/gdb/linux/constants.py" $dev/lib/modules/*/build/scripts/gdb/linux/ || echo "$buildRoot/scripts/gdb/linux/constants.py doesn't exist"
|
||||
'';
|
||||
})
|
||||
);
|
||||
boot.kernelParams = [ "nokaslr" ];
|
||||
networking.useNetworkd = true;
|
||||
virtualisation.qemu.options = [ "-s" ];
|
||||
environment.systemPackages = [
|
||||
pkgs.gdb
|
||||
package # CH
|
||||
cfg.runner
|
||||
uvmsPkgs.taps
|
||||
];
|
||||
system.build.gdbScript = gdbScript;
|
||||
systemd.services.taps = {
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
environment.TAPS_SOCK = "/run/taps/taps.sock";
|
||||
serviceConfig = {
|
||||
UMask = "0007";
|
||||
ExecStart = "${lib.getExe uvmsPkgs.taps} serve";
|
||||
RuntimeDirectory = "taps";
|
||||
DynamicUser = true;
|
||||
AmbientCapabilities = [
|
||||
"CAP_NET_BIND_SERVICE"
|
||||
"CAP_NET_ADMIN"
|
||||
];
|
||||
NoNewPrivileges = true;
|
||||
};
|
||||
};
|
||||
};
|
||||
testScript = ''
|
||||
machine.succeed("${lib.getExe cfg.runner}")
|
||||
'';
|
||||
}
|
||||
exec -a "uuvm/$GUESTNAME" "''${args[@]}"
|
||||
);
|
||||
|
||||
# NOTE: Used to be an even uglier bash script, but, for now, execline makes for easier comparisons against spectrum
|
||||
uvms.cloud-hypervisor.runner = writeElb "run-${hostName}" ''
|
||||
importas -i HOME HOME
|
||||
importas -SsD ${lib.getExe' pkgs.passt "passt"} PASST
|
||||
importas -SsD ${lib.getExe package} CH
|
||||
importas -SsD "${lib.getExe' package "ch-remote"} --api-socket=${vmmSock}" CHR
|
||||
foreground { mkdir -p "${uvmPrefix}" "${uvmPrefix}" }
|
||||
cd "${uvmPrefix}"
|
||||
background {
|
||||
s6-ipcserver-socketbinder -B ${vmmSock}
|
||||
trap { default { foreground { s6-ipcclient -q ${uvmPrefix}/cleanup.sock true } exit } }
|
||||
fdmove -c 3 0
|
||||
redirfd -r 0 /dev/null
|
||||
exec -a "uuvm/${hostName} cloud-hypervisor" $CH --api-socket fd=3
|
||||
}
|
||||
importas CH_PID !
|
||||
background {
|
||||
s6-ipcserver-socketbinder -B ${pastaSock}
|
||||
s6-ipcserverd
|
||||
trap { default { foreground { s6-ipcclient -q ${uvmPrefix}/cleanup.sock true } exit } }
|
||||
exec -a "uuvm/${hostName} passt" $PASST --vhost-user -fF 0
|
||||
}
|
||||
importas PASST_PID !
|
||||
background {
|
||||
getpid -E CLEANUP_PID
|
||||
s6-ipcserver-socketbinder -B "${uvmPrefix}/cleanup.sock"
|
||||
s6-ipcserverd
|
||||
foreground { kill $CH_PID $PASST_PID }
|
||||
foreground {
|
||||
rm -f ${vmmSock} ${uvmPrefix}/vsock.sock
|
||||
}
|
||||
kill $CLEANUP_PID
|
||||
}
|
||||
trap { default { foreground { s6-ipcclient -q ${uvmPrefix}/cleanup.sock true } exit } }
|
||||
if { $CHR create ${chSettingsFile} }
|
||||
# if { $CHR add-net "vhost_user=on,socket=${pastaSock}" }
|
||||
if { ${lib.getExe uvmsPkgs.taps} pass $CHR add-net "id=wan,fd=3,mac=00:00:00:00:00:01" }
|
||||
if { $CHR boot }
|
||||
if { $CHR info }
|
||||
'';
|
||||
}
|
||||
(lib.mkIf cfg.enable {
|
||||
|
|
@ -103,12 +334,6 @@ in
|
|||
}
|
||||
) layers
|
||||
);
|
||||
uvms.cloud-hypervisor.argv = [
|
||||
"--memory=size=1536M,hotplug_size=1536M,hotplugged_size=512M,hotplug_method=virtio-mem,mergeable=on,shared=on"
|
||||
"--cpus=boot=4"
|
||||
"--disk"
|
||||
]
|
||||
++ map (img: "path=${img},readonly=true,id=${toString img.label}") layers;
|
||||
})
|
||||
];
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue