taps: MVE

- TUNSETIFF works, the interface is indeed created.
- An fd is indeed passed over the unix socket,
- and is a valid (enough) fd because it can fed into `dup2`.
- `nix run -f . --offline pkgs.taps -- pass sleep 5` works, the
  interface exists for 5 seconds and disappears
- `nix run -f . --offline pkgs.taps -- pass ch-remote --api-socket=$HOME/uvms/nixos/vmm.sock add-net fd=3`
  obscurely fails, killing the VMM with:

  ```console
  [root@nixos:~]# cloud-hypervisor:  12.388270s: <_net1_qp0> ERROR:/build/source/net_util/src/queue_pair.rs:112 -- net: tx: failed writing to tap: Input/output error (os er
  ror 5)
  cloud-hypervisor:  12.388459s: <_net1_qp0> ERROR:virtio-devices/src/thread_helper.rs:54 -- Error running worker: HandleEvent(Error processing TX queue: NetQueuePair(Write
  Tap(Os { code: 5, kind: Uncategorized, message: "Input/output error" })))
  ```

pkgs/ch-proxy/meson.build

@@ -1,3 +1,9 @@
 project('ch-proxy', 'c')
 
-executable('ch-proxy', 'proxy.c', install: true)
+pkg = import('pkgconfig')
+
+sendfd = library('sendfd', [ 'sendfd.c', 'sendfd.h' ], install: true)
+pkg.generate(sendfd)
+install_headers('sendfd.h')
+
+executable('ch-proxy', 'proxy.c', link_with: [sendfd], install: true)

pkgs/ch-proxy/package.nix

@@ -8,6 +8,7 @@
 stdenv.mkDerivation {
   pname = "ch-proxy";
   version = "0.0.0";
+  outputs = [ "out" "lib" ];
   nativeBuildInputs = [
     meson
     ninja
@@ -19,6 +20,8 @@ stdenv.mkDerivation {
     fs.toSource {
       fileset = fs.unions [
         ./proxy.c
+        ./sendfd.c
+        ./sendfd.h
         ./meson.build
       ];
       root = ./.;

pkgs/ch-proxy/proxy.c

@@ -9,9 +9,9 @@
 
 #include <linux/vm_sockets.h>
 
-struct msghdr mk_msghdr();
+#include "sendfd.h"
+
 int ch_connect(const char*, const char*);
-ssize_t send_fd(int, int);
 
 #define _WRITE_CONFIRM(fd, buf, buflen) {if (write((fd), (buf), (buflen)) != (buflen)) { perror("ch-proxy/write/partial write"); exit(EXIT_FAILURE); }}
 
@@ -168,19 +168,13 @@ int main(int argc, char** argv) {
         exit(EXIT_FAILURE);
     }
 
-    if (send_fd(1, s) == -1) {
+    if (send_fd(1, s, NULL) == -1) {
         perror("ssh-vsock-proxy/main/send_fd");
         return EXIT_FAILURE;
     }
     return 0;
 }
 
-struct msghdr mk_msghdr() {
-    struct msghdr msg;
-    memset(&msg, 0, sizeof(msg));
-
-    return msg;
-}
 
 int ch_connect(const char *path, const char *port) {
     int s = socket(AF_UNIX, SOCK_STREAM, 0);
@@ -212,38 +206,3 @@ int ch_connect(const char *path, const char *port) {
 
     return s;
 }
-
-ssize_t send_fd(int dst_fd, int fd) {
-    struct msghdr msg = mk_msghdr();
-
-    /* openssh expects to receive a dummy length=1 iovec? */
-    char ch;
-    struct iovec vec;
-    vec.iov_base = &ch;
-    vec.iov_len = 1;
-    msg.msg_iov = &vec;
-    msg.msg_iovlen = 1;
-
-    union {
-        struct cmsghdr align;
-        char buf[CMSG_SPACE(sizeof(int))];
-    } u;
-
-    msg.msg_control = u.buf;
-    msg.msg_controllen = sizeof(u.buf);
-
-    struct cmsghdr *cmptr;
-    cmptr = CMSG_FIRSTHDR(&msg);
-
-    if (cmptr == NULL) {
-        fprintf(stderr, "ch-proxy/send_fd/CMSG_FIRSTHDR: failed to initialize msg_control\n");
-        exit(EXIT_FAILURE);
-    }
-
-    cmptr->cmsg_len = CMSG_LEN(sizeof(int));
-    cmptr->cmsg_level = SOL_SOCKET;
-    cmptr->cmsg_type = SCM_RIGHTS;
-    *((int*) CMSG_DATA(cmptr)) = fd;
-
-    return (sendmsg(dst_fd, &msg, 0));
-}

pkgs/ch-proxy/sendfd.c

@@ -0,0 +1,74 @@
+#include "sendfd.h"
+#include "sys/socket.h" /* cmsghdr */
+#include "stdio.h" /* perror */
+
+
+ssize_t send_fd(int dst_fd, int fd, const struct iovec *iov) {
+    struct msghdr msg = { 0 };
+
+    /* openssh expects to receive a dummy length=1 iovec? */
+    char ch = 0;
+    struct iovec vecDefault = { 0 };
+    vecDefault.iov_base = &ch;
+    vecDefault.iov_len = 1;
+    msg.msg_iov = iov == NULL ? &vecDefault : iov;
+    msg.msg_iovlen = 1;
+
+    union {
+        struct cmsghdr align;
+        char buf[CMSG_SPACE(sizeof(int))];
+    } u;
+
+    msg.msg_control = u.buf;
+    msg.msg_controllen = sizeof(u.buf);
+
+    struct cmsghdr *cmptr;
+    cmptr = CMSG_FIRSTHDR(&msg);
+
+    if (cmptr == NULL) {
+        perror("ch-proxy/send_fd/CMSG_FIRSTHDR: failed to initialize msg_control\n");
+    }
+
+    cmptr->cmsg_len = CMSG_LEN(sizeof(int));
+    cmptr->cmsg_level = SOL_SOCKET;
+    cmptr->cmsg_type = SCM_RIGHTS;
+    *((int*) CMSG_DATA(cmptr)) = fd;
+
+    return (sendmsg(dst_fd, &msg, 0));
+}
+
+int recv_fd(int sock, int flags) {
+    int out = -1;
+
+    struct msghdr msg = { 0 };
+    struct cmsghdr *cmsg = NULL;
+    struct iovec iov = { 0 };
+    char dummy = 0;
+
+    msg.msg_iov = &iov;
+    msg.msg_iovlen = 1;
+
+    iov.iov_base = &dummy;
+    iov.iov_len = sizeof(dummy);
+
+    union {
+        struct cmsghdr align;
+        char buf[CMSG_SPACE(sizeof(int))];
+    } u;
+
+    msg.msg_control = u.buf;
+    msg.msg_controllen = sizeof(u.buf);
+
+    int bytes = 0;
+    if ((bytes = recvmsg(sock, &msg, flags)) < 0) {
+        perror("recv_fd: recvmsg");
+        return -1;
+    }
+    for (cmsg = CMSG_FIRSTHDR(&msg); cmsg != NULL; cmsg = CMSG_NXTHDR(&msg, cmsg)) {
+        if (cmsg->cmsg_level != SOL_SOCKET) { continue; }
+        if (cmsg->cmsg_type != SCM_RIGHTS) { continue; }
+        if (CMSG_LEN(cmsg) < sizeof(out)) { continue; }
+        out = *(int*)CMSG_DATA(cmsg);
+    }
+    return out;
+}

pkgs/ch-proxy/sendfd.h

@@ -0,0 +1,27 @@
+#ifndef _CH_PROXY_SENFD
+#define _CH_PROXY_SENFD
+
+#include <stddef.h> /* size_t */
+#include <sys/types.h> /* ssize_t */
+#include <sys/uio.h> /* iovec */
+
+
+/* send_fd(chanFd, fd, *iov)
+ *
+ * chanFd: fd to sendmsg over;
+ * fd: fd to send;
+ * iov: extra data to send or NULL;
+ *
+ * returns: result of sendmsg,
+ * i.e. the number of bytes sent */
+ssize_t send_fd(int chanFd, int fd, const struct iovec *);
+
+/* recv_fd(chanFd, flags)
+ *
+ * chanFd: fd to recvmsg from;
+ * flags: recvmsg flags e.g. 0, or MSG_CMSG_CLOEXEC?
+ *
+ * returns: the received fd or -1 */
+int recv_fd(int chanFd, int flags);
+
+#endif /* _CH_PROXY_SENFD */

pkgs/taps/.envrc

@@ -0,0 +1 @@
+use nix ../../ -A pkgs.taps

pkgs/taps/main.c

@@ -0,0 +1,326 @@
+#define _GNU_SOURCE
+
+#include <stdlib.h> /* secure_getenv */
+#include <sys/socket.h>
+#include <unistd.h>
+#include <sys/un.h>
+#include <string.h>
+#include <stdbool.h>
+#include <ctype.h>
+#include <pwd.h>
+#include <fnmatch.h>
+#include <fcntl.h> /* open, O_NONBLOCK, &c */
+#include <err.h>
+#include <error.h>
+#include <errno.h>
+#include <sys/ioctl.h>
+#include <signal.h>
+
+#define __UAPI_DEF_IF_IFNAMSIZ 1
+#include <linux/if_tun.h>
+#include <linux/if.h>
+
+#include "sendfd.h"
+
+// From `man unix`
+#define SUN_PATH_SZ 108
+#define N_CONNS 16
+
+char *TEMP_PATHS[1024] = { 0 }; 
+int LAST_TEMP_PATH = -1;
+
+#define IFR_FLAGS_ALLOWED (IFF_NO_PI | IFF_TAP | IFF_TUN | IFF_VNET_HDR | IFF_MULTI_QUEUE | IFF_PERSIST)
+#define IFR_FLAGS_DEFAULT (IFF_NO_PI | IFF_TAP | IFF_VNET_HDR | IFF_PERSIST)
+
+#define PTR_OR_DIE(expr) TRUE_OR_DIE((expr) != NULL)
+#define DO_OR_DIE(expr) TRUE_OR_DIE((expr) != -1)
+#define TRUE_OR_DIE(expr, ...) TRUE_OR_(EXIT_FAILURE, expr, __VA_ARGS__)
+#define TRUE_OR_WARN(expr, ...) TRUE_OR_(0, expr, __VA_ARGS__)
+#define TRUE_OR_(status, expr, ...) \
+    do if (!(expr)) { \
+        error(status, errno, "Failed assertion: " #expr "." __VA_ARGS__); \
+    } while(false)
+
+struct allow_pattern {
+    // enum { USER = 1, GROUP = 2 } type;
+    // union { uid_t uid, gid_t gid } xid;
+    char *name;
+};
+struct allow_patterns {
+    size_t n;
+    struct allow_pattern *patterns;
+};
+
+/* Running on the same host, not caring for alignment */
+struct tap_request {
+    short ifrFlags; /* 0 to use defaults: IFF_TAP | IFF_NO_PI | IFF_VNET_HDR */
+    char name[IFNAMSIZ];
+};
+
+struct tap_reply {
+    enum { OK = 0, AUTH_ERROR = 1 } status;
+    char name[IFNAMSIZ];
+};
+
+int tuntap_alloc(char *dev, short openFlags, short ifrFlags, int *out_fd);
+
+bool match_mask(const char *test_addr, const char *expected_addr, const char *mask, int n) {
+    for (int octet = 0; octet < n; ++octet) {
+        if ((test_addr[octet] & mask[octet]) != expected_addr[octet]) {
+            return false;
+        }
+    }
+    return true;
+}
+
+/* 
+ * Adapted from spectrum's `mktuntap.c` (2019 Alyssa Ross <hi@alyssa.is>
+ * GPL-2.0-only), which in turn adapts `tun_alloc` from
+ * `linux/Documentation/networking/tuntap.rst`.
+ *
+ * ifrFlags: IFF_TUN   - TUN device (no Ethernet headers)
+ *           IFF_TAP   - TAP device
+ *
+ *           IFF_NO_PI - Do not provide packet information
+ */
+int tuntap_alloc(char *dev, short openFlags, short ifrFlags, int *out_fd) {
+  struct ifreq ifr = { 0 };
+  int fd = -1, err = 0;
+
+  DO_OR_DIE(fd = open("/dev/net/tun", openFlags));
+
+  if (dev != NULL) {
+    int devLen = strlen(dev);
+    if (devLen >= IFNAMSIZ) {
+      /* If client requests a name, we do want the entire name to fit */
+      errno = EINVAL;
+      return EINVAL;
+    }
+    strncpy(ifr.ifr_name, dev, IFNAMSIZ - 1);
+  }
+  ifr.ifr_flags = ifrFlags;
+
+  TRUE_OR_WARN((err = ioctl(fd, TUNSETIFF, (void *)&ifr)) == 0);
+  if (err != 0) {
+    close(fd);
+    return err;
+  }
+
+  strncpy(dev, ifr.ifr_name, IFNAMSIZ);
+  *out_fd = fd;
+  return 0;
+}
+
+int acceptRequests(const char *requestsPath, const struct allow_patterns *patterns) {
+    int listener;
+    struct sockaddr_un addr;
+    const int t = 1;
+
+    DO_OR_DIE(listener = socket(AF_UNIX, SOCK_SEQPACKET, 0));
+    DO_OR_DIE(setsockopt(listener, SOL_SOCKET, SO_PASSCRED, &t, sizeof(t)));
+
+    addr.sun_family = AF_UNIX;
+    strncpy(addr.sun_path, requestsPath, SUN_PATH_SZ - 1);
+    DO_OR_DIE (bind(listener, &addr, sizeof(addr)));
+    PTR_OR_DIE(TEMP_PATHS[++LAST_TEMP_PATH] = strdup(requestsPath));
+
+    DO_OR_DIE(listen(listener, N_CONNS));
+
+    for (;;) {
+        /* Already changed my mind about looking at ucred, but keeping the code around for now */
+        int sock = -1;
+        struct ucred cred = { 0 };
+        struct msghdr msg = { 0 };
+        struct cmsghdr *cmsg = NULL;
+        struct iovec iov = { 0 };
+        struct tap_request req = { 0 };
+
+        msg.msg_iov = &iov;
+        msg.msg_iovlen = 1;
+
+        iov.iov_base = &req;
+        iov.iov_len = sizeof(struct tap_request);
+
+        DO_OR_DIE((sock = accept(listener, NULL, NULL)));
+
+        TRUE_OR_DIE(recvmsg(sock, &msg, 0) > 0);
+        req.name[IFNAMSIZ] = 0;
+        for (cmsg = CMSG_FIRSTHDR(&msg); cmsg != NULL; cmsg = CMSG_NXTHDR(&msg, cmsg)) {
+            if (cmsg->cmsg_level != SOL_SOCKET) { continue; }
+            if (cmsg->cmsg_type != SCM_CREDENTIALS) { continue; }
+            if (CMSG_LEN(cmsg) < sizeof(struct ucred)) { continue; }
+            memcpy(&cred, CMSG_DATA(cmsg), sizeof(struct ucred));
+            break;
+        }
+
+        if (req.ifrFlags == 0) {
+            req.ifrFlags = IFR_FLAGS_DEFAULT;
+        }
+
+        bool allowed = false;
+        for (int i = 0; !allowed && i < patterns->n; ++i) {
+            bool ifnameOk = fnmatch(patterns->patterns[i].name, req.name, 0) == 0;
+            bool flagsOk = (req.ifrFlags & IFR_FLAGS_ALLOWED) == req.ifrFlags;
+            allowed = ifnameOk && flagsOk;
+        }
+
+        struct tap_reply reply = { 0 };
+        if (!allowed) { reply.status = AUTH_ERROR; }
+        if (allowed) {
+            /* O_CLOEXEC? */
+            int fd = -1;
+            TRUE_OR_DIE(tuntap_alloc(req.name, O_RDWR | O_NONBLOCK, req.ifrFlags, &fd) == 0);
+            struct iovec iov = { 0 };
+            iov.iov_base = &reply;
+            iov.iov_len = sizeof(struct tap_reply);
+            TRUE_OR_DIE(send_fd(sock, fd, &iov) > 0);
+            close(fd); 
+        }
+        close(sock);
+    }
+    close(listener);
+}
+
+struct allow_patterns parsePatterns(const char *raw) {
+    const size_t rawLen = strlen(raw);
+
+    size_t nPatterns = 0;
+    for (int i = 0; i < rawLen; ++i) {
+        const int start = i;
+        if (isspace(raw[i])) { continue; }
+        for (; i < rawLen && !isspace(raw[i]); ++i) { }
+        if (start < i) { ++nPatterns; }
+    }
+
+    struct allow_pattern *patterns = NULL;
+    PTR_OR_DIE(patterns = calloc(nPatterns, sizeof(struct allow_pattern)));
+
+    int iPattern = 0;
+    for (int i = 0; i < rawLen; ++i) {
+        if (isspace(raw[i])) { continue; }
+        /* used to have per-group/per-user patterns, "u:$username:$pattern", &c - gone */
+        {
+            const int start = i;
+            for (; i < rawLen && !isspace(raw[i]); ++i) { }
+            if (start < i) {
+                PTR_OR_DIE(patterns[iPattern].name = strndup(&raw[start], i - start));
+                iPattern += 1;
+            }
+        }
+    }
+    struct allow_patterns out = {
+        .n = nPatterns,
+        .patterns = patterns
+    };
+    return out;
+}
+
+int get(const char *servePath, const char *ifname, short ifrFlags) {
+    /* TODO: sock: move out */
+    int sock;
+    struct sockaddr_un addr;
+
+    DO_OR_DIE(sock = socket(AF_UNIX, SOCK_SEQPACKET, 0));
+
+    addr.sun_family = AF_UNIX;
+    strncpy(addr.sun_path, servePath, SUN_PATH_SZ - 1);
+    DO_OR_DIE (connect(sock, &addr, sizeof(addr)));
+
+    struct msghdr msg = { 0 };
+    struct cmsghdr *cmsg = NULL;
+    struct iovec iov = { 0 };
+    struct tap_request req = { 0 };
+    strncpy(req.name, ifname, IFNAMSIZ - 1);
+    req.ifrFlags = ifrFlags;
+
+    msg.msg_iov = &iov;
+    msg.msg_iovlen = 1;
+
+    iov.iov_base = &req;
+    iov.iov_len = sizeof(struct tap_request);
+
+    TRUE_OR_DIE(sendmsg(sock, &msg, 0) > 0);
+
+    int tunFd = -1;
+    DO_OR_DIE(tunFd = recv_fd(sock, 0));
+    close(sock);
+    return tunFd;
+}
+
+void cleanup(int signo, siginfo_t *info, void *_context) {
+    for (int i = 0; i <= LAST_TEMP_PATH; ++i) {
+        TRUE_OR_DIE(unlink(TEMP_PATHS[i]) != -1 || errno == ENOENT);
+    }
+    if (signo == SIGINT) {
+        exit(EXIT_SUCCESS);
+    }
+    errx(EXIT_FAILURE, "Exiting with signal %d", signo);
+}
+
+int main(int argc, char **argv) {
+    struct sigaction act = { 0 };
+    act.sa_flags = SA_SIGINFO;
+    act.sa_sigaction = cleanup;
+    DO_OR_DIE(sigaction(SIGINT, &act, NULL));
+    DO_OR_DIE(sigaction(SIGSEGV, &act, NULL));
+
+    bool cmdServe = false;
+    bool cmdPass = false;
+    char *ifname = "vt%d";
+
+    char **rest = argv + 1;
+    char **end = argv + argc;
+
+    TRUE_OR_DIE(argc > 1);
+    if (strcmp(rest[0], "serve") == 0) {
+        cmdServe = true;
+        ++rest;
+    } else if (strcmp(rest[0], "pass") == 0) {
+        cmdPass = true;
+        ++rest;
+        for (; rest != end && rest[0][0] == '-'; ++rest) {
+            if (strcmp(rest[0], "--")) { break; }
+            else if (strncmp(rest[0], "--ifname=", sizeof("--ifname="))) {
+                ifname = rest[0] + sizeof("--ifname=");
+            }
+        }
+    } else {
+        error(EINVAL, EINVAL, "no subcommand \"%s\"", rest[0]);
+    }
+
+    int nextArgc = argc - (rest - argv);
+    char * const* nextArgv = rest;
+
+    const char *patternsRaw = secure_getenv("TAPS_ALLOW");
+    if (patternsRaw == NULL) {
+        patternsRaw = "*";
+    }
+
+    struct allow_patterns patterns = { 0 };
+    if (cmdServe) {
+        PTR_OR_DIE((patterns = parsePatterns(patternsRaw)).patterns);
+    }
+
+    const char *servePath = secure_getenv("TAPS_SOCK");
+    if (servePath == NULL) {
+        servePath = "taps.sock";
+    }
+
+    if (cmdServe) {
+        acceptRequests(servePath, &patterns);
+    } else if (cmdPass) {
+        TRUE_OR_DIE(nextArgc > 0);
+        int fd = -1;
+        DO_OR_DIE(fd = get(servePath, ifname, 0));
+        if (fd != 3) {
+            DO_OR_DIE(dup2(fd, 3));
+            close(fd);
+            fd = 3;
+        }
+        DO_OR_DIE(execvp(nextArgv[0], nextArgv));
+    } else {
+        error(EINVAL, EINVAL, "subcommand args");
+    }
+
+    return 0;
+}

pkgs/taps/meson.build

@@ -0,0 +1,4 @@
+project('taps', 'c')
+
+sendfd = dependency('sendfd')
+executable('taps', 'main.c', dependencies: [sendfd], install: true)

pkgs/taps/package.nix

@@ -0,0 +1,44 @@
+{
+  lib,
+  stdenv,
+  meson,
+  pkg-config,
+  rustc,
+  ninja,
+  ch-proxy,
+}:
+
+stdenv.mkDerivation {
+  pname = "taps";
+  version = "0.0.0";
+  src =
+    let
+      fs = lib.fileset;
+    in
+    fs.toSource {
+      root = ./.;
+      fileset = fs.unions [
+        ./meson.build
+        ./main.c
+      ];
+    };
+
+  nativeBuildInputs = [
+    ninja
+    meson
+    pkg-config
+    rustc
+  ];
+  buildInputs = [ ch-proxy ];
+}
+# { lib, rustPlatform }:
+#
+# rustPlatform.buildRustPackage {
+#     pname = "taps";
+#     version = "0.0.0";
+#     src = let fs = lib.filesystem; in fs.toSource {
+#         root = ./.;
+#         fileset = fs.unions [
+#         ];
+#     };
+# };

profiles/uvms-guest.nix

@@ -41,6 +41,7 @@ in
         volumes = [
           {
             image = "swapfile.img";
+            serial = "swapfiles";
             mountPoint = "/var/swapfiles";
             size = 1024;
           }

profiles/uvms-users.nix

@@ -29,7 +29,6 @@ in
   };
   config = mergeIf cfg.enable [
     {
-      services.getty.autologinUser = "user";
       security.sudo.wheelNeedsPassword = false;
       users.mutableUsers = false;
       users.users.user = {

shell.nix

@@ -0,0 +1,13 @@
+with import <nixpkgs> { };
+
+mkShell.override { stdenv = stdenvNoCC; } {
+  packages = map lib.getBin [
+    cloud-hypervisor
+    virtiofsd
+    crosvm # virtio-gpu
+    npins
+  ] ++ [
+    man-pages
+    linux-manual
+  ];
+}