libct/nsenter: namespace the bindfd shuffle

Processes can watch /proc/self/mounts or /mountinfo, and the kernel
will notify them whenever the namespace's mount table is modified. The
notified process still needs to read and parse the mountinfo to
determine what changed once notified. Many such processes, including
udisksd and SystemD < v248, make no attempt to rate-limit their
mountinfo notifications. This tends to not be a problem on many systems,
where mount tables are small and mounting and unmounting is uncommon.
Every runC exec which successfully uses the try_bindfd container-escape
mitigation performs two mount()s and one umount() in the host's mount
namespace, causing any mount-watching processes to wake up and parse the
mountinfo file three times in a row. Consequently, using 'exec' health
checks on containers has a larger-than-expected impact on system load
when such mount-watching daemons are running. Furthermore, the size of
the mount table in the host's mount namespace tends to be proportional
to the number of OCI containers as a unique mount is required for the
rootfs of each container. Therefore, on systems with mount-watching
processes, the system load increases *quadratically* with the number of
running containers which use health checks!

Prevent runC from incidentally modifying the host's mount namespace for
container-escape mitigations by setting up the mitigation in a private
mount namespace.

Signed-off-by: Cory Snider <[email protected]>

Author: corhere

libcontainer/nsenter/cloned_binary.c

@@ -49,14 +49,20 @@
 #include <fcntl.h>
 #include <errno.h>
 
+#include <sched.h>
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <sys/statfs.h>
 #include <sys/vfs.h>
 #include <sys/mman.h>
 #include <sys/mount.h>
 #include <sys/sendfile.h>
+#include <sys/socket.h>
 #include <sys/syscall.h>
+#include <sys/wait.h>
+
+#include "ipc.h"
+#include "log.h"
 
 /* Use our own wrapper for memfd_create. */
 #ifndef SYS_memfd_create
@@ -394,6 +400,98 @@ static int seal_execfd(int *fd, int fdtype)
 	return -1;
 }
 
+struct bindfd_child_args {
+	int sockfd;
+	const char *mount_target;
+};
+
+static int bindfd_in_subprocess(void *arg)
+{
+	/*
+	 * In the interests of efficiency (read: minimizing the syscall count)
+	 * and conciseness, no attempt is made to release resources which would
+	 * be cleaned up automatically on process exit, i.e. when this function
+	 * returns. This includes filesystem mounts, as this function is
+	 * executed in a dedicated mount namespace.
+	 */
+
+	/*
+	 * For obvious reasons this won't work in rootless mode because we
+	 * haven't created a userns -- but getting that to work will be a bit
+	 * complicated and it's only worth doing if someone actually needs it.
+	 */
+	if (mount("none", "/", NULL, MS_PRIVATE | MS_REC, NULL) < 0)
+		return errno;
+	/*
+	 * The kernel refuses to bind-mount from the magic symlink when the
+	 * process has been cloned (or unshared) into a new mount namespace,
+	 * at least on Linux 4.4.
+	 */
+	char linkbuf[PATH_MAX + 1] = { 0 };
+	ssize_t linkpathlen = readlink("/proc/self/exe", linkbuf, sizeof(linkbuf));
+	if (linkpathlen < 0)
+		return errno;
+	if (linkpathlen == sizeof(linkbuf)) {
+		/*
+		 * The link path is longer than PATH_MAX, and the contents of
+		 * linkbuf might have been truncated. A truncated path could
+		 * happen to be a valid path to a different file, which could
+		 * allow for local privilege escalation if we were to exec it.
+		 * The mount syscall doesn't accept paths longer than PATH_MAX,
+		 * anyway.
+		 */
+		return ENAMETOOLONG;
+	}
+
+	const struct bindfd_child_args *args = arg;
+	if (mount(linkbuf, args->mount_target, "", MS_BIND, "") < 0)
+		return errno;
+	if (mount("", args->mount_target, "", MS_REMOUNT | MS_BIND | MS_RDONLY, "") < 0)
+		return errno;
+
+	/* Get read-only handle that we're sure can't be made read-write. */
+	int fd = open(args->mount_target, O_PATH | O_CLOEXEC);
+	if (fd < 0)
+		return errno;
+
+	/*
+	 * Make sure the MNT_DETACH works, otherwise we could get remounted
+	 * read-write and that would be quite bad (the fd would be made
+	 * read-write too, invalidating the protection).
+	 */
+	if (umount2(args->mount_target, MNT_DETACH) < 0)
+		return errno;
+
+	if (send_fd(args->sockfd, fd) < 0)
+		return errno;
+	return 0;
+}
+
+static int spawn_bindfd_child(const struct bindfd_child_args *args) __attribute__((noinline));
+static int spawn_bindfd_child(const struct bindfd_child_args *args)
+{
+	/*
+	 * Carve out a chunk of our call stack for the child process to use as
+	 * we can be sure it is correctly mapped for use as stack. (Technically
+	 * only the libc clone() wrapper writes to it; the child process
+	 * operates on a copy of the parent's virtual memory space unless we use
+	 * the CLONE_VM flag.)
+	 */
+	char stack[128 * 1024] __attribute__((aligned(16)));
+	return clone(bindfd_in_subprocess,
+		     /*
+		      * Assume stack grows down, as HP-PA, the only Linux
+		      * platform where stack grows up, is obsolete.
+		      */
+		     stack + sizeof(stack),
+		     /*
+		      * Suspend the parent process until the child has exited to
+		      * save an unnecessary context switch as we'd just be
+		      * waiting for the child process to exit anyway.
+		      */
+		     CLONE_NEWNS | CLONE_VFORK, (void *)args);
+}
+
 static int try_bindfd(void)
 {
 	int fd, ret = -1;
@@ -415,32 +513,51 @@ static int try_bindfd(void)
 	close(fd);
 
 	/*
-	 * For obvious reasons this won't work in rootless mode because we haven't
-	 * created a userns+mntns -- but getting that to work will be a bit
-	 * complicated and it's only worth doing if someone actually needs it.
+	 * Daemons such as systemd and udisks2 watch /proc/self/mountinfo and
+	 * re-parse it on every change, which gets expensive when the mount table
+	 * is large and/or changes frequently. Set up the bind-mounts in a new,
+	 * private mount namespace so as not to wake up those processes every
+	 * time we nsexec into a container. We clone a child process into a new
+	 * mount namespace to do the dirty work so the side effects of unsharing
+	 * the mount namespace do not leak into the current process.
 	 */
-	ret = -EPERM;
-	if (mount("/proc/self/exe", template, "", MS_BIND, "") < 0)
-		goto out;
-	if (mount("", template, "", MS_REMOUNT | MS_BIND | MS_RDONLY, "") < 0)
-		goto out_umount;
-
-	/* Get read-only handle that we're sure can't be made read-write. */
-	ret = open(template, O_PATH | O_CLOEXEC);
+	int sock[2];
+	if (socketpair(AF_LOCAL, SOCK_STREAM, 0, sock) < 0)
+		goto cleanup_unlink;
+
+	struct bindfd_child_args args = {
+		.sockfd = sock[0],
+		.mount_target = template,
+	};
+	int cpid = spawn_bindfd_child(&args);
+	close(sock[0]);
+	if (cpid < 0)
+		goto cleanup_socketpair;
+
+	int wstatus = 0;
+	if (waitpid(cpid, &wstatus, __WCLONE) < 0)
+		bail("error waiting for bindfd child process to exit");
+	if (WIFEXITED(wstatus)) {
+		if ((errno = WEXITSTATUS(wstatus)))
+			goto cleanup_socketpair;
+	} else if (WIFSIGNALED(wstatus)) {
+		int sig = WTERMSIG(wstatus);
+		bail("bindfd child process terminated by signal %d (%s)", sig, strsignal(sig));
+	} else {
+		/* Should never happen... */
+		bail("unexpected waitpid() status for bindfd child process: 0x%x", wstatus);
+	}
 
-out_umount:
-	/*
-	 * Make sure the MNT_DETACH works, otherwise we could get remounted
-	 * read-write and that would be quite bad (the fd would be made read-write
-	 * too, invalidating the protection).
-	 */
-	if (umount2(template, MNT_DETACH) < 0) {
-		if (ret >= 0)
-			close(ret);
-		ret = -ENOTRECOVERABLE;
+	ret = receive_fd(sock[1]);
+	if (fcntl(ret, F_SETFD, O_PATH | O_CLOEXEC) < 0) {
+		ret = -1;
+		goto cleanup_socketpair;
 	}
 
-out:
+cleanup_socketpair:
+	close(sock[1]);
+
+cleanup_unlink:
 	/*
 	 * We don't care about unlink errors, the worst that happens is that
 	 * there's an empty file left around in STATEDIR.

libcontainer/nsenter/ipc.c

@@ -1,4 +1,5 @@
 #define _GNU_SOURCE
+#include <alloca.h>
 #include <stdlib.h>
 #include <string.h>
 #include <sys/socket.h>
@@ -54,9 +55,8 @@ int receive_fd(int sockfd)
 	return ret;
 }
 
-void send_fd(int sockfd, int fd)
+int send_fd(int sockfd, int fd)
 {
-	int bytes_written;
 	struct msghdr msg = { };
 	struct cmsghdr *cmsg;
 	struct iovec iov[1] = { };
@@ -71,11 +71,7 @@ void send_fd(int sockfd, int fd)
 	/* We send only one fd as specified by cmsg->cmsg_len below, even
 	 * though msg.msg_controllen might have more space due to alignment. */
 	msg.msg_controllen = CMSG_SPACE(sizeof(int));
-	msg.msg_control = malloc(msg.msg_controllen);
-	if (msg.msg_control == NULL) {
-		bail("Can't allocate memory to send fd.");
-	}
-
+	msg.msg_control = alloca(msg.msg_controllen);
 	memset(msg.msg_control, 0, msg.msg_controllen);
 
 	cmsg = CMSG_FIRSTHDR(&msg);
@@ -84,10 +80,5 @@ void send_fd(int sockfd, int fd)
 	cmsg->cmsg_len = CMSG_LEN(sizeof(int));
 	memcpy(CMSG_DATA(cmsg), &fd, sizeof(int));
 
-	bytes_written = sendmsg(sockfd, &msg, 0);
-
-	free(msg.msg_control);
-
-	if (bytes_written != 1)
-		bail("failed to send fd %d via unix socket %d", fd, sockfd);
+	return sendmsg(sockfd, &msg, 0);
 }

libcontainer/nsenter/ipc.h

@@ -2,6 +2,11 @@
 #define NSENTER_IPC_H
 
 int receive_fd(int sockfd);
-void send_fd(int sockfd, int fd);
+
+/*
+ * send_fd passes the open file descriptor fd to another process via the UNIX
+ * domain socket sockfd. The return value of the sendmsg(2) call is returned.
+ */
+int send_fd(int sockfd, int fd);
 
 #endif /* NSENTER_IPC_H */

libcontainer/nsenter/nsexec.c

@@ -600,7 +600,8 @@ void send_mountsources(int sockfd, pid_t child, char *mountsources, size_t mount
 		if (fd < 0)
 			bail("failed to open mount source %s", mountsources);
 
-		send_fd(sockfd, fd);
+		if (send_fd(sockfd, fd) < 0)
+			bail("failed to send fd %d via unix socket %d", fd, sockfd);
 
 		ret = close(fd);
 		if (ret != 0)