runc exec: implement --cgroup

TODO:
 - commit message (this one);
 - tests for cgroup v1;
 - yet moar tests;
 - ...
 - PROFIT!

Signed-off-by: Kir Kolyshkin <[email protected]>

Author: kolyshkin

exec.go

@@ -89,6 +89,10 @@ following will output a list of processes running in the container:
 			Name:  "preserve-fds",
 			Usage: "Pass N additional file descriptors to the container (stdio + $LISTEN_FDS + N in total)",
 		},
+		cli.StringFlag{
+			Name:  "cgroup",
+			Usage: "run the process in an (existing) sub-cgroup",
+		},
 	},
 	Action: func(context *cli.Context) error {
 		if err := checkArgs(context, 1, minArgs); err != nil {
@@ -148,6 +152,7 @@ func execProcess(context *cli.Context) (int, error) {
 		init:            false,
 		preserveFDs:     context.Int("preserve-fds"),
 		logLevel:        logLevel,
+		cgroup:          context.String("cgroup"),
 	}
 	return r.run(p)
 }

libcontainer/container_linux.go

@@ -12,6 +12,7 @@ import (
 	"net"
 	"os"
 	"os/exec"
+	"path"
 	"path/filepath"
 	"reflect"
 	"strconv"
@@ -555,7 +556,7 @@ func (c *linuxContainer) newSetnsProcess(p *Process, cmd *exec.Cmd, messageSockP
 	if err != nil {
 		return nil, err
 	}
-	return &setnsProcess{
+	proc := &setnsProcess{
 		cmd:             cmd,
 		cgroupPaths:     state.CgroupPaths,
 		rootlessCgroups: c.config.RootlessCgroups,
@@ -567,7 +568,16 @@ func (c *linuxContainer) newSetnsProcess(p *Process, cmd *exec.Cmd, messageSockP
 		process:         p,
 		bootstrapData:   data,
 		initProcessPid:  state.InitProcessPid,
-	}, nil
+	}
+	if p.Cgroup != "" {
+		for k := range proc.cgroupPaths {
+			proc.cgroupPaths[k] = path.Join(proc.cgroupPaths[k], p.Cgroup)
+		}
+		// Do not try to join init process's cgroup as a fallback
+		// (see (*setnsProcess).start).
+		proc.initProcessPid = 0
+	}
+	return proc, nil
 }
 
 func (c *linuxContainer) newInitConfig(process *Process) *initConfig {

libcontainer/process.go

@@ -80,6 +80,10 @@ type Process struct {
 	ops processOperations
 
 	LogLevel string
+
+	// Cgroup specifies a (sub-cgroup of a container) to run the process in.
+	// If empty, the default top-level container's cgroup is used.
+	Cgroup string
 }
 
 // Wait waits for the process to exit.

libcontainer/process_linux.go

@@ -134,7 +134,7 @@ func (p *setnsProcess) start() (retErr error) {
 			// On cgroup v2 + nesting + domain controllers, WriteCgroupProc may fail with EBUSY.
 			// https://github.com/opencontainers/runc/issues/2356#issuecomment-621277643
 			// Try to join the cgroup of InitProcessPid.
-			if cgroups.IsCgroup2UnifiedMode() {
+			if cgroups.IsCgroup2UnifiedMode() && p.initProcessPid != 0 {
 				initProcCgroupFile := fmt.Sprintf("/proc/%d/cgroup", p.initProcessPid)
 				initCg, initCgErr := cgroups.ParseCgroupFile(initProcCgroupFile)
 				if initCgErr == nil {

man/runc-exec.8.md

@@ -59,6 +59,13 @@ multiple times.
 : Pass _N_ additional file descriptors to the container (**stdio** +
 **$LISTEN_FDS** + _N_ in total). Default is **0**.
 
+**--cgroup** _path_
+: Execute a process in a sub-cgroup. If the specified cgroup does not exist, an
+error is returned. Default is empty path, which means to use container's top
+level cgroup. Note that for cgroup v2, by default if **runc exec** can't join
+the top level cgroup, it falls back to joining whatever cgroup container's init
+is in. This fallback can be disabled by using **--cgroup /**.
+
 # EXAMPLES
 If the container can run **ps**(1) command, the following
 will output a list of processes running in the container:

tests/integration/exec.bats

@@ -167,3 +167,68 @@ function check_exec_debug() {
 	[[ "${output}" == *"level=debug"* ]]
 	check_exec_debug "$output"
 }
+
+@test "runc exec --cgroup subcgroup [v2]" {
+	requires root cgroups_v2
+
+	set_cgroups_path
+	set_cgroup_mount_writable
+
+	__runc run -d --console-socket "$CONSOLE_SOCKET" test_busybox
+	testcontainer test_busybox running
+
+	# Check we can't join non-existing subcgroup.
+	runc exec --cgroup nonexistent test_busybox cat /proc/self/cgroup
+	[ "$status" -ne 0 ]
+	[[ "$output" == *" adding pid "*"/nonexistent/cgroup.procs: no such file "* ]]
+
+	# Check we can join top-level cgroup (implicit).
+	runc exec test_busybox grep '^0::/$' /proc/self/cgroup
+	[ "$status" -eq 0 ]
+
+	# Check we can join top-level cgroup (explicit).
+	runc exec --cgroup / test_busybox grep '^0::/$' /proc/self/cgroup
+	[ "$status" -eq 0 ]
+
+	# Now move "init" to a subcgroup, and check it was moved.
+	runc exec test_busybox sh -euc "mkdir /sys/fs/cgroup/foobar \
+		&& echo 1 > /sys/fs/cgroup/foobar/cgroup.procs \
+		&& grep -w foobar /proc/1/cgroup"
+	[ "$status" -eq 0 ]
+
+	# The following part is taken from
+	# @test "runc exec (cgroup v2 + init process in non-root cgroup) succeeds"
+
+	# The init process is now in "/foo", but an exec process can still
+	# join "/" because we haven't enabled any domain controller yet.
+	runc exec test_busybox grep '^0::/$' /proc/self/cgroup
+	[ "$status" -eq 0 ]
+
+	# Turn on a domain controller (memory).
+	runc exec test_busybox sh -euc 'echo $$ > /sys/fs/cgroup/foobar/cgroup.procs; echo +memory > /sys/fs/cgroup/cgroup.subtree_control'
+	[ "$status" -eq 0 ]
+
+	# An exec process can no longer join "/" after turning on a domain
+	# controller.  Check that cgroup v2 fallback to init cgroup works.
+	runc exec test_busybox sh -euc "cat /proc/self/cgroup && grep '^0::/foobar$' /proc/self/cgroup"
+	[ "$status" -eq 0 ]
+
+	# Check that --cgroup / disables the init cgroup fallback.
+	runc exec --cgroup / test_busybox true
+	[ "$status" -ne 0 ]
+	[[ "$output" == *" adding pid "*" to cgroups"*"/cgroup.procs: device or resource busy"* ]]
+
+	# Check that explicit --cgroup foobar works.
+	runc exec --cgroup foobar test_busybox grep '^0::/foobar$' /proc/self/cgroup
+	[ "$status" -eq 0 ]
+
+	# Check all processes is in foobar (this check is redundant).
+	runc exec --cgroup foobar test_busybox sh -euc '! grep -vwH foobar /proc/*/cgroup'
+	[ "$status" -eq 0 ]
+
+	# Add a second subcgroup, check we're in it.
+	runc exec --cgroup foobar test_busybox mkdir /sys/fs/cgroup/second
+	[ "$status" -eq 0 ]
+	runc exec --cgroup second test_busybox grep -w second /proc/self/cgroup
+	[ "$status" -eq 0 ]
+}

utils_linux.go

@@ -258,6 +258,7 @@ type runner struct {
 	notifySocket    *notifySocket
 	criuOpts        *libcontainer.CriuOpts
 	logLevel        string
+	cgroup          string
 }
 
 func (r *runner) run(config *specs.Process) (int, error) {
@@ -277,6 +278,7 @@ func (r *runner) run(config *specs.Process) (int, error) {
 	// Populate the fields that come from runner.
 	process.Init = r.init
 	process.LogLevel = r.logLevel
+	process.Cgroup = r.cgroup
 	if len(r.listenFDs) > 0 {
 		process.Env = append(process.Env, "LISTEN_FDS="+strconv.Itoa(len(r.listenFDs)), "LISTEN_PID=1")
 		process.ExtraFiles = append(process.ExtraFiles, r.listenFDs...)