Skip to content

Commit 0147921

Browse files
mrunalpmrunalp
authored
Merge pull request #3546 from kolyshkin/criu-add-ignore-cgroup
checkpoint/restore: implement --manage-cgroups-mode ignore
2 parents a1c51c5 + c4aa452 commit 0147921

File tree

7 files changed

+145
-101
lines changed

7 files changed

+145
-101
lines changed

‎checkpoint.go‎

Lines changed: 57 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ checkpointed.`,
3838
cli.StringFlag{Name: "page-server", Value: "", Usage: "ADDRESS:PORT of the page server"},
3939
cli.BoolFlag{Name: "file-locks", Usage: "handle file locks, for safety"},
4040
cli.BoolFlag{Name: "pre-dump", Usage: "dump container's memory information only, leave the container running after this"},
41-
cli.StringFlag{Name: "manage-cgroups-mode", Value: "", Usage: "cgroups mode: 'soft' (default), 'full' and 'strict'"},
41+
cli.StringFlag{Name: "manage-cgroups-mode", Value: "", Usage: "cgroups mode: soft|full|strict|ignore (default: soft)"},
4242
cli.StringSliceFlag{Name: "empty-ns", Usage: "create a namespace, but don't restore its properties"},
4343
cli.BoolFlag{Name: "auto-dedup", Usage: "enable auto deduplication of memory images"},
4444
},
@@ -67,17 +67,6 @@ checkpointed.`,
6767
return err
6868
}
6969

70-
// these are the mandatory criu options for a container
71-
if err := setPageServer(context, options); err != nil {
72-
return err
73-
}
74-
if err := setManageCgroupsMode(context, options); err != nil {
75-
return err
76-
}
77-
if err := setEmptyNsMask(context, options); err != nil {
78-
return err
79-
}
80-
8170
err = container.Checkpoint(options)
8271
if err == nil && !(options.LeaveRunning || options.PreDump) {
8372
// Destroy the container unless we tell CRIU to keep it.
@@ -119,59 +108,80 @@ func prepareImagePaths(context *cli.Context) (string, string, error) {
119108
return imagePath, parentPath, nil
120109
}
121110

122-
func setPageServer(context *cli.Context, options *libcontainer.CriuOpts) error {
123-
// xxx following criu opts are optional
124-
// The dump image can be sent to a criu page server
111+
func criuOptions(context *cli.Context) (*libcontainer.CriuOpts, error) {
112+
imagePath, parentPath, err := prepareImagePaths(context)
113+
if err != nil {
114+
return nil, err
115+
}
116+
117+
opts := &libcontainer.CriuOpts{
118+
ImagesDirectory: imagePath,
119+
WorkDirectory: context.String("work-path"),
120+
ParentImage: parentPath,
121+
LeaveRunning: context.Bool("leave-running"),
122+
TcpEstablished: context.Bool("tcp-established"),
123+
ExternalUnixConnections: context.Bool("ext-unix-sk"),
124+
ShellJob: context.Bool("shell-job"),
125+
FileLocks: context.Bool("file-locks"),
126+
PreDump: context.Bool("pre-dump"),
127+
AutoDedup: context.Bool("auto-dedup"),
128+
LazyPages: context.Bool("lazy-pages"),
129+
StatusFd: context.Int("status-fd"),
130+
LsmProfile: context.String("lsm-profile"),
131+
LsmMountContext: context.String("lsm-mount-context"),
132+
}
133+
134+
// CRIU options below may or may not be set.
135+
125136
if psOpt := context.String("page-server"); psOpt != "" {
126137
address, port, err := net.SplitHostPort(psOpt)
127138

128139
if err != nil || address == "" || port == "" {
129-
return errors.New("Use --page-server ADDRESS:PORT to specify page server")
140+
return nil, errors.New("Use --page-server ADDRESS:PORT to specify page server")
130141
}
131142
portInt, err := strconv.Atoi(port)
132143
if err != nil {
133-
return errors.New("Invalid port number")
144+
return nil, errors.New("Invalid port number")
134145
}
135-
options.PageServer = libcontainer.CriuPageServerInfo{
146+
opts.PageServer = libcontainer.CriuPageServerInfo{
136147
Address: address,
137148
Port: int32(portInt),
138149
}
139150
}
140-
return nil
141-
}
142151

143-
func setManageCgroupsMode(context *cli.Context, options *libcontainer.CriuOpts) error {
144-
if cgOpt := context.String("manage-cgroups-mode"); cgOpt != "" {
145-
switch cgOpt {
146-
case "soft":
147-
options.ManageCgroupsMode = criu.CriuCgMode_SOFT
148-
case "full":
149-
options.ManageCgroupsMode = criu.CriuCgMode_FULL
150-
case "strict":
151-
options.ManageCgroupsMode = criu.CriuCgMode_STRICT
152-
default:
153-
return errors.New("Invalid manage cgroups mode")
154-
}
152+
switch context.String("manage-cgroups-mode") {
153+
case "":
154+
// do nothing
155+
case "soft":
156+
opts.ManageCgroupsMode = criu.CriuCgMode_SOFT
157+
case "full":
158+
opts.ManageCgroupsMode = criu.CriuCgMode_FULL
159+
case "strict":
160+
opts.ManageCgroupsMode = criu.CriuCgMode_STRICT
161+
case "ignore":
162+
opts.ManageCgroupsMode = criu.CriuCgMode_IGNORE
163+
default:
164+
return nil, errors.New("Invalid manage-cgroups-mode value")
155165
}
156-
return nil
157-
}
158166

159-
var namespaceMapping = map[specs.LinuxNamespaceType]int{
160-
specs.NetworkNamespace: unix.CLONE_NEWNET,
161-
}
162-
163-
func setEmptyNsMask(context *cli.Context, options *libcontainer.CriuOpts) error {
164-
/* Runc doesn't manage network devices and their configuration */
167+
// runc doesn't manage network devices and their configuration.
165168
nsmask := unix.CLONE_NEWNET
166169

167-
for _, ns := range context.StringSlice("empty-ns") {
168-
f, exists := namespaceMapping[specs.LinuxNamespaceType(ns)]
169-
if !exists {
170-
return fmt.Errorf("namespace %q is not supported", ns)
170+
if context.IsSet("empty-ns") {
171+
namespaceMapping := map[specs.LinuxNamespaceType]int{
172+
specs.NetworkNamespace: unix.CLONE_NEWNET,
173+
}
174+
175+
for _, ns := range context.StringSlice("empty-ns") {
176+
f, exists := namespaceMapping[specs.LinuxNamespaceType(ns)]
177+
if !exists {
178+
return nil, fmt.Errorf("namespace %q is not supported", ns)
179+
}
180+
nsmask |= f
171181
}
172-
nsmask |= f
173182
}
174183

175-
options.EmptyNs = uint32(nsmask)
176-
return nil
184+
opts.EmptyNs = uint32(nsmask)
185+
186+
return opts, nil
177187
}

‎libcontainer/container_linux.go‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1560,11 +1560,8 @@ func (c *Container) criuApplyCgroups(pid int, req *criurpc.CriuReq) error {
15601560
return err
15611561
}
15621562

1563-
if cgroups.IsCgroup2UnifiedMode() {
1564-
return nil
1565-
}
1566-
// the stuff below is cgroupv1-specific
1567-
1563+
// TODO(@kolyshkin): should we use c.cgroupManager.GetPaths()
1564+
// instead of reading /proc/pid/cgroup?
15681565
path := fmt.Sprintf("/proc/%d/cgroup", pid)
15691566
cgroupsPaths, err := cgroups.ParseCgroupFile(path)
15701567
if err != nil {

‎man/runc-checkpoint.8.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ together with **criu lazy-pages**. See
5757
: Do a pre-dump, i.e. dump container's memory information only, leaving the
5858
container running. See [criu iterative migration](https://criu.org/Iterative_migration).
5959

60-
**--manage-cgroups-mode** **soft**|**full**|**strict**.
60+
**--manage-cgroups-mode** **soft**|**full**|**strict**|**ignore**.
6161
: Cgroups mode. Default is **soft**. See
6262
[criu --manage-cgroups option](https://criu.org/CLI/opt/--manage-cgroups).
6363

‎man/runc-restore.8.md‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,10 +37,15 @@ image files directory.
3737
: Allow checkpoint/restore of file locks. See
3838
[criu --file-locks option](https://criu.org/CLI/opt/--file-locks).
3939

40-
**--manage-cgroups-mode** **soft**|**full**|**strict**.
40+
**--manage-cgroups-mode** **soft**|**full**|**strict**|**ignore**.
4141
: Cgroups mode. Default is **soft**. See
4242
[criu --manage-cgroups option](https://criu.org/CLI/opt/--manage-cgroups).
4343

44+
: In particular, to restore the container into a different cgroup,
45+
**--manage-cgroups-mode ignore** must be used during both
46+
**checkpoint** and **restore**, and the _container_id_ (or
47+
**cgroupsPath** property in OCI config, if set) must be changed.
48+
4449
**--bundle**|**-b** _path_
4550
: Path to the root of the bundle directory. Default is current directory.
4651

‎restore.go‎

Lines changed: 1 addition & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ package main
33
import (
44
"os"
55

6-
"github.com/opencontainers/runc/libcontainer"
76
"github.com/opencontainers/runc/libcontainer/userns"
87
"github.com/sirupsen/logrus"
98
"github.com/urfave/cli"
@@ -53,7 +52,7 @@ using the runc checkpoint command.`,
5352
cli.StringFlag{
5453
Name: "manage-cgroups-mode",
5554
Value: "",
56-
Usage: "cgroups mode: 'soft' (default), 'full' and 'strict'",
55+
Usage: "cgroups mode: soft|full|strict|ignore (default: soft)",
5756
},
5857
cli.StringFlag{
5958
Name: "bundle, b",
@@ -113,9 +112,6 @@ using the runc checkpoint command.`,
113112
if err != nil {
114113
return err
115114
}
116-
if err := setEmptyNsMask(context, options); err != nil {
117-
return err
118-
}
119115
status, err := startContainer(context, CT_ACT_RESTORE, options)
120116
if err != nil {
121117
return err
@@ -126,27 +122,3 @@ using the runc checkpoint command.`,
126122
return nil
127123
},
128124
}
129-
130-
func criuOptions(context *cli.Context) (*libcontainer.CriuOpts, error) {
131-
imagePath, parentPath, err := prepareImagePaths(context)
132-
if err != nil {
133-
return nil, err
134-
}
135-
136-
return &libcontainer.CriuOpts{
137-
ImagesDirectory: imagePath,
138-
WorkDirectory: context.String("work-path"),
139-
ParentImage: parentPath,
140-
LeaveRunning: context.Bool("leave-running"),
141-
TcpEstablished: context.Bool("tcp-established"),
142-
ExternalUnixConnections: context.Bool("ext-unix-sk"),
143-
ShellJob: context.Bool("shell-job"),
144-
FileLocks: context.Bool("file-locks"),
145-
PreDump: context.Bool("pre-dump"),
146-
AutoDedup: context.Bool("auto-dedup"),
147-
LazyPages: context.Bool("lazy-pages"),
148-
StatusFd: context.Int("status-fd"),
149-
LsmProfile: context.String("lsm-profile"),
150-
LsmMountContext: context.String("lsm-mount-context"),
151-
}, nil
152-
}

‎tests/integration/checkpoint.bats‎

Lines changed: 59 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -224,7 +224,14 @@ function simple_cr() {
224224
# TCP port for lazy migration
225225
port=27277
226226

227-
__runc checkpoint --lazy-pages --page-server 0.0.0.0:${port} --status-fd ${lazy_w} --work-path ./work-dir --image-path ./image-dir test_busybox &
227+
__runc checkpoint \
228+
--lazy-pages \
229+
--page-server 0.0.0.0:${port} \
230+
--status-fd ${lazy_w} \
231+
--manage-cgroups-mode=ignore \
232+
--work-path ./work-dir \
233+
--image-path ./image-dir \
234+
test_busybox &
228235
cpt_pid=$!
229236

230237
# wait for lazy page server to be ready
@@ -246,14 +253,18 @@ function simple_cr() {
246253
lp_pid=$!
247254

248255
# Restore lazily from checkpoint.
249-
# The restored container needs a different name (as well as systemd
250-
# unit name, in case systemd cgroup driver is used) as the checkpointed
251-
# container is not yet destroyed. It is only destroyed at that point
252-
# in time when the last page is lazily transferred to the destination.
256+
#
257+
# The restored container needs a different name and a different cgroup
258+
# (and a different systemd unit name, in case systemd cgroup driver is
259+
# used) as the checkpointed container is not yet destroyed. It is only
260+
# destroyed at that point in time when the last page is lazily
261+
# transferred to the destination.
262+
#
253263
# Killing the CRIU on the checkpoint side will let the container
254264
# continue to run if the migration failed at some point.
255-
[ -v RUNC_USE_SYSTEMD ] && set_cgroups_path
256-
runc_restore_with_pipes ./image-dir test_busybox_restore --lazy-pages
265+
runc_restore_with_pipes ./image-dir test_busybox_restore \
266+
--lazy-pages \
267+
--manage-cgroups-mode=ignore
257268

258269
wait $cpt_pid
259270

@@ -405,3 +416,44 @@ function simple_cr() {
405416
# busybox should be back up and running
406417
testcontainer test_busybox running
407418
}
419+
420+
@test "checkpoint then restore into a different cgroup (via --manage-cgroups-mode ignore)" {
421+
set_resources_limit
422+
set_cgroups_path
423+
runc run -d --console-socket "$CONSOLE_SOCKET" test_busybox
424+
[ "$status" -eq 0 ]
425+
testcontainer test_busybox running
426+
427+
local orig_path
428+
orig_path=$(get_cgroup_path "pids")
429+
# Check that the cgroup exists.
430+
test -d "$orig_path"
431+
432+
runc checkpoint --work-path ./work-dir --manage-cgroups-mode ignore test_busybox
433+
grep -B 5 Error ./work-dir/dump.log || true
434+
[ "$status" -eq 0 ]
435+
testcontainer test_busybox checkpointed
436+
# Check that the cgroup is gone.
437+
! test -d "$orig_path"
438+
439+
# Restore into a different cgroup.
440+
set_cgroups_path # Changes the path.
441+
runc restore -d --manage-cgroups-mode ignore --pid-file pid \
442+
--work-path ./work-dir --console-socket "$CONSOLE_SOCKET" test_busybox
443+
grep -B 5 Error ./work-dir/restore.log || true
444+
[ "$status" -eq 0 ]
445+
testcontainer test_busybox running
446+
447+
# Check that the old cgroup path doesn't exist.
448+
! test -d "$orig_path"
449+
450+
# Check that the new path exists.
451+
local new_path
452+
new_path=$(get_cgroup_path "pids")
453+
test -d "$new_path"
454+
455+
# Check that container's init is in the new cgroup.
456+
local pid
457+
pid=$(cat "pid")
458+
grep -q "${REL_CGROUPS_PATH}$" "/proc/$pid/cgroup"
459+
}

‎tests/integration/helpers.bash‎

Lines changed: 19 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -232,19 +232,27 @@ function set_cgroups_path() {
232232
update_config '.linux.cgroupsPath |= "'"${OCI_CGROUPS_PATH}"'"'
233233
}
234234

235-
# Get a value from a cgroup file.
236-
function get_cgroup_value() {
237-
local source=$1
238-
local cgroup var current
239-
235+
# Get a path to cgroup directory, based on controller name.
236+
# Parameters:
237+
# $1: controller name (like "pids") or a file name (like "pids.max").
238+
function get_cgroup_path() {
240239
if [ -v CGROUP_V2 ]; then
241-
cgroup=$CGROUP_PATH
242-
else
243-
var=${source%%.*} # controller name (e.g. memory)
244-
var=CGROUP_${var^^}_BASE_PATH # variable name (e.g. CGROUP_MEMORY_BASE_PATH)
245-
eval cgroup=\$"${var}${REL_CGROUPS_PATH}"
240+
echo "$CGROUP_PATH"
241+
return
246242
fi
247-
cat "$cgroup/$source"
243+
244+
local var cgroup
245+
var=${1%%.*} # controller name (e.g. memory)
246+
var=CGROUP_${var^^}_BASE_PATH # variable name (e.g. CGROUP_MEMORY_BASE_PATH)
247+
eval cgroup=\$"${var}${REL_CGROUPS_PATH}"
248+
echo "$cgroup"
249+
}
250+
251+
# Get a value from a cgroup file.
252+
function get_cgroup_value() {
253+
local cgroup
254+
cgroup="$(get_cgroup_path "$1")"
255+
cat "$cgroup/$1"
248256
}
249257

250258
# Helper to check a if value in a cgroup file matches the expected one.

0 commit comments

Comments
 (0)