Skip to content

Commit d2b4e6b

Browse files
author
Paweł Szulik
committed
intelrdt: add mbm stats
Signed-off-by: Paweł Szulik <[email protected]>
1 parent 7fa13b2 commit d2b4e6b

File tree

6 files changed

+305
-1
lines changed

6 files changed

+305
-1
lines changed

‎events.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,9 @@ func convertLibcontainerStats(ls *libcontainer.Stats) *types.Stats {
161161
s.IntelRdt.MemBwSchemaRoot = is.MemBwSchemaRoot
162162
s.IntelRdt.MemBwSchema = is.MemBwSchema
163163
}
164+
if intelrdt.IsMbmEnabled() {
165+
s.IntelRdt.MBMStats = is.MBMStats
166+
}
164167
}
165168

166169
s.NetworkInterfaces = ls.Interfaces

‎libcontainer/intelrdt/intelrdt.go‎

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,10 @@ import (
5555
* | | |-- cbm_mask
5656
* | | |-- min_cbm_bits
5757
* | | |-- num_closids
58+
* | |-- L3_MON
59+
* | | |-- max_threshold_occupancy
60+
* | | |-- mon_features
61+
* | | |-- num_rmids
5862
* | |-- MB
5963
* | |-- bandwidth_gran
6064
* | |-- delay_linear
@@ -221,6 +225,17 @@ func init() {
221225
isMbaEnabled = true
222226
}
223227
}
228+
229+
if flagsSet.MBMTotal || flagsSet.MBMLocal {
230+
if _, err := os.Stat(filepath.Join(intelRdtRoot, "info", "L3_MON")); err == nil {
231+
isMbmEnabled = true
232+
}
233+
234+
enabledMonFeatures, err = getMonFeatures(intelRdtRoot)
235+
if err != nil {
236+
return
237+
}
238+
}
224239
}
225240

226241
// Return the mount point path of Intel RDT "resource control" filesysem
@@ -300,6 +315,10 @@ func isIntelRdtMounted() bool {
300315
type cpuInfoFlags struct {
301316
CAT bool // Cache Allocation Technology
302317
MBA bool // Memory Bandwidth Allocation
318+
319+
// Memory Bandwidth Monitoring related.
320+
MBMTotal bool
321+
MBMLocal bool
303322
}
304323

305324
func parseCpuInfoFile(path string) (cpuInfoFlags, error) {
@@ -325,6 +344,10 @@ func parseCpuInfoFile(path string) (cpuInfoFlags, error) {
325344
infoFlags.CAT = true
326345
case "mba":
327346
infoFlags.MBA = true
347+
case "cqm_mbm_total":
348+
infoFlags.MBMTotal = true
349+
case "cqm_mbm_local":
350+
infoFlags.MBMLocal = true
328351
}
329352
}
330353
return infoFlags, nil
@@ -589,7 +612,8 @@ func (m *IntelRdtManager) GetStats() (*Stats, error) {
589612
schemaRootStrings := strings.Split(tmpRootStrings, "\n")
590613

591614
// The L3 cache and memory bandwidth schemata in 'container_id' group
592-
tmpStrings, err := getIntelRdtParamString(m.GetPath(), "schemata")
615+
containerPath := m.GetPath()
616+
tmpStrings, err := getIntelRdtParamString(containerPath, "schemata")
593617
if err != nil {
594618
return nil, err
595619
}
@@ -641,6 +665,14 @@ func (m *IntelRdtManager) GetStats() (*Stats, error) {
641665
}
642666
}
643667

668+
if IsMbmEnabled() {
669+
mbmStats, err := getMBMStats(containerPath)
670+
if err != nil {
671+
return stats, err
672+
}
673+
stats.MBMStats = mbmStats
674+
}
675+
644676
return stats, nil
645677
}
646678

‎libcontainer/intelrdt/mbm.go‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
// +build linux
2+
3+
package intelrdt
4+
5+
import (
6+
"bufio"
7+
"fmt"
8+
"io"
9+
"io/ioutil"
10+
"os"
11+
"path/filepath"
12+
13+
"github.com/sirupsen/logrus"
14+
)
15+
16+
var (
17+
// The flag to indicate if Intel RDT/MBM is enabled
18+
isMbmEnabled bool
19+
20+
enabledMonFeatures monFeatures
21+
)
22+
23+
type monFeatures struct {
24+
mbmTotalBytes bool
25+
mbmLocalBytes bool
26+
llcOccupancy bool
27+
}
28+
29+
// Check if Intel RDT/MBM is enabled
30+
func IsMbmEnabled() bool {
31+
return isMbmEnabled
32+
}
33+
34+
func getMonFeatures(intelRdtRoot string) (monFeatures, error) {
35+
file, err := os.Open(filepath.Join(intelRdtRoot, "info", "L3_MON", "mon_features"))
36+
defer file.Close()
37+
if err != nil {
38+
return monFeatures{}, err
39+
}
40+
return parseMonFeatures(file)
41+
}
42+
43+
func parseMonFeatures(reader io.Reader) (monFeatures, error) {
44+
scanner := bufio.NewScanner(reader)
45+
46+
monFeatures := monFeatures{}
47+
48+
for scanner.Scan() {
49+
50+
switch feature := scanner.Text(); feature {
51+
52+
case "mbm_total_bytes":
53+
monFeatures.mbmTotalBytes = true
54+
case "mbm_local_bytes":
55+
monFeatures.mbmLocalBytes = true
56+
case "llc_occupancy":
57+
monFeatures.llcOccupancy = true
58+
default:
59+
logrus.Warnf(fmt.Sprintf("Unsupported RDT Memory Bandwidth Monitoring (MBM) feature: %s", feature))
60+
}
61+
}
62+
63+
return monFeatures, scanner.Err()
64+
}
65+
66+
func getMBMStats(containerPath string) (*[]MBMNumaNodeStats, error) {
67+
mbmStats := []MBMNumaNodeStats{}
68+
69+
numaFiles, err := ioutil.ReadDir(filepath.Join(containerPath, "mon_data"))
70+
if err != nil {
71+
return &mbmStats, err
72+
}
73+
74+
for _, file := range numaFiles {
75+
if file.IsDir() {
76+
numaStats, err := getMBMNumaNodeStats(filepath.Join(containerPath, "mon_data", file.Name()))
77+
if err != nil {
78+
return &mbmStats, nil
79+
}
80+
mbmStats = append(mbmStats, *numaStats)
81+
}
82+
}
83+
84+
return &mbmStats, nil
85+
}
86+
87+
func getMBMNumaNodeStats(numaPath string) (*MBMNumaNodeStats, error) {
88+
stats := &MBMNumaNodeStats{}
89+
if enabledMonFeatures.mbmTotalBytes {
90+
mbmTotalBytes, err := getIntelRdtParamUint(numaPath, "mbm_total_bytes")
91+
if err != nil {
92+
return nil, err
93+
}
94+
stats.MBMTotalBytes = mbmTotalBytes
95+
}
96+
97+
if enabledMonFeatures.mbmLocalBytes {
98+
mbmLocalBytes, err := getIntelRdtParamUint(numaPath, "mbm_local_bytes")
99+
if err != nil {
100+
return nil, err
101+
}
102+
stats.MBMLocalBytes = mbmLocalBytes
103+
}
104+
105+
if enabledMonFeatures.llcOccupancy {
106+
llcOccupancy, err := getIntelRdtParamUint(numaPath, "llc_occupancy")
107+
if err != nil {
108+
return nil, err
109+
}
110+
stats.LLCOccupancy = llcOccupancy
111+
}
112+
return stats, nil
113+
}

‎libcontainer/intelrdt/mbm_test.go‎

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
// +build linux
2+
3+
package intelrdt
4+
5+
import (
6+
"io/ioutil"
7+
"os"
8+
"path/filepath"
9+
"strconv"
10+
"strings"
11+
"testing"
12+
)
13+
14+
func TestParseMonFeatures(t *testing.T) {
15+
t.Run("All features available", func(t *testing.T) {
16+
parsedMonFeatures, err := parseMonFeatures(
17+
strings.NewReader("mbm_total_bytes\nmbm_local_bytes\nllc_occupancy"))
18+
if err != nil {
19+
t.Errorf("Error while parsing mon features err = %v", err)
20+
}
21+
22+
expectedMonFeatures := monFeatures{true, true, true}
23+
24+
if parsedMonFeatures != expectedMonFeatures {
25+
t.Error("Cannot gather all features!")
26+
}
27+
})
28+
29+
t.Run("No features available", func(t *testing.T) {
30+
parsedMonFeatures, err := parseMonFeatures(strings.NewReader(""))
31+
32+
if err != nil {
33+
t.Errorf("Error while parsing mon features err = %v", err)
34+
}
35+
36+
expectedMonFeatures := monFeatures{false, false, false}
37+
38+
if parsedMonFeatures != expectedMonFeatures {
39+
t.Error("Expected no features available but there is any!")
40+
}
41+
})
42+
}
43+
44+
func mockMBM(NUMANodes []string, mocks map[string]uint64) (string, error) {
45+
testDir, err := ioutil.TempDir("", "rdt_mbm_test")
46+
if err != nil {
47+
return "", err
48+
}
49+
monDataPath := filepath.Join(testDir, "mon_data")
50+
51+
for _, numa := range NUMANodes {
52+
numaPath := filepath.Join(monDataPath, numa)
53+
err = os.MkdirAll(numaPath, os.ModePerm)
54+
if err != nil {
55+
return "", err
56+
}
57+
58+
for fileName, value := range mocks {
59+
err := ioutil.WriteFile(filepath.Join(numaPath, fileName), []byte(strconv.FormatUint(value, 10)), 777)
60+
if err != nil {
61+
return "", err
62+
}
63+
}
64+
65+
}
66+
67+
return testDir, nil
68+
}
69+
70+
func TestGetMbmStats(t *testing.T) {
71+
mocksNUMANodesToCreate := []string{"mon_l3_00", "mon_l3_01"}
72+
73+
mocksFilesToCreate := map[string]uint64{
74+
"mbm_total_bytes": 9123911,
75+
"mbm_local_bytes": 2361361,
76+
"llc_occupancy": 123013,
77+
}
78+
79+
mockedMBM, err := mockMBM(mocksNUMANodesToCreate, mocksFilesToCreate)
80+
81+
defer func() {
82+
err := os.RemoveAll(mockedMBM)
83+
if err != nil {
84+
t.Fatal(err)
85+
}
86+
}()
87+
88+
if err != nil {
89+
t.Fatal(err)
90+
}
91+
92+
t.Run("Gather mbm", func(t *testing.T) {
93+
enabledMonFeatures.mbmTotalBytes = true
94+
enabledMonFeatures.mbmLocalBytes = true
95+
enabledMonFeatures.llcOccupancy = true
96+
97+
stats, err := getMBMStats(mockedMBM)
98+
if err != nil {
99+
t.Fatal(err)
100+
}
101+
102+
if len(*stats) != len(mocksNUMANodesToCreate) {
103+
t.Fatalf("Wrong number of stats slices from NUMA nodes. Expected: %v but got: %v",
104+
len(mocksNUMANodesToCreate), len(*stats))
105+
}
106+
107+
checkStatCorrection := func(got MBMNumaNodeStats, expected MBMNumaNodeStats, t *testing.T) {
108+
if got.MBMTotalBytes != expected.MBMTotalBytes {
109+
t.Fatalf("Wrong value of mbm_total_bytes. Expected: %v but got: %v",
110+
expected.MBMTotalBytes,
111+
got.MBMTotalBytes)
112+
}
113+
114+
if got.MBMLocalBytes != expected.MBMLocalBytes {
115+
t.Fatalf("Wrong value of mbm_local_bytes. Expected: %v but got: %v",
116+
expected.MBMLocalBytes,
117+
got.MBMLocalBytes)
118+
}
119+
120+
if got.LLCOccupancy != expected.LLCOccupancy {
121+
t.Fatalf("Wrong value of llc_occupancy. Expected: %v but got: %v",
122+
expected.LLCOccupancy,
123+
got.LLCOccupancy)
124+
}
125+
}
126+
127+
expectedStats := MBMNumaNodeStats{
128+
MBMTotalBytes: mocksFilesToCreate["mbm_total_bytes"],
129+
MBMLocalBytes: mocksFilesToCreate["mbm_local_bytes"],
130+
LLCOccupancy: mocksFilesToCreate["llc_occupancy"],
131+
}
132+
133+
checkStatCorrection((*stats)[0], expectedStats, t)
134+
checkStatCorrection((*stats)[1], expectedStats, t)
135+
})
136+
137+
}

‎libcontainer/intelrdt/stats.go‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,17 @@ type MemBwInfo struct {
1515
NumClosids uint64 `json:"num_closids,omitempty"`
1616
}
1717

18+
type MBMNumaNodeStats struct {
19+
// The 'mbm_total_bytes' in 'container_id' group
20+
MBMTotalBytes uint64 `json:"mbm_total_bytes,omitempty"`
21+
22+
// The 'mbm_local_bytes' in 'container_id' group
23+
MBMLocalBytes uint64 `json:"mbm_local_bytes,omitempty"`
24+
25+
// The 'llc_occupancy' in 'container_id' group
26+
LLCOccupancy uint64 `json:"llc_occupancy,omitempty"`
27+
}
28+
1829
type Stats struct {
1930
// The read-only L3 cache information
2031
L3CacheInfo *L3CacheInfo `json:"l3_cache_info,omitempty"`
@@ -33,6 +44,9 @@ type Stats struct {
3344

3445
// The memory bandwidth schema in 'container_id' group
3546
MemBwSchema string `json:"mem_bw_schema,omitempty"`
47+
48+
// The memory bandwidth monitoring statistics from NUMA nodes in 'container_id' group
49+
MBMStats *[]MBMNumaNodeStats `json:"mbm_statistics,omitempty"`
3650
}
3751

3852
func NewStats() *Stats {

‎types/events.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
package types
22

3+
import "github.com/opencontainers/runc/libcontainer/intelrdt"
4+
35
// Event struct for encoding the event data to json.
46
type Event struct {
57
Type string `json:"type"`
@@ -113,6 +115,9 @@ type IntelRdt struct {
113115

114116
// The memory bandwidth schema in 'container_id' group
115117
MemBwSchema string `json:"mem_bw_schema,omitempty"`
118+
119+
// The memory bandwidth monitoring statistics from NUMA nodes in 'container_id' group
120+
MBMStats *[]intelrdt.MBMNumaNodeStats `json:"mbm_statistics,omitempty"`
116121
}
117122

118123
type NetworkInterface struct {

0 commit comments

Comments
 (0)