fix: cpu stat in cgroup v2 (#3857)

core/stat/internal/cgroup_linux.go

@@ -2,6 +2,7 @@ package internal
 
 import (
 	"bufio"
+	"errors"
 	"fmt"
 	"math"
 	"os"
@@ -18,6 +19,7 @@ import (
 
 const (
 	cgroupDir   = "/sys/fs/cgroup"
+	cpuMaxFile  = cgroupDir + "/cpu.max"
 	cpuStatFile = cgroupDir + "/cpu.stat"
 	cpusetFile  = cgroupDir + "/cpuset.cpus.effective"
 )
@@ -30,10 +32,9 @@ var (
 )
 
 type cgroup interface {
-	cpuQuotaUs() (int64, error)
+	cpuQuota() (float64, error)
-	cpuPeriodUs() (uint64, error)
+	cpuUsage() (uint64, error)
-	cpus() ([]uint64, error)
+	effectiveCpus() (int, error)
-	usageAllCpus() (uint64, error)
 }
 
 func currentCgroup() (cgroup, error) {
@@ -48,13 +49,22 @@ type cgroupV1 struct {
 	cgroups map[string]string
 }
 
-func (c *cgroupV1) cpuQuotaUs() (int64, error) {
+func (c *cgroupV1) cpuQuota() (float64, error) {
-	data, err := iox.ReadText(path.Join(c.cgroups["cpu"], "cpu.cfs_quota_us"))
+	quotaUs, err := c.cpuQuotaUs()
 	if err != nil {
 		return 0, err
 	}
 
-	return strconv.ParseInt(data, 10, 64)
+	if quotaUs == -1 {
+		return -1, nil
+	}
+
+	periodUs, err := c.cpuPeriodUs()
+	if err != nil {
+		return 0, err
+	}
+
+	return float64(quotaUs) / float64(periodUs), nil
 }
 
 func (c *cgroupV1) cpuPeriodUs() (uint64, error) {
@@ -66,16 +76,16 @@ func (c *cgroupV1) cpuPeriodUs() (uint64, error) {
 	return parseUint(data)
 }
 
-func (c *cgroupV1) cpus() ([]uint64, error) {
+func (c *cgroupV1) cpuQuotaUs() (int64, error) {
-	data, err := iox.ReadText(path.Join(c.cgroups["cpuset"], "cpuset.cpus"))
+	data, err := iox.ReadText(path.Join(c.cgroups["cpu"], "cpu.cfs_quota_us"))
 	if err != nil {
-		return nil, err
+		return 0, err
 	}
 
-	return parseUints(data)
+	return strconv.ParseInt(data, 10, 64)
 }
 
-func (c *cgroupV1) usageAllCpus() (uint64, error) {
+func (c *cgroupV1) cpuUsage() (uint64, error) {
 	data, err := iox.ReadText(path.Join(c.cgroups["cpuacct"], "cpuacct.usage"))
 	if err != nil {
 		return 0, err
@@ -84,38 +94,53 @@ func (c *cgroupV1) usageAllCpus() (uint64, error) {
 	return parseUint(data)
 }
 
-type cgroupV2 struct {
+func (c *cgroupV1) effectiveCpus() (int, error) {
-	cgroups map[string]string
+	data, err := iox.ReadText(path.Join(c.cgroups["cpuset"], "cpuset.cpus"))
+	if err != nil {
+		return 0, err
 	}
 
-func (c *cgroupV2) cpuQuotaUs() (int64, error) {
+	cpus, err := parseUints(data)
-	data, err := iox.ReadText(path.Join(cgroupDir, "cpu.cfs_quota_us"))
 	if err != nil {
 		return 0, err
 	}
 
-	return strconv.ParseInt(data, 10, 64)
+	return len(cpus), nil
+}
+
+type cgroupV2 struct {
+	cgroups map[string]string
 }
 
-func (c *cgroupV2) cpuPeriodUs() (uint64, error) {
+func (c *cgroupV2) cpuQuota() (float64, error) {
-	data, err := iox.ReadText(path.Join(cgroupDir, "cpu.cfs_period_us"))
+	data, err := iox.ReadText(cpuMaxFile)
 	if err != nil {
 		return 0, err
 	}
 
-	return parseUint(data)
+	fields := strings.Fields(data)
+	if len(fields) != 2 {
+		return 0, fmt.Errorf("cgroup: bad /sys/fs/cgroup/cpu.max file: %s", data)
 	}
 
-func (c *cgroupV2) cpus() ([]uint64, error) {
+	if fields[0] == "max" {
-	data, err := iox.ReadText(cpusetFile)
+		return -1, nil
+	}
+
+	quotaUs, err := strconv.ParseInt(fields[0], 10, 64)
 	if err != nil {
-		return nil, err
+		return 0, err
+	}
+
+	periodUs, err := strconv.ParseUint(fields[1], 10, 64)
+	if err != nil {
+		return 0, err
 	}
 
-	return parseUints(data)
+	return float64(quotaUs) / float64(periodUs), nil
 }
 
-func (c *cgroupV2) usageAllCpus() (uint64, error) {
+func (c *cgroupV2) cpuUsage() (uint64, error) {
 	usec, err := parseUint(c.cgroups["usage_usec"])
 	if err != nil {
 		return 0, err
@@ -124,6 +149,20 @@ func (c *cgroupV2) usageAllCpus() (uint64, error) {
 	return usec * uint64(time.Microsecond), nil
 }
 
+func (c *cgroupV2) effectiveCpus() (int, error) {
+	data, err := iox.ReadText(cpusetFile)
+	if err != nil {
+		return 0, err
+	}
+
+	cpus, err := parseUints(data)
+	if err != nil {
+		return 0, err
+	}
+
+	return len(cpus), nil
+}
+
 func currentCgroupV1() (cgroup, error) {
 	cgroupFile := fmt.Sprintf("/proc/%d/cgroup", os.Getpid())
 	lines, err := iox.ReadTextLines(cgroupFile, iox.WithoutBlank())
@@ -200,7 +239,7 @@ func isCgroup2UnifiedMode() bool {
 func parseUint(s string) (uint64, error) {
 	v, err := strconv.ParseInt(s, 10, 64)
 	if err != nil {
-		if err.(*strconv.NumError).Err == strconv.ErrRange {
+		if errors.Is(err, strconv.ErrRange) {
 			return 0, nil
 		}
 
@@ -225,21 +264,21 @@ func parseUints(val string) ([]uint64, error) {
 	for _, r := range cols {
 		if strings.Contains(r, "-") {
 			fields := strings.SplitN(r, "-", 2)
-			min, err := parseUint(fields[0])
+			minimum, err := parseUint(fields[0])
 			if err != nil {
 				return nil, fmt.Errorf("cgroup: bad int list format: %s", val)
 			}
 
-			max, err := parseUint(fields[1])
+			maximum, err := parseUint(fields[1])
 			if err != nil {
 				return nil, fmt.Errorf("cgroup: bad int list format: %s", val)
 			}
 
-			if max < min {
+			if maximum < minimum {
 				return nil, fmt.Errorf("cgroup: bad int list format: %s", val)
 			}
 
-			for i := min; i <= max; i++ {
+			for i := minimum; i <= maximum; i++ {
 				if _, ok := ints[i]; !ok {
 					ints[i] = lang.Placeholder
 					sets = append(sets, i)

core/stat/internal/cgroup_linux_test.go

@@ -16,15 +16,23 @@ func TestCgroupV1(t *testing.T) {
 	if isCgroup2UnifiedMode() {
 		cg, err := currentCgroupV1()
 		assert.NoError(t, err)
-		_, err = cg.cpus()
+		_, err = cg.effectiveCpus()
 		assert.Error(t, err)
-		_, err = cg.cpuPeriodUs()
+		_, err = cg.cpuQuota()
 		assert.Error(t, err)
-		_, err = cg.cpuQuotaUs()
+		_, err = cg.cpuUsage()
-		assert.Error(t, err)
-		_, err = cg.usageAllCpus()
 		assert.Error(t, err)
 	}
+
+	// test cgroup v2
+	cg, err := currentCgroupV2()
+	assert.NoError(t, err)
+	_, err = cg.effectiveCpus()
+	assert.NoError(t, err)
+	_, err = cg.cpuQuota()
+	assert.Error(t, err)
+	_, err = cg.cpuUsage()
+	assert.NoError(t, err)
 }
 
 func TestParseUint(t *testing.T) {

core/stat/internal/cpu_linux.go

@@ -15,40 +15,31 @@ const (
 	cpuTicks  = 100
 	cpuFields = 8
 	cpuMax    = 1000
-	statDir   = "/proc/stat"
+	statFile  = "/proc/stat"
 )
 
 var (
 	preSystem uint64
 	preTotal  uint64
-	quota     float64
+	limit     float64
 	cores     uint64
 	initOnce  sync.Once
 )
 
 // if /proc not present, ignore the cpu calculation, like wsl linux
 func initialize() {
-	cpus, err := cpuSets()
+	cpus, err := effectiveCpus()
 	if err != nil {
 		logx.Error(err)
 		return
 	}
 
-	cores = uint64(len(cpus))
+	cores = uint64(cpus)
-	quota = float64(len(cpus))
+	limit = float64(cpus)
-	cq, err := cpuQuota()
+	quota, err := cpuQuota()
-	if err == nil {
+	if err == nil && quota > 0 {
-		if cq != -1 {
+		if quota < limit {
-			period, err := cpuPeriod()
+			limit = quota
-			if err != nil {
-				logx.Error(err)
-				return
-			}
-
-			limit := float64(cq) / float64(period)
-			if limit < quota {
-				quota = limit
-			}
 		}
 	}
 
@@ -58,7 +49,7 @@ func initialize() {
 		return
 	}
 
-	preTotal, err = totalCpuUsage()
+	preTotal, err = cpuUsage()
 	if err != nil {
 		logx.Error(err)
 		return
@@ -69,7 +60,7 @@ func initialize() {
 func RefreshCpu() uint64 {
 	initOnce.Do(initialize)
 
-	total, err := totalCpuUsage()
+	total, err := cpuUsage()
 	if err != nil {
 		return 0
 	}
@@ -83,7 +74,7 @@ func RefreshCpu() uint64 {
 	cpuDelta := total - preTotal
 	systemDelta := system - preSystem
 	if cpuDelta > 0 && systemDelta > 0 {
-		usage = uint64(float64(cpuDelta*cores*cpuMax) / (float64(systemDelta) * quota))
+		usage = uint64(float64(cpuDelta*cores*cpuMax) / (float64(systemDelta) * limit))
 		if usage > cpuMax {
 			usage = cpuMax
 		}
@@ -94,35 +85,35 @@ func RefreshCpu() uint64 {
 	return usage
 }
 
-func cpuQuota() (int64, error) {
+func cpuQuota() (float64, error) {
 	cg, err := currentCgroup()
 	if err != nil {
 		return 0, err
 	}
 
-	return cg.cpuQuotaUs()
+	return cg.cpuQuota()
 }
 
-func cpuPeriod() (uint64, error) {
+func cpuUsage() (uint64, error) {
 	cg, err := currentCgroup()
 	if err != nil {
 		return 0, err
 	}
 
-	return cg.cpuPeriodUs()
+	return cg.cpuUsage()
 }
 
-func cpuSets() ([]uint64, error) {
+func effectiveCpus() (int, error) {
 	cg, err := currentCgroup()
 	if err != nil {
-		return nil, err
+		return 0, err
 	}
 
-	return cg.cpus()
+	return cg.effectiveCpus()
 }
 
 func systemCpuUsage() (uint64, error) {
-	lines, err := iox.ReadTextLines(statDir, iox.WithoutBlank())
+	lines, err := iox.ReadTextLines(statFile, iox.WithoutBlank())
 	if err != nil {
 		return 0, err
 	}
@@ -150,12 +141,3 @@ func systemCpuUsage() (uint64, error) {
 
 	return 0, errors.New("bad stats format")
 }
-
-func totalCpuUsage() (usage uint64, err error) {
-	var cg cgroup
-	if cg, err = currentCgroup(); err != nil {
-		return
-	}
-
-	return cg.usageAllCpus()
-}