From aa2a1c4101f31e583375c1ad7abcd4a450a9fc8d Mon Sep 17 00:00:00 2001 From: Oleg Zhurakivskyy Date: Mon, 8 Jun 2026 00:17:52 +0300 Subject: [PATCH] topology-aware: Add PoC of gofmbt based test for libmem allocations Signed-off-by: Oleg Zhurakivskyy --- .../topology-aware/policy/libmem_test.go | 377 +++++++++++++++++- go.mod | 2 +- go.sum | 10 +- .../n4c16/test06-fuzz/generate.go | 2 +- 4 files changed, 374 insertions(+), 17 deletions(-) diff --git a/cmd/plugins/topology-aware/policy/libmem_test.go b/cmd/plugins/topology-aware/policy/libmem_test.go index df1ff3431..9695fd7e8 100644 --- a/cmd/plugins/topology-aware/policy/libmem_test.go +++ b/cmd/plugins/topology-aware/policy/libmem_test.go @@ -15,8 +15,12 @@ package topologyaware import ( + "flag" + "fmt" "os" "path" + "runtime" + "sort" "strings" "testing" @@ -24,27 +28,53 @@ import ( policyapi "github.com/containers/nri-plugins/pkg/resmgr/policy" system "github.com/containers/nri-plugins/pkg/sysfs" "github.com/containers/nri-plugins/pkg/utils" + "github.com/go-logr/logr" + m "github.com/ozhuraki/gofmbt/gofmbt" + "k8s.io/klog/v2" ) +// LibmemState is the abstract model state for TestLibmemGofmbt2. It +// tracks how many bytes are free and which named allocations are live. +type LibmemState struct { + freeBytes int64 + allocs map[string]int64 // abstract name -> allocated size +} + // setupTestPolicy creates a policy from the server sysfs testdata. +// If testdata/sysfs/server/sys already exists in the current directory it is +// used directly and the returned dir is empty (caller must not delete it). +// Otherwise the tarball is unpacked into a temp dir and that dir is returned +// so the caller can clean it up with removeAll. func setupTestPolicy(t *testing.T) (*policy, string) { t.Helper() - dir, err := os.MkdirTemp("", "nri-libmem-test-") - if err != nil { - t.Fatalf("failed to create temp dir: %v", err) - } - if err := utils.UncompressTbz2(path.Join("testdata", "sysfs.tar.bz2"), dir); err != nil { - if rerr := os.RemoveAll(dir); rerr != nil { - t.Logf("failed to remove temp dir %q: %v", dir, rerr) + + const preUnpacked = "testdata/sysfs/server/sys" + var sysPath string + var dir string + + if _, err := os.Stat(preUnpacked); err == nil { + sysPath = preUnpacked + } else { + var err error + dir, err = os.MkdirTemp("", "nri-libmem-test-") + if err != nil { + t.Fatalf("failed to create temp dir: %v", err) + } + if err := utils.UncompressTbz2(path.Join("testdata", "sysfs.tar.bz2"), dir); err != nil { + if rerr := os.RemoveAll(dir); rerr != nil { + t.Logf("failed to remove temp dir %q: %v", dir, rerr) + } + t.Fatalf("failed to uncompress testdata: %v", err) } - t.Fatalf("failed to uncompress testdata: %v", err) + sysPath = path.Join(dir, "sysfs", "server", "sys") } - sysPath := path.Join(dir, "sysfs", "server", "sys") sys, err := system.DiscoverSystemAt(sysPath) if err != nil { - if rerr := os.RemoveAll(dir); rerr != nil { - t.Logf("failed to remove temp dir %q: %v", dir, rerr) + if dir != "" { + if rerr := os.RemoveAll(dir); rerr != nil { + t.Logf("failed to remove temp dir %q: %v", dir, rerr) + } } t.Fatalf("failed to discover system: %v", err) } @@ -57,14 +87,49 @@ func setupTestPolicy(t *testing.T) (*policy, string) { ReservedResources: cfgapi.Constraints{cfgapi.CPU: "750m"}, }, }); err != nil { - if rerr := os.RemoveAll(dir); rerr != nil { - t.Logf("failed to remove temp dir %q: %v", dir, rerr) + if dir != "" { + if rerr := os.RemoveAll(dir); rerr != nil { + t.Logf("failed to remove temp dir %q: %v", dir, rerr) + } } t.Fatalf("failed to setup policy: %v", err) } + printSystemDRAM(sys) return p, dir } +// printSystemDRAM prints DRAM capacity per NUMA node and the total. +func printSystemDRAM(sys system.System) { + var total uint64 + for _, id := range sys.NodeIDs() { + n := sys.Node(id) + if n.GetMemoryType() != system.MemoryTypeDRAM { + continue + } + info, err := n.MemoryInfo() + if err != nil || info == nil { + continue + } + fmt.Printf(" NUMA node %d DRAM: %s\n", id, formatBytes(info.MemTotal)) + total += info.MemTotal + } + fmt.Printf(" DRAM total: %s\n", formatBytes(total)) +} + +// formatBytes formats a byte count in a human-readable form (GiB/MiB/KiB/B). +func formatBytes(b uint64) string { + switch { + case b >= 1<<30: + return fmt.Sprintf("%.1f GiB", float64(b)/float64(1<<30)) + case b >= 1<<20: + return fmt.Sprintf("%.1f MiB", float64(b)/float64(1<<20)) + case b >= 1<<10: + return fmt.Sprintf("%.1f KiB", float64(b)/float64(1<<10)) + default: + return fmt.Sprintf("%d B", b) + } +} + // TestLibmemGetMemOfferByHintsMemoryPreserve verifies that getMemOfferByHints // returns an error immediately when memoryPreserve is requested. func TestLibmemGetMemOfferByHintsMemoryPreserve(t *testing.T) { @@ -118,6 +183,106 @@ func TestLibmemGetMemOfferByHintsNoHints(t *testing.T) { } } +// mallocSeq is used to generate unique container IDs in malloc. +var mallocSeq int + +// mallocSizeByID tracks the allocated size per container ID so free() can print it. +var mallocSizeByID = map[string]int64{} + +// malloc allocates memory of the given size on a leaf DRAM node of the policy +// and returns the container ID of the committed allocation. +func malloc(p *policy, size int64) (string, error) { + mallocSeq++ + id := fmt.Sprintf("%d", mallocSeq) + + fmt.Printf("malloc(%dGB)", size>>30) + if fmbtV >= 1 { + fmt.Printf(" id=%s", id) + } + for i := 1; i <= callerDepth; i++ { + pc, _, _, ok := runtime.Caller(i) + if !ok { + break + } + full := runtime.FuncForPC(pc).Name() + short := full[strings.LastIndex(full, "/")+1:] + short = short[strings.Index(short, ".")+1:] + fmt.Printf(" %s()", short) + } + fmt.Println() + + var pool Node + for _, n := range p.pools { + if n.IsLeafNode() && n.HasMemoryType(memoryDRAM) { + pool = n + break + } + } + if pool == nil { + return "", fmt.Errorf("no leaf DRAM node found in test system") + } + ctr := &mockContainer{returnValueForGetID: id} + req := &request{ + memType: memoryDRAM, + memReq: size, + container: ctr, + } + offer, err := p.getMemOffer(pool, req) + if err != nil { + return "", fmt.Errorf("getMemOffer failed: %w", err) + } + if _, _, err := offer.Commit(); err != nil { + return "", fmt.Errorf("Offer.Commit() failed: %w", err) + } + mallocSizeByID[id] = size + return id, nil +} + +// free releases a previously committed memory allocation for the given container ID. +func free(p *policy, id string) error { + fmt.Printf("free(%dGB)", mallocSizeByID[id]>>30) + if fmbtV >= 1 { + fmt.Printf(" id=%s", id) + } + for i := 1; i <= callerDepth; i++ { + pc, _, _, ok := runtime.Caller(i) + if !ok { + break + } + full := runtime.FuncForPC(pc).Name() + short := full[strings.LastIndex(full, "/")+1:] + short = short[strings.Index(short, ".")+1:] + fmt.Printf(" %s()", short) + } + fmt.Println() + err := p.releaseMem(id) + if err == nil { + delete(mallocSizeByID, id) + } + return err +} + +// TestLibmemReleaseMem verifies that releaseMem releases a previously committed +// memory allocation, and returns an error for an unknown ID. +func TestLibmemReleaseMem(t *testing.T) { + p, dir := setupTestPolicy(t) + defer removeAll(t, dir) + + id, err := malloc(p, 64*1024*1024) // 64 MiB + if err != nil { + t.Fatalf("malloc failed: %v", err) + } + + if err := free(p, id); err != nil { + t.Errorf("free failed for known ID: %v", err) + } + + // Releasing the same ID again should return an error (unknown request). + if err := free(p, id); err == nil { + t.Error("expected error releasing unknown ID, got nil") + } +} + // TestLibmemPoolZoneCapacityAndFree verifies that poolZoneCapacity returns a // positive value and that poolZoneFree does not exceed it. func TestLibmemPoolZoneCapacityAndFree(t *testing.T) { @@ -145,3 +310,189 @@ func TestLibmemPoolZoneCapacityAndFree(t *testing.T) { t.Errorf("expected 0 <= free (%d) <= capacity (%d)", free, capacity) } } + +func (s *LibmemState) String() string { + names := make([]string, 0, len(s.allocs)) + for name := range s.allocs { + names = append(names, name) + } + sort.Strings(names) + return fmt.Sprintf("[free:%dGB allocs:[%s]]", s.freeBytes>>30, strings.Join(names, " ")) +} + +var ( + maxLibmem2Steps int + libmem2Search int + callerDepth int + fmbtV int +) + +// init registers flags and switches CommandLine to ContinueOnError so that +// flags passed via -args on the command line are accepted. +func init() { + flag.CommandLine.Init(os.Args[0], flag.ContinueOnError) + flag.IntVar(&maxLibmem2Steps, "libmem2-steps", 1000, "number of test steps for TestLibmemGofmbt2") + flag.IntVar(&libmem2Search, "libmem2-search-depth", 4, "look-ahead depth for TestLibmemGofmbt2") + flag.IntVar(&callerDepth, "caller-depth", 1, "number of caller frames printed by malloc() and free()") + flag.IntVar(&fmbtV, "fmbt-v", 0, "verbosity for TestLibmemGofmbt2: 1=basic, 2=include caller info in mallocFn/freeFn") +} + +// TestLibmemGofmbt uses gofmbt model-based testing to drive malloc/free +// sequences against the policy, verifying that all operations succeed. +func TestLibmemGofmbt(t *testing.T) { + klog.SetLogger(logr.Discard()) + p, dir := setupTestPolicy(t) + klog.ClearLogger() + defer removeAll(t, dir) + + allocNames := []string{"a0", "a1", "a2", "a3", "a4"} + allocSizes := map[string]int64{ + "a0": 2 << 30, + "a1": 4 << 30, + "a2": 8 << 30, + "a3": 16 << 30, + "a4": 32 << 30, + } + + var totalAllocBytes int64 + for _, size := range allocSizes { + totalAllocBytes += size + } + + allocIDs := map[string]string{} // abstract name -> real container ID + var execute bool // true only during step execution; guards doMalloc/doFree from BestPath exploration calls + + doMalloc := func(name string) (string, error) { + if !execute { + return "", nil + } + id, err := malloc(p, allocSizes[name]) + if err == nil { + allocIDs[name] = id + } + return id, err + } + + doFree := func(name string) error { + if !execute { + return nil + } + id, ok := allocIDs[name] + if !ok { + return nil + } + err := free(p, id) + if err == nil { + delete(allocIDs, name) + } + return err + } + + mallocFn := func(name string, size int64) m.StateChange { + return func(curr m.State) m.State { + s := curr.(*LibmemState) + if _, ok := s.allocs[name]; ok || s.freeBytes < size { + return nil + } + newAllocs := make(map[string]int64, len(s.allocs)+1) + for k, v := range s.allocs { + newAllocs[k] = v + } + newAllocs[name] = size + if fmbtV >= 2 { + pc, _, _, _ := runtime.Caller(1) + fmt.Printf("mallocFn(%dGB) called from %s\n", size>>30, runtime.FuncForPC(pc).Name()) + } else if fmbtV == 1 { + fmt.Printf("mallocFn(%dGB)\n", size>>30) + } + return &LibmemState{freeBytes: s.freeBytes - size, allocs: newAllocs} + } + } + + freeFn := func(name string) m.StateChange { + return func(curr m.State) m.State { + s := curr.(*LibmemState) + size, ok := s.allocs[name] + if !ok { + return nil + } + newAllocs := make(map[string]int64, len(s.allocs)) + for k, v := range s.allocs { + if k != name { + newAllocs[k] = v + } + } + if fmbtV >= 2 { + pc, _, _, _ := runtime.Caller(1) + fmt.Printf("freeFn(%dGB) called from %s\n", size>>30, runtime.FuncForPC(pc).Name()) + } else if fmbtV == 1 { + fmt.Printf("freeFn(%dGB)\n", size>>30) + } + return &LibmemState{freeBytes: s.freeBytes + size, allocs: newAllocs} + } + } + + model := m.NewModel() + + model.From(func(curr m.State) []*m.Transition { + s := curr.(*LibmemState) + var ts []*m.Transition + for _, name := range allocNames { + if _, ok := s.allocs[name]; !ok && s.freeBytes >= allocSizes[name] { + ts = append(ts, m.OnAction("malloc %s", name).Register(doMalloc, name).Do(mallocFn(name, allocSizes[name]))...) + } + } + return ts + }) + + model.From(func(curr m.State) []*m.Transition { + s := curr.(*LibmemState) + var ts []*m.Transition + for _, name := range allocNames { + if _, ok := s.allocs[name]; ok { + ts = append(ts, m.OnAction("free %s", name).Register(doFree, name).Do(freeFn(name))...) + } + } + return ts + }) + + coverer := m.NewCoverer() + coverer.CoverActionCombinations(3) + + state := m.State(&LibmemState{ + freeBytes: totalAllocBytes, + allocs: map[string]int64{}, + }) + + testStep := 0 + for testStep < maxLibmem2Steps { + path, covStats := coverer.BestPath(model, state, libmem2Search) + if len(path) == 0 { + break + } + for i := 0; i <= covStats.MaxStep; i++ { + testStep++ + step := path[i] + fmt.Printf("step:%d coverage:%d state:%v\n", testStep, coverer.Coverage(), state) + pc, _, _, _ := runtime.Caller(0) + full := runtime.FuncForPC(pc).Name() + short := full[strings.LastIndex(full, "/")+1:] + short = short[strings.Index(short, ".")+1:] + fmt.Printf("%s %s()\n", step.Action(), short) + execute = true + results := step.Action().Execute() + execute = false + if len(results) > 0 { + if err, _ := results[len(results)-1].(error); err != nil { + t.Errorf("step %d: %s failed: %v", testStep, step.Action(), err) + } + } + state = step.EndState() + coverer.MarkCovered(step) + coverer.UpdateCoverage() + if testStep >= maxLibmem2Steps { + break + } + } + } +} diff --git a/go.mod b/go.mod index 56e3c99ec..a1f622f38 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module github.com/containers/nri-plugins go 1.25.0 require ( - github.com/askervin/gofmbt v0.0.0-20250119175120-506d925f666f github.com/containerd/nri v0.11.0 github.com/containerd/otelttrpc v0.0.0-20240305015340-ea5083fda723 github.com/containerd/ttrpc v1.2.7 @@ -75,6 +74,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opencontainers/runtime-spec v1.3.0 // indirect + github.com/ozhuraki/gofmbt v0.0.0-20260602230753-ae0116f9519d // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.2 // indirect diff --git a/go.sum b/go.sum index 7aabb344a..d669e2d64 100644 --- a/go.sum +++ b/go.sum @@ -608,8 +608,6 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0= github.com/apache/arrow/go/v11 v11.0.0/go.mod h1:Eg5OsL5H+e299f7u5ssuXsuHQVEGC4xei5aX110hRiI= github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= -github.com/askervin/gofmbt v0.0.0-20250119175120-506d925f666f h1:AKRIaPPDqBRhpWnvxhvtdbVtkV/3XrboabuFaLyp1kw= -github.com/askervin/gofmbt v0.0.0-20250119175120-506d925f666f/go.mod h1:1rWH2fCHPoGz1ApWyGyEV9YhZ2ZHeeCPaHcicW3b6uk= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= @@ -899,6 +897,14 @@ github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= github.com/opencontainers/runtime-spec v1.3.0 h1:YZupQUdctfhpZy3TM39nN9Ika5CBWT5diQ8ibYCRkxg= github.com/opencontainers/runtime-spec v1.3.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/ozhuraki/gofmbt v0.0.0-20260423111600-db0a7cbedb3e h1:k3/s6DVcjtHql7QoRchvRdSs5Rm2xAkpnlbEdBqbVkA= +github.com/ozhuraki/gofmbt v0.0.0-20260423111600-db0a7cbedb3e/go.mod h1:Aml7A4F6EQMuEO5eFcJ86QqdWEEwBtn09Wtc663NTp4= +github.com/ozhuraki/gofmbt v0.0.0-20260602210616-4013dc18018f h1:Dfwe9yTSw/i/WMZlg/YQwOJGRjbz4/Rl2AkhSmoBSok= +github.com/ozhuraki/gofmbt v0.0.0-20260602210616-4013dc18018f/go.mod h1:Aml7A4F6EQMuEO5eFcJ86QqdWEEwBtn09Wtc663NTp4= +github.com/ozhuraki/gofmbt v0.0.0-20260602221104-49324efa15a6 h1:EyScTFoN0krWfUV82AthBIM8zWJwAsHI+6RxHr96SxU= +github.com/ozhuraki/gofmbt v0.0.0-20260602221104-49324efa15a6/go.mod h1:Aml7A4F6EQMuEO5eFcJ86QqdWEEwBtn09Wtc663NTp4= +github.com/ozhuraki/gofmbt v0.0.0-20260602230753-ae0116f9519d h1:X3c9UGncPCimFNeWbH/ZIqyCEaZrczhU2aVvGbZ+y8A= +github.com/ozhuraki/gofmbt v0.0.0-20260602230753-ae0116f9519d/go.mod h1:Aml7A4F6EQMuEO5eFcJ86QqdWEEwBtn09Wtc663NTp4= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY= diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test06-fuzz/generate.go b/test/e2e/policies.test-suite/topology-aware/n4c16/test06-fuzz/generate.go index 7bed90940..19e22bb6c 100644 --- a/test/e2e/policies.test-suite/topology-aware/n4c16/test06-fuzz/generate.go +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test06-fuzz/generate.go @@ -7,7 +7,7 @@ import ( "sort" "strings" - m "github.com/askervin/gofmbt/gofmbt" + m "github.com/ozhuraki/gofmbt/gofmbt" ) type PodResources struct {