Skip to content

Commit 76adce9

Browse files
committed
feat(agent): wire NetManager + IP allocator into FirecrackerDriver
- Add Net (NetManager) and Alloc (*Allocator) fields to FirecrackerDriver - Add netInfo field to fcProc to carry per-VM network state - Add setupNetwork: fetches ImpNetwork, allocates IP, creates bridge+TAP, installs NAT rules (best-effort) - Start calls setupNetwork when vm.Spec.NetworkRef != nil && d.Net != nil - buildConfig gains a netInfo arg and populates NetworkInterfaces when set - Stop calls Net.TeardownVM and Alloc.Release when proc.netInfo != nil - Inspect returns IP from proc.netInfo.IP when present - NewFirecrackerDriver initialises Alloc via network.NewAllocator() - Add four new unit tests covering the above paths
1 parent b80fc89 commit 76adce9

2 files changed

Lines changed: 253 additions & 19 deletions

File tree

internal/agent/firecracker_driver.go

Lines changed: 143 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"context"
88
"errors"
99
"fmt"
10+
gonet "net"
1011
"os"
1112
"os/exec"
1213
"path/filepath"
@@ -17,8 +18,10 @@ import (
1718
firecracker "github.com/firecracker-microvm/firecracker-go-sdk"
1819
"github.com/firecracker-microvm/firecracker-go-sdk/client/models"
1920
ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
21+
logf "sigs.k8s.io/controller-runtime/pkg/log"
2022

2123
impdevv1alpha1 "github.com/syscode-labs/imp/api/v1alpha1"
24+
"github.com/syscode-labs/imp/internal/agent/network"
2225
"github.com/syscode-labs/imp/internal/agent/rootfs"
2326
)
2427

@@ -31,6 +34,7 @@ type fcProc struct {
3134
machine *firecracker.Machine
3235
pid int64
3336
socket string
37+
netInfo *network.NetworkInfo // nil when NetworkRef is absent
3438
}
3539

3640
// FirecrackerDriver is a VMDriver that launches real Firecracker microVMs.
@@ -48,6 +52,11 @@ type FirecrackerDriver struct {
4852
Cache *rootfs.Builder
4953
// Client is the controller-runtime Kubernetes client.
5054
Client ctrlclient.Client
55+
// Net manages host-level networking (bridge, TAP, NAT).
56+
// May be nil for VMs that do not reference an ImpNetwork.
57+
Net network.NetManager
58+
// Alloc manages in-memory IP allocation per ImpNetwork.
59+
Alloc *network.Allocator
5160

5261
// mu guards procs. Must be held for any read or write of the procs map.
5362
mu sync.Mutex
@@ -97,13 +106,14 @@ func NewFirecrackerDriver(client ctrlclient.Client) (*FirecrackerDriver, error)
97106
KernelArgs: kernelArgs,
98107
Cache: &rootfs.Builder{CacheDir: cacheDir},
99108
Client: client,
109+
Alloc: network.NewAllocator(),
100110
procs: make(map[string]*fcProc),
101111
}, nil
102112
}
103113

104114
// Start implements VMDriver. Fetches the ImpVMClass for compute specs, builds
105-
// an ext4 rootfs from the OCI image, then boots a Firecracker microVM.
106-
// Phase 1: loopback networking only. Returns the VMM process PID.
115+
// an ext4 rootfs from the OCI image, sets up networking if NetworkRef is set,
116+
// then boots a Firecracker microVM. Returns the VMM process PID.
107117
func (d *FirecrackerDriver) Start(ctx context.Context, vm *impdevv1alpha1.ImpVM) (int64, error) {
108118
if vm.Spec.ClassRef == nil {
109119
return 0, fmt.Errorf("vm %s/%s has no classRef", vm.Namespace, vm.Name)
@@ -121,19 +131,29 @@ func (d *FirecrackerDriver) Start(ctx context.Context, vm *impdevv1alpha1.ImpVM)
121131
return 0, fmt.Errorf("build rootfs for %s: %w", vm.Spec.Image, err)
122132
}
123133

124-
// 3. Ensure socket directory exists.
134+
// 3. Set up networking if a NetworkRef is specified.
135+
var netInfo *network.NetworkInfo
136+
if vm.Spec.NetworkRef != nil && d.Net != nil {
137+
ni, err := d.setupNetwork(ctx, vm)
138+
if err != nil {
139+
return 0, fmt.Errorf("setup network: %w", err)
140+
}
141+
netInfo = ni
142+
}
143+
144+
// 4. Ensure socket directory exists.
125145
if err := os.MkdirAll(d.SocketDir, 0o750); err != nil {
126146
return 0, fmt.Errorf("socket dir: %w", err)
127147
}
128148
sockPath := d.socketPath(vm)
129149

130-
// 4. Build Firecracker config.
131-
cfg := d.buildConfig(&class, rootfsPath, sockPath)
150+
// 5. Build Firecracker config.
151+
cfg := d.buildConfig(&class, rootfsPath, sockPath, netInfo)
132152

133-
// 5. Build the VMM command.
153+
// 6. Build the VMM command.
134154
cmd := exec.CommandContext(ctx, d.BinPath, "--api-sock", sockPath) //nolint:gosec // G204: BinPath validated in NewFirecrackerDriver
135155

136-
// 6. Create and start the machine.
156+
// 7. Create and start the machine.
137157
m, err := firecracker.NewMachine(ctx, cfg, firecracker.WithProcessRunner(cmd))
138158
if err != nil {
139159
return 0, fmt.Errorf("create machine: %w", err)
@@ -152,7 +172,7 @@ func (d *FirecrackerDriver) Start(ctx context.Context, vm *impdevv1alpha1.ImpVM)
152172
}
153173

154174
d.mu.Lock()
155-
d.procs[vmKey(vm)] = &fcProc{machine: m, pid: int64(pid), socket: sockPath}
175+
d.procs[vmKey(vm)] = &fcProc{machine: m, pid: int64(pid), socket: sockPath, netInfo: netInfo}
156176
d.mu.Unlock()
157177

158178
return int64(pid), nil
@@ -172,17 +192,32 @@ func (d *FirecrackerDriver) Stop(ctx context.Context, vm *impdevv1alpha1.ImpVM)
172192
return nil // already stopped or never started
173193
}
174194

175-
// Attempt graceful ACPI shutdown with a timeout.
176-
shutdownCtx, cancel := context.WithTimeout(ctx, shuttingDownTimeout)
177-
defer cancel()
178-
_ = proc.machine.Shutdown(shutdownCtx) //nolint:errcheck // best-effort graceful stop
195+
if proc.machine != nil {
196+
// Attempt graceful ACPI shutdown with a timeout.
197+
shutdownCtx, cancel := context.WithTimeout(ctx, shuttingDownTimeout)
198+
defer cancel()
199+
_ = proc.machine.Shutdown(shutdownCtx) //nolint:errcheck // best-effort graceful stop
179200

180-
// Force-kill the VMM process regardless of shutdown result.
181-
_ = proc.machine.StopVMM() //nolint:errcheck // best-effort force stop
201+
// Force-kill the VMM process regardless of shutdown result.
202+
_ = proc.machine.StopVMM() //nolint:errcheck // best-effort force stop
203+
}
182204

183205
// Remove the API socket file.
184206
_ = os.Remove(proc.socket) //nolint:errcheck // best-effort cleanup
185207

208+
// Tear down networking if this VM had a network attached.
209+
// NOTE: NAT rules are not torn down in Phase 1 (they are shared across VMs).
210+
if proc.netInfo != nil {
211+
if d.Net != nil {
212+
if err := d.Net.TeardownVM(ctx, proc.netInfo.TAPName); err != nil {
213+
logf.FromContext(ctx).Error(err, "TeardownVM failed", "tap", proc.netInfo.TAPName)
214+
}
215+
}
216+
if d.Alloc != nil {
217+
d.Alloc.Release(proc.netInfo.NetworkKey, proc.netInfo.IP)
218+
}
219+
}
220+
186221
d.mu.Lock()
187222
delete(d.procs, key)
188223
d.mu.Unlock()
@@ -192,7 +227,7 @@ func (d *FirecrackerDriver) Stop(ctx context.Context, vm *impdevv1alpha1.ImpVM)
192227

193228
// Inspect implements VMDriver. Uses kill(pid,0) to check if the Firecracker
194229
// process is still alive. Returns Running=false for VMs not launched by this driver.
195-
// Phase 1: IP is always empty (loopback only, no TAP).
230+
// IP is populated from NetworkInfo when the VM was started with a NetworkRef.
196231
func (d *FirecrackerDriver) Inspect(_ context.Context, vm *impdevv1alpha1.ImpVM) (VMState, error) {
197232
key := vmKey(vm)
198233

@@ -224,8 +259,11 @@ func (d *FirecrackerDriver) Inspect(_ context.Context, vm *impdevv1alpha1.ImpVM)
224259
return VMState{}, fmt.Errorf("inspect %s: %w", key, err)
225260
}
226261

227-
// Phase 1: IP is empty (no TAP networking).
228-
return VMState{Running: true, PID: proc.pid}, nil
262+
ip := ""
263+
if proc.netInfo != nil {
264+
ip = proc.netInfo.IP
265+
}
266+
return VMState{Running: true, PID: proc.pid, IP: ip}, nil
229267
}
230268

231269
// socketPath returns the Unix socket path for the given VM.
@@ -237,8 +275,9 @@ func (d *FirecrackerDriver) socketPath(vm *impdevv1alpha1.ImpVM) string {
237275
func (d *FirecrackerDriver) buildConfig(
238276
class *impdevv1alpha1.ImpVMClass,
239277
rootfsPath, socketPath string,
278+
netInfo *network.NetworkInfo,
240279
) firecracker.Config {
241-
return firecracker.Config{
280+
cfg := firecracker.Config{
242281
SocketPath: socketPath,
243282
KernelImagePath: d.KernelPath,
244283
KernelArgs: d.KernelArgs,
@@ -255,6 +294,92 @@ func (d *FirecrackerDriver) buildConfig(
255294
MemSizeMib: firecracker.Int64(int64(class.Spec.MemoryMiB)),
256295
},
257296
}
297+
if netInfo != nil {
298+
cfg.NetworkInterfaces = firecracker.NetworkInterfaces{{
299+
StaticConfiguration: &firecracker.StaticNetworkConfiguration{
300+
MacAddress: netInfo.MACAddr,
301+
HostDevName: netInfo.TAPName,
302+
IPConfiguration: &firecracker.IPConfiguration{
303+
IPAddr: gonet.IPNet{
304+
IP: gonet.ParseIP(netInfo.IP).To4(),
305+
Mask: gonet.CIDRMask(netInfo.PrefixLen, 32),
306+
},
307+
Gateway: gonet.ParseIP(netInfo.Gateway),
308+
Nameservers: netInfo.DNS,
309+
},
310+
},
311+
}}
312+
}
313+
return cfg
314+
}
315+
316+
// setupNetwork fetches the ImpNetwork, allocates an IP, creates the bridge+TAP,
317+
// and optionally installs NAT rules. Returns the NetworkInfo for this VM.
318+
func (d *FirecrackerDriver) setupNetwork(ctx context.Context, vm *impdevv1alpha1.ImpVM) (*network.NetworkInfo, error) {
319+
var impNet impdevv1alpha1.ImpNetwork
320+
if err := d.Client.Get(ctx, ctrlclient.ObjectKey{
321+
Namespace: vm.Namespace,
322+
Name: vm.Spec.NetworkRef.Name,
323+
}, &impNet); err != nil {
324+
return nil, fmt.Errorf("get network %q: %w", vm.Spec.NetworkRef.Name, err)
325+
}
326+
327+
netKey := impNet.Namespace + "/" + impNet.Name
328+
vKey := vmKey(vm)
329+
bridgeName := network.BridgeName(netKey)
330+
tapName := network.TAPName(vKey)
331+
macAddr := network.MACAddr(vKey)
332+
333+
_, cidr, err := gonet.ParseCIDR(impNet.Spec.Subnet)
334+
if err != nil {
335+
return nil, fmt.Errorf("parse subnet %q: %w", impNet.Spec.Subnet, err)
336+
}
337+
prefixLen, _ := cidr.Mask.Size()
338+
339+
gateway := impNet.Spec.Gateway
340+
if gateway == "" {
341+
gw := make(gonet.IP, 4)
342+
copy(gw, cidr.IP.To4())
343+
gw[3]++
344+
gateway = gw.String()
345+
}
346+
347+
// Allocate VM IP.
348+
ip, err := d.Alloc.Allocate(netKey, impNet.Spec.Subnet, gateway)
349+
if err != nil {
350+
return nil, fmt.Errorf("allocate IP: %w", err)
351+
}
352+
353+
// Ensure bridge exists with gateway IP.
354+
if err := d.Net.EnsureNetwork(ctx, bridgeName, gateway, prefixLen); err != nil {
355+
d.Alloc.Release(netKey, ip)
356+
return nil, fmt.Errorf("ensure bridge: %w", err)
357+
}
358+
359+
// Create TAP and attach to bridge.
360+
if err := d.Net.SetupVM(ctx, tapName, bridgeName, macAddr); err != nil {
361+
d.Alloc.Release(netKey, ip)
362+
return nil, fmt.Errorf("setup tap: %w", err)
363+
}
364+
365+
// Install NAT if requested (best-effort — don't block VM start on NAT failure).
366+
if impNet.Spec.NAT.Enabled {
367+
if natErr := d.Net.EnsureNAT(ctx, impNet.Spec.Subnet, impNet.Spec.NAT.EgressInterface); natErr != nil {
368+
logf.FromContext(ctx).Error(natErr, "EnsureNAT failed — VM will start without NAT")
369+
}
370+
}
371+
372+
return &network.NetworkInfo{
373+
TAPName: tapName,
374+
BridgeName: bridgeName,
375+
MACAddr: macAddr,
376+
IP: ip,
377+
PrefixLen: prefixLen,
378+
Gateway: gateway,
379+
DNS: impNet.Spec.DNS,
380+
Subnet: impNet.Spec.Subnet,
381+
NetworkKey: netKey,
382+
}, nil
258383
}
259384

260385
// compile-time interface check.

internal/agent/firecracker_driver_test.go

Lines changed: 110 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
"sigs.k8s.io/controller-runtime/pkg/client/fake"
1313

1414
impdevv1alpha1 "github.com/syscode-labs/imp/api/v1alpha1"
15+
"github.com/syscode-labs/imp/internal/agent/network"
1516
"github.com/syscode-labs/imp/internal/agent/rootfs"
1617
)
1718

@@ -132,7 +133,7 @@ func TestFirecrackerDriver_BuildConfig(t *testing.T) {
132133
class.Spec.VCPU = 2
133134
class.Spec.MemoryMiB = 512
134135

135-
cfg := d.buildConfig(class, "/cache/abc.ext4", "/run/imp/sockets/default-vm.sock")
136+
cfg := d.buildConfig(class, "/cache/abc.ext4", "/run/imp/sockets/default-vm.sock", nil)
136137

137138
if cfg.SocketPath != "/run/imp/sockets/default-vm.sock" {
138139
t.Errorf("SocketPath = %q, want %q", cfg.SocketPath, "/run/imp/sockets/default-vm.sock")
@@ -239,3 +240,111 @@ func TestFirecrackerDriver_Start_Integration(t *testing.T) {
239240
}
240241
t.Skip("integration test — run manually on KVM-capable node")
241242
}
243+
244+
func TestFirecrackerDriver_BuildConfig_WithNetInfo(t *testing.T) {
245+
d := &FirecrackerDriver{
246+
KernelPath: "/boot/vmlinux",
247+
KernelArgs: "console=ttyS0 reboot=k panic=1 pci=off",
248+
}
249+
250+
class := &impdevv1alpha1.ImpVMClass{}
251+
class.Spec.VCPU = 1
252+
class.Spec.MemoryMiB = 256
253+
254+
ni := &network.NetworkInfo{
255+
TAPName: "imptap-aabbccdd",
256+
MACAddr: "02:aa:bb:cc:dd:ee",
257+
IP: "192.168.100.2",
258+
PrefixLen: 24,
259+
Gateway: "192.168.100.1",
260+
DNS: []string{"8.8.8.8"},
261+
}
262+
263+
cfg := d.buildConfig(class, "/cache/root.ext4", "/run/imp/s/vm.sock", ni)
264+
265+
if len(cfg.NetworkInterfaces) != 1 {
266+
t.Fatalf("len(NetworkInterfaces) = %d, want 1", len(cfg.NetworkInterfaces))
267+
}
268+
iface := cfg.NetworkInterfaces[0]
269+
if iface.StaticConfiguration == nil {
270+
t.Fatal("StaticConfiguration is nil")
271+
}
272+
if iface.StaticConfiguration.HostDevName != "imptap-aabbccdd" {
273+
t.Errorf("HostDevName = %q, want %q", iface.StaticConfiguration.HostDevName, "imptap-aabbccdd")
274+
}
275+
if iface.StaticConfiguration.MacAddress != "02:aa:bb:cc:dd:ee" {
276+
t.Errorf("MacAddress = %q, want %q", iface.StaticConfiguration.MacAddress, "02:aa:bb:cc:dd:ee")
277+
}
278+
if iface.StaticConfiguration.IPConfiguration == nil {
279+
t.Fatal("IPConfiguration is nil")
280+
}
281+
if iface.StaticConfiguration.IPConfiguration.IPAddr.IP.String() != "192.168.100.2" {
282+
t.Errorf("IP = %q, want 192.168.100.2", iface.StaticConfiguration.IPConfiguration.IPAddr.IP)
283+
}
284+
}
285+
286+
func TestFirecrackerDriver_BuildConfig_WithoutNetInfo(t *testing.T) {
287+
d := &FirecrackerDriver{KernelPath: "/boot/vmlinux", KernelArgs: "console=ttyS0"}
288+
class := &impdevv1alpha1.ImpVMClass{}
289+
class.Spec.VCPU = 1
290+
class.Spec.MemoryMiB = 256
291+
292+
cfg := d.buildConfig(class, "/cache/root.ext4", "/run/imp/s/vm.sock", nil)
293+
294+
if len(cfg.NetworkInterfaces) != 0 {
295+
t.Errorf("expected no NetworkInterfaces when netInfo is nil, got %d", len(cfg.NetworkInterfaces))
296+
}
297+
}
298+
299+
func TestFirecrackerDriver_Inspect_ReturnsNetworkIP(t *testing.T) {
300+
d := &FirecrackerDriver{procs: make(map[string]*fcProc)}
301+
302+
vm := &impdevv1alpha1.ImpVM{}
303+
vm.Namespace = "default"
304+
vm.Name = "net-vm"
305+
306+
d.procs[vmKey(vm)] = &fcProc{
307+
pid: int64(os.Getpid()),
308+
netInfo: &network.NetworkInfo{IP: "192.168.1.5"},
309+
}
310+
311+
state, err := d.Inspect(context.Background(), vm)
312+
if err != nil {
313+
t.Fatalf("unexpected error: %v", err)
314+
}
315+
if !state.Running {
316+
t.Error("expected Running=true")
317+
}
318+
if state.IP != "192.168.1.5" {
319+
t.Errorf("IP = %q, want %q", state.IP, "192.168.1.5")
320+
}
321+
}
322+
323+
func TestFirecrackerDriver_Stop_TeardownVMCalled(t *testing.T) {
324+
stub := &network.StubNetManager{}
325+
d := &FirecrackerDriver{
326+
Net: stub,
327+
Alloc: network.NewAllocator(),
328+
procs: make(map[string]*fcProc),
329+
}
330+
331+
vm := &impdevv1alpha1.ImpVM{}
332+
vm.Namespace = "default"
333+
vm.Name = "net-vm-stop"
334+
335+
d.procs[vmKey(vm)] = &fcProc{
336+
pid: 99999, // not running, but we are only testing teardown logic
337+
netInfo: &network.NetworkInfo{
338+
TAPName: "imptap-deadbeef",
339+
NetworkKey: "default/mynet",
340+
IP: "10.0.0.2",
341+
},
342+
}
343+
344+
if err := d.Stop(context.Background(), vm); err != nil {
345+
t.Fatalf("unexpected Stop error: %v", err)
346+
}
347+
if len(stub.TeardownVMCalls) != 1 || stub.TeardownVMCalls[0] != "imptap-deadbeef" {
348+
t.Errorf("TeardownVMCalls = %v, want [imptap-deadbeef]", stub.TeardownVMCalls)
349+
}
350+
}

0 commit comments

Comments
 (0)