From 0600136c2484639a883638a9fd914c78a68a5060 Mon Sep 17 00:00:00 2001 From: Kei Kamikawa Date: Thu, 29 Oct 2026 12:50:44 +0900 Subject: [PATCH 1/2] feat(vmnet): expose interface offload settings --- vmnet/interface.go | 44 +++++++++++++++++++++++++++++++---------- vmnet/interface_test.go | 2 +- 2 files changed, 35 insertions(+), 11 deletions(-) diff --git a/vmnet/interface.go b/vmnet/interface.go index 6c5d68d7..2b4ede74 100644 --- a/vmnet/interface.go +++ b/vmnet/interface.go @@ -90,15 +90,17 @@ func WithInterfaceIsolation(enabled bool) InterfaceOption { type Interface struct{ owner *interfaceOwner } type interfaceOwner struct{ resource *interfaceResource } type interfaceResource struct { - mu sync.Mutex - session unsafe.Pointer - handle cgo.Handle - wake, done chan struct{} - stopping bool - stopErr error - maxSize int - mac [6]byte - readers chan struct{} + mu sync.Mutex + session unsafe.Pointer + handle cgo.Handle + wake, done chan struct{} + stopping bool + stopErr error + maxSize int + virtioHeader bool + tso bool + mac [6]byte + readers chan struct{} } var interfaceWake = purego.NewCallback(func(handle uint64) { @@ -163,7 +165,13 @@ func startInterface(n *Network, name string, opts ...InterfaceOption) (*Interfac if c.enableVirtioHeader && c.enableChecksumOffload { return nil, fmt.Errorf("vmnet virtio header and checksum offload cannot both be enabled") } - r := &interfaceResource{wake: make(chan struct{}, 1), done: make(chan struct{}), readers: make(chan struct{}, 1)} + r := &interfaceResource{ + wake: make(chan struct{}, 1), + done: make(chan struct{}), + readers: make(chan struct{}, 1), + virtioHeader: c.enableVirtioHeader, + tso: c.enableTSO, + } r.handle = cgo.NewHandle(r.wake) var size uint64 var errorPtr unsafe.Pointer @@ -237,6 +245,22 @@ func (i *Interface) MaxPacketSize() int { return i.owner.resource.maxSize } +// VirtioHeaderEnabled reports whether the interface was started with raw Virtio +// headers. It returns false for a nil or uninitialized interface. +// +// See https://developer.apple.com/documentation/vmnet/vmnet_enable_virtio_header_key +func (i *Interface) VirtioHeaderEnabled() bool { + return i != nil && i.owner != nil && i.owner.resource.virtioHeader +} + +// TSOEnabled reports whether the interface was started with TCP segmentation +// offload. It returns false for a nil or uninitialized interface. +// +// See https://developer.apple.com/documentation/vmnet/vmnet_enable_tso_key +func (i *Interface) TSOEnabled() bool { + return i != nil && i.owner != nil && i.owner.resource.tso +} + // ReadPacket waits for one frame. The buffer must hold MaxPacketSize bytes. // [WithInterfaceVirtioHeader] includes the Virtio header in the buffer and byte count. // A canceled context or Stop interrupts the wait. Concurrent readers share packets. diff --git a/vmnet/interface_test.go b/vmnet/interface_test.go index 4f466098..d48f2d45 100644 --- a/vmnet/interface_test.go +++ b/vmnet/interface_test.go @@ -91,7 +91,7 @@ func TestVMNetInterfaceBoundaries(t *testing.T) { if err := i.Stop(); err == nil { t.Fatal("invalid interface stopped") } - if i.MaxPacketSize() != 0 || i.MACAddress() != nil { + if i.MaxPacketSize() != 0 || i.MACAddress() != nil || i.VirtioHeaderEnabled() || i.TSOEnabled() { t.Fatal("invalid interface metadata") } } From 17472d2333ae4f83ab623f89911bc927efc7ef90 Mon Sep 17 00:00:00 2001 From: Kei Kamikawa Date: Thu, 29 Oct 2026 12:50:49 +0900 Subject: [PATCH 2/2] feat(vmnetsock): convert Virtio headers for Ethernet peers --- vmnetsock/adapter.go | 91 ++++++++++++++++++++++++++++++++++----- vmnetsock/adapter_test.go | 61 +++++++++++++++++++++++--- vmnetsock/virtio.go | 64 +++++++++++++++++++++++++++ vmnetsock/virtio_test.go | 69 +++++++++++++++++++++++++++++ 4 files changed, 268 insertions(+), 17 deletions(-) create mode 100644 vmnetsock/virtio.go create mode 100644 vmnetsock/virtio_test.go diff --git a/vmnetsock/adapter.go b/vmnetsock/adapter.go index 118fef4b..5c96b5f4 100644 --- a/vmnetsock/adapter.go +++ b/vmnetsock/adapter.go @@ -25,6 +25,26 @@ const ( Datagram ) +// Option configures a socket adapter. +type Option func(*adapterConfiguration) + +type adapterConfiguration struct { + convertVirtioHeader bool +} + +// WithVirtioHeaderConversion connects a raw Virtio vmnet interface to a peer +// that exchanges plain Ethernet frames, such as QEMU stream/socket networking. +// Start iface with vmnet.WithInterfaceVirtioHeader(true) and leave TSO disabled. +// The adapter completes partial checksums and removes incoming Virtio headers. +// The peer must send fully checksummed Ethernet frames; the adapter adds +// zeroed headers to them. Unsupported segmentation or RSC checksum metadata +// stops forwarding. Close returns the error. +func WithVirtioHeaderConversion() Option { + return func(config *adapterConfiguration) { + config.convertVirtioHeader = true + } +} + // Adapter forwards packets until Close, context cancellation, or a forwarding error. // Copies share a single lifetime. Adapter borrows the Interface and never calls Stop. type Adapter struct { @@ -44,9 +64,10 @@ type adapterState struct { // New creates a socket pair and starts forwarding immediately. // The Adapter owns both sockets and borrows iface without stopping it. // Dup returns a caller-owned socket to pass through exec.Cmd.ExtraFiles. -// Packets pass through unchanged. If iface uses Virtio headers, the peer must -// support that format and the offload features enabled on iface. -func New(ctx context.Context, iface *vmnet.Interface, transport Transport) (*Adapter, error) { +// Packets pass through unchanged unless WithVirtioHeaderConversion is set. +// Without conversion, a peer of an interface using Virtio headers must support that +// format and the offload features enabled on iface. +func New(ctx context.Context, iface *vmnet.Interface, transport Transport, opts ...Option) (*Adapter, error) { if iface == nil || iface.MaxPacketSize() == 0 { return nil, fmt.Errorf("vmnet interface is nil or uninitialized") } @@ -57,6 +78,22 @@ func New(ctx context.Context, iface *vmnet.Interface, transport Transport) (*Ada return nil, err } + var config adapterConfiguration + for _, opt := range opts { + opt(&config) + } + if config.convertVirtioHeader { + if !iface.VirtioHeaderEnabled() { + return nil, fmt.Errorf("Virtio header conversion requires interface Virtio headers") + } + if iface.TSOEnabled() { + return nil, fmt.Errorf("Virtio header conversion requires interface TSO to be disabled") + } + if iface.MaxPacketSize() <= virtioHeaderSize { + return nil, fmt.Errorf("vmnet packet size cannot hold a Virtio header") + } + } + conn, file, err := socketPair(transport, iface.MaxPacketSize()) if err != nil { return nil, err @@ -73,10 +110,10 @@ func New(ctx context.Context, iface *vmnet.Interface, transport Transport) (*Ada var pumps sync.WaitGroup pumps.Go(func() { - s.finish(s.forwardToInterface(run, iface, transport)) + s.finish(s.forwardToInterface(run, iface, transport, config)) }) pumps.Go(func() { - s.finish(s.forwardToPeer(run, iface, transport)) + s.finish(s.forwardToPeer(run, iface, transport, config)) }) go func() { <-run.Done() @@ -112,13 +149,16 @@ func (a *Adapter) Dup() (*os.File, error) { return os.NewFile(uintptr(fd), "vmnet-peer"), nil } -func (s *adapterState) forwardToInterface(ctx context.Context, iface *vmnet.Interface, transport Transport) error { +func (s *adapterState) forwardToInterface(ctx context.Context, iface *vmnet.Interface, transport Transport, config adapterConfiguration) error { + size := min(iface.MaxPacketSize(), packetBatchBytes) + if config.convertVirtioHeader { + size -= virtioHeaderSize + } var readPackets func() ([][]byte, error) if transport == Stream { - reader := newStreamPacketReader(s.conn, iface.MaxPacketSize()) + reader := newStreamPacketReader(s.conn, size) readPackets = reader.readPackets } else { - size := min(iface.MaxPacketSize(), packetBatchBytes) count := packetBatchSize(size) raw, err := s.conn.SyscallConn() if err != nil { @@ -134,11 +174,26 @@ func (s *adapterState) forwardToInterface(ctx context.Context, iface *vmnet.Inte } } + var nativeFrames [][]byte + if config.convertVirtioHeader { + nativeFrames = make([][]byte, packetBatchSize(size)) + for i := range nativeFrames { + nativeFrames[i] = make([]byte, size+virtioHeaderSize) + } + } + for { if err := ctx.Err(); err != nil { return err } frames, err := readPackets() + if config.convertVirtioHeader { + for i, frame := range frames { + nativeFrames[i] = nativeFrames[i][:virtioHeaderSize+len(frame)] + copy(nativeFrames[i][virtioHeaderSize:], frame) + } + frames = nativeFrames[:len(frames)] + } if len(frames) > 0 { if _, writeErr := iface.WritePackets(frames); writeErr != nil && !vmnetBufferExhausted(writeErr) { return writeErr @@ -153,13 +208,17 @@ func (s *adapterState) forwardToInterface(ctx context.Context, iface *vmnet.Inte } } -func (s *adapterState) forwardToPeer(ctx context.Context, iface *vmnet.Interface, transport Transport) error { +func (s *adapterState) forwardToPeer(ctx context.Context, iface *vmnet.Interface, transport Transport, config adapterConfiguration) error { size := iface.MaxPacketSize() count := packetBatchSize(size) frames := make([][]byte, count) for j := range frames { frames[j] = make([]byte, size) } + var peerFrames [][]byte + if config.convertVirtioHeader { + peerFrames = make([][]byte, count) + } var headers [][4]byte var vectors net.Buffers var raw syscall.RawConn @@ -178,14 +237,24 @@ func (s *adapterState) forwardToPeer(ctx context.Context, iface *vmnet.Interface if err != nil { return err } + packets := frames[:n] + if config.convertVirtioHeader { + for i, frame := range packets { + peerFrames[i], err = virtioEthernetPacket(frame) + if err != nil { + return err + } + } + packets = peerFrames[:n] + } if transport == Stream { - if err := writeStreamFrames(s.conn, frames[:n], headers, vectors); err != nil { + if err := writeStreamFrames(s.conn, packets, headers, vectors); err != nil { return err } } else { // Drop the unsent suffix on ENOBUFS. Socket writability does not // guarantee space in the peer's datagram receive buffer. - if err := writeDatagramPackets(raw, frames[:n]); err != nil && !errors.Is(err, unix.ENOBUFS) { + if err := writeDatagramPackets(raw, packets); err != nil && !errors.Is(err, unix.ENOBUFS) { return err } } diff --git a/vmnetsock/adapter_test.go b/vmnetsock/adapter_test.go index 115e35cc..d53029dc 100644 --- a/vmnetsock/adapter_test.go +++ b/vmnetsock/adapter_test.go @@ -19,9 +19,9 @@ import ( "github.com/Code-Hex/vz/v4/vmnetsock" ) -func newInterface(t *testing.T, n *vmnet.Network) *vmnet.Interface { +func newInterface(t *testing.T, n *vmnet.Network, opts ...vmnet.InterfaceOption) *vmnet.Interface { t.Helper() - i, err := vmnet.StartInterfaceWithNetwork(n) + i, err := vmnet.StartInterfaceWithNetwork(n, opts...) if err != nil { t.Fatal(err) } @@ -37,8 +37,18 @@ func TestAdapterFramesAndLifetime(t *testing.T) { if macos.Available(26, vzbridge.SDKVersion) != nil { t.Skip("requires macOS 26 SDK") } - for _, transport := range []vmnetsock.Transport{vmnetsock.Stream, vmnetsock.Datagram} { - t.Run(string(rune('0'+transport)), func(t *testing.T) { + for _, test := range []struct { + name string + transport vmnetsock.Transport + convert bool + }{ + {name: "stream", transport: vmnetsock.Stream}, + {name: "datagram", transport: vmnetsock.Datagram}, + {name: "stream Virtio conversion", transport: vmnetsock.Stream, convert: true}, + {name: "datagram Virtio conversion", transport: vmnetsock.Datagram, convert: true}, + } { + t.Run(test.name, func(t *testing.T) { + transport := test.transport c, err := vmnet.NewNetworkConfiguration(vmnet.ModeHost) if err != nil { t.Fatal(err) @@ -49,10 +59,21 @@ func TestAdapterFramesAndLifetime(t *testing.T) { t.Fatal(err) } t.Cleanup(func() { objc.Release((*objc.Pointer)(n)) }) - a, b := newInterface(t, n), newInterface(t, n) + a := newInterface(t, n, vmnet.WithInterfaceVirtioHeader(test.convert)) + b := newInterface(t, n) + if a.VirtioHeaderEnabled() != test.convert || a.TSOEnabled() { + t.Fatal("unexpected interface offload configuration") + } + var opts []vmnetsock.Option + if test.convert { + opts = append(opts, vmnetsock.WithVirtioHeaderConversion()) + } else if adapter, err := vmnetsock.New(context.Background(), a, transport, vmnetsock.WithVirtioHeaderConversion()); err == nil { + adapter.Close() + t.Fatal("conversion accepted an interface without Virtio headers") + } ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - adapter, err := vmnetsock.New(ctx, a, transport) + adapter, err := vmnetsock.New(ctx, a, transport, opts...) if err != nil { t.Fatal(err) } @@ -126,6 +147,9 @@ func TestAdapterFramesAndLifetime(t *testing.T) { if _, err := peer.Write(frame); err == nil { t.Fatal("peer write succeeded after adapter closed") } + if test.convert { + frame = append(make([]byte, 12), frame...) + } if err := a.WritePacket(frame); err != nil { t.Fatalf("borrowed interface stopped: %v", err) } @@ -336,3 +360,28 @@ func TestAdapterQueuedBurst(t *testing.T) { }) } } + +func TestAdapterVirtioConversionRejectsTSO(t *testing.T) { + if macos.Available(26, vzbridge.SDKVersion) != nil { + t.Skip("requires macOS 26 SDK") + } + c, err := vmnet.NewNetworkConfiguration(vmnet.ModeHost, vmnet.WithMTU(1500)) + if err != nil { + t.Fatal(err) + } + defer objc.Release(c) + n, err := vmnet.NewNetwork(c) + if err != nil { + t.Fatal(err) + } + defer objc.Release((*objc.Pointer)(n)) + iface := newInterface(t, n, vmnet.WithInterfaceVirtioHeader(true), vmnet.WithInterfaceTSO(true)) + if !iface.VirtioHeaderEnabled() || !iface.TSOEnabled() { + t.Fatal("interface did not retain offload configuration") + } + adapter, err := vmnetsock.New(context.Background(), iface, vmnetsock.Stream, vmnetsock.WithVirtioHeaderConversion()) + if err == nil { + adapter.Close() + t.Fatal("conversion accepted an interface with TSO") + } +} diff --git a/vmnetsock/virtio.go b/vmnetsock/virtio.go new file mode 100644 index 00000000..979e6a57 --- /dev/null +++ b/vmnetsock/virtio.go @@ -0,0 +1,64 @@ +package vmnetsock + +import ( + "encoding/binary" + "fmt" +) + +// vmnet uses the 12-byte Virtio 1.1 network header, including num_buffers. +// See https://developer.apple.com/documentation/vmnet/vmnet_enable_virtio_header_key +const virtioHeaderSize = 12 + +const ( + virtioNeedsChecksum byte = 1 + virtioRSCInfo byte = 4 +) + +// virtioEthernetPacket completes checksum offload in place and returns the +// Ethernet frame. The caller retains the original slice for the next read. +func virtioEthernetPacket(packet []byte) ([]byte, error) { + if len(packet) < virtioHeaderSize+14 { + return nil, fmt.Errorf("virtio packet is shorter than its header and Ethernet frame") + } + flags := packet[0] + if packet[1] != 0 { + return nil, fmt.Errorf("unsupported virtio GSO type %#x: disable interface TSO", packet[1]) + } + frame := packet[virtioHeaderSize:] + // Virtio 1.1 section 5.1.6.4.2 requires ignoring unknown flags. + // Checksum fields apply only when NEEDS_CSUM is set without RSC_INFO. + // See https://docs.oasis-open.org/virtio/virtio/v1.1/virtio-v1.1.html + if flags&virtioNeedsChecksum == 0 { + return frame, nil + } + if flags&virtioRSCInfo != 0 { + return nil, fmt.Errorf("cannot complete virtio checksum with RSC metadata") + } + start := int(binary.LittleEndian.Uint16(packet[6:8])) + offset := int(binary.LittleEndian.Uint16(packet[8:10])) + if start+offset+2 > len(frame) { + return nil, fmt.Errorf("virtio checksum field lies outside the Ethernet frame") + } + + // NEEDS_CSUM includes the pseudo-header seed in the checksum field. + // Sum it with the payload, then replace it with the completed checksum. + // See https://docs.oasis-open.org/virtio/virtio/v1.1/virtio-v1.1.html + var sum uint64 + data := frame[start:] + for len(data) >= 2 { + sum += uint64(binary.BigEndian.Uint16(data)) + data = data[2:] + } + if len(data) != 0 { + sum += uint64(data[0]) << 8 + } + for sum>>16 != 0 { + sum = sum&0xffff + sum>>16 + } + checksum := ^uint16(sum) + if checksum == 0 { + checksum = 0xffff + } + binary.BigEndian.PutUint16(frame[start+offset:], checksum) + return frame, nil +} diff --git a/vmnetsock/virtio_test.go b/vmnetsock/virtio_test.go new file mode 100644 index 00000000..9ebeb5b3 --- /dev/null +++ b/vmnetsock/virtio_test.go @@ -0,0 +1,69 @@ +package vmnetsock + +import ( + "bytes" + "encoding/binary" + "testing" +) + +func TestVirtioEthernetPacket(t *testing.T) { + for _, flags := range []byte{0, 2, 4, 6, 0x80} { + packet := make([]byte, 12+14) + packet[0] = flags + binary.LittleEndian.PutUint16(packet[6:], 0xffff) + binary.LittleEndian.PutUint16(packet[8:], 0xffff) + copy(packet[12:], []byte("ethernet frame")) + got, err := virtioEthernetPacket(packet) + if err != nil || !bytes.Equal(got, []byte("ethernet frame")) { + t.Fatalf("flags %d: packet = %x, error = %v", flags, got, err) + } + } + + // The checksum field already contains the pseudo-header seed 0x1234. + // The five checksum bytes sum to 0x1234 + 0x5678 + 0x9a00 = 0x102ac. + packet := make([]byte, 12+19) + packet[0] = 1 + binary.LittleEndian.PutUint16(packet[6:], 14) + copy(packet[12+14:], []byte{0x12, 0x34, 0x56, 0x78, 0x9a}) + got, err := virtioEthernetPacket(packet) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got[14:], []byte{0xfd, 0x52, 0x56, 0x78, 0x9a}) { + t.Fatalf("checksum completion = %x", got[14:]) + } + + packet = make([]byte, 12+18) + packet[0] = 1 + binary.LittleEndian.PutUint16(packet[6:], 14) + binary.LittleEndian.PutUint16(packet[8:], 2) + copy(packet[12+14:], []byte{0x12, 0x34, 0xed, 0xcb}) + got, err = virtioEthernetPacket(packet) + if err != nil || !bytes.Equal(got[14:], []byte{0x12, 0x34, 0xff, 0xff}) { + t.Fatalf("zero checksum = %x, error = %v", got, err) + } +} + +func TestVirtioEthernetPacketRejectsInvalidMetadata(t *testing.T) { + for _, test := range []struct { + name string + header []byte + size int + }{ + {name: "short header", size: 11}, + {name: "short Ethernet frame", size: 25}, + {name: "RSC with incomplete checksum", header: []byte{5}, size: 26}, + {name: "GSO", header: []byte{1, 1}, size: 26}, + {name: "checksum start past frame", header: []byte{1, 0, 0, 0, 0, 0, 14}, size: 26}, + {name: "checksum field past frame", header: []byte{1, 0, 0, 0, 0, 0, 12, 0, 1}, size: 26}, + {name: "large checksum offset", header: []byte{1, 0, 0, 0, 0, 0, 0xff, 0xff, 0xff, 0xff}, size: 26}, + } { + t.Run(test.name, func(t *testing.T) { + packet := make([]byte, test.size) + copy(packet, test.header) + if _, err := virtioEthernetPacket(packet); err == nil { + t.Fatal("invalid metadata accepted") + } + }) + } +}