512 default queue size, user can set it

Author: CMGS

README.md

@@ -10,7 +10,7 @@ Lightweight MicroVM engine with dual hypervisor backends: [Cloud Hypervisor](htt
 - **UEFI boot** — CLOUDHV.fd firmware by default; direct kernel boot for OCI images (auto-detected)
 - **COW overlays** — copy-on-write disks backed by shared base images (raw for OCI, qcow2 for cloud images)
 - **CNI networking** — automatic NIC creation via CNI plugins, multi-NIC support, per-VM IP allocation
-- **Multi-queue virtio-net** — TAP devices created with per-vCPU queue pairs; TSO/UFO/csum offload enabled by default
+- **Multi-queue virtio-net** — TAP devices created with per-vCPU queue pairs; configurable ring depth (`--queue-size`, default 512); TSO/UFO/csum offload enabled by default
 - **TC redirect I/O path** — veth ↔ TAP wired via ingress qdisc + mirred redirect (no bridge in the data path)
 - **DNS configuration** — custom DNS servers injected into VMs via kernel cmdline (OCI) or cloud-init network-config (cloudimg)
 - **Cloud-init metadata** — automatic NoCloud cidata FAT12 disk for cloudimg VMs (hostname, root password, multi-NIC Netplan v2 network-config); cidata is automatically skipped on subsequent boots
@@ -178,6 +178,7 @@ Applies to `cocoon vm create`, `cocoon vm run`, and `cocoon vm debug`:
 | `--memory`  | `1G`             | Memory size (e.g., 512M, 2G)                  |
 | `--storage` | `10G`            | COW disk size (e.g., 10G, 20G)                |
 | `--nics`    | `1`              | Number of network interfaces (0 = no network) |
+| `--queue-size` | `0` (default 512) | Virtio-net ring depth per queue (larger = better bulk throughput, smaller = better RPC latency; CH only, ignored by FC) |
 | `--network` | empty (default)  | CNI conflist name (empty = first conflist)     |
 | `--windows` | `false`          | Windows guest (UEFI boot, kvm_hyperv=on, no cidata) |
 
@@ -192,6 +193,7 @@ Applies to `cocoon vm clone`:
 | `--memory`  | empty (inherit)          | Memory size (must be >= snapshot value)                  |
 | `--storage` | empty (inherit)          | COW disk size (must be >= snapshot value)                |
 | `--nics`    | `0` (inherit)            | Number of NICs (must be >= snapshot value)               |
+| `--queue-size` | `0` (inherit)         | Virtio-net ring depth per queue (0 = inherit from snapshot) |
 | `--network` | empty (inherit)          | CNI conflist name (empty = inherit from source VM)       |
 
 ### Snapshot Flags
@@ -273,7 +275,7 @@ Cocoon uses [CNI](https://www.cni.dev/) for VM networking. Each NIC is backed by
 Guest virtio-net  ←→  TAP (multi-queue)  ←TC redirect→  veth  ←→  CNI bridge/overlay
 ```
 
-- **Multi-queue**: each TAP device is created with one queue pair per boot vCPU (`num_queues = 2 × vCPU` in Cloud Hypervisor), enabling per-CPU TX/RX rings for better throughput
+- **Multi-queue**: each TAP device is created with one queue pair per boot vCPU (`num_queues = 2 × vCPU` in Cloud Hypervisor), enabling per-CPU TX/RX rings for better throughput. Ring depth per queue is configurable via `--queue-size` (default 512; larger values improve bulk download throughput, smaller values improve RPC latency)
 - **Offload**: TSO, UFO, and checksum offload are enabled on the virtio-net device; TAP uses `VNET_HDR` for zero-copy GSO passthrough
 - **MAC passthrough**: the guest NIC inherits the CNI veth's MAC address, satisfying anti-spoofing requirements of Cilium, Calico eBPF, and VPC ENI plugins
 - **MTU sync**: TAP MTU is automatically synced to the veth to prevent silent large-packet drops in overlay or jumbo-frame setups
@@ -284,6 +286,7 @@ Guest virtio-net  ←→  TAP (multi-queue)  ←TC redirect→  veth  ←→  CN
 - **No network**: `--nics 0` creates a VM with no network interfaces
 - **Multi-NIC**: `--nics N` creates N interfaces; for cloudimg VMs all NICs are auto-configured via Netplan, for OCI images all NICs are auto-configured via kernel `ip=` parameters
 - **Multi-network**: `--network <name>` selects a specific CNI conflist by name (e.g., `--network macvlan`); omitting uses the first conflist alphabetically. The network name is stored in the VM record for recovery after host reboot. Clone allows `--network` override; restore reuses the existing network.
+- **Bridge mode**: `--bridge <device>` creates TAP devices directly on an existing Linux bridge (e.g., `--bridge cni0`), bypassing CNI and TC redirect. VMs get IP via DHCP from the bridge. Mutually exclusive with `--network`
 - **DNS**: Use `--dns` to set custom DNS servers (comma separated)
 
 ### CNI Configuration

cmd/core/helpers.go

@@ -250,6 +250,7 @@ func VMConfigFromFlags(cmd *cobra.Command, image string) (*types.VMConfig, error
 	cpu, _ := cmd.Flags().GetInt("cpu")
 	memStr, _ := cmd.Flags().GetString("memory")
 	storStr, _ := cmd.Flags().GetString("storage")
+	queueSize, _ := cmd.Flags().GetInt("queue-size")
 	network, _ := cmd.Flags().GetString("network")
 	windows, _ := cmd.Flags().GetBool("windows")
 
@@ -267,13 +268,14 @@ func VMConfigFromFlags(cmd *cobra.Command, image string) (*types.VMConfig, error
 	}
 
 	cfg := &types.VMConfig{
-		Name:    vmName,
-		CPU:     cpu,
-		Memory:  memBytes,
-		Storage: storBytes,
-		Image:   image,
-		Network: network,
-		Windows: windows,
+		Name:      vmName,
+		CPU:       cpu,
+		Memory:    memBytes,
+		Storage:   storBytes,
+		QueueSize: queueSize,
+		Image:     image,
+		Network:   network,
+		Windows:   windows,
 	}
 	if err := cfg.Validate(); err != nil {
 		return nil, err
@@ -290,20 +292,25 @@ func CloneVMConfigFromFlags(cmd *cobra.Command, snapCfg *types.SnapshotConfig) (
 	if network == "" {
 		network = snapCfg.Network
 	}
+	queueSize, _ := cmd.Flags().GetInt("queue-size")
+	if queueSize == 0 {
+		queueSize = snapCfg.QueueSize
+	}
 
 	cpu, memBytes, storBytes, err := mergeResourceFlags(cmd, snapCfg.CPU, snapCfg.Memory, snapCfg.Storage, snapCfg)
 	if err != nil {
 		return nil, err
 	}
 
 	cfg := &types.VMConfig{
-		Name:    vmName,
-		CPU:     cpu,
-		Memory:  memBytes,
-		Storage: storBytes,
-		Image:   snapCfg.Image,
-		Network: network,
-		Windows: snapCfg.Windows,
+		Name:      vmName,
+		CPU:       cpu,
+		Memory:    memBytes,
+		Storage:   storBytes,
+		QueueSize: queueSize,
+		Image:     snapCfg.Image,
+		Network:   network,
+		Windows:   snapCfg.Windows,
 	}
 	if err := cfg.Validate(); err != nil {
 		return nil, err

cmd/vm/commands.go

@@ -155,6 +155,7 @@ func addVMFlags(cmd *cobra.Command) {
 	cmd.Flags().String("memory", "1G", "memory size")     //nolint:mnd
 	cmd.Flags().String("storage", "10G", "COW disk size") //nolint:mnd
 	cmd.Flags().Int("nics", 1, "number of network interfaces (0 = no network); multiple NICs with auto IP config only works for cloudimg; OCI images auto-configure only the last NIC, others require manual setup inside the guest")
+	cmd.Flags().Int("queue-size", 0, "virtio-net ring depth per queue (0 = default 512; tradeoff: larger improves download throughput, smaller improves RPC latency)") //nolint:mnd
 	cmd.Flags().String("network", "", "CNI conflist name (empty = default); mutually exclusive with --bridge")
 	cmd.Flags().String("bridge", "", "use TAP-on-bridge instead of CNI (value is bridge device, e.g. cni0); VM gets IP via DHCP from the bridge")
 	cmd.Flags().Bool("windows", false, "Windows guest (UEFI boot, kvm_hyperv=on, no cidata)")
@@ -166,6 +167,7 @@ func addCloneFlags(cmd *cobra.Command) {
 	cmd.Flags().String("memory", "", "memory size (empty = inherit from snapshot)")
 	cmd.Flags().String("storage", "", "COW disk size (empty = inherit from snapshot)")
 	cmd.Flags().Int("nics", 0, "number of NICs (0 = inherit from snapshot)")
+	cmd.Flags().Int("queue-size", 0, "virtio-net ring depth per queue (0 = inherit from snapshot)") //nolint:mnd
 	cmd.Flags().String("network", "", "CNI conflist name (empty = inherit from source VM)")
 	cmd.Flags().String("bridge", "", "use TAP-on-bridge instead of CNI (value is bridge device, e.g. cni0)")
 }

hypervisor/cloudhypervisor/snapshot.go

@@ -131,6 +131,7 @@ func (ch *CloudHypervisor) Snapshot(ctx context.Context, ref string) (*types.Sna
 		Memory:     rec.Config.Memory,
 		Storage:    rec.Config.Storage,
 		NICs:       len(rec.NetworkConfigs),
+		QueueSize:  rec.Config.QueueSize,
 		Network:    rec.Config.Network,
 		Windows:    rec.Config.Windows,
 	}

hypervisor/firecracker/snapshot.go

@@ -123,6 +123,7 @@ func (fc *Firecracker) Snapshot(ctx context.Context, ref string) (*types.Snapsho
 		Memory:     rec.Config.Memory,
 		Storage:    rec.Config.Storage,
 		NICs:       len(rec.NetworkConfigs),
+		QueueSize:  rec.Config.QueueSize,
 		Network:    rec.Config.Network,
 	}
 	if rec.ImageBlobIDs != nil {

network/bridge/bridge_linux.go

@@ -122,7 +122,7 @@ func (b *Bridge) Config(ctx context.Context, vmID string, numNICs int, vmCfg *ty
 			Tap:       name,
 			Mac:       mac,
 			NumQueues: queues,
-			QueueSize: network.NetQueueSize,
+			QueueSize: network.ResolveQueueSize(vmCfg.QueueSize),
 			Backend:   typ,
 			BridgeDev: b.bridgeDev,
 			// NetnsPath: empty — TAP is in host netns.

network/cni/create.go

@@ -98,7 +98,7 @@ func (c *CNI) Config(ctx context.Context, vmID string, numNICs int, vmCfg *types
 			Tap:       tapName,
 			Mac:       mac,
 			NumQueues: network.NetNumQueues(vmCfg.CPU),
-			QueueSize: network.NetQueueSize,
+			QueueSize: network.ResolveQueueSize(vmCfg.QueueSize),
 			Backend:   typ,
 			NetnsPath: nsPath,
 			Network:   netInfo,

network/utils.go

@@ -4,9 +4,9 @@ const (
 	vmIDPrefixLen = 8
 
 	// NetQueueSize is the default virtio-net ring depth per queue.
-	// 1024 doubles the CH default (256) to allow more in-flight descriptors
-	// per epoll wakeup, reducing eventfd round-trips under high throughput.
-	NetQueueSize = 1024
+	// 512 balances download throughput (favors larger rings) against
+	// request-response latency (favors smaller rings).
+	NetQueueSize = 512
 )
 
 // NetNumQueues returns the virtio-net queue count for the given CPU count.
@@ -18,6 +18,14 @@ func NetNumQueues(cpu int) int {
 	return cpu * 2 //nolint:mnd
 }
 
+// ResolveQueueSize returns qs if positive, otherwise the default NetQueueSize.
+func ResolveQueueSize(qs int) int {
+	if qs > 0 {
+		return qs
+	}
+	return NetQueueSize
+}
+
 // VMIDPrefix returns the first 8 characters of a VM ID, matching the
 // truncation used by both bridge and CNI TAP device naming.
 func VMIDPrefix(vmID string) string {

snapshot/localfile/localfile.go

@@ -271,6 +271,7 @@ func snapshotRecordToConfig(rec *snapshot.SnapshotRecord) *types.SnapshotConfig
 		Memory:       rec.Memory,
 		Storage:      rec.Storage,
 		NICs:         rec.NICs,
+		QueueSize:    rec.QueueSize,
 		Network:      rec.Network,
 		Windows:      rec.Windows,
 	}

types/snapshot.go

@@ -14,12 +14,13 @@ type SnapshotConfig struct {
 
 	// Original VM resource config, populated during snapshot creation.
 	// Used by clone for parameter inheritance and validation.
-	CPU     int    `json:"cpu,omitempty"`
-	Memory  int64  `json:"memory,omitempty"`  // bytes
-	Storage int64  `json:"storage,omitempty"` // bytes
-	NICs    int    `json:"nics,omitempty"`
-	Network string `json:"network,omitempty"`
-	Windows bool   `json:"windows,omitempty"`
+	CPU       int    `json:"cpu,omitempty"`
+	Memory    int64  `json:"memory,omitempty"`  // bytes
+	Storage   int64  `json:"storage,omitempty"` // bytes
+	NICs      int    `json:"nics,omitempty"`
+	QueueSize int    `json:"queue_size,omitempty"`
+	Network   string `json:"network,omitempty"`
+	Windows   bool   `json:"windows,omitempty"`
 }
 
 // Snapshot is the public record for a snapshot.