feat: default network policy per SandboxTemplate (kubernetes-sigs#287)

* feat: default network policy

nit

update

update to fix e2e tests

policy per template

nit

updated api comments

update

update

update

nit

updated

update

update

update

nit

nit

nit

nit

update api

udpdate

nit

* creation of sandboxtemplate controller to handle networkpolicy lifecycle

update

nit

Author: vicentefb

cmd/agent-sandbox-controller/main.go

@@ -60,6 +60,7 @@ func main() {
 	var sandboxConcurrentWorkers int
 	var sandboxClaimConcurrentWorkers int
 	var sandboxWarmPoolConcurrentWorkers int
+	var sandboxTemplateConcurrentWorkers int
 	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	flag.BoolVar(&enableLeaderElection, "leader-elect", true,
@@ -84,6 +85,7 @@ func main() {
 	flag.IntVar(&sandboxConcurrentWorkers, "sandbox-concurrent-workers", 1, "Max concurrent reconciles for the Sandbox controller")
 	flag.IntVar(&sandboxClaimConcurrentWorkers, "sandbox-claim-concurrent-workers", 1, "Max concurrent reconciles for the SandboxClaim controller")
 	flag.IntVar(&sandboxWarmPoolConcurrentWorkers, "sandbox-warm-pool-concurrent-workers", 1, "Max concurrent reconciles for the SandboxWarmPool controller")
+	flag.IntVar(&sandboxTemplateConcurrentWorkers, "sandbox-template-concurrent-workers", 1, "Max concurrent reconciles for the SandboxTemplate controller")
 	opts := zap.Options{
 		Development: true,
 	}
@@ -98,7 +100,7 @@ func main() {
 		os.Exit(1)
 	}
 	// A logical maximum (too much will create unnecessary load on the API server)
-	totalWorkers := sandboxConcurrentWorkers + sandboxClaimConcurrentWorkers + sandboxWarmPoolConcurrentWorkers
+	totalWorkers := sandboxConcurrentWorkers + sandboxClaimConcurrentWorkers + sandboxWarmPoolConcurrentWorkers + sandboxTemplateConcurrentWorkers
 	if totalWorkers > 1000 {
 		setupLog.Info("Warning: total concurrent workers exceeds 1000, which could lead to resource exhaustion", "total", totalWorkers)
 	}
@@ -220,6 +222,16 @@ func main() {
 			os.Exit(1)
 		}
 
+		if err = (&extensionscontrollers.SandboxTemplateReconciler{
+			Client:   mgr.GetClient(),
+			Scheme:   mgr.GetScheme(),
+			Recorder: mgr.GetEventRecorderFor("sandboxtemplate-controller"),
+			Tracer:   instrumenter,
+		}).SetupWithManager(mgr, sandboxTemplateConcurrentWorkers); err != nil {
+			setupLog.Error(err, "unable to create controller", "controller", "SandboxTemplate")
+			os.Exit(1)
+		}
+
 		if err = (&extensionscontrollers.SandboxWarmPoolReconciler{
 			Client: mgr.GetClient(),
 		}).SetupWithManager(mgr, sandboxWarmPoolConcurrentWorkers); err != nil {

extensions/api/v1alpha1/sandboxtemplate_types.go

@@ -23,11 +23,24 @@ import (
 // NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.
 // Important: Run "make" to regenerate code after modifying this file
 
+// NetworkPolicyManagement defines whether the controller automatically generates
+// and manages a shared NetworkPolicy for this template.
+type NetworkPolicyManagement string
+
 const (
 	// SandboxIDLabel is the label key applied to the Pod to identify the owning Claim UID.
 	// The SandboxClaim controller injects this label into the Pod
 	// System-injected labels/annotations shouldn't be touched
 	SandboxIDLabel = "agents.x-k8s.io/claim-uid"
+
+	// NetworkPolicyManagementManaged means the controller will ensure a shared NetworkPolicy exists.
+	// This shared NetworkPolicy will be a user provide one or a default controller created policy.
+	// This is the default behavior if the field is omitted.
+	NetworkPolicyManagementManaged NetworkPolicyManagement = "Managed"
+
+	// NetworkPolicyManagementUnmanaged means the controller will skip NetworkPolicy
+	// creation entirely, allowing external systems (like Cilium) to manage networking.
+	NetworkPolicyManagementUnmanaged NetworkPolicyManagement = "Unmanaged"
 )
 
 // NetworkPolicySpec defines the desired state of the NetworkPolicy.
@@ -55,7 +68,19 @@ type SandboxTemplateSpec struct {
 	PodTemplate sandboxv1alpha1.PodTemplate `json:"podTemplate" protobuf:"bytes,3,opt,name=podTemplate"`
 
 	// NetworkPolicy defines the network policy to be applied to the sandboxes
-	// created from this template.
+	// created from this template. A single shared NetworkPolicy is created per Template.
+	// Behavior is dictated by the NetworkPolicyManagement field:
+	// - If Management is "Unmanaged": This field is completely ignored.
+	// - If Management is "Managed" (default) and this field is omitted (nil): The controller
+	//   automatically applies a strict Secure Default policy:
+	//     * Ingress: Allow traffic only from the Sandbox Router.
+	//     * Egress: Allow Public Internet only. Blocks internal IPs (RFC1918), Metadata Server, etc.
+	// - If Management is "Managed" and this field is provided: The controller applies your custom rules.
+	// Update Behavior:
+	// Because the NetworkPolicy is shared at the template level, any updates to these rules
+	// will be applied to the single shared policy object. The underlying Kubernetes CNI will then
+	// dynamically enforce the updated rules across all existing and future sandboxes
+	// referencing this template.
 	// NOTE: This is a restricted subset of the standard Kubernetes NetworkPolicySpec.
 	// Fields like 'PodSelector' and 'PolicyTypes' are intentionally excluded because
 	// they are managed by the controller to ensure strict isolation and default-deny posture.
@@ -66,6 +91,13 @@ type SandboxTemplateSpec struct {
 	// otherwise the sidecars may fail health checks.
 	// +optional
 	NetworkPolicy *NetworkPolicySpec `json:"networkPolicy,omitempty"`
+
+	// NetworkPolicyManagement defines whether the controller manages the NetworkPolicy.
+	// Valid values are "Managed" (default) or "Unmanaged".
+	// +kubebuilder:validation:Enum=Managed;Unmanaged
+	// +kubebuilder:default=Managed
+	// +optional
+	NetworkPolicyManagement NetworkPolicyManagement `json:"networkPolicyManagement,omitempty"`
 }
 
 // SandboxTemplateStatus defines the observed state of Sandbox.

extensions/controllers/sandboxclaim_controller.go

@@ -18,12 +18,11 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"reflect"
 	"sort"
 	"time"
 
 	corev1 "k8s.io/api/core/v1"
-	networkingv1 "k8s.io/api/networking/v1"
+	"k8s.io/apimachinery/pkg/api/equality"
 	k8errors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -45,8 +44,9 @@ import (
 
 // TODO: These constants should be imported from the main controller package Issue #216
 const (
-	sandboxLabel = "agents.x-k8s.io/sandbox-name-hash"
-	poolNameNone = "none"
+	sandboxLabel         = "agents.x-k8s.io/sandbox-name-hash"
+	poolNameNone         = "none"
+	sandboxTemplateLabel = "agents.x-k8s.io/sandbox-template-ref-hash"
 )
 
 // ErrTemplateNotFound is a sentinel error indicating a SandboxTemplate was not found.
@@ -66,7 +66,6 @@ type SandboxClaimReconciler struct {
 //+kubebuilder:rbac:groups=agents.x-k8s.io,resources=sandboxes,verbs=get;list;watch;create;update;patch;delete
 //+kubebuilder:rbac:groups=extensions.agents.x-k8s.io,resources=sandboxtemplates,verbs=get;list;watch
 //+kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;update;patch
-//+kubebuilder:rbac:groups=networking.k8s.io,resources=networkpolicies,verbs=get;list;watch;create;update;patch;delete
 //+kubebuilder:rbac:groups="",resources=events,verbs=create;patch
 //+kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;list;watch;create;update;patch;delete
 
@@ -194,13 +193,6 @@ func (r *SandboxClaimReconciler) reconcileActive(ctx context.Context, claim *ext
 
 	// Only attempt network policy reconciliation if template was found.
 	if templateErr == nil || k8errors.IsNotFound(templateErr) {
-		// This ensures the firewall is up before the pod starts.
-		if template != nil {
-			if npErr := r.reconcileNetworkPolicy(ctx, claim, template); npErr != nil {
-				return nil, fmt.Errorf("failed to reconcile network policy: %w", npErr)
-			}
-		}
-
 		// Try getting sandbox even if template is not found
 		// It is possible that the template was deleted after the sandbox was created
 		sandbox, err := r.getOrCreateSandbox(ctx, claim, template)
@@ -249,7 +241,7 @@ func (r *SandboxClaimReconciler) updateStatus(ctx context.Context, oldStatus *ex
 		return claim.Status.Conditions[i].Type < claim.Status.Conditions[j].Type
 	})
 
-	if reflect.DeepEqual(oldStatus, &claim.Status) {
+	if equality.Semantic.DeepEqual(oldStatus, &claim.Status) {
 		return nil
 	}
 
@@ -399,7 +391,6 @@ func (r *SandboxClaimReconciler) tryAdoptPodFromPool(ctx context.Context, claim
 
 	// Remove the pool labels
 	delete(pod.Labels, poolLabel)
-	delete(pod.Labels, sandboxTemplateRefHash)
 
 	// Remove existing owner references (from SandboxWarmPool)
 	pod.OwnerReferences = nil
@@ -414,6 +405,8 @@ func (r *SandboxClaimReconciler) tryAdoptPodFromPool(ctx context.Context, claim
 	// Label required by NetworkPolicy
 	// We add the new label with the Claim UID for unique targeting.
 	pod.Labels[extensionsv1alpha1.SandboxIDLabel] = string(claim.UID)
+	// Adopted pods must have the template hash label to ensure they are selected by the correct NetworkPolicy.
+	pod.Labels[sandboxTemplateLabel] = sandboxcontrollers.NameHash(claim.Spec.TemplateRef.Name)
 
 	// Update the pod
 	if err := r.Update(ctx, pod); err != nil {
@@ -441,6 +434,11 @@ func (r *SandboxClaimReconciler) createSandbox(ctx context.Context, claim *exten
 		},
 	}
 
+	// Determine if we are in "Secure By Default" mode
+	management := template.Spec.NetworkPolicyManagement
+	isManaged := management == "" || management == extensionsv1alpha1.NetworkPolicyManagementManaged
+	isSecureByDefault := isManaged && template.Spec.NetworkPolicy == nil
+
 	// Propagate the trace context annotation to the Sandbox resource
 	if sandbox.Annotations == nil {
 		sandbox.Annotations = make(map[string]string)
@@ -460,10 +458,23 @@ func (r *SandboxClaimReconciler) createSandbox(ctx context.Context, claim *exten
 		automount := false
 		sandbox.Spec.PodTemplate.Spec.AutomountServiceAccountToken = &automount
 	}
+	// To prevent internal DNS enumeration while still allowing public domain resolution,
+	// we explicitly override the Pod's DNS config to use external public resolvers.
+	// We only inject this if using the strict "Secure by Default" policy. If the user
+	// provides custom rules or is Unmanaged, we leave DNS alone for air-gapped/proxy compatibility.
+	if isSecureByDefault && sandbox.Spec.PodTemplate.Spec.DNSPolicy == "" {
+		sandbox.Spec.PodTemplate.Spec.DNSPolicy = corev1.DNSNone
+		sandbox.Spec.PodTemplate.Spec.DNSConfig = &corev1.PodDNSConfig{
+			Nameservers: []string{"8.8.8.8", "1.1.1.1"}, // Google & Cloudflare public DNS
+		}
+	}
+
 	if sandbox.Spec.PodTemplate.ObjectMeta.Labels == nil {
 		sandbox.Spec.PodTemplate.ObjectMeta.Labels = make(map[string]string)
 	}
 	sandbox.Spec.PodTemplate.ObjectMeta.Labels[extensionsv1alpha1.SandboxIDLabel] = string(claim.UID)
+	// This handles the scenario where the Warm Pool is empty (or disabled), and you have to create a brand new Pod from scratch.
+	sandbox.Spec.PodTemplate.ObjectMeta.Labels[sandboxTemplateLabel] = sandboxcontrollers.NameHash(template.Name)
 
 	if err := controllerutil.SetControllerReference(claim, sandbox, r.Scheme); err != nil {
 		err = fmt.Errorf("failed to set controller reference for sandbox: %w", err)
@@ -578,69 +589,6 @@ func (r *SandboxClaimReconciler) SetupWithManager(mgr ctrl.Manager, concurrentWo
 		Complete(r)
 }
 
-// reconcileNetworkPolicy ensures a NetworkPolicy exists for the claimed Sandbox.
-func (r *SandboxClaimReconciler) reconcileNetworkPolicy(ctx context.Context, claim *extensionsv1alpha1.SandboxClaim, template *extensionsv1alpha1.SandboxTemplate) error {
-	logger := log.FromContext(ctx)
-
-	// 1. Cleanup Check: If missing, delete existing policy
-	if template == nil || template.Spec.NetworkPolicy == nil {
-		existingNP := &networkingv1.NetworkPolicy{
-			ObjectMeta: metav1.ObjectMeta{
-				Name:      claim.Name + "-network-policy",
-				Namespace: claim.Namespace,
-			},
-		}
-		if err := r.Delete(ctx, existingNP); err != nil {
-			if !k8errors.IsNotFound(err) {
-				logger.Error(err, "Failed to clean up disabled NetworkPolicy")
-				return err
-			}
-		} else {
-			logger.Info("Deleted disabled NetworkPolicy", "name", existingNP.Name)
-		}
-		return nil
-	}
-
-	np := &networkingv1.NetworkPolicy{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      claim.Name + "-network-policy",
-			Namespace: claim.Namespace,
-		},
-	}
-
-	_, err := controllerutil.CreateOrUpdate(ctx, r.Client, np, func() error {
-		np.Spec.PodSelector = metav1.LabelSelector{
-			MatchLabels: map[string]string{
-				extensionsv1alpha1.SandboxIDLabel: string(claim.UID),
-			},
-		}
-		np.Spec.PolicyTypes = []networkingv1.PolicyType{
-			networkingv1.PolicyTypeIngress,
-			networkingv1.PolicyTypeEgress,
-		}
-
-		templateNP := template.Spec.NetworkPolicy
-
-		if len(templateNP.Ingress) > 0 {
-			np.Spec.Ingress = templateNP.Ingress
-		}
-
-		if len(templateNP.Egress) > 0 {
-			np.Spec.Egress = templateNP.Egress
-		}
-
-		return controllerutil.SetControllerReference(claim, np, r.Scheme)
-	})
-
-	if err != nil {
-		logger.Error(err, "Failed to create or update NetworkPolicy for claim")
-		return err
-	}
-
-	logger.Info("Successfully reconciled NetworkPolicy for claim", "NetworkPolicy.Name", np.Name)
-	return nil
-}
-
 // recordCreationLatencyMetric detects and records transitions to Ready state.
 func (r *SandboxClaimReconciler) recordCreationLatencyMetric(
 	claim *extensionsv1alpha1.SandboxClaim,