Skip to content

Commit f8ea540

Browse files
committed
feat: refactor /etc mounts
Stop using individual bind mounts for `/etc` files and use an overlay backed by rootfs `/etc` and an anonymous tmpfs. This allows in future to support files under `/etc` as machine config documents. Drop all bind mount support, containers now also follow same pattern. Also move `/etc/cni` and `/etc/kuberentes` as tmpfs, since we cannot have a nested overlay over size of two, when system has an extension(s), `/etc` being an overlay means, it's already two levels in, so `/etc/cni` and `/etc/kubernetes` cannot be overlayed on top again. Signed-off-by: Noel Georgi <git@frezbo.dev>
1 parent 5d4ba70 commit f8ea540

18 files changed

Lines changed: 450 additions & 198 deletions

File tree

Dockerfile

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -789,8 +789,8 @@ END
789789
# symlinks to avoid accidentally cleaning them up.
790790
RUN --mount=type=bind,source=hack/cleanup.sh,target=/usr/bin/cleanup.sh <<END
791791
cleanup.sh /rootfs
792-
mkdir -pv /rootfs/{boot/EFI,etc/{iscsi,nvme,cri/conf.d/hosts},usr/lib/firmware,usr/etc,usr/local/share,usr/share/zoneinfo/Etc,mnt,system,opt,.extra}
793-
mkdir -pv /rootfs/{etc/kubernetes/manifests,etc/cni/net.d,etc/ssl/certs,usr/libexec/kubernetes,/usr/local/lib/kubelet/credentialproviders,etc/selinux/targeted/contexts/files}
792+
mkdir -pv /rootfs/{boot/EFI,usr/lib/firmware,usr/etc,usr/local/share,usr/share/zoneinfo/Etc,mnt,system,opt,.extra}
793+
mkdir -pv /rootfs/{etc/ssl/certs,usr/libexec/kubernetes,/usr/local/lib/kubelet/credentialproviders}
794794
mkdir -pv /rootfs/opt/{containerd/bin,containerd/lib}
795795
# Go standard library is shipped with Talos, thus it must be tracked in SBOM
796796
install -D /usr/share/spdx/golang.spdx.json /rootfs/usr/share/spdx/golang.spdx.json
@@ -806,7 +806,7 @@ COPY --chmod=0644 hack/lvm.conf /rootfs/etc/lvm/lvm.conf
806806
COPY --link --chmod=0644 --from=base /src/pkg/machinery/version/os-release /rootfs/etc/os-release
807807
RUN <<END
808808
ln -s /usr/share/zoneinfo/Etc/UTC /rootfs/etc/localtime
809-
touch /rootfs/etc/{extensions.yaml,resolv.conf,hosts,machine-id,cri/conf.d/cri.toml,cri/conf.d/01-registries.part,cri/conf.d/20-customization.part,cri/conf.d/base-spec.json,ssl/certs/ca-certificates.crt,selinux/targeted/contexts/files/file_contexts,iscsi/initiatorname.iscsi,nvme/{hostid,hostnqn}}
809+
touch /rootfs/etc/extensions.yaml
810810
ln -s ca-certificates.crt /rootfs/etc/ssl/certs/ca-certificates
811811
ln -s /etc/ssl /rootfs/etc/pki
812812
ln -s /etc/ssl /rootfs/usr/share/ca-certificates
@@ -878,8 +878,8 @@ END
878878
# symlinks to avoid accidentally cleaning them up.
879879
RUN --mount=type=bind,source=hack/cleanup.sh,target=/usr/bin/cleanup.sh <<END
880880
cleanup.sh /rootfs
881-
mkdir -pv /rootfs/{boot/EFI,etc/{iscsi,nvme,cri/conf.d/hosts},usr/lib/firmware,usr/etc,usr/local/share,usr/share/zoneinfo/Etc,mnt,system,opt,.extra}
882-
mkdir -pv /rootfs/{etc/kubernetes/manifests,etc/cni/net.d,etc/ssl/certs,usr/libexec/kubernetes,/usr/local/lib/kubelet/credentialproviders,etc/selinux/targeted/contexts/files}
881+
mkdir -pv /rootfs/{boot/EFI,usr/lib/firmware,usr/etc,usr/local/share,usr/share/zoneinfo/Etc,mnt,system,opt,.extra}
882+
mkdir -pv /rootfs/{etc/ssl/certs,usr/libexec/kubernetes,/usr/local/lib/kubelet/credentialproviders}
883883
mkdir -pv /rootfs/opt/{containerd/bin,containerd/lib}
884884
# Go standard library is shipped with Talos, thus it must be tracked in SBOM
885885
install -D /usr/share/spdx/golang.spdx.json /rootfs/usr/share/spdx/golang.spdx.json
@@ -895,7 +895,7 @@ COPY --chmod=0644 hack/lvm.conf /rootfs/etc/lvm/lvm.conf
895895
COPY --link --chmod=0644 --from=base /src/pkg/machinery/version/os-release /rootfs/etc/os-release
896896
RUN <<END
897897
ln -s /usr/share/zoneinfo/Etc/UTC /rootfs/etc/localtime
898-
touch /rootfs/etc/{extensions.yaml,resolv.conf,hosts,machine-id,cri/conf.d/cri.toml,cri/conf.d/01-registries.part,cri/conf.d/20-customization.part,cri/conf.d/base-spec.json,ssl/certs/ca-certificates.crt,selinux/targeted/contexts/files/file_contexts,iscsi/initiatorname.iscsi,nvme/{hostid,hostnqn}}
898+
touch /rootfs/etc/extensions.yaml
899899
ln -s ca-certificates.crt /rootfs/etc/ssl/certs/ca-certificates
900900
ln -s /etc/ssl /rootfs/etc/pki
901901
ln -s /etc/ssl /rootfs/usr/share/ca-certificates

internal/app/machined/pkg/controllers/block/internal/volumes/volumeconfig/system_volumes.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,7 @@ func GetOverlayVolumesTransformer(inContainer bool) func(configconfig.Config) ([
174174
}
175175

176176
var resources []VolumeResource
177+
177178
for _, overlay := range constants.Overlays {
178179
resources = append(resources, VolumeResource{
179180
VolumeID: overlay.Path,

internal/app/machined/pkg/controllers/block/internal/volumes/volumeconfig/system_volumes_test.go

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,6 @@ func TestGetSystemVolumeTransformers(t *testing.T) {
6363
constants.EphemeralPartitionLabel,
6464
"/var/run",
6565
"/var/log",
66-
"/etc/cni",
6766
} {
6867
assert.Condition(t, func() bool {
6968
for _, r := range allResources {

internal/app/machined/pkg/controllers/files/cri_registry_config.go

Lines changed: 2 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -27,12 +27,11 @@ import (
2727

2828
// CRIRegistryConfigController generates parts of the CRI config for registry configuration.
2929
type CRIRegistryConfigController struct {
30-
// Path to /etc directory, read-only filesystem.
30+
// Path to /etc directory; the host path that containerd reads. Files written to
31+
// EtcRoot surface here read-only through the composed /etc overlay (see setupEtcOverlay).
3132
EtcPath string
3233
// EtcRoot is the root for /etc filesystem operations.
3334
EtcRoot xfs.Root
34-
35-
bindMountCreated bool
3635
}
3736

3837
// Name implements controller.Controller interface.
@@ -69,22 +68,6 @@ func (ctrl *CRIRegistryConfigController) Run(ctx context.Context, r controller.R
6968
src := filepath.Join(constants.CRIConfdPath, "hosts")
7069
dest := filepath.Join(ctrl.EtcPath, src)
7170

72-
if !ctrl.bindMountCreated {
73-
if ctrl.EtcRoot.FSType() == "os" {
74-
shadowPath := filepath.Join(ctrl.EtcRoot.Source(), src)
75-
76-
if err := createBindMountDir(shadowPath, dest); err != nil {
77-
return fmt.Errorf("bind mount failed for %q -> %q: %w", shadowPath, dest, err)
78-
}
79-
} else {
80-
if err := createBindMountDirFd(ctrl.EtcRoot, src, dest); err != nil {
81-
return fmt.Errorf("bind mount failed for %q: %w", dest, err)
82-
}
83-
}
84-
85-
ctrl.bindMountCreated = true
86-
}
87-
8871
for {
8972
select {
9073
case <-ctx.Done():

internal/app/machined/pkg/controllers/files/etcfile.go

Lines changed: 11 additions & 119 deletions
Original file line numberDiff line numberDiff line change
@@ -16,23 +16,18 @@ import (
1616
"github.com/cosi-project/runtime/pkg/resource"
1717
"github.com/cosi-project/runtime/pkg/safe"
1818
"go.uber.org/zap"
19-
"golang.org/x/sys/unix"
2019

21-
"github.com/siderolabs/talos/internal/pkg/mount/v3"
2220
"github.com/siderolabs/talos/internal/pkg/selinux"
2321
"github.com/siderolabs/talos/pkg/machinery/resources/files"
2422
"github.com/siderolabs/talos/pkg/xfs"
2523
)
2624

2725
// EtcFileController watches EtcFileSpecs, creates/updates files.
2826
type EtcFileController struct {
29-
// Path to /etc directory, read-only filesystem.
30-
EtcPath string
31-
// EtcRoot is the root for /etc filesystem operations.
27+
// EtcRoot is the root for /etc filesystem operations. Files written here surface
28+
// read-only at /etc through the composed /etc overlay (see setupEtcOverlay), so no
29+
// per-file bind mounts are needed.
3230
EtcRoot xfs.Root
33-
34-
// Cache of bind mounts created.
35-
bindMounts map[string]any
3631
}
3732

3833
// Name implements controller.Controller interface.
@@ -63,12 +58,8 @@ func (ctrl *EtcFileController) Outputs() []controller.Output {
6358

6459
// Run implements controller.Controller interface.
6560
//
66-
//nolint:gocyclo,cyclop
61+
//nolint:gocyclo
6762
func (ctrl *EtcFileController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error {
68-
if ctrl.bindMounts == nil {
69-
ctrl.bindMounts = make(map[string]any)
70-
}
71-
7263
for {
7364
select {
7465
case <-ctx.Done():
@@ -96,56 +87,24 @@ func (ctrl *EtcFileController) Run(ctx context.Context, r controller.Runtime, lo
9687

9788
for spec := range list.All() {
9889
filename := spec.Metadata().ID()
99-
_, mountExists := ctrl.bindMounts[filename]
100-
101-
dst := filepath.Join(ctrl.EtcPath, filename)
102-
src := filename
10390

10491
switch spec.Metadata().Phase() {
10592
case resource.PhaseTearingDown:
106-
if mountExists {
107-
logger.Debug("removing bind mount", zap.String("src", src), zap.String("dst", dst))
108-
109-
if err = unix.Unmount(dst, 0); err != nil && !errors.Is(err, os.ErrNotExist) {
110-
return fmt.Errorf("failed to unmount bind mount %q: %w", dst, err)
111-
}
93+
logger.Debug("removing file", zap.String("name", filename))
11294

113-
delete(ctrl.bindMounts, filename)
95+
if err = xfs.Remove(ctrl.EtcRoot, filename); err != nil && !errors.Is(err, os.ErrNotExist) {
96+
return fmt.Errorf("failed to remove %q: %w", filename, err)
11497
}
11598

116-
logger.Debug("removing file", zap.String("src", src))
117-
118-
if err = xfs.Remove(ctrl.EtcRoot, src); err != nil && !errors.Is(err, os.ErrNotExist) {
119-
return fmt.Errorf("failed to remove %q: %w", src, err)
120-
}
121-
122-
// now remove finalizer as the link was deleted
99+
// now remove finalizer as the file was deleted
123100
if err = r.RemoveFinalizer(ctx, spec.Metadata(), ctrl.Name()); err != nil {
124101
return fmt.Errorf("error removing finalizer: %w", err)
125102
}
126103
case resource.PhaseRunning:
127-
if !mountExists {
128-
logger.Debug("creating bind mount", zap.String("src", src), zap.String("dst", dst))
129-
130-
if ctrl.EtcRoot.FSType() == "os" {
131-
shadow := filepath.Join(ctrl.EtcRoot.Source(), src)
104+
logger.Debug("writing file contents", zap.String("name", filename), zap.Stringer("version", spec.Metadata().Version()))
132105

133-
if err = createBindMountFile(shadow, dst, spec.TypedSpec().Mode); err != nil {
134-
return fmt.Errorf("failed to create shadow bind mount %q -> %q (mode: os): %w", shadow, dst, err)
135-
}
136-
} else {
137-
if err = createBindMountFileFd(ctrl.EtcRoot, src, dst, spec.TypedSpec().Mode); err != nil {
138-
return fmt.Errorf("failed to create shadow bind mount %q -> %q (mode: fd): %w", src, dst, err)
139-
}
140-
}
141-
142-
ctrl.bindMounts[filename] = struct{}{}
143-
}
144-
145-
logger.Debug("writing file contents", zap.String("src", src), zap.Stringer("version", spec.Metadata().Version()))
146-
147-
if err = UpdateFile(ctrl.EtcRoot, src, spec.TypedSpec().Contents, spec.TypedSpec().Mode, spec.TypedSpec().SelinuxLabel); err != nil {
148-
return fmt.Errorf("error updating %q: %w", src, err)
106+
if err = UpdateFile(ctrl.EtcRoot, filename, spec.TypedSpec().Contents, spec.TypedSpec().Mode, spec.TypedSpec().SelinuxLabel); err != nil {
107+
return fmt.Errorf("error updating %q: %w", filename, err)
149108
}
150109

151110
if err = safe.WriterModify(ctx, r, files.NewEtcFileStatus(files.NamespaceName, filename), func(r *files.EtcFileStatus) error {
@@ -178,73 +137,6 @@ func (ctrl *EtcFileController) Run(ctx context.Context, r controller.Runtime, lo
178137
}
179138
}
180139

181-
// createBindMountFile creates a common way to create a writable source file with a
182-
// bind mounted destination. This is most commonly used for well known files
183-
// under /etc that need to be adjusted during startup.
184-
func createBindMountFile(src, dst string, mode os.FileMode) (err error) {
185-
if err = os.MkdirAll(filepath.Dir(src), 0o755); err != nil {
186-
return fmt.Errorf("mkdir all failed: %w", err)
187-
}
188-
189-
var f *os.File
190-
191-
if f, err = os.OpenFile(src, os.O_WRONLY|os.O_CREATE, mode); err != nil {
192-
return fmt.Errorf("open file failed: %w", err)
193-
}
194-
195-
if err = f.Close(); err != nil {
196-
return fmt.Errorf("close file failed: %w", err)
197-
}
198-
199-
return mount.BindReadonly(src, dst)
200-
}
201-
202-
// createBindMountDir creates a common way to create a writable source dir with a
203-
// bind mounted destination. This is most commonly used for well known directories
204-
// under /etc that need to be adjusted during startup.
205-
func createBindMountDir(src, dst string) (err error) {
206-
if err = os.MkdirAll(src, 0o755); err != nil {
207-
return err
208-
}
209-
210-
return mount.BindReadonly(src, dst)
211-
}
212-
213-
// createBindMountFileFd creates a common way to create a writable source file with a
214-
// bind mounted destination. This is most commonly used for well known files
215-
// under /etc that need to be adjusted during startup.
216-
func createBindMountFileFd(root xfs.Root, src, dst string, mode os.FileMode) (err error) {
217-
if err = xfs.MkdirAll(root, filepath.Dir(src), 0o755); err != nil {
218-
return err
219-
}
220-
221-
fsrc, err := xfs.OpenFile(root, src, os.O_WRONLY|os.O_CREATE, mode)
222-
if err != nil {
223-
return err
224-
}
225-
defer fsrc.Close() //nolint:errcheck
226-
227-
return mount.BindReadonlyFd(int(fsrc.Fd()), dst)
228-
}
229-
230-
// createBindMountDirFd creates a common way to create a writable source dir with a
231-
// bind mounted destination. This is most commonly used for well known directories
232-
// under /etc that need to be adjusted during startup.
233-
func createBindMountDirFd(root xfs.Root, src, dst string) (err error) {
234-
if err = xfs.MkdirAll(root, src, 0o755); err != nil {
235-
return fmt.Errorf("mkdir all failed: %w", err)
236-
}
237-
238-
f, err := xfs.OpenFile(root, src, os.O_RDONLY, 0)
239-
if err != nil {
240-
return fmt.Errorf("open file failed: %w", err)
241-
}
242-
243-
defer f.Close() //nolint:errcheck
244-
245-
return mount.BindReadonlyFd(int(f.Fd()), dst)
246-
}
247-
248140
// UpdateFile is like `os.WriteFile`, but it will only update the file if the
249141
// contents have changed.
250142
func UpdateFile(root xfs.Root, filename string, contents []byte, mode os.FileMode, selinuxLabel string) error {

internal/app/machined/pkg/controllers/files/etcfile_test.go

Lines changed: 2 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ package files_test
66

77
import (
88
"os"
9-
"path/filepath"
109
"testing"
1110
"time"
1211

@@ -25,7 +24,6 @@ import (
2524
type EtcFileSuite struct {
2625
ctest.DefaultSuite
2726

28-
etcPath string
2927
etcRoot xfs.Root
3028
}
3129

@@ -34,25 +32,19 @@ func (suite *EtcFileSuite) TestFiles() {
3432
etcFileSpec.TypedSpec().Contents = []byte("foo")
3533
etcFileSpec.TypedSpec().Mode = 0o644
3634

37-
// create "read-only" mock (in Talos it's part of rootfs)
38-
suite.T().Logf("mock created %q", filepath.Join(suite.etcPath, etcFileSpec.Metadata().ID()))
39-
suite.Require().NoError(os.WriteFile(filepath.Join(suite.etcPath, etcFileSpec.Metadata().ID()), nil, 0o644))
40-
4135
suite.Create(etcFileSpec)
4236

4337
// controller should put a finalizer on the spec, bumping the version
4438
expectedVersion := etcFileSpec.Metadata().Version().Next()
4539

40+
// the controller writes into the managed EtcRoot tmpfs; /etc surfacing is the overlay's
41+
// job (composed in machined, not exercised here).
4642
ctest.AssertResource(suite, "test1", func(r *files.EtcFileStatus, asrt *assert.Assertions) {
4743
asrt.Equal(expectedVersion.String(), r.TypedSpec().SpecVersion)
4844

4945
rwb, err := xfs.ReadFile(suite.etcRoot, "test1")
5046
asrt.NoError(err)
5147
asrt.Equal("foo", string(rwb))
52-
53-
rob, err := os.ReadFile(filepath.Join(suite.etcPath, "test1"))
54-
asrt.NoError(err)
55-
asrt.Equal("foo", string(rob))
5648
})
5749

5850
rtestutils.Destroy[*files.EtcFileSpec](suite.Ctx(), suite.T(), suite.State(), []string{etcFileSpec.Metadata().ID()})
@@ -72,8 +64,6 @@ func TestEtcFileSuite(t *testing.T) {
7264
ok, err := runtime.KernelCapabilities().OpentreeOnAnonymousFS()
7365
s.Require().NoError(err)
7466

75-
etcSuite.etcPath = s.T().TempDir()
76-
7767
if ok {
7868
etcSuite.etcRoot = &xfs.UnixRoot{FS: opentree.NewFromPath(s.T().TempDir())}
7969
} else {
@@ -83,7 +73,6 @@ func TestEtcFileSuite(t *testing.T) {
8373
s.Require().NoError(etcSuite.etcRoot.OpenFS())
8474

8575
s.Require().NoError(s.Runtime().RegisterController(&filesctrl.EtcFileController{
86-
EtcPath: etcSuite.etcPath,
8776
EtcRoot: etcSuite.etcRoot,
8877
}))
8978
},

0 commit comments

Comments
 (0)