diff --git a/integration/client/client_test.go b/integration/client/client_test.go index 37a1b479c..69a1dd0b0 100644 --- a/integration/client/client_test.go +++ b/integration/client/client_test.go @@ -36,7 +36,6 @@ import ( "github.com/containerd/containerd/namespaces" "github.com/containerd/containerd/pkg/testutil" "github.com/containerd/containerd/platforms" - "github.com/containerd/containerd/sys" "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/identity" "github.com/sirupsen/logrus" @@ -71,7 +70,7 @@ func TestMain(m *testing.M) { defer cancel() if !noDaemon { - sys.ForceRemoveAll(defaultRoot) + _ = forceRemoveAll(defaultRoot) stdioFile, err := os.CreateTemp("", "") if err != nil { @@ -173,7 +172,7 @@ func TestMain(m *testing.M) { } } - if err := sys.ForceRemoveAll(defaultRoot); err != nil { + if err := forceRemoveAll(defaultRoot); err != nil { fmt.Fprintln(os.Stderr, "failed to remove test root dir", err) os.Exit(1) } diff --git a/integration/client/container_fuzzer.go b/integration/client/container_fuzzer.go index 9afdc7cf2..e176603d3 100644 --- a/integration/client/container_fuzzer.go +++ b/integration/client/container_fuzzer.go @@ -33,7 +33,6 @@ import ( fuzz "github.com/AdaLogics/go-fuzz-headers" "github.com/containerd/containerd" "github.com/containerd/containerd/oci" - "github.com/containerd/containerd/sys" exec "golang.org/x/sys/execabs" ) @@ -125,7 +124,7 @@ func tearDown() error { return err } } - if err := sys.ForceRemoveAll(defaultRoot); err != nil { + if err := forceRemoveAll(defaultRoot); err != nil { return err } diff --git a/integration/client/helpers_unix.go b/integration/client/helpers_unix.go new file mode 100644 index 000000000..e9786268a --- /dev/null +++ b/integration/client/helpers_unix.go @@ -0,0 +1,27 @@ +//go:build !windows +// +build !windows + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package client + +import "os" + +// forceRemoveAll on unix is just a wrapper for os.RemoveAll +func forceRemoveAll(path string) error { + return os.RemoveAll(path) +} diff --git a/integration/client/helpers_windows.go b/integration/client/helpers_windows.go new file mode 100644 index 000000000..3d8afb10c --- /dev/null +++ b/integration/client/helpers_windows.go @@ -0,0 +1,116 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package client + +import ( + "fmt" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "syscall" + + "github.com/Microsoft/hcsshim" + "golang.org/x/sys/windows" +) + +// forceRemoveAll is the same as os.RemoveAll, but is aware of io.containerd.snapshotter.v1.windows +// and uses hcsshim to unmount and delete container layers contained therein, in the correct order, +// when passed a containerd root data directory (i.e. the `--root` directory for containerd). +func forceRemoveAll(path string) error { + // snapshots/windows/windows.go init() + const snapshotPlugin = "io.containerd.snapshotter.v1" + "." + "windows" + // snapshots/windows/windows.go NewSnapshotter() + snapshotDir := filepath.Join(path, snapshotPlugin, "snapshots") + if stat, err := os.Stat(snapshotDir); err == nil && stat.IsDir() { + if err := cleanupWCOWLayers(snapshotDir); err != nil { + return fmt.Errorf("failed to cleanup WCOW layers in %s: %w", snapshotDir, err) + } + } + + return os.RemoveAll(path) +} + +func cleanupWCOWLayers(root string) error { + // See snapshots/windows/windows.go getSnapshotDir() + var layerNums []int + var rmLayerNums []int + if err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if path != root && info.IsDir() { + name := filepath.Base(path) + if strings.HasPrefix(name, "rm-") { + layerNum, err := strconv.Atoi(strings.TrimPrefix(name, "rm-")) + if err != nil { + return err + } + rmLayerNums = append(rmLayerNums, layerNum) + } else { + layerNum, err := strconv.Atoi(name) + if err != nil { + return err + } + layerNums = append(layerNums, layerNum) + } + return filepath.SkipDir + } + + return nil + }); err != nil { + return err + } + + sort.Sort(sort.Reverse(sort.IntSlice(rmLayerNums))) + for _, rmLayerNum := range rmLayerNums { + if err := cleanupWCOWLayer(filepath.Join(root, "rm-"+strconv.Itoa(rmLayerNum))); err != nil { + return err + } + } + + sort.Sort(sort.Reverse(sort.IntSlice(layerNums))) + for _, layerNum := range layerNums { + if err := cleanupWCOWLayer(filepath.Join(root, strconv.Itoa(layerNum))); err != nil { + return err + } + } + + return nil +} + +func cleanupWCOWLayer(layerPath string) error { + info := hcsshim.DriverInfo{ + HomeDir: filepath.Dir(layerPath), + } + + // ERROR_DEV_NOT_EXIST is returned if the layer is not currently prepared or activated. + // ERROR_FLT_INSTANCE_NOT_FOUND is returned if the layer is currently activated but not prepared. + if err := hcsshim.UnprepareLayer(info, filepath.Base(layerPath)); err != nil { + if hcserror, ok := err.(*hcsshim.HcsError); !ok || (hcserror.Err != windows.ERROR_DEV_NOT_EXIST && hcserror.Err != syscall.Errno(windows.ERROR_FLT_INSTANCE_NOT_FOUND)) { + return fmt.Errorf("failed to unprepare %s: %w", layerPath, err) + } + } + + if err := hcsshim.DeactivateLayer(info, filepath.Base(layerPath)); err != nil { + return fmt.Errorf("failed to deactivate %s: %w", layerPath, err) + } + + if err := hcsshim.DestroyLayer(info, filepath.Base(layerPath)); err != nil { + return fmt.Errorf("failed to destroy %s: %w", layerPath, err) + } + + return nil +} diff --git a/integration/client/restart_monitor_test.go b/integration/client/restart_monitor_test.go index a1bf94499..f80930719 100644 --- a/integration/client/restart_monitor_test.go +++ b/integration/client/restart_monitor_test.go @@ -36,7 +36,6 @@ import ( "github.com/containerd/containerd/pkg/testutil" "github.com/containerd/containerd/runtime/restart" srvconfig "github.com/containerd/containerd/services/server/config" - "github.com/containerd/containerd/sys" "github.com/containerd/typeurl" exec "golang.org/x/sys/execabs" ) @@ -107,7 +106,7 @@ func newDaemonWithConfig(t *testing.T, configTOML string) (*Client, *daemon, fun t.Errorf("failed to wait for: %v", err) } } - if err := sys.ForceRemoveAll(tempDir); err != nil { + if err := forceRemoveAll(tempDir); err != nil { t.Errorf("failed to remove %s: %v", tempDir, err) } if t.Failed() { diff --git a/sys/filesys_unix.go b/sys/filesys_unix.go index 805a7a736..4624f7253 100644 --- a/sys/filesys_unix.go +++ b/sys/filesys_unix.go @@ -21,11 +21,6 @@ package sys import "os" -// ForceRemoveAll on unix is just a wrapper for os.RemoveAll -func ForceRemoveAll(path string) error { - return os.RemoveAll(path) -} - // MkdirAllWithACL is a wrapper for os.MkdirAll on Unix systems. func MkdirAllWithACL(path string, perm os.FileMode) error { return os.MkdirAll(path, perm) diff --git a/sys/filesys_windows.go b/sys/filesys_windows.go index 54433f257..3d76be672 100644 --- a/sys/filesys_windows.go +++ b/sys/filesys_windows.go @@ -17,17 +17,13 @@ package sys import ( - "fmt" "os" "path/filepath" "regexp" - "sort" - "strconv" "strings" "syscall" "unsafe" - "github.com/Microsoft/hcsshim" "golang.org/x/sys/windows" ) @@ -154,89 +150,3 @@ func IsAbs(path string) bool { } return true } - -// ForceRemoveAll is the same as os.RemoveAll, but is aware of io.containerd.snapshotter.v1.windows -// and uses hcsshim to unmount and delete container layers contained therein, in the correct order, -// when passed a containerd root data directory (i.e. the `--root` directory for containerd). -func ForceRemoveAll(path string) error { - // snapshots/windows/windows.go init() - const snapshotPlugin = "io.containerd.snapshotter.v1" + "." + "windows" - // snapshots/windows/windows.go NewSnapshotter() - snapshotDir := filepath.Join(path, snapshotPlugin, "snapshots") - if stat, err := os.Stat(snapshotDir); err == nil && stat.IsDir() { - if err := cleanupWCOWLayers(snapshotDir); err != nil { - return fmt.Errorf("failed to cleanup WCOW layers in %s: %w", snapshotDir, err) - } - } - - return os.RemoveAll(path) -} - -func cleanupWCOWLayers(root string) error { - // See snapshots/windows/windows.go getSnapshotDir() - var layerNums []int - var rmLayerNums []int - if err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { - if path != root && info.IsDir() { - name := filepath.Base(path) - if strings.HasPrefix(name, "rm-") { - layerNum, err := strconv.Atoi(strings.TrimPrefix(name, "rm-")) - if err != nil { - return err - } - rmLayerNums = append(rmLayerNums, layerNum) - } else { - layerNum, err := strconv.Atoi(name) - if err != nil { - return err - } - layerNums = append(layerNums, layerNum) - } - return filepath.SkipDir - } - - return nil - }); err != nil { - return err - } - - sort.Sort(sort.Reverse(sort.IntSlice(rmLayerNums))) - for _, rmLayerNum := range rmLayerNums { - if err := cleanupWCOWLayer(filepath.Join(root, "rm-"+strconv.Itoa(rmLayerNum))); err != nil { - return err - } - } - - sort.Sort(sort.Reverse(sort.IntSlice(layerNums))) - for _, layerNum := range layerNums { - if err := cleanupWCOWLayer(filepath.Join(root, strconv.Itoa(layerNum))); err != nil { - return err - } - } - - return nil -} - -func cleanupWCOWLayer(layerPath string) error { - info := hcsshim.DriverInfo{ - HomeDir: filepath.Dir(layerPath), - } - - // ERROR_DEV_NOT_EXIST is returned if the layer is not currently prepared or activated. - // ERROR_FLT_INSTANCE_NOT_FOUND is returned if the layer is currently activated but not prepared. - if err := hcsshim.UnprepareLayer(info, filepath.Base(layerPath)); err != nil { - if hcserror, ok := err.(*hcsshim.HcsError); !ok || (hcserror.Err != windows.ERROR_DEV_NOT_EXIST && hcserror.Err != syscall.Errno(windows.ERROR_FLT_INSTANCE_NOT_FOUND)) { - return fmt.Errorf("failed to unprepare %s: %w", layerPath, err) - } - } - - if err := hcsshim.DeactivateLayer(info, filepath.Base(layerPath)); err != nil { - return fmt.Errorf("failed to deactivate %s: %w", layerPath, err) - } - - if err := hcsshim.DestroyLayer(info, filepath.Base(layerPath)); err != nil { - return fmt.Errorf("failed to destroy %s: %w", layerPath, err) - } - - return nil -}