From 0c386e2ebda373d5ac9ce1e6e6f78bdf9329dd21 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Tue, 18 Apr 2023 23:07:50 -0700 Subject: [PATCH 1/4] Add blockfile snapshotter Signed-off-by: Derek McGowan --- snapshots/blockfile/blockfile.go | 389 ++++++++++++++++++ .../blockfile/blockfile_loopsetup_test.go | 86 ++++ snapshots/blockfile/blockfile_other_test.go | 26 ++ snapshots/blockfile/blockfile_test.go | 47 +++ snapshots/blockfile/plugin/plugin.go | 67 +++ 5 files changed, 615 insertions(+) create mode 100644 snapshots/blockfile/blockfile.go create mode 100644 snapshots/blockfile/blockfile_loopsetup_test.go create mode 100644 snapshots/blockfile/blockfile_other_test.go create mode 100644 snapshots/blockfile/blockfile_test.go create mode 100644 snapshots/blockfile/plugin/plugin.go diff --git a/snapshots/blockfile/blockfile.go b/snapshots/blockfile/blockfile.go new file mode 100644 index 000000000..9886272e2 --- /dev/null +++ b/snapshots/blockfile/blockfile.go @@ -0,0 +1,389 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package blockfile + +import ( + "context" + "fmt" + "os" + "path/filepath" + + "github.com/containerd/containerd/log" + "github.com/containerd/containerd/mount" + "github.com/containerd/containerd/plugin" + "github.com/containerd/containerd/snapshots" + "github.com/containerd/containerd/snapshots/storage" + + "github.com/containerd/continuity/fs" +) + +// SnapshotterConfig holds the configurable properties for the blockfile snapshotter +type SnapshotterConfig struct { + // recreateScratch is whether scratch should be recreated even + // if already exists + recreateScratch bool + + scratchGenerator func(string) error + + // fsType is the filesystem type for the mount (defaults to ext4) + fsType string +} + +// Opt is an option to configure the overlay snapshotter +type Opt func(string, *SnapshotterConfig) + +// WithScratchFile provides a scratch file which will get copied on startup +// if the scratch file needs to be generated. +func WithScratchFile(src string) Opt { + return func(root string, config *SnapshotterConfig) { + config.scratchGenerator = func(dst string) error { + // Copy src to dst + scratch := filepath.Join(root, "scratch") + if err := fs.CopyFile(scratch, src); err != nil { + return fmt.Errorf("failed to copy scratch: %w", err) + } + return nil + } + } +} + +// WithFSType defines the filesystem type to apply to mounts of the blockfile +func WithFSType(fsType string) Opt { + return func(root string, config *SnapshotterConfig) { + config.fsType = fsType + } +} + +type snapshotter struct { + root string + scratch string + fsType string + ms *storage.MetaStore +} + +// NewSnapshotter returns a Snapshotter which copies layers on the underlying +// file system. A metadata file is stored under the root. +func NewSnapshotter(root string, opts ...Opt) (snapshots.Snapshotter, error) { + var config SnapshotterConfig + if err := os.MkdirAll(root, 0700); err != nil { + return nil, err + } + + for _, opt := range opts { + opt(root, &config) + } + + scratch := filepath.Join(root, "scratch") + createScratch := config.recreateScratch + if !createScratch { + if _, err := os.Stat(scratch); err != nil { + if !os.IsNotExist(err) { + return nil, fmt.Errorf("unable to stat scratch file: %w", err) + } + createScratch = true + } + } + if createScratch { + if config.scratchGenerator == nil { + return nil, fmt.Errorf("no scratch file generator: %w", plugin.ErrSkipPlugin) + } + if err := config.scratchGenerator(scratch); err != nil { + return nil, fmt.Errorf("failed to generate scratch file: %w", err) + } + } + + if config.fsType == "" { + config.fsType = "ext4" + } + + ms, err := storage.NewMetaStore(filepath.Join(root, "metadata.db")) + if err != nil { + return nil, err + } + + if err := os.Mkdir(filepath.Join(root, "snapshots"), 0700); err != nil && !os.IsExist(err) { + return nil, err + } + + return &snapshotter{ + root: root, + scratch: scratch, + fsType: config.fsType, + ms: ms, + }, nil +} + +// Stat returns the info for an active or committed snapshot by name or +// key. +// +// Should be used for parent resolution, existence checks and to discern +// the kind of snapshot. +func (o *snapshotter) Stat(ctx context.Context, key string) (info snapshots.Info, err error) { + err = o.ms.WithTransaction(ctx, false, func(ctx context.Context) error { + _, info, _, err = storage.GetInfo(ctx, key) + return err + }) + if err != nil { + return snapshots.Info{}, err + } + + return info, nil +} + +func (o *snapshotter) Update(ctx context.Context, info snapshots.Info, fieldpaths ...string) (_ snapshots.Info, err error) { + err = o.ms.WithTransaction(ctx, true, func(ctx context.Context) error { + info, err = storage.UpdateInfo(ctx, info, fieldpaths...) + return err + }) + if err != nil { + return snapshots.Info{}, err + } + + return info, nil +} + +func (o *snapshotter) Usage(ctx context.Context, key string) (usage snapshots.Usage, err error) { + var ( + id string + info snapshots.Info + ) + + err = o.ms.WithTransaction(ctx, false, func(ctx context.Context) error { + id, info, usage, err = storage.GetInfo(ctx, key) + if err != nil { + return err + } + + // Current usage calculation is an approximation based on the size + // of the block file - the size of its parent. This does not consider + // that the filesystem may not support shared extents between the block + // file and its parents, in which case the accurate calculation would just + // be the size of the block file. Additionally, this does not take into + // consideration that file may have been removed before being adding, + // making the number of shared extents between the parent and the block + // file smaller than the parent, under reporting actual usage. + // + // A more ideal calculation would look like: + // size(block) - usage(extent_intersection(block,parent)) + // OR + // usage(extent_union(block,parent)) - size(parent) + + if info.Kind == snapshots.KindActive { + // TODO: Use size calculator from fs package + st, err := os.Stat(o.getBlockFile(id)) + if err != nil { + return err + } + usage.Size = st.Size() + usage.Inodes = 1 + } + + if info.Parent != "" { + // GetInfo returns total number of bytes used by a snapshot (including parent). + // So subtract parent usage in order to get delta consumed by layer itself. + _, _, parentUsage, err := storage.GetInfo(ctx, info.Parent) + if err != nil { + return err + } + + usage.Size -= parentUsage.Size + } + + return err + }) + if err != nil { + return snapshots.Usage{}, err + } + + return usage, nil +} + +func (o *snapshotter) Prepare(ctx context.Context, key, parent string, opts ...snapshots.Opt) ([]mount.Mount, error) { + return o.createSnapshot(ctx, snapshots.KindActive, key, parent, opts) +} + +func (o *snapshotter) View(ctx context.Context, key, parent string, opts ...snapshots.Opt) ([]mount.Mount, error) { + return o.createSnapshot(ctx, snapshots.KindView, key, parent, opts) +} + +// Mounts returns the mounts for the transaction identified by key. Can be +// called on an read-write or readonly transaction. +// +// This can be used to recover mounts after calling View or Prepare. +func (o *snapshotter) Mounts(ctx context.Context, key string) (_ []mount.Mount, err error) { + var s storage.Snapshot + err = o.ms.WithTransaction(ctx, false, func(ctx context.Context) error { + s, err = storage.GetSnapshot(ctx, key) + if err != nil { + return fmt.Errorf("failed to get snapshot mount: %w", err) + } + + return nil + }) + if err != nil { + return nil, err + } + + return o.mounts(s), nil +} + +func (o *snapshotter) Commit(ctx context.Context, name, key string, opts ...snapshots.Opt) error { + return o.ms.WithTransaction(ctx, true, func(ctx context.Context) error { + id, _, _, err := storage.GetInfo(ctx, key) + if err != nil { + return err + } + + st, err := os.Stat(o.getBlockFile(id)) + if err != nil { + return err + } + + usage := snapshots.Usage{ + Size: st.Size(), + Inodes: 1, + } + + if _, err = storage.CommitActive(ctx, key, name, usage, opts...); err != nil { + return fmt.Errorf("failed to commit snapshot: %w", err) + } + return nil + }) +} + +// Remove abandons the transaction identified by key. All resources +// associated with the key will be removed. +func (o *snapshotter) Remove(ctx context.Context, key string) (err error) { + var ( + renamed, path string + restore bool + ) + + err = o.ms.WithTransaction(ctx, true, func(ctx context.Context) error { + id, _, err := storage.Remove(ctx, key) + if err != nil { + return fmt.Errorf("failed to remove: %w", err) + } + + path = o.getBlockFile(id) + renamed = filepath.Join(o.root, "snapshots", "rm-"+id) + if err = os.Rename(path, renamed); err != nil { + if !os.IsNotExist(err) { + return fmt.Errorf("failed to rename: %w", err) + } + renamed = "" + } + + restore = true + return nil + }) + + if err != nil { + if renamed != "" && restore { + if err1 := os.Rename(renamed, path); err1 != nil { + // May cause inconsistent data on disk + log.G(ctx).WithError(err1).WithField("path", renamed).Error("failed to rename after failed commit") + } + } + return err + } + if renamed != "" { + if err := os.Remove(renamed); err != nil { + // Must be cleaned up, any "rm-*" could be removed if no active transactions + log.G(ctx).WithError(err).WithField("path", renamed).Warnf("failed to remove root filesystem") + } + } + + return nil +} + +// Walk the committed snapshots. +func (o *snapshotter) Walk(ctx context.Context, fn snapshots.WalkFunc, fs ...string) error { + return o.ms.WithTransaction(ctx, false, func(ctx context.Context) error { + return storage.WalkInfo(ctx, fn, fs...) + }) +} + +func (o *snapshotter) createSnapshot(ctx context.Context, kind snapshots.Kind, key, parent string, opts []snapshots.Opt) (_ []mount.Mount, err error) { + var s storage.Snapshot + + err = o.ms.WithTransaction(ctx, true, func(ctx context.Context) error { + s, err = storage.CreateSnapshot(ctx, kind, key, parent, opts...) + if err != nil { + return fmt.Errorf("failed to create snapshot: %w", err) + } + + if len(s.ParentIDs) == 0 || s.Kind == snapshots.KindActive { + path := o.getBlockFile(s.ID) + + if len(s.ParentIDs) > 0 { + if err = fs.CopyFile(path, o.getBlockFile(s.ParentIDs[0])); err != nil { + return fmt.Errorf("copying of parent failed: %w", err) + } + } else { + if err = fs.CopyFile(path, o.scratch); err != nil { + return fmt.Errorf("copying of scratch failed: %w", err) + } + } + } + + return nil + }) + if err != nil { + return nil, err + } + + return o.mounts(s), nil +} + +func (o *snapshotter) getBlockFile(id string) string { + return filepath.Join(o.root, "snapshots", id) +} + +func (o *snapshotter) mounts(s storage.Snapshot) []mount.Mount { + var ( + mountOptions = []string{ + "loop", + } + source string + ) + + if s.Kind == snapshots.KindView { + mountOptions = append(mountOptions, "ro") + } else { + mountOptions = append(mountOptions, "rw") + } + + if len(s.ParentIDs) == 0 || s.Kind == snapshots.KindActive { + source = o.getBlockFile(s.ID) + } else { + source = o.getBlockFile(s.ParentIDs[0]) + } + + return []mount.Mount{ + { + Source: source, + Type: o.fsType, + Options: mountOptions, + }, + } +} + +// Close closes the snapshotter +func (o *snapshotter) Close() error { + return o.ms.Close() +} diff --git a/snapshots/blockfile/blockfile_loopsetup_test.go b/snapshots/blockfile/blockfile_loopsetup_test.go new file mode 100644 index 000000000..816f12ffd --- /dev/null +++ b/snapshots/blockfile/blockfile_loopsetup_test.go @@ -0,0 +1,86 @@ +//go:build !windows && !darwin + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package blockfile + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/containerd/containerd/mount" + "github.com/containerd/continuity/fs" + "github.com/containerd/continuity/testutil/loopback" + "golang.org/x/sys/unix" +) + +func setupSnapshotter(t *testing.T) ([]Opt, error) { + mkfs, err := exec.LookPath("mkfs.ext4") + if err != nil { + t.Skipf("Could not find mkfs.ext4: %v", err) + } + + loopbackSize := int64(128 << 20) // 128 MB + if os.Getpagesize() > 4096 { + loopbackSize = int64(650 << 20) // 650 MB + } + + loop, err := loopback.New(loopbackSize) + if err != nil { + return nil, err + } + defer loop.Close() + + if out, err := exec.Command(mkfs, loop.Device).CombinedOutput(); err != nil { + return nil, fmt.Errorf("failed to make ext4 filesystem (out: %q): %w", out, err) + } + // sync after a mkfs on the loopback before trying to mount the device + unix.Sync() + + if err := testMount(t, loop.Device); err != nil { + return nil, err + } + + scratch := filepath.Join(t.TempDir(), "scratch") + err = fs.CopyFile(scratch, loop.File) + if err != nil { + return nil, err + } + + return []Opt{ + WithScratchFile(scratch), + }, nil +} + +func testMount(t *testing.T, device string) error { + root, err := os.MkdirTemp(t.TempDir(), "") + if err != nil { + return err + } + defer os.RemoveAll(root) + + if out, err := exec.Command("mount", device, root).CombinedOutput(); err != nil { + return fmt.Errorf("failed to mount device %s (out: %q): %w", device, out, err) + } + if err := os.Remove(filepath.Join(root, "lost+found")); err != nil { + return err + } + return mount.UnmountAll(root, unix.MNT_DETACH) +} diff --git a/snapshots/blockfile/blockfile_other_test.go b/snapshots/blockfile/blockfile_other_test.go new file mode 100644 index 000000000..0bfff2cc4 --- /dev/null +++ b/snapshots/blockfile/blockfile_other_test.go @@ -0,0 +1,26 @@ +//go:build windows || darwin + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package blockfile + +import "testing" + +func setupSnapshotter(t *testing.T) ([]Opt, error) { + t.Skip("No support for loopback mounts") + return nil, nil +} diff --git a/snapshots/blockfile/blockfile_test.go b/snapshots/blockfile/blockfile_test.go new file mode 100644 index 000000000..7cf0c9c49 --- /dev/null +++ b/snapshots/blockfile/blockfile_test.go @@ -0,0 +1,47 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package blockfile + +import ( + "context" + "testing" + + "github.com/containerd/containerd/pkg/testutil" + "github.com/containerd/containerd/snapshots" + "github.com/containerd/containerd/snapshots/testsuite" +) + +func newSnapshotter(t *testing.T) func(ctx context.Context, root string) (snapshots.Snapshotter, func() error, error) { + opts, err := setupSnapshotter(t) + if err != nil { + t.Fatal("failed to get snapshotter options:", err) + } + + return func(ctx context.Context, root string) (snapshots.Snapshotter, func() error, error) { + snapshotter, err := NewSnapshotter(root, opts...) + if err != nil { + return nil, nil, err + } + + return snapshotter, func() error { return snapshotter.Close() }, nil + } +} + +func TestBlockfile(t *testing.T) { + testutil.RequiresRoot(t) + testsuite.SnapshotterSuite(t, "Blockfile", newSnapshotter(t)) +} diff --git a/snapshots/blockfile/plugin/plugin.go b/snapshots/blockfile/plugin/plugin.go new file mode 100644 index 000000000..a3c0f0d52 --- /dev/null +++ b/snapshots/blockfile/plugin/plugin.go @@ -0,0 +1,67 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package plugin + +import ( + "errors" + + "github.com/containerd/containerd/platforms" + "github.com/containerd/containerd/plugin" + "github.com/containerd/containerd/snapshots/blockfile" +) + +// Config represents configuration for the native plugin. +type Config struct { + // Root directory for the plugin + RootPath string `toml:"root_path"` + + // ScratchFile is the scratch block file to use as an empty block + ScratchFile string `toml:"scratch_file"` + + // FSType is the filesystem type for the mount + FSType string `toml:"fs_type"` +} + +func init() { + plugin.Register(&plugin.Registration{ + Type: plugin.SnapshotPlugin, + ID: "blockfile", + Config: &Config{}, + InitFn: func(ic *plugin.InitContext) (interface{}, error) { + ic.Meta.Platforms = append(ic.Meta.Platforms, platforms.DefaultSpec()) + + config, ok := ic.Config.(*Config) + if !ok { + return nil, errors.New("invalid blockfile configuration") + } + + var opts []blockfile.Opt + root := ic.Root + if len(config.RootPath) != 0 { + root = config.RootPath + } + if config.ScratchFile != "" { + opts = append(opts, blockfile.WithScratchFile(config.ScratchFile)) + } + if config.FSType != "" { + opts = append(opts, blockfile.WithFSType(config.FSType)) + } + + return blockfile.NewSnapshotter(root, opts...) + }, + }) +} From 798dba9faa8020083bd4f7bb9975b162cb683c7b Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Mon, 24 Apr 2023 14:25:02 -0700 Subject: [PATCH 2/4] Add blockfile as a builtin Signed-off-by: Derek McGowan --- cmd/containerd/builtins/builtins_linux.go | 1 + cmd/containerd/builtins/builtins_unix.go | 1 + 2 files changed, 2 insertions(+) diff --git a/cmd/containerd/builtins/builtins_linux.go b/cmd/containerd/builtins/builtins_linux.go index 8df06891b..fead87916 100644 --- a/cmd/containerd/builtins/builtins_linux.go +++ b/cmd/containerd/builtins/builtins_linux.go @@ -20,6 +20,7 @@ import ( _ "github.com/containerd/containerd/metrics/cgroups" _ "github.com/containerd/containerd/metrics/cgroups/v2" _ "github.com/containerd/containerd/runtime/v2/runc/options" + _ "github.com/containerd/containerd/snapshots/blockfile/plugin" _ "github.com/containerd/containerd/snapshots/native/plugin" _ "github.com/containerd/containerd/snapshots/overlay/plugin" ) diff --git a/cmd/containerd/builtins/builtins_unix.go b/cmd/containerd/builtins/builtins_unix.go index f10915e4f..348e2ccbb 100644 --- a/cmd/containerd/builtins/builtins_unix.go +++ b/cmd/containerd/builtins/builtins_unix.go @@ -19,5 +19,6 @@ package builtins import ( + _ "github.com/containerd/containerd/snapshots/blockfile/plugin" _ "github.com/containerd/containerd/snapshots/native/plugin" ) From b729962e3e5ce3c796ee83807a0a4f6f84e17074 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Thu, 11 May 2023 19:16:07 -0700 Subject: [PATCH 3/4] Add blockfile snapshotter to snapshotters doc Signed-off-by: Derek McGowan --- docs/snapshotters/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/snapshotters/README.md b/docs/snapshotters/README.md index c7fd35e7e..17d928c2f 100644 --- a/docs/snapshotters/README.md +++ b/docs/snapshotters/README.md @@ -10,10 +10,13 @@ Generic: - `overlayfs` (default): OverlayFS. This driver is akin to Docker/Moby's "overlay2" storage driver, but containerd's implementation is not called "overlay2". - `native`: Native file copying driver. Akin to Docker/Moby's "vfs" driver. +Block-based: +- `blockfile`: A driver using raw block files for each snapshot. Block files are copied from a parent or base empty block file. Mounting requires a virtual machine or support for loopback mounts. +- `devmapper`: ext4/xfs device mapper. See [`devmapper.md`](./devmapper.md). + Filesystem-specific: - `btrfs`: btrfs. Needs the plugin root (`/var/lib/containerd/io.containerd.snapshotter.v1.btrfs`) to be mounted as btrfs. - `zfs`: ZFS. Needs the plugin root (`/var/lib/containerd/io.containerd.snapshotter.v1.zfs`) to be mounted as ZFS. See also https://github.com/containerd/zfs . -- `devmapper`: ext4/xfs device mapper. See [`devmapper.md`](./devmapper.md). [Deprecated](https://github.com/containerd/containerd/blob/main/RELEASES.md#deprecated-features): - `aufs`: AUFS. Deprecated since containerd 1.5. Removed in containerd 2.0. See also https://github.com/containerd/aufs . From 1be571b14f964dd709389ef84e8d587f4cf5ad02 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Sat, 13 May 2023 00:11:04 -0700 Subject: [PATCH 4/4] Add sync before unmount on snapshotter layer test Signed-off-by: Derek McGowan --- snapshots/testsuite/testsuite.go | 1 + snapshots/testsuite/testsuite_unix.go | 10 +++++++++- snapshots/testsuite/testsuite_windows.go | 2 ++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/snapshots/testsuite/testsuite.go b/snapshots/testsuite/testsuite.go index 36982adab..7afbd953e 100644 --- a/snapshots/testsuite/testsuite.go +++ b/snapshots/testsuite/testsuite.go @@ -959,6 +959,7 @@ func check128LayersMount(name string) func(ctx context.Context, t *testing.T, sn t.Fatalf("[layer %d] preparing doesn't equal to flat after apply: %+v", i, err) } + sync() testutil.Unmount(t, preparing) parent = filepath.Join(work, fmt.Sprintf("committed-%d", i)) diff --git a/snapshots/testsuite/testsuite_unix.go b/snapshots/testsuite/testsuite_unix.go index 46a974b71..8a274ce24 100644 --- a/snapshots/testsuite/testsuite_unix.go +++ b/snapshots/testsuite/testsuite_unix.go @@ -18,7 +18,11 @@ package testsuite -import "syscall" +import ( + "syscall" + + "golang.org/x/sys/unix" +) func clearMask() func() { oldumask := syscall.Umask(0) @@ -26,3 +30,7 @@ func clearMask() func() { syscall.Umask(oldumask) } } + +func sync() { + unix.Sync() +} diff --git a/snapshots/testsuite/testsuite_windows.go b/snapshots/testsuite/testsuite_windows.go index a3cb82fbf..382a85978 100644 --- a/snapshots/testsuite/testsuite_windows.go +++ b/snapshots/testsuite/testsuite_windows.go @@ -19,3 +19,5 @@ package testsuite func clearMask() func() { return func() {} } + +func sync() {}