diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 30a34d489..827ae181a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -322,6 +322,7 @@ jobs: ENABLE_CRI_SANDBOXES: ${{ matrix.enable_cri_sandboxes }} GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-serial-junit.xml GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-serial-gotest.json + EXTRA_TESTFLAGS: "-timeout=20m" run: mingw32-make.exe integration - run: if [ -f *-gotest.json ]; then echo '# Integration 1' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi if: always() diff --git a/diff/windows/windows.go b/diff/windows/windows.go index d59ef1cff..2cfdfd71a 100644 --- a/diff/windows/windows.go +++ b/diff/windows/windows.go @@ -320,6 +320,7 @@ func mountsToLayerAndParents(mounts []mount.Mount) (string, []string, error) { return "", nil, fmt.Errorf("number of mounts should always be 1 for Windows layers: %w", errdefs.ErrInvalidArgument) } mnt := mounts[0] + if mnt.Type != "windows-layer" { // This is a special case error. When this is received the diff service // will attempt the next differ in the chain which for Windows is the @@ -332,6 +333,17 @@ func mountsToLayerAndParents(mounts []mount.Mount) (string, []string, error) { return "", nil, err } + if mnt.ReadOnly() { + if len(parentLayerPaths) == 0 { + // rootfs.CreateDiff creates a new, empty View to diff against, + // when diffing something with no parent. + // This makes perfect sense for a walking Diff, but for WCOW, + // we have to recognise this as "diff against nothing" + return "", nil, nil + } + // Ignore the dummy sandbox. + return parentLayerPaths[0], parentLayerPaths[1:], nil + } return mnt.Source, parentLayerPaths, nil } @@ -346,8 +358,16 @@ func mountPairToLayerStack(lower, upper []mount.Mount) ([]string, error) { return nil, fmt.Errorf("Upper mount invalid: %w", err) } + lowerLayer, lowerParentLayerPaths, err := mountsToLayerAndParents(lower) + if errdefs.IsNotImplemented(err) { + // Upper was a windows-layer, lower is not. We can't handle that. + return nil, fmt.Errorf("windowsDiff cannot diff a windows-layer against a non-windows-layer: %w", errdefs.ErrInvalidArgument) + } else if err != nil { + return nil, fmt.Errorf("Lower mount invalid: %w", err) + } + // Trivial case, diff-against-nothing - if len(lower) == 0 { + if lowerLayer == "" { if len(upperParentLayerPaths) != 0 { return nil, fmt.Errorf("windowsDiff cannot diff a layer with parents against a null layer: %w", errdefs.ErrInvalidArgument) } @@ -358,14 +378,6 @@ func mountPairToLayerStack(lower, upper []mount.Mount) ([]string, error) { return nil, fmt.Errorf("windowsDiff cannot diff a layer with no parents against another layer: %w", errdefs.ErrInvalidArgument) } - lowerLayer, lowerParentLayerPaths, err := mountsToLayerAndParents(lower) - if errdefs.IsNotImplemented(err) { - // Upper was a windows-layer, lower is not. We can't handle that. - return nil, fmt.Errorf("windowsDiff cannot diff a windows-layer against a non-windows-layer: %w", errdefs.ErrInvalidArgument) - } else if err != nil { - return nil, fmt.Errorf("Lower mount invalid: %w", err) - } - if upperParentLayerPaths[0] != lowerLayer { return nil, fmt.Errorf("windowsDiff cannot diff a layer against a layer other than its own parent: %w", errdefs.ErrInvalidArgument) } diff --git a/go.mod b/go.mod index bf116f5c5..b2587e3ff 100644 --- a/go.mod +++ b/go.mod @@ -5,13 +5,13 @@ go 1.19 require ( github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1 github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20221215162035-5330a85ea652 - github.com/Microsoft/go-winio v0.6.0 + github.com/Microsoft/go-winio v0.6.1-0.20230228163719-dd5de6900b62 github.com/Microsoft/hcsshim v0.10.0-rc.7 github.com/container-orchestrated-devices/container-device-interface v0.5.4 github.com/containerd/btrfs/v2 v2.0.0 github.com/containerd/cgroups/v3 v3.0.1 github.com/containerd/console v1.0.3 - github.com/containerd/continuity v0.3.0 + github.com/containerd/continuity v0.3.1-0.20230307035957-72c70feb3081 github.com/containerd/fifo v1.1.0 github.com/containerd/go-cni v1.1.9 github.com/containerd/go-runc v1.0.0 @@ -125,13 +125,13 @@ require ( go.opentelemetry.io/otel/metric v0.37.0 // indirect go.opentelemetry.io/proto/otlp v0.19.0 // indirect golang.org/x/crypto v0.1.0 // indirect - golang.org/x/mod v0.7.0 // indirect + golang.org/x/mod v0.8.0 // indirect golang.org/x/net v0.7.0 // indirect golang.org/x/oauth2 v0.4.0 // indirect golang.org/x/term v0.5.0 // indirect golang.org/x/text v0.7.0 // indirect golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect - golang.org/x/tools v0.5.0 // indirect + golang.org/x/tools v0.6.0 // indirect google.golang.org/appengine v1.6.7 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/square/go-jose.v2 v2.5.1 // indirect diff --git a/go.sum b/go.sum index 3b88e57b0..89c6fcb11 100644 --- a/go.sum +++ b/go.sum @@ -75,8 +75,8 @@ github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3/go.mod h1:JP github.com/Microsoft/go-winio v0.4.17-0.20210324224401-5516f17a5958/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= -github.com/Microsoft/go-winio v0.6.0 h1:slsWYD/zyx7lCXoZVlvQrj0hPTM1HI4+v1sIda2yDvg= -github.com/Microsoft/go-winio v0.6.0/go.mod h1:cTAf44im0RAYeL23bpB+fzCyDH2MJiz2BO69KH/soAE= +github.com/Microsoft/go-winio v0.6.1-0.20230228163719-dd5de6900b62 h1:PNDnNt0QOfCBd3bmdl9bhAt4+/PRCZpthE3PL0CMMtI= +github.com/Microsoft/go-winio v0.6.1-0.20230228163719-dd5de6900b62/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= github.com/Microsoft/hcsshim v0.8.7-0.20190325164909-8abdbb8205e4/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ= @@ -232,8 +232,8 @@ github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ= github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM= github.com/containerd/continuity v0.2.2/go.mod h1:pWygW9u7LtS1o4N/Tn0FoCFDIXZ7rxcMX7HX1Dmibvk= -github.com/containerd/continuity v0.3.0 h1:nisirsYROK15TAMVukJOUyGJjz4BNQJBVsNvAXZJ/eg= -github.com/containerd/continuity v0.3.0/go.mod h1:wJEAIwKOm/pBZuBd0JmeTvnLquTB1Ag8espWhkykbPM= +github.com/containerd/continuity v0.3.1-0.20230307035957-72c70feb3081 h1:1/vHscvA9Q2uXev+au1072As9m7pa9C4OaFDs6NgEBk= +github.com/containerd/continuity v0.3.1-0.20230307035957-72c70feb3081/go.mod h1:X1lYRPhc8/FifyvZvm13dkXxT6X+MhWvZykoSnMPyWU= github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= github.com/containerd/fifo v0.0.0-20200410184934-f15a3290365b/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= @@ -1115,8 +1115,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.7.0 h1:LapD9S96VoQRhi/GrNTqeBJFrUjs5UHCAtTlgwA5oZA= -golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1405,8 +1405,8 @@ golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.5.0 h1:+bSpV5HIeWkuvgaMfI3UmKRThoTA5ODJTUd8T17NO+4= -golang.org/x/tools v0.5.0/go.mod h1:N+Kgy78s5I24c24dU8OfWNEotWjutIs8SnJvn5IDq+k= +golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/integration/client/go.mod b/integration/client/go.mod index 7075cc60a..db78ee998 100644 --- a/integration/client/go.mod +++ b/integration/client/go.mod @@ -4,11 +4,12 @@ go 1.19 require ( github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1 + github.com/Microsoft/go-winio v0.6.1-0.20230228163719-dd5de6900b62 // indirect github.com/Microsoft/hcsshim v0.10.0-rc.7 github.com/Microsoft/hcsshim/test v0.0.0-20210408205431-da33ecd607e1 github.com/containerd/cgroups/v3 v3.0.1 github.com/containerd/containerd v1.7.0-beta.0 // see replace; the actual version of containerd is replaced with the code at the root of this repository - github.com/containerd/continuity v0.3.0 + github.com/containerd/continuity v0.3.1-0.20230307035957-72c70feb3081 github.com/containerd/go-runc v1.0.0 github.com/containerd/ttrpc v1.2.1 github.com/containerd/typeurl/v2 v2.1.0 @@ -24,7 +25,6 @@ require ( require ( github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20221215162035-5330a85ea652 // indirect - github.com/Microsoft/go-winio v0.6.0 // indirect github.com/cilium/ebpf v0.9.1 // indirect github.com/containerd/cgroups v1.1.0 // indirect github.com/containerd/console v1.0.3 // indirect @@ -55,11 +55,11 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/otel/trace v1.14.0 // indirect - golang.org/x/mod v0.7.0 // indirect + golang.org/x/mod v0.8.0 // indirect golang.org/x/net v0.7.0 // indirect golang.org/x/sync v0.1.0 // indirect golang.org/x/text v0.7.0 // indirect - golang.org/x/tools v0.5.0 // indirect + golang.org/x/tools v0.6.0 // indirect google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect google.golang.org/grpc v1.53.0 // indirect google.golang.org/protobuf v1.28.1 // indirect diff --git a/integration/client/go.sum b/integration/client/go.sum index caec2f416..38c5dbd5c 100644 --- a/integration/client/go.sum +++ b/integration/client/go.sum @@ -538,8 +538,9 @@ github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3/go.mod h1:JP github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= -github.com/Microsoft/go-winio v0.6.0 h1:slsWYD/zyx7lCXoZVlvQrj0hPTM1HI4+v1sIda2yDvg= github.com/Microsoft/go-winio v0.6.0/go.mod h1:cTAf44im0RAYeL23bpB+fzCyDH2MJiz2BO69KH/soAE= +github.com/Microsoft/go-winio v0.6.1-0.20230228163719-dd5de6900b62 h1:PNDnNt0QOfCBd3bmdl9bhAt4+/PRCZpthE3PL0CMMtI= +github.com/Microsoft/go-winio v0.6.1-0.20230228163719-dd5de6900b62/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn69iY6URG00= github.com/Microsoft/hcsshim v0.9.4/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc= github.com/Microsoft/hcsshim v0.9.6/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc= @@ -652,8 +653,9 @@ github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8a github.com/containerd/console v1.0.3 h1:lIr7SlA5PxZyMV30bDW0MGbiOPXwc63yRuCP0ARubLw= github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ= -github.com/containerd/continuity v0.3.0 h1:nisirsYROK15TAMVukJOUyGJjz4BNQJBVsNvAXZJ/eg= github.com/containerd/continuity v0.3.0/go.mod h1:wJEAIwKOm/pBZuBd0JmeTvnLquTB1Ag8espWhkykbPM= +github.com/containerd/continuity v0.3.1-0.20230307035957-72c70feb3081 h1:1/vHscvA9Q2uXev+au1072As9m7pa9C4OaFDs6NgEBk= +github.com/containerd/continuity v0.3.1-0.20230307035957-72c70feb3081/go.mod h1:X1lYRPhc8/FifyvZvm13dkXxT6X+MhWvZykoSnMPyWU= github.com/containerd/fifo v1.0.0/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= github.com/containerd/fifo v1.1.0 h1:4I2mbh5stb1u6ycIABlBw9zgtlK8viPI9QkQNRQEEmY= github.com/containerd/fifo v1.1.0/go.mod h1:bmC4NWMbXlt2EZ0Hc7Fx7QzTFxgPID13eH0Qu+MAb2o= @@ -1521,8 +1523,9 @@ golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= -golang.org/x/mod v0.7.0 h1:LapD9S96VoQRhi/GrNTqeBJFrUjs5UHCAtTlgwA5oZA= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1878,8 +1881,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= -golang.org/x/tools v0.5.0 h1:+bSpV5HIeWkuvgaMfI3UmKRThoTA5ODJTUd8T17NO+4= -golang.org/x/tools v0.5.0/go.mod h1:N+Kgy78s5I24c24dU8OfWNEotWjutIs8SnJvn5IDq+k= +golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/integration/client/snapshot_test.go b/integration/client/snapshot_test.go index 1aca97147..4086bd42d 100644 --- a/integration/client/snapshot_test.go +++ b/integration/client/snapshot_test.go @@ -18,7 +18,6 @@ package client import ( "context" - "runtime" "testing" . "github.com/containerd/containerd" @@ -44,8 +43,6 @@ func TestSnapshotterClient(t *testing.T) { if testing.Short() { t.Skip() } - if runtime.GOOS == "windows" { - t.Skip("snapshots not yet supported on Windows") - } + testsuite.SnapshotterSuite(t, DefaultSnapshotter, newSnapshotter) } diff --git a/mount/mount.go b/mount/mount.go index 24bfc7d01..ae7520f98 100644 --- a/mount/mount.go +++ b/mount/mount.go @@ -68,6 +68,17 @@ func UnmountMounts(mounts []Mount, target string, flags int) error { return nil } +// ReadOnly returns a boolean value indicating whether this mount has the "ro" +// option set. +func (m *Mount) ReadOnly() bool { + for _, option := range m.Options { + if option == "ro" { + return true + } + } + return false +} + // Mount to the provided target path. func (m *Mount) Mount(target string) error { target, err := fs.RootPath(target, m.Target) diff --git a/mount/mount_windows.go b/mount/mount_windows.go index b73fe3646..7c24fa600 100644 --- a/mount/mount_windows.go +++ b/mount/mount_windows.go @@ -17,17 +17,29 @@ package mount import ( + "context" "encoding/json" + "errors" "fmt" "os" "path/filepath" "strings" + "github.com/Microsoft/go-winio/pkg/bindfilter" "github.com/Microsoft/hcsshim" + "github.com/containerd/containerd/log" + "golang.org/x/sys/windows" +) + +const sourceStreamName = "containerd.io-source" + +var ( + // ErrNotImplementOnWindows is returned when an action is not implemented for windows + ErrNotImplementOnWindows = errors.New("not implemented under windows") ) // Mount to the provided target. -func (m *Mount) mount(target string) error { +func (m *Mount) mount(target string) (retErr error) { if m.Type != "windows-layer" { return fmt.Errorf("invalid windows mount type: '%s'", m.Type) } @@ -43,25 +55,60 @@ func (m *Mount) mount(target string) error { HomeDir: home, } - if err = hcsshim.ActivateLayer(di, layerID); err != nil { + if err := hcsshim.ActivateLayer(di, layerID); err != nil { return fmt.Errorf("failed to activate layer %s: %w", m.Source, err) } + defer func() { + if retErr != nil { + if layerErr := hcsshim.DeactivateLayer(di, layerID); layerErr != nil { + log.G(context.TODO()).WithError(layerErr).Error("failed to deactivate layer during mount failure cleanup") + } + } + }() - if err = hcsshim.PrepareLayer(di, layerID, parentLayerPaths); err != nil { + if err := hcsshim.PrepareLayer(di, layerID, parentLayerPaths); err != nil { return fmt.Errorf("failed to prepare layer %s: %w", m.Source, err) } - // We can link the layer mount path to the given target. It is an UNC path, and it needs - // a trailing backslash. - mountPath, err := hcsshim.GetLayerMountPath(di, layerID) - if err != nil { - return fmt.Errorf("failed to get layer mount path for %s: %w", m.Source, err) - } - mountPath = mountPath + `\` + defer func() { + if retErr != nil { + if layerErr := hcsshim.UnprepareLayer(di, layerID); layerErr != nil { + log.G(context.TODO()).WithError(layerErr).Error("failed to unprepare layer during mount failure cleanup") + } + } + }() - if err = os.Symlink(mountPath, target); err != nil { - return fmt.Errorf("failed to link mount to target %s: %w", target, err) + volume, err := hcsshim.GetLayerMountPath(di, layerID) + if err != nil { + return fmt.Errorf("failed to get volume path for layer %s: %w", m.Source, err) } + + if len(parentLayerPaths) == 0 { + // this is a base layer. It gets mounted without going through WCIFS. We need to mount the Files + // folder, not the actual source, or the client may inadvertently remove metadata files. + volume = filepath.Join(volume, "Files") + if _, err := os.Stat(volume); err != nil { + return fmt.Errorf("no Files folder in layer %s", layerID) + } + } + if err := bindfilter.ApplyFileBinding(target, volume, m.ReadOnly()); err != nil { + return fmt.Errorf("failed to set volume mount path for layer %s: %w", m.Source, err) + } + defer func() { + if retErr != nil { + if bindErr := bindfilter.RemoveFileBinding(target); bindErr != nil { + log.G(context.TODO()).WithError(bindErr).Error("failed to remove binding during mount failure cleanup") + } + } + }() + + // Add an Alternate Data Stream to record the layer source. + // See https://docs.microsoft.com/en-au/archive/blogs/askcore/alternate-data-streams-in-ntfs + // for details on Alternate Data Streams. + if err := os.WriteFile(filepath.Clean(target)+":"+sourceStreamName, []byte(m.Source), 0666); err != nil { + return fmt.Errorf("failed to record source for layer %s: %w", m.Source, err) + } + return nil } @@ -85,25 +132,55 @@ func (m *Mount) GetParentPaths() ([]string, error) { // Unmount the mount at the provided path func Unmount(mount string, flags int) error { - var ( - home, layerID = filepath.Split(mount) - di = hcsshim.DriverInfo{ - HomeDir: home, + mount = filepath.Clean(mount) + adsFile := mount + ":" + sourceStreamName + var layerPath string + + if _, err := os.Lstat(adsFile); err == nil { + layerPathb, err := os.ReadFile(mount + ":" + sourceStreamName) + if err != nil { + return fmt.Errorf("failed to retrieve source for layer %s: %w", mount, err) } - ) - - if err := hcsshim.UnprepareLayer(di, layerID); err != nil { - return fmt.Errorf("failed to unprepare layer %s: %w", mount, err) - } - if err := hcsshim.DeactivateLayer(di, layerID); err != nil { - return fmt.Errorf("failed to deactivate layer %s: %w", mount, err) + layerPath = string(layerPathb) } + if err := bindfilter.RemoveFileBinding(mount); err != nil { + if errors.Is(err, windows.ERROR_INVALID_PARAMETER) || errors.Is(err, windows.ERROR_NOT_FOUND) { + // not a mount point + return nil + } + return fmt.Errorf("removing mount: %w", err) + } + + if layerPath != "" { + var ( + home, layerID = filepath.Split(layerPath) + di = hcsshim.DriverInfo{ + HomeDir: home, + } + ) + + if err := hcsshim.UnprepareLayer(di, layerID); err != nil { + return fmt.Errorf("failed to unprepare layer %s: %w", mount, err) + } + + if err := hcsshim.DeactivateLayer(di, layerID); err != nil { + return fmt.Errorf("failed to deactivate layer %s: %w", mount, err) + } + } return nil } // UnmountAll unmounts from the provided path func UnmountAll(mount string, flags int) error { + if mount == "" { + // This isn't an error, per the EINVAL handling in the Linux version + return nil + } + if _, err := os.Stat(mount); os.IsNotExist(err) { + return nil + } + return Unmount(mount, flags) } diff --git a/pkg/cri/opts/container.go b/pkg/cri/opts/container.go index 04a3c1777..0f0c0d9b3 100644 --- a/pkg/cri/opts/container.go +++ b/pkg/cri/opts/container.go @@ -21,8 +21,6 @@ import ( "errors" "fmt" "os" - "path/filepath" - goruntime "runtime" "strings" "github.com/containerd/continuity/fs" @@ -86,53 +84,34 @@ func WithVolumes(volumeMounts map[string]string) containerd.NewContainerOpts { // https://github.com/containerd/containerd/pull/1785 defer os.Remove(root) - unmounter := func(mountPath string) { - if uerr := mount.Unmount(mountPath, 0); uerr != nil { + if err := mount.All(mounts, root); err != nil { + return fmt.Errorf("failed to mount: %w", err) + } + defer func() { + if uerr := mount.Unmount(root, 0); uerr != nil { log.G(ctx).WithError(uerr).Errorf("Failed to unmount snapshot %q", root) if err == nil { err = uerr } } - } - - var mountPaths []string - if goruntime.GOOS == "windows" { - for _, m := range mounts { - // appending the layerID to the root. - mountPath := filepath.Join(root, filepath.Base(m.Source)) - mountPaths = append(mountPaths, mountPath) - if err := m.Mount(mountPath); err != nil { - return err - } - - defer unmounter(m.Source) - } - } else { - mountPaths = append(mountPaths, root) - if err := mount.All(mounts, root); err != nil { - return fmt.Errorf("failed to mount: %w", err) - } - defer unmounter(root) - } + }() for host, volume := range volumeMounts { // The volume may have been defined with a C: prefix, which we can't use here. volume = strings.TrimPrefix(volume, "C:") - for _, mountPath := range mountPaths { - src, err := fs.RootPath(mountPath, volume) - if err != nil { - return fmt.Errorf("rootpath on mountPath %s, volume %s: %w", mountPath, volume, err) - } - if _, err := os.Stat(src); err != nil { - if os.IsNotExist(err) { - // Skip copying directory if it does not exist. - continue - } - return fmt.Errorf("stat volume in rootfs: %w", err) - } - if err := copyExistingContents(src, host); err != nil { - return fmt.Errorf("taking runtime copy of volume: %w", err) + src, err := fs.RootPath(root, volume) + if err != nil { + return fmt.Errorf("rootpath on mountPath %s, volume %s: %w", root, volume, err) + } + if _, err := os.Stat(src); err != nil { + if os.IsNotExist(err) { + // Skip copying directory if it does not exist. + continue } + return fmt.Errorf("stat volume in rootfs: %w", err) + } + if err := copyExistingContents(src, host); err != nil { + return fmt.Errorf("taking runtime copy of volume: %w", err) } } return nil diff --git a/pkg/testutil/helpers.go b/pkg/testutil/helpers.go index 9a9f857e2..850f51f83 100644 --- a/pkg/testutil/helpers.go +++ b/pkg/testutil/helpers.go @@ -23,6 +23,9 @@ import ( "path/filepath" "strconv" "testing" + + "github.com/containerd/containerd/mount" + "github.com/stretchr/testify/assert" ) var rootEnabled bool @@ -79,3 +82,10 @@ func DumpDirOnFailure(t *testing.T, root string) { DumpDir(t, root) } } + +// Unmount unmounts a given mountPoint and sets t.Error if it fails +func Unmount(t testing.TB, mountPoint string) { + t.Log("unmount", mountPoint) + err := mount.UnmountAll(mountPoint, umountflags) + assert.NoError(t, err) +} diff --git a/pkg/testutil/helpers_unix.go b/pkg/testutil/helpers_unix.go index 6e03a07f9..f402e44a3 100644 --- a/pkg/testutil/helpers_unix.go +++ b/pkg/testutil/helpers_unix.go @@ -23,17 +23,9 @@ import ( "os" "testing" - "github.com/containerd/containerd/mount" "github.com/stretchr/testify/assert" ) -// Unmount unmounts a given mountPoint and sets t.Error if it fails -func Unmount(t testing.TB, mountPoint string) { - t.Log("unmount", mountPoint) - err := mount.UnmountAll(mountPoint, umountflags) - assert.NoError(t, err) -} - // RequiresRoot skips tests that require root, unless the test.root flag has // been set func RequiresRoot(t testing.TB) { diff --git a/pkg/testutil/helpers_windows.go b/pkg/testutil/helpers_windows.go index 203d98771..501c3d803 100644 --- a/pkg/testutil/helpers_windows.go +++ b/pkg/testutil/helpers_windows.go @@ -25,8 +25,3 @@ func RequiresRoot(t testing.TB) { // RequiresRootM is similar to RequiresRoot but intended to be called from *testing.M. func RequiresRootM() { } - -// Unmount unmounts a given mountPoint and sets t.Error if it fails -// Does nothing on Windows -func Unmount(t *testing.T, mountPoint string) { -} diff --git a/pkg/testutil/mount_other.go b/pkg/testutil/mount_other.go index 163073bd0..b1aa7d460 100644 --- a/pkg/testutil/mount_other.go +++ b/pkg/testutil/mount_other.go @@ -1,4 +1,5 @@ -//go:build !linux && !windows +//go:build !linux +// +build !linux /* Copyright The containerd Authors. diff --git a/snapshots/native/native_test.go b/snapshots/native/native_test.go index 79e02637d..4f48e6a76 100644 --- a/snapshots/native/native_test.go +++ b/snapshots/native/native_test.go @@ -35,10 +35,10 @@ func newSnapshotter(ctx context.Context, root string) (snapshots.Snapshotter, fu return snapshotter, func() error { return snapshotter.Close() }, nil } -func TestNaive(t *testing.T) { +func TestNative(t *testing.T) { if runtime.GOOS == "windows" { - t.Skip("snapshotter not implemented on windows") + t.Skip("Native snapshotter not implemented on windows") } testutil.RequiresRoot(t) - testsuite.SnapshotterSuite(t, "Naive", newSnapshotter) + testsuite.SnapshotterSuite(t, "Native", newSnapshotter) } diff --git a/snapshots/testsuite/issues.go b/snapshots/testsuite/issues.go index eea87767e..df6eb4848 100644 --- a/snapshots/testsuite/issues.go +++ b/snapshots/testsuite/issues.go @@ -19,6 +19,7 @@ package testsuite import ( "context" "fmt" + "runtime" "strings" "testing" "time" @@ -94,6 +95,9 @@ func checkRemoveDirectoryInLowerLayer(ctx context.Context, t *testing.T, sn snap // See https://github.com/docker/docker/issues/24913 overlay // see https://github.com/docker/docker/issues/28391 overlay2 func checkChown(ctx context.Context, t *testing.T, sn snapshots.Snapshotter, work string) { + if runtime.GOOS == "windows" { + t.Skip("Chown is not supported on Windows") + } l1Init := fstest.Apply( fstest.CreateDir("/opt", 0700), fstest.CreateDir("/opt/a", 0700), @@ -147,6 +151,9 @@ func checkRename(ss string) func(ctx context.Context, t *testing.T, sn snapshots // checkDirectoryPermissionOnCommit // https://github.com/docker/docker/issues/27298 func checkDirectoryPermissionOnCommit(ctx context.Context, t *testing.T, sn snapshots.Snapshotter, work string) { + if runtime.GOOS == "windows" { + t.Skip("Chown is not supported on WCOW") + } l1Init := fstest.Apply( fstest.CreateDir("/dir1", 0700), fstest.CreateDir("/dir2", 0700), diff --git a/snapshots/testsuite/testsuite.go b/snapshots/testsuite/testsuite.go index cd2bfc913..36982adab 100644 --- a/snapshots/testsuite/testsuite.go +++ b/snapshots/testsuite/testsuite.go @@ -23,6 +23,7 @@ import ( "fmt" "os" "path/filepath" + "runtime" "sort" "testing" "time" @@ -162,6 +163,7 @@ func checkSnapshotterBasic(ctx context.Context, t *testing.T, snapshotter snapsh } if err := initialApplier.Apply(preparing); err != nil { + testutil.Unmount(t, preparing) t.Fatalf("failure reason: %+v", err) } // unmount before commit @@ -199,10 +201,12 @@ func checkSnapshotterBasic(ctx context.Context, t *testing.T, snapshotter snapsh } if err := fstest.CheckDirectoryEqualWithApplier(next, initialApplier); err != nil { + testutil.Unmount(t, next) t.Fatalf("failure reason: %+v", err) } if err := diffApplier.Apply(next); err != nil { + testutil.Unmount(t, next) t.Fatalf("failure reason: %+v", err) } // unmount before commit @@ -386,11 +390,12 @@ func checkSnapshotterTransitivity(ctx context.Context, t *testing.T, snapshotter if err != nil { t.Fatal(err) } - defer testutil.Unmount(t, preparing) if err = os.WriteFile(filepath.Join(preparing, "foo"), []byte("foo\n"), 0777); err != nil { + testutil.Unmount(t, preparing) t.Fatal(err) } + testutil.Unmount(t, preparing) snapA := filepath.Join(work, "snapA") if err = snapshotter.Commit(ctx, snapA, preparing, opt); err != nil { @@ -401,11 +406,12 @@ func checkSnapshotterTransitivity(ctx context.Context, t *testing.T, snapshotter if err != nil { t.Fatal(err) } - defer testutil.Unmount(t, next) if err = os.WriteFile(filepath.Join(next, "foo"), []byte("foo bar\n"), 0777); err != nil { + testutil.Unmount(t, next) t.Fatal(err) } + testutil.Unmount(t, next) snapB := filepath.Join(work, "snapB") if err = snapshotter.Commit(ctx, snapB, next, opt); err != nil { @@ -440,7 +446,7 @@ func checkSnapshotterPrepareView(ctx context.Context, t *testing.T, snapshotter if err != nil { t.Fatal(err) } - defer testutil.Unmount(t, preparing) + testutil.Unmount(t, preparing) snapA := filepath.Join(work, "snapA") if err = snapshotter.Commit(ctx, snapA, preparing, opt); err != nil { @@ -517,6 +523,7 @@ func checkRemoveIntermediateSnapshot(ctx context.Context, t *testing.T, snapshot if err != nil { t.Fatal(err) } + testutil.Unmount(t, base) committedBase := filepath.Join(work, "committed-base") if err = snapshotter.Commit(ctx, committedBase, base, opt); err != nil { @@ -555,7 +562,6 @@ func checkRemoveIntermediateSnapshot(ctx context.Context, t *testing.T, snapshot if err != nil { t.Fatal(err) } - testutil.Unmount(t, base) err = snapshotter.Remove(ctx, committedBase) if err != nil { t.Fatal(err) @@ -814,12 +820,13 @@ func checkSnapshotterViewReadonly(ctx context.Context, t *testing.T, snapshotter } testfile := filepath.Join(viewMountPoint, "testfile") - if err := os.WriteFile(testfile, []byte("testcontent"), 0777); err != nil { + err = os.WriteFile(testfile, []byte("testcontent"), 0777) + testutil.Unmount(t, viewMountPoint) + if err != nil { t.Logf("write to %q failed with %v (EROFS is expected but can be other error code)", testfile, err) } else { t.Fatalf("write to %q should fail (EROFS) but did not fail", testfile) } - testutil.Unmount(t, viewMountPoint) assert.Nil(t, snapshotter.Remove(ctx, view)) assert.Nil(t, snapshotter.Remove(ctx, committed)) } @@ -869,6 +876,10 @@ func closeTwice(ctx context.Context, t *testing.T, snapshotter snapshots.Snapsho } func checkRootPermission(ctx context.Context, t *testing.T, snapshotter snapshots.Snapshotter, work string) { + if runtime.GOOS == "windows" { + t.Skip("Filesystem permissions are not supported on Windows") + } + preparing, err := snapshotterPrepareMount(ctx, snapshotter, "preparing", "", work) if err != nil { t.Fatal(err) @@ -888,19 +899,19 @@ func check128LayersMount(name string) func(ctx context.Context, t *testing.T, sn lowestApply := fstest.Apply( fstest.CreateFile("/bottom", []byte("way at the bottom\n"), 0777), fstest.CreateFile("/overwriteme", []byte("FIRST!\n"), 0777), - fstest.CreateDir("/ADDHERE", 0755), - fstest.CreateDir("/ONLYME", 0755), - fstest.CreateFile("/ONLYME/bottom", []byte("bye!\n"), 0777), + fstest.CreateDir("/addhere", 0755), + fstest.CreateDir("/onlyme", 0755), + fstest.CreateFile("/onlyme/bottom", []byte("bye!\n"), 0777), ) appliers := []fstest.Applier{lowestApply} for i := 1; i <= 127; i++ { appliers = append(appliers, fstest.Apply( fstest.CreateFile("/overwriteme", []byte(fmt.Sprintf("%d WAS HERE!\n", i)), 0777), - fstest.CreateFile(fmt.Sprintf("/ADDHERE/file-%d", i), []byte("same\n"), 0755), - fstest.RemoveAll("/ONLYME"), - fstest.CreateDir("/ONLYME", 0755), - fstest.CreateFile(fmt.Sprintf("/ONLYME/file-%d", i), []byte("only me!\n"), 0777), + fstest.CreateFile(fmt.Sprintf("/addhere/file-%d", i), []byte("same\n"), 0755), + fstest.RemoveAll("/onlyme"), + fstest.CreateDir("/onlyme", 0755), + fstest.CreateFile(fmt.Sprintf("/onlyme/file-%d", i), []byte("only me!\n"), 0777), )) } @@ -926,7 +937,9 @@ func check128LayersMount(name string) func(ctx context.Context, t *testing.T, sn t.Fatalf("[layer %d] failed to mount on the target(%s): %+v", i, preparing, err) } - if err := fstest.CheckDirectoryEqual(preparing, flat); err != nil { + t.Log("mount", preparing) + + if err := fstest.CheckDirectoryEqual(flat, preparing); err != nil { testutil.Unmount(t, preparing) t.Fatalf("[layer %d] preparing doesn't equal to flat before apply: %+v", i, err) } @@ -941,7 +954,7 @@ func check128LayersMount(name string) func(ctx context.Context, t *testing.T, sn t.Fatalf("[layer %d] failed to apply on preparing dir: %+v", i, err) } - if err := fstest.CheckDirectoryEqual(preparing, flat); err != nil { + if err := fstest.CheckDirectoryEqual(flat, preparing); err != nil { testutil.Unmount(t, preparing) t.Fatalf("[layer %d] preparing doesn't equal to flat after apply: %+v", i, err) } @@ -970,7 +983,7 @@ func check128LayersMount(name string) func(ctx context.Context, t *testing.T, sn } defer testutil.Unmount(t, view) - if err := fstest.CheckDirectoryEqual(view, flat); err != nil { + if err := fstest.CheckDirectoryEqual(flat, view); err != nil { t.Fatalf("fullview should equal to flat: %+v", err) } } diff --git a/snapshots/windows/windows.go b/snapshots/windows/windows.go index 310e4380b..8e79f10e7 100644 --- a/snapshots/windows/windows.go +++ b/snapshots/windows/windows.go @@ -187,7 +187,7 @@ func (s *snapshotter) Mounts(ctx context.Context, key string) (_ []mount.Mount, return nil, err } - return s.mounts(snapshot), nil + return s.mounts(snapshot, key), nil } func (s *snapshotter) Commit(ctx context.Context, name, key string, opts ...snapshots.Opt) (retErr error) { @@ -208,7 +208,11 @@ func (s *snapshotter) Commit(ctx context.Context, name, key string, opts ...snap // If (windowsDiff).Apply was used to populate this layer, then it's already in the 'committed' state. // See createSnapshot below for more details if !strings.Contains(key, snapshots.UnpackKeyPrefix) { - if err := s.convertScratchToReadOnlyLayer(ctx, snapshot, path); err != nil { + if len(snapshot.ParentIDs) == 0 { + if err = hcsshim.ConvertToBaseLayer(path); err != nil { + return err + } + } else if err := s.convertScratchToReadOnlyLayer(ctx, snapshot, path); err != nil { return err } } @@ -299,11 +303,9 @@ func (s *snapshotter) Close() error { return s.ms.Close() } -func (s *snapshotter) mounts(sn storage.Snapshot) []mount.Mount { +func (s *snapshotter) mounts(sn storage.Snapshot, key string) []mount.Mount { var ( - roFlag string - source string - parentLayerPaths []string + roFlag string ) if sn.Kind == snapshots.KindView { @@ -312,27 +314,28 @@ func (s *snapshotter) mounts(sn storage.Snapshot) []mount.Mount { roFlag = "rw" } - if len(sn.ParentIDs) == 0 || sn.Kind == snapshots.KindActive { - source = s.getSnapshotDir(sn.ID) - parentLayerPaths = s.parentIDsToParentPaths(sn.ParentIDs) - } else { - source = s.getSnapshotDir(sn.ParentIDs[0]) - parentLayerPaths = s.parentIDsToParentPaths(sn.ParentIDs[1:]) - } + source := s.getSnapshotDir(sn.ID) + parentLayerPaths := s.parentIDsToParentPaths(sn.ParentIDs) + + mountType := "windows-layer" // error is not checked here, as a string array will never fail to Marshal parentLayersJSON, _ := json.Marshal(parentLayerPaths) parentLayersOption := mount.ParentLayerPathsFlag + string(parentLayersJSON) - var mounts []mount.Mount - mounts = append(mounts, mount.Mount{ - Source: source, - Type: "windows-layer", - Options: []string{ - roFlag, - parentLayersOption, + options := []string{ + roFlag, + } + if len(sn.ParentIDs) != 0 { + options = append(options, parentLayersOption) + } + mounts := []mount.Mount{ + { + Source: source, + Type: mountType, + Options: options, }, - }) + } return mounts } @@ -349,73 +352,80 @@ func (s *snapshotter) createSnapshot(ctx context.Context, kind snapshots.Kind, k return fmt.Errorf("failed to create snapshot: %w", err) } - if kind != snapshots.KindActive { - return nil - } - - log.G(ctx).Debug("createSnapshot active") + log.G(ctx).Debug("createSnapshot") // Create the new snapshot dir snDir := s.getSnapshotDir(newSnapshot.ID) if err = os.MkdirAll(snDir, 0700); err != nil { - return err + return fmt.Errorf("failed to create snapshot dir %s: %w", snDir, err) } - // IO/disk space optimization - // - // We only need one sandbox.vhdx for the container. Skip making one for this - // snapshot if this isn't the snapshot that just houses the final sandbox.vhd - // that will be mounted as the containers scratch. Currently the key for a snapshot - // where a layer will be extracted to will have the string `extract-` in it. - if !strings.Contains(key, snapshots.UnpackKeyPrefix) { - parentLayerPaths := s.parentIDsToParentPaths(newSnapshot.ParentIDs) + if strings.Contains(key, snapshots.UnpackKeyPrefix) { + // IO/disk space optimization: Do nothing + // + // We only need one sandbox.vhdx for the container. Skip making one for this + // snapshot if this isn't the snapshot that just houses the final sandbox.vhd + // that will be mounted as the containers scratch. Currently the key for a snapshot + // where a layer will be extracted to will have the string `extract-` in it. + return nil + } - var snapshotInfo snapshots.Info - for _, o := range opts { - o(&snapshotInfo) + if len(newSnapshot.ParentIDs) == 0 { + // A parentless snapshot a new base layer. Valid base layers must have a "Files" folder. + // When committed, there'll be some post-processing to fill in the rest + // of the metadata. + filesDir := filepath.Join(snDir, "Files") + if err := os.MkdirAll(filesDir, 0700); err != nil { + return fmt.Errorf("creating Files dir: %w", err) } + return nil + } - var sizeInBytes uint64 - if sizeGBstr, ok := snapshotInfo.Labels[rootfsSizeInGBLabel]; ok { - log.G(ctx).Warnf("%q label is deprecated, please use %q instead.", rootfsSizeInGBLabel, rootfsSizeInBytesLabel) + parentLayerPaths := s.parentIDsToParentPaths(newSnapshot.ParentIDs) + var snapshotInfo snapshots.Info + for _, o := range opts { + o(&snapshotInfo) + } - sizeInGB, err := strconv.ParseUint(sizeGBstr, 10, 32) - if err != nil { - return fmt.Errorf("failed to parse label %q=%q: %w", rootfsSizeInGBLabel, sizeGBstr, err) - } - sizeInBytes = sizeInGB * 1024 * 1024 * 1024 + var sizeInBytes uint64 + if sizeGBstr, ok := snapshotInfo.Labels[rootfsSizeInGBLabel]; ok { + log.G(ctx).Warnf("%q label is deprecated, please use %q instead.", rootfsSizeInGBLabel, rootfsSizeInBytesLabel) + + sizeInGB, err := strconv.ParseUint(sizeGBstr, 10, 32) + if err != nil { + return fmt.Errorf("failed to parse label %q=%q: %w", rootfsSizeInGBLabel, sizeGBstr, err) } + sizeInBytes = sizeInGB * 1024 * 1024 * 1024 + } - // Prefer the newer label in bytes over the deprecated Windows specific GB variant. - if sizeBytesStr, ok := snapshotInfo.Labels[rootfsSizeInBytesLabel]; ok { - sizeInBytes, err = strconv.ParseUint(sizeBytesStr, 10, 64) - if err != nil { - return fmt.Errorf("failed to parse label %q=%q: %w", rootfsSizeInBytesLabel, sizeBytesStr, err) - } - } - - var makeUVMScratch bool - if _, ok := snapshotInfo.Labels[uvmScratchLabel]; ok { - makeUVMScratch = true - } - - // This has to be run first to avoid clashing with the containers sandbox.vhdx. - if makeUVMScratch { - if err = s.createUVMScratchLayer(ctx, snDir, parentLayerPaths); err != nil { - return fmt.Errorf("failed to make UVM's scratch layer: %w", err) - } - } - if err = s.createScratchLayer(ctx, snDir, parentLayerPaths, sizeInBytes); err != nil { - return fmt.Errorf("failed to create scratch layer: %w", err) + // Prefer the newer label in bytes over the deprecated Windows specific GB variant. + if sizeBytesStr, ok := snapshotInfo.Labels[rootfsSizeInBytesLabel]; ok { + sizeInBytes, err = strconv.ParseUint(sizeBytesStr, 10, 64) + if err != nil { + return fmt.Errorf("failed to parse label %q=%q: %w", rootfsSizeInBytesLabel, sizeBytesStr, err) } } + var makeUVMScratch bool + if _, ok := snapshotInfo.Labels[uvmScratchLabel]; ok { + makeUVMScratch = true + } + + // This has to be run first to avoid clashing with the containers sandbox.vhdx. + if makeUVMScratch { + if err = s.createUVMScratchLayer(ctx, snDir, parentLayerPaths); err != nil { + return fmt.Errorf("failed to make UVM's scratch layer: %w", err) + } + } + if err = s.createScratchLayer(ctx, snDir, parentLayerPaths, sizeInBytes); err != nil { + return fmt.Errorf("failed to create scratch layer: %w", err) + } return nil }) if err != nil { return nil, err } - return s.mounts(newSnapshot), nil + return s.mounts(newSnapshot, key), nil } func (s *snapshotter) parentIDsToParentPaths(parentIDs []string) []string { diff --git a/snapshots/windows/windows_test.go b/snapshots/windows/windows_test.go new file mode 100644 index 000000000..d8d9eca82 --- /dev/null +++ b/snapshots/windows/windows_test.go @@ -0,0 +1,43 @@ +//go:build windows +// +build windows + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package windows + +import ( + "context" + "testing" + + "github.com/containerd/containerd/pkg/testutil" + "github.com/containerd/containerd/snapshots" + "github.com/containerd/containerd/snapshots/testsuite" +) + +func newSnapshotter(ctx context.Context, root string) (snapshots.Snapshotter, func() error, error) { + snapshotter, err := NewSnapshotter(root) + if err != nil { + return nil, nil, err + } + + return snapshotter, func() error { return snapshotter.Close() }, nil +} + +func TestWindows(t *testing.T) { + testutil.RequiresRoot(t) + testsuite.SnapshotterSuite(t, "Windows", newSnapshotter) +} diff --git a/vendor/github.com/Microsoft/go-winio/.golangci.yml b/vendor/github.com/Microsoft/go-winio/.golangci.yml index af403bb13..2bf84e321 100644 --- a/vendor/github.com/Microsoft/go-winio/.golangci.yml +++ b/vendor/github.com/Microsoft/go-winio/.golangci.yml @@ -8,12 +8,8 @@ linters: - containedctx # struct contains a context - dupl # duplicate code - errname # erorrs are named correctly - - goconst # strings that should be constants - - godot # comments end in a period - - misspell - nolintlint # "//nolint" directives are properly explained - revive # golint replacement - - stylecheck # golint replacement, less configurable than revive - unconvert # unnecessary conversions - wastedassign @@ -23,10 +19,7 @@ linters: - exhaustive # check exhaustiveness of enum switch statements - gofmt # files are gofmt'ed - gosec # security - - nestif # deeply nested ifs - nilerr # returns nil even with non-nil error - - prealloc # slices that can be pre-allocated - - structcheck # unused struct fields - unparam # unused function params issues: @@ -56,6 +49,8 @@ issues: linters-settings: + exhaustive: + default-signifies-exhaustive: true govet: enable-all: true disable: @@ -98,6 +93,8 @@ linters-settings: disabled: true - name: flag-parameter # excessive, and a common idiom we use disabled: true + - name: unhandled-error # warns over common fmt.Print* and io.Close; rely on errcheck instead + disabled: true # general config - name: line-length-limit arguments: @@ -138,7 +135,3 @@ linters-settings: - VPCI - WCOW - WIM - stylecheck: - checks: - - "all" - - "-ST1003" # use revive's var naming diff --git a/vendor/github.com/Microsoft/go-winio/pkg/bindfilter/bind_filter.go b/vendor/github.com/Microsoft/go-winio/pkg/bindfilter/bind_filter.go new file mode 100644 index 000000000..7ac377ae4 --- /dev/null +++ b/vendor/github.com/Microsoft/go-winio/pkg/bindfilter/bind_filter.go @@ -0,0 +1,308 @@ +//go:build windows +// +build windows + +package bindfilter + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "syscall" + "unsafe" + + "golang.org/x/sys/windows" +) + +//go:generate go run github.com/Microsoft/go-winio/tools/mkwinsyscall -output zsyscall_windows.go ./bind_filter.go +//sys bfSetupFilter(jobHandle windows.Handle, flags uint32, virtRootPath string, virtTargetPath string, virtExceptions **uint16, virtExceptionPathCount uint32) (hr error) = bindfltapi.BfSetupFilter? +//sys bfRemoveMapping(jobHandle windows.Handle, virtRootPath string) (hr error) = bindfltapi.BfRemoveMapping? +//sys bfGetMappings(flags uint32, jobHandle windows.Handle, virtRootPath *uint16, sid *windows.SID, bufferSize *uint32, outBuffer *byte) (hr error) = bindfltapi.BfGetMappings? + +// BfSetupFilter flags. See: +// https://github.com/microsoft/BuildXL/blob/a6dce509f0d4f774255e5fbfb75fa6d5290ed163/Public/Src/Utilities/Native/Processes/Windows/NativeContainerUtilities.cs#L193-L240 +// +//nolint:revive // var-naming: ALL_CAPS +const ( + BINDFLT_FLAG_READ_ONLY_MAPPING uint32 = 0x00000001 + // Tells bindflt to fail mapping with STATUS_INVALID_PARAMETER if a mapping produces + // multiple targets. + BINDFLT_FLAG_NO_MULTIPLE_TARGETS uint32 = 0x00000040 +) + +//nolint:revive // var-naming: ALL_CAPS +const ( + BINDFLT_GET_MAPPINGS_FLAG_VOLUME uint32 = 0x00000001 + BINDFLT_GET_MAPPINGS_FLAG_SILO uint32 = 0x00000002 + BINDFLT_GET_MAPPINGS_FLAG_USER uint32 = 0x00000004 +) + +// ApplyFileBinding creates a global mount of the source in root, with an optional +// read only flag. +// The bind filter allows us to create mounts of directories and volumes. By default it allows +// us to mount multiple sources inside a single root, acting as an overlay. Files from the +// second source will superscede the first source that was mounted. +// This function disables this behavior and sets the BINDFLT_FLAG_NO_MULTIPLE_TARGETS flag +// on the mount. +func ApplyFileBinding(root, source string, readOnly bool) error { + // The parent directory needs to exist for the bind to work. MkdirAll stats and + // returns nil if the directory exists internally so we should be fine to mkdirall + // every time. + if err := os.MkdirAll(filepath.Dir(root), 0); err != nil { + return err + } + + if strings.Contains(source, "Volume{") && !strings.HasSuffix(source, "\\") { + // Add trailing slash to volumes, otherwise we get an error when binding it to + // a folder. + source = source + "\\" + } + + flags := BINDFLT_FLAG_NO_MULTIPLE_TARGETS + if readOnly { + flags |= BINDFLT_FLAG_READ_ONLY_MAPPING + } + + // Set the job handle to 0 to create a global mount. + if err := bfSetupFilter( + 0, + flags, + root, + source, + nil, + 0, + ); err != nil { + return fmt.Errorf("failed to bind target %q to root %q: %w", source, root, err) + } + return nil +} + +// RemoveFileBinding removes a mount from the root path. +func RemoveFileBinding(root string) error { + if err := bfRemoveMapping(0, root); err != nil { + return fmt.Errorf("removing file binding: %w", err) + } + return nil +} + +// GetBindMappings returns a list of bind mappings that have their root on a +// particular volume. The volumePath parameter can be any path that exists on +// a volume. For example, if a number of mappings are created in C:\ProgramData\test, +// to get a list of those mappings, the volumePath parameter would have to be set to +// C:\ or the VOLUME_NAME_GUID notation of C:\ (\\?\Volume{GUID}\), or any child +// path that exists. +func GetBindMappings(volumePath string) ([]BindMapping, error) { + rootPtr, err := windows.UTF16PtrFromString(volumePath) + if err != nil { + return nil, err + } + + flags := BINDFLT_GET_MAPPINGS_FLAG_VOLUME + // allocate a large buffer for results + var outBuffSize uint32 = 256 * 1024 + buf := make([]byte, outBuffSize) + + if err := bfGetMappings(flags, 0, rootPtr, nil, &outBuffSize, &buf[0]); err != nil { + return nil, err + } + + if outBuffSize < 12 { + return nil, fmt.Errorf("invalid buffer returned") + } + + result := buf[:outBuffSize] + + // The first 12 bytes are the three uint32 fields in getMappingsResponseHeader{} + headerBuffer := result[:12] + // The alternative to using unsafe and casting it to the above defined structures, is to manually + // parse the fields. Not too terrible, but not sure it'd worth the trouble. + header := *(*getMappingsResponseHeader)(unsafe.Pointer(&headerBuffer[0])) + + if header.MappingCount == 0 { + // no mappings + return []BindMapping{}, nil + } + + mappingsBuffer := result[12 : int(unsafe.Sizeof(mappingEntry{}))*int(header.MappingCount)] + // Get a pointer to the first mapping in the slice + mappingsPointer := (*mappingEntry)(unsafe.Pointer(&mappingsBuffer[0])) + // Get slice of mappings + mappings := unsafe.Slice(mappingsPointer, header.MappingCount) + + mappingEntries := make([]BindMapping, header.MappingCount) + for i := 0; i < int(header.MappingCount); i++ { + bindMapping, err := getBindMappingFromBuffer(result, mappings[i]) + if err != nil { + return nil, fmt.Errorf("fetching bind mappings: %w", err) + } + mappingEntries[i] = bindMapping + } + + return mappingEntries, nil +} + +// mappingEntry holds information about where in the response buffer we can +// find information about the virtual root (the mount point) and the targets (sources) +// that get mounted, as well as the flags used to bind the targets to the virtual root. +type mappingEntry struct { + VirtRootLength uint32 + VirtRootOffset uint32 + Flags uint32 + NumberOfTargets uint32 + TargetEntriesOffset uint32 +} + +type mappingTargetEntry struct { + TargetRootLength uint32 + TargetRootOffset uint32 +} + +// getMappingsResponseHeader represents the first 12 bytes of the BfGetMappings() response. +// It gives us the size of the buffer, the status of the call and the number of mappings. +// A response +type getMappingsResponseHeader struct { + Size uint32 + Status uint32 + MappingCount uint32 +} + +type BindMapping struct { + MountPoint string + Flags uint32 + Targets []string +} + +func decodeEntry(buffer []byte) (string, error) { + name := make([]uint16, len(buffer)/2) + err := binary.Read(bytes.NewReader(buffer), binary.LittleEndian, &name) + if err != nil { + return "", fmt.Errorf("decoding name: %w", err) + } + return windows.UTF16ToString(name), nil +} + +func getTargetsFromBuffer(buffer []byte, offset, count int) ([]string, error) { + if len(buffer) < offset+count*6 { + return nil, fmt.Errorf("invalid buffer") + } + + targets := make([]string, count) + for i := 0; i < count; i++ { + entryBuf := buffer[offset+i*8 : offset+i*8+8] + tgt := *(*mappingTargetEntry)(unsafe.Pointer(&entryBuf[0])) + if len(buffer) < int(tgt.TargetRootOffset)+int(tgt.TargetRootLength) { + return nil, fmt.Errorf("invalid buffer") + } + decoded, err := decodeEntry(buffer[tgt.TargetRootOffset : tgt.TargetRootOffset+tgt.TargetRootLength]) + if err != nil { + return nil, fmt.Errorf("decoding name: %w", err) + } + decoded, err = getFinalPath(decoded) + if err != nil { + return nil, fmt.Errorf("fetching final path: %w", err) + } + + targets[i] = decoded + } + return targets, nil +} + +func getFinalPath(pth string) (string, error) { + // BfGetMappings returns VOLUME_NAME_NT paths like \Device\HarddiskVolume2\ProgramData. + // These can be accessed by prepending \\.\GLOBALROOT to the path. We use this to get the + // DOS paths for these files. + if strings.HasPrefix(pth, `\Device`) { + pth = `\\.\GLOBALROOT` + pth + } + + han, err := openPath(pth) + if err != nil { + return "", fmt.Errorf("fetching file handle: %w", err) + } + defer func() { + _ = windows.CloseHandle(han) + }() + + buf := make([]uint16, 100) + var flags uint32 = 0x0 + for { + n, err := windows.GetFinalPathNameByHandle(han, &buf[0], uint32(len(buf)), flags) + if err != nil { + // if we mounted a volume that does not also have a drive letter assigned, attempting to + // fetch the VOLUME_NAME_DOS will fail with os.ErrNotExist. Attempt to get the VOLUME_NAME_GUID. + if errors.Is(err, os.ErrNotExist) && flags != 0x1 { + flags = 0x1 + continue + } + return "", fmt.Errorf("getting final path name: %w", err) + } + if n < uint32(len(buf)) { + break + } + buf = make([]uint16, n) + } + finalPath := syscall.UTF16ToString(buf) + // We got VOLUME_NAME_DOS, we need to strip away some leading slashes. + // Leave unchanged if we ended up requesting VOLUME_NAME_GUID + if len(finalPath) > 4 && finalPath[:4] == `\\?\` && flags == 0x0 { + finalPath = finalPath[4:] + if len(finalPath) > 3 && finalPath[:3] == `UNC` { + // return path like \\server\share\... + finalPath = `\` + finalPath[3:] + } + } + + return finalPath, nil +} + +func getBindMappingFromBuffer(buffer []byte, entry mappingEntry) (BindMapping, error) { + if len(buffer) < int(entry.VirtRootOffset)+int(entry.VirtRootLength) { + return BindMapping{}, fmt.Errorf("invalid buffer") + } + + src, err := decodeEntry(buffer[entry.VirtRootOffset : entry.VirtRootOffset+entry.VirtRootLength]) + if err != nil { + return BindMapping{}, fmt.Errorf("decoding entry: %w", err) + } + targets, err := getTargetsFromBuffer(buffer, int(entry.TargetEntriesOffset), int(entry.NumberOfTargets)) + if err != nil { + return BindMapping{}, fmt.Errorf("fetching targets: %w", err) + } + + src, err = getFinalPath(src) + if err != nil { + return BindMapping{}, fmt.Errorf("fetching final path: %w", err) + } + + return BindMapping{ + Flags: entry.Flags, + Targets: targets, + MountPoint: src, + }, nil +} + +func openPath(path string) (windows.Handle, error) { + u16, err := windows.UTF16PtrFromString(path) + if err != nil { + return 0, err + } + h, err := windows.CreateFile( + u16, + 0, + windows.FILE_SHARE_READ|windows.FILE_SHARE_WRITE|windows.FILE_SHARE_DELETE, + nil, + windows.OPEN_EXISTING, + windows.FILE_FLAG_BACKUP_SEMANTICS, // Needed to open a directory handle. + 0) + if err != nil { + return 0, &os.PathError{ + Op: "CreateFile", + Path: path, + Err: err, + } + } + return h, nil +} diff --git a/vendor/github.com/Microsoft/go-winio/pkg/bindfilter/zsyscall_windows.go b/vendor/github.com/Microsoft/go-winio/pkg/bindfilter/zsyscall_windows.go new file mode 100644 index 000000000..45c45c96e --- /dev/null +++ b/vendor/github.com/Microsoft/go-winio/pkg/bindfilter/zsyscall_windows.go @@ -0,0 +1,116 @@ +//go:build windows + +// Code generated by 'go generate' using "github.com/Microsoft/go-winio/tools/mkwinsyscall"; DO NOT EDIT. + +package bindfilter + +import ( + "syscall" + "unsafe" + + "golang.org/x/sys/windows" +) + +var _ unsafe.Pointer + +// Do the interface allocations only once for common +// Errno values. +const ( + errnoERROR_IO_PENDING = 997 +) + +var ( + errERROR_IO_PENDING error = syscall.Errno(errnoERROR_IO_PENDING) + errERROR_EINVAL error = syscall.EINVAL +) + +// errnoErr returns common boxed Errno values, to prevent +// allocations at runtime. +func errnoErr(e syscall.Errno) error { + switch e { + case 0: + return errERROR_EINVAL + case errnoERROR_IO_PENDING: + return errERROR_IO_PENDING + } + // TODO: add more here, after collecting data on the common + // error values see on Windows. (perhaps when running + // all.bat?) + return e +} + +var ( + modbindfltapi = windows.NewLazySystemDLL("bindfltapi.dll") + + procBfGetMappings = modbindfltapi.NewProc("BfGetMappings") + procBfRemoveMapping = modbindfltapi.NewProc("BfRemoveMapping") + procBfSetupFilter = modbindfltapi.NewProc("BfSetupFilter") +) + +func bfGetMappings(flags uint32, jobHandle windows.Handle, virtRootPath *uint16, sid *windows.SID, bufferSize *uint32, outBuffer *byte) (hr error) { + hr = procBfGetMappings.Find() + if hr != nil { + return + } + r0, _, _ := syscall.Syscall6(procBfGetMappings.Addr(), 6, uintptr(flags), uintptr(jobHandle), uintptr(unsafe.Pointer(virtRootPath)), uintptr(unsafe.Pointer(sid)), uintptr(unsafe.Pointer(bufferSize)), uintptr(unsafe.Pointer(outBuffer))) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + +func bfRemoveMapping(jobHandle windows.Handle, virtRootPath string) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(virtRootPath) + if hr != nil { + return + } + return _bfRemoveMapping(jobHandle, _p0) +} + +func _bfRemoveMapping(jobHandle windows.Handle, virtRootPath *uint16) (hr error) { + hr = procBfRemoveMapping.Find() + if hr != nil { + return + } + r0, _, _ := syscall.Syscall(procBfRemoveMapping.Addr(), 2, uintptr(jobHandle), uintptr(unsafe.Pointer(virtRootPath)), 0) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + +func bfSetupFilter(jobHandle windows.Handle, flags uint32, virtRootPath string, virtTargetPath string, virtExceptions **uint16, virtExceptionPathCount uint32) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(virtRootPath) + if hr != nil { + return + } + var _p1 *uint16 + _p1, hr = syscall.UTF16PtrFromString(virtTargetPath) + if hr != nil { + return + } + return _bfSetupFilter(jobHandle, flags, _p0, _p1, virtExceptions, virtExceptionPathCount) +} + +func _bfSetupFilter(jobHandle windows.Handle, flags uint32, virtRootPath *uint16, virtTargetPath *uint16, virtExceptions **uint16, virtExceptionPathCount uint32) (hr error) { + hr = procBfSetupFilter.Find() + if hr != nil { + return + } + r0, _, _ := syscall.Syscall6(procBfSetupFilter.Addr(), 6, uintptr(jobHandle), uintptr(flags), uintptr(unsafe.Pointer(virtRootPath)), uintptr(unsafe.Pointer(virtTargetPath)), uintptr(unsafe.Pointer(virtExceptions)), uintptr(virtExceptionPathCount)) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} diff --git a/vendor/github.com/containerd/continuity/.golangci.yml b/vendor/github.com/containerd/continuity/.golangci.yml index 2924bc4cf..afebdb4e6 100644 --- a/vendor/github.com/containerd/continuity/.golangci.yml +++ b/vendor/github.com/containerd/continuity/.golangci.yml @@ -1,7 +1,5 @@ linters: enable: - - structcheck - - varcheck - staticcheck - unconvert - gofmt diff --git a/vendor/github.com/containerd/continuity/context.go b/vendor/github.com/containerd/continuity/context.go index f92299c22..f22ea8678 100644 --- a/vendor/github.com/containerd/continuity/context.go +++ b/vendor/github.com/containerd/continuity/context.go @@ -18,6 +18,7 @@ package continuity import ( "bytes" + "errors" "fmt" "io" "os" @@ -151,7 +152,7 @@ func (c *context) Resource(p string, fi os.FileInfo) (Resource, error) { } base.xattrs, err = c.resolveXAttrs(fp, fi, base) - if err != nil && err != ErrNotSupported { + if err != nil && !errors.Is(err, ErrNotSupported) { return nil, err } @@ -410,7 +411,7 @@ func (c *context) Apply(resource Resource) error { return fmt.Errorf("resource %v escapes root", resource) } - var chmod = true + chmod := true fi, err := c.driver.Lstat(fp) if err != nil { if !os.IsNotExist(err) { diff --git a/vendor/github.com/containerd/continuity/driver/utils.go b/vendor/github.com/containerd/continuity/driver/utils.go index d122a3f73..90bfcc3f6 100644 --- a/vendor/github.com/containerd/continuity/driver/utils.go +++ b/vendor/github.com/containerd/continuity/driver/utils.go @@ -56,7 +56,7 @@ func WriteFile(r Driver, filename string, data []byte, perm os.FileMode) error { return nil } -// ReadDir works the same as ioutil.ReadDir with the Driver abstraction +// ReadDir works the same as os.ReadDir with the Driver abstraction func ReadDir(r Driver, dirname string) ([]os.FileInfo, error) { f, err := r.Open(dirname) if err != nil { diff --git a/vendor/github.com/containerd/continuity/fs/copy.go b/vendor/github.com/containerd/continuity/fs/copy.go index 6982a761b..a5c210810 100644 --- a/vendor/github.com/containerd/continuity/fs/copy.go +++ b/vendor/github.com/containerd/continuity/fs/copy.go @@ -18,7 +18,6 @@ package fs import ( "fmt" - "io/ioutil" "os" "path/filepath" "sync" @@ -111,7 +110,7 @@ func copyDirectory(dst, src string, inodes map[uint64]string, o *copyDirOpts) er } } - fis, err := ioutil.ReadDir(src) + entries, err := os.ReadDir(src) if err != nil { return fmt.Errorf("failed to read %s: %w", src, err) } @@ -124,18 +123,23 @@ func copyDirectory(dst, src string, inodes map[uint64]string, o *copyDirOpts) er return fmt.Errorf("failed to copy xattrs: %w", err) } - for _, fi := range fis { - source := filepath.Join(src, fi.Name()) - target := filepath.Join(dst, fi.Name()) + for _, entry := range entries { + source := filepath.Join(src, entry.Name()) + target := filepath.Join(dst, entry.Name()) + + fileInfo, err := entry.Info() + if err != nil { + return fmt.Errorf("failed to get file info for %s: %w", entry.Name(), err) + } switch { - case fi.IsDir(): + case entry.IsDir(): if err := copyDirectory(target, source, inodes, o); err != nil { return err } continue - case (fi.Mode() & os.ModeType) == 0: - link, err := getLinkSource(target, fi, inodes) + case (fileInfo.Mode() & os.ModeType) == 0: + link, err := getLinkSource(target, fileInfo, inodes) if err != nil { return fmt.Errorf("failed to get hardlink: %w", err) } @@ -146,7 +150,7 @@ func copyDirectory(dst, src string, inodes map[uint64]string, o *copyDirOpts) er } else if err := CopyFile(target, source); err != nil { return fmt.Errorf("failed to copy files: %w", err) } - case (fi.Mode() & os.ModeSymlink) == os.ModeSymlink: + case (fileInfo.Mode() & os.ModeSymlink) == os.ModeSymlink: link, err := os.Readlink(source) if err != nil { return fmt.Errorf("failed to read link: %s: %w", source, err) @@ -154,18 +158,18 @@ func copyDirectory(dst, src string, inodes map[uint64]string, o *copyDirOpts) er if err := os.Symlink(link, target); err != nil { return fmt.Errorf("failed to create symlink: %s: %w", target, err) } - case (fi.Mode() & os.ModeDevice) == os.ModeDevice, - (fi.Mode() & os.ModeNamedPipe) == os.ModeNamedPipe, - (fi.Mode() & os.ModeSocket) == os.ModeSocket: - if err := copyIrregular(target, fi); err != nil { + case (fileInfo.Mode() & os.ModeDevice) == os.ModeDevice, + (fileInfo.Mode() & os.ModeNamedPipe) == os.ModeNamedPipe, + (fileInfo.Mode() & os.ModeSocket) == os.ModeSocket: + if err := copyIrregular(target, fileInfo); err != nil { return fmt.Errorf("failed to create irregular file: %w", err) } default: - logrus.Warnf("unsupported mode: %s: %s", source, fi.Mode()) + logrus.Warnf("unsupported mode: %s: %s", source, fileInfo.Mode()) continue } - if err := copyFileInfo(fi, source, target); err != nil { + if err := copyFileInfo(fileInfo, source, target); err != nil { return fmt.Errorf("failed to copy file info: %w", err) } diff --git a/vendor/github.com/containerd/continuity/fs/copy_windows.go b/vendor/github.com/containerd/continuity/fs/copy_windows.go index 4dad9441d..6ec13b989 100644 --- a/vendor/github.com/containerd/continuity/fs/copy_windows.go +++ b/vendor/github.com/containerd/continuity/fs/copy_windows.go @@ -49,7 +49,6 @@ func copyFileInfo(fi os.FileInfo, src, name string) error { secInfo, err := windows.GetNamedSecurityInfo( src, windows.SE_FILE_OBJECT, windows.OWNER_SECURITY_INFORMATION|windows.DACL_SECURITY_INFORMATION) - if err != nil { return err } @@ -68,7 +67,6 @@ func copyFileInfo(fi os.FileInfo, src, name string) error { name, windows.SE_FILE_OBJECT, windows.OWNER_SECURITY_INFORMATION|windows.DACL_SECURITY_INFORMATION, sid, nil, dacl, nil); err != nil { - return err } return nil diff --git a/vendor/github.com/containerd/continuity/fs/diff.go b/vendor/github.com/containerd/continuity/fs/diff.go index 3cd4eee6f..d2c3c568e 100644 --- a/vendor/github.com/containerd/continuity/fs/diff.go +++ b/vendor/github.com/containerd/continuity/fs/diff.go @@ -80,12 +80,13 @@ type ChangeFunc func(ChangeKind, string, os.FileInfo, error) error // // The change callback is called by the order of path names and // should be appliable in that order. -// Due to this apply ordering, the following is true -// - Removed directory trees only create a single change for the root -// directory removed. Remaining changes are implied. -// - A directory which is modified to become a file will not have -// delete entries for sub-path items, their removal is implied -// by the removal of the parent directory. +// +// Due to this apply ordering, the following is true +// - Removed directory trees only create a single change for the root +// directory removed. Remaining changes are implied. +// - A directory which is modified to become a file will not have +// delete entries for sub-path items, their removal is implied +// by the removal of the parent directory. // // Opaque directories will not be treated specially and each file // removed from the base directory will show up as a removal. diff --git a/vendor/github.com/containerd/continuity/fs/dtype_linux.go b/vendor/github.com/containerd/continuity/fs/dtype_linux.go index a8eab1db8..9f55e7980 100644 --- a/vendor/github.com/containerd/continuity/fs/dtype_linux.go +++ b/vendor/github.com/containerd/continuity/fs/dtype_linux.go @@ -21,14 +21,13 @@ package fs import ( "fmt" - "io/ioutil" "os" "syscall" "unsafe" ) func locateDummyIfEmpty(path string) (string, error) { - children, err := ioutil.ReadDir(path) + children, err := os.ReadDir(path) if err != nil { return "", err } diff --git a/vendor/github.com/containerd/continuity/fs/du_unix.go b/vendor/github.com/containerd/continuity/fs/du_unix.go index bf33c42d7..51a08a1d7 100644 --- a/vendor/github.com/containerd/continuity/fs/du_unix.go +++ b/vendor/github.com/containerd/continuity/fs/du_unix.go @@ -28,10 +28,11 @@ import ( // blocksUnitSize is the unit used by `st_blocks` in `stat` in bytes. // See https://man7.org/linux/man-pages/man2/stat.2.html -// st_blocks -// This field indicates the number of blocks allocated to the -// file, in 512-byte units. (This may be smaller than -// st_size/512 when the file has holes.) +// +// st_blocks +// This field indicates the number of blocks allocated to the +// file, in 512-byte units. (This may be smaller than +// st_size/512 when the file has holes.) const blocksUnitSize = 512 type inode struct { @@ -48,7 +49,6 @@ func newInode(stat *syscall.Stat_t) inode { } func diskUsage(ctx context.Context, roots ...string) (Usage, error) { - var ( size int64 inodes = map[inode]struct{}{} // expensive! diff --git a/vendor/github.com/containerd/continuity/fs/du_windows.go b/vendor/github.com/containerd/continuity/fs/du_windows.go index 08fb28333..ea721f826 100644 --- a/vendor/github.com/containerd/continuity/fs/du_windows.go +++ b/vendor/github.com/containerd/continuity/fs/du_windows.go @@ -26,9 +26,7 @@ import ( ) func diskUsage(ctx context.Context, roots ...string) (Usage, error) { - var ( - size int64 - ) + var size int64 // TODO(stevvooe): Support inodes (or equivalent) for windows. @@ -57,9 +55,7 @@ func diskUsage(ctx context.Context, roots ...string) (Usage, error) { } func diffUsage(ctx context.Context, a, b string) (Usage, error) { - var ( - size int64 - ) + var size int64 if err := Changes(ctx, a, b, func(kind ChangeKind, _ string, fi os.FileInfo, err error) error { if err != nil { diff --git a/vendor/github.com/containerd/continuity/fs/fstest/compare_windows.go b/vendor/github.com/containerd/continuity/fs/fstest/compare_windows.go index a35781999..0cf3dbe95 100644 --- a/vendor/github.com/containerd/continuity/fs/fstest/compare_windows.go +++ b/vendor/github.com/containerd/continuity/fs/fstest/compare_windows.go @@ -16,9 +16,22 @@ package fstest -// TODO: Any more metadata files generated by Windows layers? -// TODO: Also skip Recycle Bin contents in Windows layers which is used to store deleted files in some cases var metadataFiles = map[string]bool{ - "\\System Volume Information": true, - "\\WcSandboxState": true, + "\\System Volume Information": true, + "\\WcSandboxState": true, + "\\WcSandboxState\\Hives": true, + "\\WcSandboxState\\Hives\\DefaultUser_Delta": true, + "\\WcSandboxState\\Hives\\Sam_Delta": true, + "\\WcSandboxState\\Hives\\Security_Delta": true, + "\\WcSandboxState\\Hives\\Software_Delta": true, + "\\WcSandboxState\\Hives\\System_Delta": true, + "\\WcSandboxState\\initialized": true, + "\\Windows": true, + "\\Windows\\System32": true, + "\\Windows\\System32\\config": true, + "\\Windows\\System32\\config\\DEFAULT": true, + "\\Windows\\System32\\config\\SAM": true, + "\\Windows\\System32\\config\\SECURITY": true, + "\\Windows\\System32\\config\\SOFTWARE": true, + "\\Windows\\System32\\config\\SYSTEM": true, } diff --git a/vendor/github.com/containerd/continuity/fs/fstest/continuity_util.go b/vendor/github.com/containerd/continuity/fs/fstest/continuity_util.go index 3b687a64a..45dd57413 100644 --- a/vendor/github.com/containerd/continuity/fs/fstest/continuity_util.go +++ b/vendor/github.com/containerd/continuity/fs/fstest/continuity_util.go @@ -129,7 +129,6 @@ func compareResource(r1, r2 continuity.Resource) bool { // TODO(dmcgowan): Check if is XAttrer return compareResourceTypes(r1, r2) - } func compareResourceTypes(r1, r2 continuity.Resource) bool { diff --git a/vendor/github.com/containerd/continuity/fs/fstest/file_windows.go b/vendor/github.com/containerd/continuity/fs/fstest/file_windows.go index 45fd9f611..6fd9ee8d9 100644 --- a/vendor/github.com/containerd/continuity/fs/fstest/file_windows.go +++ b/vendor/github.com/containerd/continuity/fs/fstest/file_windows.go @@ -32,13 +32,13 @@ func Lchtimes(name string, atime, mtime time.Time) Applier { // that the filter will mount. It is used for testing the snapshotter func Base() Applier { return Apply( - CreateDir("Windows", 0755), - CreateDir("Windows/System32", 0755), - CreateDir("Windows/System32/Config", 0755), - CreateFile("Windows/System32/Config/SYSTEM", []byte("foo\n"), 0777), - CreateFile("Windows/System32/Config/SOFTWARE", []byte("foo\n"), 0777), - CreateFile("Windows/System32/Config/SAM", []byte("foo\n"), 0777), - CreateFile("Windows/System32/Config/SECURITY", []byte("foo\n"), 0777), - CreateFile("Windows/System32/Config/DEFAULT", []byte("foo\n"), 0777), + CreateDir("Windows", 0o755), + CreateDir("Windows/System32", 0o755), + CreateDir("Windows/System32/Config", 0o755), + CreateFile("Windows/System32/Config/SYSTEM", []byte("foo\n"), 0o777), + CreateFile("Windows/System32/Config/SOFTWARE", []byte("foo\n"), 0o777), + CreateFile("Windows/System32/Config/SAM", []byte("foo\n"), 0o777), + CreateFile("Windows/System32/Config/SECURITY", []byte("foo\n"), 0o777), + CreateFile("Windows/System32/Config/DEFAULT", []byte("foo\n"), 0o777), ) } diff --git a/vendor/github.com/containerd/continuity/fs/fstest/testsuite.go b/vendor/github.com/containerd/continuity/fs/fstest/testsuite.go index 420126e6c..cda8a0fb0 100644 --- a/vendor/github.com/containerd/continuity/fs/fstest/testsuite.go +++ b/vendor/github.com/containerd/continuity/fs/fstest/testsuite.go @@ -81,14 +81,14 @@ var ( // baseApplier creates a basic filesystem layout // with multiple types of files for basic tests. baseApplier = Apply( - CreateDir("/etc/", 0755), - CreateFile("/etc/hosts", []byte("127.0.0.1 localhost"), 0644), + CreateDir("/etc/", 0o755), + CreateFile("/etc/hosts", []byte("127.0.0.1 localhost"), 0o644), Link("/etc/hosts", "/etc/hosts.allow"), - CreateDir("/usr/local/lib", 0755), - CreateFile("/usr/local/lib/libnothing.so", []byte{0x00, 0x00}, 0755), + CreateDir("/usr/local/lib", 0o755), + CreateFile("/usr/local/lib/libnothing.so", []byte{0x00, 0x00}, 0o755), Symlink("libnothing.so", "/usr/local/lib/libnothing.so.2"), - CreateDir("/home", 0755), - CreateDir("/home/derek", 0700), + CreateDir("/home", 0o755), + CreateDir("/home/derek", 0o700), // TODO: CreateSocket: how should Sockets be handled in continuity? ) @@ -96,10 +96,10 @@ var ( basicTest = []Applier{ baseApplier, Apply( - CreateFile("/etc/hosts", []byte("127.0.0.1 localhost.localdomain"), 0644), - CreateFile("/etc/fstab", []byte("/dev/sda1\t/\text4\tdefaults 1 1\n"), 0600), - CreateFile("/etc/badfile", []byte(""), 0666), - CreateFile("/home/derek/.zshrc", []byte("#ZSH is just better\n"), 0640), + CreateFile("/etc/hosts", []byte("127.0.0.1 localhost.localdomain"), 0o644), + CreateFile("/etc/fstab", []byte("/dev/sda1\t/\text4\tdefaults 1 1\n"), 0o600), + CreateFile("/etc/badfile", []byte(""), 0o666), + CreateFile("/home/derek/.zshrc", []byte("#ZSH is just better\n"), 0o640), ), Apply( Remove("/etc/badfile"), @@ -111,8 +111,8 @@ var ( ), Apply( RemoveAll("/home"), - CreateDir("/home/derek", 0700), - CreateFile("/home/derek/.bashrc", []byte("#not going away\n"), 0640), + CreateDir("/home/derek", 0o700), + CreateFile("/home/derek/.bashrc", []byte("#not going away\n"), 0o640), Link("/etc/hosts", "/etc/hosts.allow"), ), } @@ -121,58 +121,58 @@ var ( // deletions are properly picked up and applied deletionTest = []Applier{ Apply( - CreateDir("/test/somedir", 0755), - CreateDir("/lib", 0700), - CreateFile("/lib/hidden", []byte{}, 0644), + CreateDir("/test/somedir", 0o755), + CreateDir("/lib", 0o700), + CreateFile("/lib/hidden", []byte{}, 0o644), ), Apply( - CreateFile("/test/a", []byte{}, 0644), - CreateFile("/test/b", []byte{}, 0644), - CreateDir("/test/otherdir", 0755), - CreateFile("/test/otherdir/.empty", []byte{}, 0644), + CreateFile("/test/a", []byte{}, 0o644), + CreateFile("/test/b", []byte{}, 0o644), + CreateDir("/test/otherdir", 0o755), + CreateFile("/test/otherdir/.empty", []byte{}, 0o644), RemoveAll("/lib"), - CreateDir("/lib", 0700), - CreateFile("/lib/not-hidden", []byte{}, 0644), + CreateDir("/lib", 0o700), + CreateFile("/lib/not-hidden", []byte{}, 0o644), ), Apply( Remove("/test/a"), Remove("/test/b"), RemoveAll("/test/otherdir"), - CreateFile("/lib/newfile", []byte{}, 0644), + CreateFile("/lib/newfile", []byte{}, 0o644), ), } // updateTest covers file updates for content and permission updateTest = []Applier{ Apply( - CreateDir("/d1", 0755), - CreateDir("/d2", 0700), - CreateFile("/d1/f1", []byte("something..."), 0644), - CreateFile("/d1/f2", []byte("else..."), 0644), - CreateFile("/d1/f3", []byte("entirely..."), 0644), + CreateDir("/d1", 0o755), + CreateDir("/d2", 0o700), + CreateFile("/d1/f1", []byte("something..."), 0o644), + CreateFile("/d1/f2", []byte("else..."), 0o644), + CreateFile("/d1/f3", []byte("entirely..."), 0o644), ), Apply( - CreateFile("/d1/f1", []byte("file content of a different length"), 0664), + CreateFile("/d1/f1", []byte("file content of a different length"), 0o664), Remove("/d1/f3"), - CreateFile("/d1/f3", []byte("updated content"), 0664), - Chmod("/d1/f2", 0766), - Chmod("/d2", 0777), + CreateFile("/d1/f3", []byte("updated content"), 0o664), + Chmod("/d1/f2", 0o766), + Chmod("/d2", 0o777), ), } // directoryPermissionsTest covers directory permissions on update directoryPermissionsTest = []Applier{ Apply( - CreateDir("/d1", 0700), - CreateDir("/d2", 0751), - CreateDir("/d3", 0777), + CreateDir("/d1", 0o700), + CreateDir("/d2", 0o751), + CreateDir("/d3", 0o777), ), Apply( - CreateFile("/d1/f", []byte("irrelevant"), 0644), - CreateDir("/d1/d", 0700), - CreateFile("/d1/d/f", []byte("irrelevant"), 0644), - CreateFile("/d2/f", []byte("irrelevant"), 0644), - CreateFile("/d3/f", []byte("irrelevant"), 0644), + CreateFile("/d1/f", []byte("irrelevant"), 0o644), + CreateDir("/d1/d", 0o700), + CreateFile("/d1/d/f", []byte("irrelevant"), 0o644), + CreateFile("/d2/f", []byte("irrelevant"), 0o644), + CreateFile("/d3/f", []byte("irrelevant"), 0o644), ), } @@ -180,28 +180,28 @@ var ( // files parentDirectoryPermissionsTest = []Applier{ Apply( - CreateDir("/d1", 0700), - CreateDir("/d1/a", 0700), - CreateDir("/d1/a/b", 0700), - CreateDir("/d1/a/b/c", 0700), - CreateFile("/d1/a/b/f", []byte("content1"), 0644), - CreateDir("/d2", 0751), - CreateDir("/d2/a/b", 0751), - CreateDir("/d2/a/b/c", 0751), - CreateFile("/d2/a/b/f", []byte("content1"), 0644), + CreateDir("/d1", 0o700), + CreateDir("/d1/a", 0o700), + CreateDir("/d1/a/b", 0o700), + CreateDir("/d1/a/b/c", 0o700), + CreateFile("/d1/a/b/f", []byte("content1"), 0o644), + CreateDir("/d2", 0o751), + CreateDir("/d2/a/b", 0o751), + CreateDir("/d2/a/b/c", 0o751), + CreateFile("/d2/a/b/f", []byte("content1"), 0o644), ), Apply( - CreateFile("/d1/a/b/f", []byte("content1"), 0644), - Chmod("/d1/a/b/c", 0700), - CreateFile("/d2/a/b/f", []byte("content2"), 0644), - Chmod("/d2/a/b/c", 0751), + CreateFile("/d1/a/b/f", []byte("content1"), 0o644), + Chmod("/d1/a/b/c", 0o700), + CreateFile("/d2/a/b/f", []byte("content2"), 0o644), + Chmod("/d2/a/b/c", 0o751), ), } hardlinkUnmodified = []Applier{ baseApplier, Apply( - CreateFile("/etc/hosts", []byte("127.0.0.1 localhost.localdomain"), 0644), + CreateFile("/etc/hosts", []byte("127.0.0.1 localhost.localdomain"), 0o644), ), Apply( Link("/etc/hosts", "/etc/hosts.deny"), @@ -213,7 +213,7 @@ var ( hardlinkBeforeUnmodified = []Applier{ baseApplier, Apply( - CreateFile("/etc/hosts", []byte("127.0.0.1 localhost.localdomain"), 0644), + CreateFile("/etc/hosts", []byte("127.0.0.1 localhost.localdomain"), 0o644), ), Apply( Link("/etc/hosts", "/etc/before-hosts"), @@ -225,11 +225,11 @@ var ( hardlinkBeforeModified = []Applier{ baseApplier, Apply( - CreateFile("/etc/hosts", []byte("127.0.0.1 localhost.localdomain"), 0644), + CreateFile("/etc/hosts", []byte("127.0.0.1 localhost.localdomain"), 0o644), ), Apply( Remove("/etc/hosts"), - CreateFile("/etc/hosts", []byte("127.0.0.1 localhost"), 0644), + CreateFile("/etc/hosts", []byte("127.0.0.1 localhost"), 0o644), Link("/etc/hosts", "/etc/before-hosts"), ), } diff --git a/vendor/github.com/containerd/continuity/fs/path.go b/vendor/github.com/containerd/continuity/fs/path.go index 97313e2b8..ec6e6a2fa 100644 --- a/vendor/github.com/containerd/continuity/fs/path.go +++ b/vendor/github.com/containerd/continuity/fs/path.go @@ -25,9 +25,7 @@ import ( "path/filepath" ) -var ( - errTooManyLinks = errors.New("too many links") -) +var errTooManyLinks = errors.New("too many links") type currentPath struct { path string diff --git a/vendor/github.com/containerd/continuity/hardlinks.go b/vendor/github.com/containerd/continuity/hardlinks.go index 1df07f54a..9cff20b86 100644 --- a/vendor/github.com/containerd/continuity/hardlinks.go +++ b/vendor/github.com/containerd/continuity/hardlinks.go @@ -21,9 +21,7 @@ import ( "os" ) -var ( - errNotAHardLink = fmt.Errorf("invalid hardlink") -) +var errNotAHardLink = fmt.Errorf("invalid hardlink") type hardlinkManager struct { hardlinks map[hardlinkKey][]Resource diff --git a/vendor/github.com/containerd/continuity/ioutils.go b/vendor/github.com/containerd/continuity/ioutils.go index 392c407fa..8b0e72013 100644 --- a/vendor/github.com/containerd/continuity/ioutils.go +++ b/vendor/github.com/containerd/continuity/ioutils.go @@ -37,26 +37,32 @@ func atomicWriteFile(filename string, r io.Reader, dataSize int64, perm os.FileM if err != nil { return err } + needClose := true + defer func() { + if needClose { + f.Close() + } + }() + err = os.Chmod(f.Name(), perm) if err != nil { - f.Close() return err } n, err := io.Copy(f, r) if err == nil && n < dataSize { - f.Close() return io.ErrShortWrite } if err != nil { - f.Close() return err } - if err := f.Sync(); err != nil { - f.Close() + if err = f.Sync(); err != nil { return err } + + needClose = false if err := f.Close(); err != nil { return err } + return os.Rename(f.Name(), filename) } diff --git a/vendor/golang.org/x/tools/go/packages/packages.go b/vendor/golang.org/x/tools/go/packages/packages.go index 1d5f0e45b..0f1505b80 100644 --- a/vendor/golang.org/x/tools/go/packages/packages.go +++ b/vendor/golang.org/x/tools/go/packages/packages.go @@ -878,12 +878,19 @@ func (ld *loader) loadPackage(lpkg *loaderPackage) { // never has to create a types.Package for an indirect dependency, // which would then require that such created packages be explicitly // inserted back into the Import graph as a final step after export data loading. + // (Hence this return is after the Types assignment.) // The Diamond test exercises this case. if !lpkg.needtypes && !lpkg.needsrc { return } if !lpkg.needsrc { - ld.loadFromExportData(lpkg) + if err := ld.loadFromExportData(lpkg); err != nil { + lpkg.Errors = append(lpkg.Errors, Error{ + Pos: "-", + Msg: err.Error(), + Kind: UnknownError, // e.g. can't find/open/parse export data + }) + } return // not a source package, don't get syntax trees } @@ -970,7 +977,8 @@ func (ld *loader) loadPackage(lpkg *loaderPackage) { // The config requested loading sources and types, but sources are missing. // Add an error to the package and fall back to loading from export data. appendError(Error{"-", fmt.Sprintf("sources missing for package %s", lpkg.ID), ParseError}) - ld.loadFromExportData(lpkg) + _ = ld.loadFromExportData(lpkg) // ignore any secondary errors + return // can't get syntax trees for this package } @@ -1194,9 +1202,10 @@ func sameFile(x, y string) bool { return false } -// loadFromExportData returns type information for the specified +// loadFromExportData ensures that type information is present for the specified // package, loading it from an export data file on the first request. -func (ld *loader) loadFromExportData(lpkg *loaderPackage) (*types.Package, error) { +// On success it sets lpkg.Types to a new Package. +func (ld *loader) loadFromExportData(lpkg *loaderPackage) error { if lpkg.PkgPath == "" { log.Fatalf("internal error: Package %s has no PkgPath", lpkg) } @@ -1207,8 +1216,8 @@ func (ld *loader) loadFromExportData(lpkg *loaderPackage) (*types.Package, error // must be sequential. (Finer-grained locking would require // changes to the gcexportdata API.) // - // The exportMu lock guards the Package.Pkg field and the - // types.Package it points to, for each Package in the graph. + // The exportMu lock guards the lpkg.Types field and the + // types.Package it points to, for each loaderPackage in the graph. // // Not all accesses to Package.Pkg need to be protected by exportMu: // graph ordering ensures that direct dependencies of source @@ -1217,18 +1226,18 @@ func (ld *loader) loadFromExportData(lpkg *loaderPackage) (*types.Package, error defer ld.exportMu.Unlock() if tpkg := lpkg.Types; tpkg != nil && tpkg.Complete() { - return tpkg, nil // cache hit + return nil // cache hit } lpkg.IllTyped = true // fail safe if lpkg.ExportFile == "" { // Errors while building export data will have been printed to stderr. - return nil, fmt.Errorf("no export data file") + return fmt.Errorf("no export data file") } f, err := os.Open(lpkg.ExportFile) if err != nil { - return nil, err + return err } defer f.Close() @@ -1240,7 +1249,7 @@ func (ld *loader) loadFromExportData(lpkg *loaderPackage) (*types.Package, error // queries.) r, err := gcexportdata.NewReader(f) if err != nil { - return nil, fmt.Errorf("reading %s: %v", lpkg.ExportFile, err) + return fmt.Errorf("reading %s: %v", lpkg.ExportFile, err) } // Build the view. @@ -1284,7 +1293,7 @@ func (ld *loader) loadFromExportData(lpkg *loaderPackage) (*types.Package, error // (May modify incomplete packages in view but not create new ones.) tpkg, err := gcexportdata.Read(r, ld.Fset, view, lpkg.PkgPath) if err != nil { - return nil, fmt.Errorf("reading %s: %v", lpkg.ExportFile, err) + return fmt.Errorf("reading %s: %v", lpkg.ExportFile, err) } if _, ok := view["go.shape"]; ok { // Account for the pseudopackage "go.shape" that gets @@ -1297,8 +1306,7 @@ func (ld *loader) loadFromExportData(lpkg *loaderPackage) (*types.Package, error lpkg.Types = tpkg lpkg.IllTyped = false - - return tpkg, nil + return nil } // impliedLoadMode returns loadMode with its dependencies. diff --git a/vendor/golang.org/x/tools/internal/gcimporter/iexport.go b/vendor/golang.org/x/tools/internal/gcimporter/iexport.go index 7d90f00f3..ba53cdcdd 100644 --- a/vendor/golang.org/x/tools/internal/gcimporter/iexport.go +++ b/vendor/golang.org/x/tools/internal/gcimporter/iexport.go @@ -22,6 +22,7 @@ import ( "strconv" "strings" + "golang.org/x/tools/internal/tokeninternal" "golang.org/x/tools/internal/typeparams" ) @@ -138,6 +139,17 @@ func iexportCommon(out io.Writer, fset *token.FileSet, bundle, shallow bool, ver p.doDecl(p.declTodo.popHead()) } + // Produce index of offset of each file record in files. + var files intWriter + var fileOffset []uint64 // fileOffset[i] is offset in files of file encoded as i + if p.shallow { + fileOffset = make([]uint64, len(p.fileInfos)) + for i, info := range p.fileInfos { + fileOffset[i] = uint64(files.Len()) + p.encodeFile(&files, info.file, info.needed) + } + } + // Append indices to data0 section. dataLen := uint64(p.data0.Len()) w := p.newWriter() @@ -163,16 +175,75 @@ func iexportCommon(out io.Writer, fset *token.FileSet, bundle, shallow bool, ver } hdr.uint64(uint64(p.version)) hdr.uint64(uint64(p.strings.Len())) + if p.shallow { + hdr.uint64(uint64(files.Len())) + hdr.uint64(uint64(len(fileOffset))) + for _, offset := range fileOffset { + hdr.uint64(offset) + } + } hdr.uint64(dataLen) // Flush output. io.Copy(out, &hdr) io.Copy(out, &p.strings) + if p.shallow { + io.Copy(out, &files) + } io.Copy(out, &p.data0) return nil } +// encodeFile writes to w a representation of the file sufficient to +// faithfully restore position information about all needed offsets. +// Mutates the needed array. +func (p *iexporter) encodeFile(w *intWriter, file *token.File, needed []uint64) { + _ = needed[0] // precondition: needed is non-empty + + w.uint64(p.stringOff(file.Name())) + + size := uint64(file.Size()) + w.uint64(size) + + // Sort the set of needed offsets. Duplicates are harmless. + sort.Slice(needed, func(i, j int) bool { return needed[i] < needed[j] }) + + lines := tokeninternal.GetLines(file) // byte offset of each line start + w.uint64(uint64(len(lines))) + + // Rather than record the entire array of line start offsets, + // we save only a sparse list of (index, offset) pairs for + // the start of each line that contains a needed position. + var sparse [][2]int // (index, offset) pairs +outer: + for i, lineStart := range lines { + lineEnd := size + if i < len(lines)-1 { + lineEnd = uint64(lines[i+1]) + } + // Does this line contains a needed offset? + if needed[0] < lineEnd { + sparse = append(sparse, [2]int{i, lineStart}) + for needed[0] < lineEnd { + needed = needed[1:] + if len(needed) == 0 { + break outer + } + } + } + } + + // Delta-encode the columns. + w.uint64(uint64(len(sparse))) + var prev [2]int + for _, pair := range sparse { + w.uint64(uint64(pair[0] - prev[0])) + w.uint64(uint64(pair[1] - prev[1])) + prev = pair + } +} + // writeIndex writes out an object index. mainIndex indicates whether // we're writing out the main index, which is also read by // non-compiler tools and includes a complete package description @@ -255,6 +326,12 @@ type iexporter struct { strings intWriter stringIndex map[string]uint64 + // In shallow mode, object positions are encoded as (file, offset). + // Each file is recorded as a line-number table. + // Only the lines of needed positions are saved faithfully. + fileInfo map[*token.File]uint64 // value is index in fileInfos + fileInfos []*filePositions + data0 intWriter declIndex map[types.Object]uint64 tparamNames map[types.Object]string // typeparam->exported name @@ -263,6 +340,11 @@ type iexporter struct { indent int // for tracing support } +type filePositions struct { + file *token.File + needed []uint64 // unordered list of needed file offsets +} + func (p *iexporter) trace(format string, args ...interface{}) { if !trace { // Call sites should also be guarded, but having this check here allows @@ -286,6 +368,25 @@ func (p *iexporter) stringOff(s string) uint64 { return off } +// fileIndexAndOffset returns the index of the token.File and the byte offset of pos within it. +func (p *iexporter) fileIndexAndOffset(file *token.File, pos token.Pos) (uint64, uint64) { + index, ok := p.fileInfo[file] + if !ok { + index = uint64(len(p.fileInfo)) + p.fileInfos = append(p.fileInfos, &filePositions{file: file}) + if p.fileInfo == nil { + p.fileInfo = make(map[*token.File]uint64) + } + p.fileInfo[file] = index + } + // Record each needed offset. + info := p.fileInfos[index] + offset := uint64(file.Offset(pos)) + info.needed = append(info.needed, offset) + + return index, offset +} + // pushDecl adds n to the declaration work queue, if not already present. func (p *iexporter) pushDecl(obj types.Object) { // Package unsafe is known to the compiler and predeclared. @@ -346,7 +447,13 @@ func (p *iexporter) doDecl(obj types.Object) { case *types.Func: sig, _ := obj.Type().(*types.Signature) if sig.Recv() != nil { - panic(internalErrorf("unexpected method: %v", sig)) + // We shouldn't see methods in the package scope, + // but the type checker may repair "func () F() {}" + // to "func (Invalid) F()" and then treat it like "func F()", + // so allow that. See golang/go#57729. + if sig.Recv().Type() != types.Typ[types.Invalid] { + panic(internalErrorf("unexpected method: %v", sig)) + } } // Function. @@ -458,13 +565,30 @@ func (w *exportWriter) tag(tag byte) { } func (w *exportWriter) pos(pos token.Pos) { - if w.p.version >= iexportVersionPosCol { + if w.p.shallow { + w.posV2(pos) + } else if w.p.version >= iexportVersionPosCol { w.posV1(pos) } else { w.posV0(pos) } } +// posV2 encoding (used only in shallow mode) records positions as +// (file, offset), where file is the index in the token.File table +// (which records the file name and newline offsets) and offset is a +// byte offset. It effectively ignores //line directives. +func (w *exportWriter) posV2(pos token.Pos) { + if pos == token.NoPos { + w.uint64(0) + return + } + file := w.p.fset.File(pos) // fset must be non-nil + index, offset := w.p.fileIndexAndOffset(file, pos) + w.uint64(1 + index) + w.uint64(offset) +} + func (w *exportWriter) posV1(pos token.Pos) { if w.p.fset == nil { w.int64(0) diff --git a/vendor/golang.org/x/tools/internal/gcimporter/iimport.go b/vendor/golang.org/x/tools/internal/gcimporter/iimport.go index a1c469653..448f903e8 100644 --- a/vendor/golang.org/x/tools/internal/gcimporter/iimport.go +++ b/vendor/golang.org/x/tools/internal/gcimporter/iimport.go @@ -137,12 +137,23 @@ func iimportCommon(fset *token.FileSet, imports map[string]*types.Package, data } sLen := int64(r.uint64()) + var fLen int64 + var fileOffset []uint64 + if insert != nil { + // Shallow mode uses a different position encoding. + fLen = int64(r.uint64()) + fileOffset = make([]uint64, r.uint64()) + for i := range fileOffset { + fileOffset[i] = r.uint64() + } + } dLen := int64(r.uint64()) whence, _ := r.Seek(0, io.SeekCurrent) stringData := data[whence : whence+sLen] - declData := data[whence+sLen : whence+sLen+dLen] - r.Seek(sLen+dLen, io.SeekCurrent) + fileData := data[whence+sLen : whence+sLen+fLen] + declData := data[whence+sLen+fLen : whence+sLen+fLen+dLen] + r.Seek(sLen+fLen+dLen, io.SeekCurrent) p := iimporter{ version: int(version), @@ -151,6 +162,9 @@ func iimportCommon(fset *token.FileSet, imports map[string]*types.Package, data stringData: stringData, stringCache: make(map[uint64]string), + fileOffset: fileOffset, + fileData: fileData, + fileCache: make([]*token.File, len(fileOffset)), pkgCache: make(map[uint64]*types.Package), declData: declData, @@ -280,6 +294,9 @@ type iimporter struct { stringData []byte stringCache map[uint64]string + fileOffset []uint64 // fileOffset[i] is offset in fileData for info about file encoded as i + fileData []byte + fileCache []*token.File // memoized decoding of file encoded as i pkgCache map[uint64]*types.Package declData []byte @@ -352,6 +369,55 @@ func (p *iimporter) stringAt(off uint64) string { return s } +func (p *iimporter) fileAt(index uint64) *token.File { + file := p.fileCache[index] + if file == nil { + off := p.fileOffset[index] + file = p.decodeFile(intReader{bytes.NewReader(p.fileData[off:]), p.ipath}) + p.fileCache[index] = file + } + return file +} + +func (p *iimporter) decodeFile(rd intReader) *token.File { + filename := p.stringAt(rd.uint64()) + size := int(rd.uint64()) + file := p.fake.fset.AddFile(filename, -1, size) + + // SetLines requires a nondecreasing sequence. + // Because it is common for clients to derive the interval + // [start, start+len(name)] from a start position, and we + // want to ensure that the end offset is on the same line, + // we fill in the gaps of the sparse encoding with values + // that strictly increase by the largest possible amount. + // This allows us to avoid having to record the actual end + // offset of each needed line. + + lines := make([]int, int(rd.uint64())) + var index, offset int + for i, n := 0, int(rd.uint64()); i < n; i++ { + index += int(rd.uint64()) + offset += int(rd.uint64()) + lines[index] = offset + + // Ensure monotonicity between points. + for j := index - 1; j > 0 && lines[j] == 0; j-- { + lines[j] = lines[j+1] - 1 + } + } + + // Ensure monotonicity after last point. + for j := len(lines) - 1; j > 0 && lines[j] == 0; j-- { + size-- + lines[j] = size + } + + if !file.SetLines(lines) { + errorf("SetLines failed: %d", lines) // can't happen + } + return file +} + func (p *iimporter) pkgAt(off uint64) *types.Package { if pkg, ok := p.pkgCache[off]; ok { return pkg @@ -645,6 +711,9 @@ func (r *importReader) qualifiedIdent() (*types.Package, string) { } func (r *importReader) pos() token.Pos { + if r.p.insert != nil { // shallow mode + return r.posv2() + } if r.p.version >= iexportVersionPosCol { r.posv1() } else { @@ -681,6 +750,15 @@ func (r *importReader) posv1() { } } +func (r *importReader) posv2() token.Pos { + file := r.uint64() + if file == 0 { + return token.NoPos + } + tf := r.p.fileAt(file - 1) + return tf.Pos(int(r.uint64())) +} + func (r *importReader) typ() types.Type { return r.p.typAt(r.uint64(), nil) } diff --git a/vendor/golang.org/x/tools/internal/pkgbits/decoder.go b/vendor/golang.org/x/tools/internal/pkgbits/decoder.go index 3205c9a16..b92e8e6eb 100644 --- a/vendor/golang.org/x/tools/internal/pkgbits/decoder.go +++ b/vendor/golang.org/x/tools/internal/pkgbits/decoder.go @@ -373,7 +373,7 @@ func (r *Decoder) Int64() int64 { return r.rawVarint() } -// Int64 decodes and returns a uint64 value from the element bitstream. +// Uint64 decodes and returns a uint64 value from the element bitstream. func (r *Decoder) Uint64() uint64 { r.Sync(SyncUint64) return r.rawUvarint() diff --git a/vendor/golang.org/x/tools/internal/pkgbits/encoder.go b/vendor/golang.org/x/tools/internal/pkgbits/encoder.go index e98e41171..6482617a4 100644 --- a/vendor/golang.org/x/tools/internal/pkgbits/encoder.go +++ b/vendor/golang.org/x/tools/internal/pkgbits/encoder.go @@ -293,7 +293,7 @@ func (w *Encoder) Len(x int) { assert(x >= 0); w.Uint64(uint64(x)) } // Int encodes and writes an int value into the element bitstream. func (w *Encoder) Int(x int) { w.Int64(int64(x)) } -// Len encodes and writes a uint value into the element bitstream. +// Uint encodes and writes a uint value into the element bitstream. func (w *Encoder) Uint(x uint) { w.Uint64(uint64(x)) } // Reloc encodes and writes a relocation for the given (section, diff --git a/vendor/golang.org/x/tools/internal/tokeninternal/tokeninternal.go b/vendor/golang.org/x/tools/internal/tokeninternal/tokeninternal.go new file mode 100644 index 000000000..a3fb2d4f2 --- /dev/null +++ b/vendor/golang.org/x/tools/internal/tokeninternal/tokeninternal.go @@ -0,0 +1,59 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// package tokeninternal provides access to some internal features of the token +// package. +package tokeninternal + +import ( + "go/token" + "sync" + "unsafe" +) + +// GetLines returns the table of line-start offsets from a token.File. +func GetLines(file *token.File) []int { + // token.File has a Lines method on Go 1.21 and later. + if file, ok := (interface{})(file).(interface{ Lines() []int }); ok { + return file.Lines() + } + + // This declaration must match that of token.File. + // This creates a risk of dependency skew. + // For now we check that the size of the two + // declarations is the same, on the (fragile) assumption + // that future changes would add fields. + type tokenFile119 struct { + _ string + _ int + _ int + mu sync.Mutex // we're not complete monsters + lines []int + _ []struct{} + } + type tokenFile118 struct { + _ *token.FileSet // deleted in go1.19 + tokenFile119 + } + + type uP = unsafe.Pointer + switch unsafe.Sizeof(*file) { + case unsafe.Sizeof(tokenFile118{}): + var ptr *tokenFile118 + *(*uP)(uP(&ptr)) = uP(file) + ptr.mu.Lock() + defer ptr.mu.Unlock() + return ptr.lines + + case unsafe.Sizeof(tokenFile119{}): + var ptr *tokenFile119 + *(*uP)(uP(&ptr)) = uP(file) + ptr.mu.Lock() + defer ptr.mu.Unlock() + return ptr.lines + + default: + panic("unexpected token.File size") + } +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 413c80ed5..9fd9fef03 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -4,11 +4,12 @@ github.com/AdaLogics/go-fuzz-headers # github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20221215162035-5330a85ea652 ## explicit; go 1.18 github.com/AdamKorcz/go-118-fuzz-build/testing -# github.com/Microsoft/go-winio v0.6.0 +# github.com/Microsoft/go-winio v0.6.1-0.20230228163719-dd5de6900b62 ## explicit; go 1.17 github.com/Microsoft/go-winio github.com/Microsoft/go-winio/backuptar github.com/Microsoft/go-winio/internal/socket +github.com/Microsoft/go-winio/pkg/bindfilter github.com/Microsoft/go-winio/pkg/etw github.com/Microsoft/go-winio/pkg/etwlogrus github.com/Microsoft/go-winio/pkg/fs @@ -97,8 +98,8 @@ github.com/containerd/cgroups/v3/cgroup2/stats # github.com/containerd/console v1.0.3 ## explicit; go 1.13 github.com/containerd/console -# github.com/containerd/continuity v0.3.0 -## explicit; go 1.17 +# github.com/containerd/continuity v0.3.1-0.20230307035957-72c70feb3081 +## explicit; go 1.19 github.com/containerd/continuity github.com/containerd/continuity/devices github.com/containerd/continuity/driver @@ -496,7 +497,7 @@ golang.org/x/crypto/openpgp/errors golang.org/x/crypto/openpgp/packet golang.org/x/crypto/openpgp/s2k golang.org/x/crypto/pbkdf2 -# golang.org/x/mod v0.7.0 +# golang.org/x/mod v0.8.0 ## explicit; go 1.17 golang.org/x/mod/semver # golang.org/x/net v0.7.0 @@ -544,7 +545,7 @@ golang.org/x/text/unicode/norm # golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 ## explicit golang.org/x/time/rate -# golang.org/x/tools v0.5.0 +# golang.org/x/tools v0.6.0 ## explicit; go 1.18 golang.org/x/tools/cmd/stringer golang.org/x/tools/go/gcexportdata @@ -558,6 +559,7 @@ golang.org/x/tools/internal/gcimporter golang.org/x/tools/internal/gocommand golang.org/x/tools/internal/packagesinternal golang.org/x/tools/internal/pkgbits +golang.org/x/tools/internal/tokeninternal golang.org/x/tools/internal/typeparams golang.org/x/tools/internal/typesinternal # google.golang.org/appengine v1.6.7