From 112497bc36dbf21b8d127d964d735b40cb6587d1 Mon Sep 17 00:00:00 2001 From: Kazuyoshi Kato Date: Wed, 15 Jun 2022 16:43:26 +0000 Subject: [PATCH 1/5] Make oss_fuzz_build.sh quiet Signed-off-by: Kazuyoshi Kato --- contrib/fuzz/oss_fuzz_build.sh | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/contrib/fuzz/oss_fuzz_build.sh b/contrib/fuzz/oss_fuzz_build.sh index 8010b132c..a65e52376 100755 --- a/contrib/fuzz/oss_fuzz_build.sh +++ b/contrib/fuzz/oss_fuzz_build.sh @@ -16,11 +16,10 @@ set -o nounset set -o pipefail set -o errexit -set -x apt-get update && apt-get install -y wget cd $SRC -wget https://go.dev/dl/go1.18.3.linux-amd64.tar.gz +wget --quiet https://go.dev/dl/go1.18.3.linux-amd64.tar.gz mkdir temp-go rm -rf /root/.go/* @@ -87,7 +86,7 @@ compile_go_fuzzer github.com/containerd/containerd/contrib/fuzz FuzzContentStore # The below fuzzers require more setup than the fuzzers above. # We need the binaries from "make". -wget -c https://github.com/protocolbuffers/protobuf/releases/download/v3.11.4/protoc-3.11.4-linux-x86_64.zip +wget --quiet https://github.com/protocolbuffers/protobuf/releases/download/v3.11.4/protoc-3.11.4-linux-x86_64.zip unzip protoc-3.11.4-linux-x86_64.zip -d /usr/local export CGO_ENABLED=1 @@ -102,10 +101,7 @@ make install # Build static containerd cd $SRC/containerd -make EXTRA_FLAGS="-buildmode pie" \ - EXTRA_LDFLAGS='-linkmode external -extldflags "-fno-PIC -static"' \ - BUILDTAGS="netgo osusergo static_build" - +make STATIC=1 mkdir $OUT/containerd-binaries || true cd $SRC/containerd/bin && cp * $OUT/containerd-binaries/ && cd - From 50f1a4e426337a918e28ba88749798a27655bf53 Mon Sep 17 00:00:00 2001 From: Kazuyoshi Kato Date: Wed, 15 Jun 2022 19:02:33 +0000 Subject: [PATCH 2/5] Don't log "ignored xattr ..." warnings It is too noisy and not really useful if the input is random. Signed-off-by: Kazuyoshi Kato --- contrib/fuzz/archive_fuzzer.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/contrib/fuzz/archive_fuzzer.go b/contrib/fuzz/archive_fuzzer.go index 1fd87a1c5..941467b7e 100644 --- a/contrib/fuzz/archive_fuzzer.go +++ b/contrib/fuzz/archive_fuzzer.go @@ -22,15 +22,25 @@ import ( "os" fuzz "github.com/AdaLogics/go-fuzz-headers" + "github.com/sirupsen/logrus" "github.com/containerd/containerd/archive" "github.com/containerd/containerd/content/local" imageArchive "github.com/containerd/containerd/images/archive" + "github.com/containerd/containerd/log" ) // FuzzApply implements a fuzzer that applies // a fuzzed tar archive on a directory func FuzzApply(data []byte) int { + ctx := context.Background() + + // Apply() is logging the below message, which is too noisy and not really useful + // if the input is random. + // + // level=warning msg="ignored xattr ... in archive" error="operation not supported" + log.G(ctx).Logger.SetLevel(logrus.PanicLevel) + f := fuzz.NewConsumer(data) iters, err := f.GetInt() if err != nil { @@ -48,7 +58,7 @@ func FuzzApply(data []byte) int { return 0 } r := bytes.NewReader(rBytes) - _, _ = archive.Apply(context.Background(), tmpDir, r) + _, _ = archive.Apply(ctx, tmpDir, r) } return 1 } From c9c5fee37e28836e0fd20d3c0a2bf4c978fdf1dd Mon Sep 17 00:00:00 2001 From: Kazuyoshi Kato Date: Wed, 15 Jun 2022 21:22:26 +0000 Subject: [PATCH 3/5] Do not hardcode fuzzing targets Signed-off-by: Kazuyoshi Kato --- contrib/fuzz/oss_fuzz_build.sh | 36 ++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/contrib/fuzz/oss_fuzz_build.sh b/contrib/fuzz/oss_fuzz_build.sh index a65e52376..f69620077 100755 --- a/contrib/fuzz/oss_fuzz_build.sh +++ b/contrib/fuzz/oss_fuzz_build.sh @@ -17,6 +17,25 @@ set -o nounset set -o pipefail set -o errexit +IFS=$'\n' + +compile_fuzzers() { + local regex=$1 + local compile_fuzzer=$2 + + for line in $(git grep "$regex" | grep -v vendor) + do + if [[ "$line" =~ (.*)/.*:.*(Fuzz[A-Za-z0-9]+) ]]; then + local pkg=${BASH_REMATCH[1]} + local func=${BASH_REMATCH[2]} + "$compile_fuzzer" "github.com/containerd/containerd/$pkg" "$func" "fuzz_$func" + else + echo "failed to parse: $line" + exit 1 + fi + done +} + apt-get update && apt-get install -y wget cd $SRC wget --quiet https://go.dev/dl/go1.18.3.linux-amd64.tar.gz @@ -67,22 +86,9 @@ mv $SRC/cmd-containerd-backup $SRC/containerd/cmd/containerd # Compile more fuzzers mv $SRC/containerd/filters/filter_test.go $SRC/containerd/filters/filter_test_fuzz.go go get github.com/AdamKorcz/go-118-fuzz-build/utils -compile_native_go_fuzzer github.com/containerd/containerd/filters FuzzFiltersParse fuzz_filters_parse -compile_native_go_fuzzer github.com/containerd/containerd/pkg/cap FuzzParseProcPIDStatus fuzz_parse_proc_pid_status -compile_native_go_fuzzer github.com/containerd/containerd/platforms FuzzPlatformsParse fuzz_platforms_parse - -compile_go_fuzzer github.com/containerd/containerd/remotes/docker FuzzFetcher fuzz_fetcher -compile_go_fuzzer github.com/containerd/containerd/remotes/docker FuzzParseDockerRef fuzz_parse_docker_ref -compile_go_fuzzer github.com/containerd/containerd/contrib/fuzz FuzzApply fuzz_apply -compile_go_fuzzer github.com/containerd/containerd/contrib/fuzz FuzzImportIndex fuzz_import_index -compile_go_fuzzer github.com/containerd/containerd/contrib/fuzz FuzzCSWalk fuzz_cs_walk -compile_go_fuzzer github.com/containerd/containerd/contrib/fuzz FuzzArchiveExport fuzz_archive_export -compile_go_fuzzer github.com/containerd/containerd/contrib/fuzz FuzzParseAuth fuzz_parse_auth -compile_go_fuzzer github.com/containerd/containerd/contrib/fuzz FuzzImageStore fuzz_image_store -compile_go_fuzzer github.com/containerd/containerd/contrib/fuzz FuzzLeaseManager fuzz_lease_manager -compile_go_fuzzer github.com/containerd/containerd/contrib/fuzz FuzzContainerStore fuzz_container_store -compile_go_fuzzer github.com/containerd/containerd/contrib/fuzz FuzzContentStore fuzz_content_store +compile_fuzzers '^func Fuzz.*testing\.F' compile_native_go_fuzzer +compile_fuzzers '^func Fuzz.*data' compile_go_fuzzer # The below fuzzers require more setup than the fuzzers above. # We need the binaries from "make". From f91616e45ee023adeff643cedb4d1a8e09c9c22c Mon Sep 17 00:00:00 2001 From: Kazuyoshi Kato Date: Thu, 16 Jun 2022 05:03:54 +0000 Subject: [PATCH 4/5] Copy FuzzConvertManifest from cncf/cncf-fuzzing This test is one of the noisiest and should be moved from cncf/cncf-fuzzing first. Signed-off-by: Kazuyoshi Kato --- integration/client/go.mod | 1 + integration/client/go.sum | 1 + remotes/docker/converter_fuzz.go | 56 ++++++++++++++++++++++++++++++++ 3 files changed, 58 insertions(+) create mode 100644 remotes/docker/converter_fuzz.go diff --git a/integration/client/go.mod b/integration/client/go.mod index 865ef4a47..d8b13b321 100644 --- a/integration/client/go.mod +++ b/integration/client/go.mod @@ -20,6 +20,7 @@ require ( ) require ( + github.com/AdaLogics/go-fuzz-headers v0.0.0-20210715213245-6c3934b029d8 // indirect github.com/Microsoft/go-winio v0.5.2 // indirect github.com/blang/semver v3.5.1+incompatible // indirect github.com/cilium/ebpf v0.7.0 // indirect diff --git a/integration/client/go.sum b/integration/client/go.sum index dea8d6a24..f2604b24c 100644 --- a/integration/client/go.sum +++ b/integration/client/go.sum @@ -38,6 +38,7 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20210715213245-6c3934b029d8 h1:V8krnnfGj4pV65YLUm3C0/8bl7V5Nry2Pwvy3ru/wLc= github.com/AdaLogics/go-fuzz-headers v0.0.0-20210715213245-6c3934b029d8/go.mod h1:CzsSbkDixRphAF5hS6wbMKq0eI6ccJRb7/A0M6JBnwg= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= diff --git a/remotes/docker/converter_fuzz.go b/remotes/docker/converter_fuzz.go new file mode 100644 index 000000000..f2a88b20a --- /dev/null +++ b/remotes/docker/converter_fuzz.go @@ -0,0 +1,56 @@ +//go:build gofuzz +// +build gofuzz + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package docker + +import ( + "context" + "os" + + fuzz "github.com/AdaLogics/go-fuzz-headers" + "github.com/containerd/containerd/content/local" + "github.com/containerd/containerd/log" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/sirupsen/logrus" +) + +func FuzzConvertManifest(data []byte) int { + ctx := context.Background() + + // Do not log the message below + // level=warning msg="do nothing for media type: ..." + log.G(ctx).Logger.SetLevel(logrus.PanicLevel) + + f := fuzz.NewConsumer(data) + desc := ocispec.Descriptor{} + err := f.GenerateStruct(&desc) + if err != nil { + return 0 + } + tmpdir, err := os.MkdirTemp("", "fuzzing-") + if err != nil { + return 0 + } + cs, err := local.NewStore(tmpdir) + if err != nil { + return 0 + } + _, _ = ConvertManifest(ctx, cs, desc) + return 1 +} From b27a229df631ac9c0ecc691a916d05b48692e1b3 Mon Sep 17 00:00:00 2001 From: Kazuyoshi Kato Date: Mon, 20 Jun 2022 17:40:33 +0000 Subject: [PATCH 5/5] Move contrib/fuzz/docker_fuzzer.go to remotes/docker Signed-off-by: Kazuyoshi Kato --- contrib/fuzz/oss_fuzz_build.sh | 1 - .../docker_fuzzer.go => remotes/docker/fetcher_fuzz.go | 10 +++------- 2 files changed, 3 insertions(+), 8 deletions(-) rename contrib/fuzz/docker_fuzzer.go => remotes/docker/fetcher_fuzz.go (91%) diff --git a/contrib/fuzz/oss_fuzz_build.sh b/contrib/fuzz/oss_fuzz_build.sh index f69620077..71e7bf1b2 100755 --- a/contrib/fuzz/oss_fuzz_build.sh +++ b/contrib/fuzz/oss_fuzz_build.sh @@ -54,7 +54,6 @@ cd "$(dirname "${BASH_SOURCE[0]}")" cd ../../ # Move all fuzzers that don't have the "fuzz" package out of this dir -mv contrib/fuzz/docker_fuzzer.go remotes/docker/ mv contrib/fuzz/container_fuzzer.go integration/client/ rm -r vendor diff --git a/contrib/fuzz/docker_fuzzer.go b/remotes/docker/fetcher_fuzz.go similarity index 91% rename from contrib/fuzz/docker_fuzzer.go rename to remotes/docker/fetcher_fuzz.go index 7f4213b78..e126ebbd8 100644 --- a/contrib/fuzz/docker_fuzzer.go +++ b/remotes/docker/fetcher_fuzz.go @@ -3,10 +3,13 @@ /* Copyright The containerd Authors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,13 +17,6 @@ limitations under the License. */ -/* - This fuzzer is run continuously by OSS-fuzz. - It is stored in contrib/fuzz for organization, - but in order to execute it, it must be moved to - remotes/docker first. -*/ - package docker import (