github/containerd
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Use Intel ISA-L's igzip if available

Browse Source 
Intel ISA-L is Intel's open source (BSD) library that outperforms both
gzip and pigz. This commit checks and uses igzip if available.

Signed-off-by: Kazuyoshi Kato <kaz@fly.io>
This commit is contained in:
Kazuyoshi Kato 2023-10-05 22:11:14 -07:00
parent 66aab638da
commit 34378ec9b4
3 changed files with 46 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
archive/compression/benchmark_test.go
View File

@ -20,6 +20,7 @@ import (
"fmt"
"io"
"net/http"
"os/exec"
"testing"
"github.com/stretchr/testify/require"
@ -49,19 +50,30 @@ func BenchmarkDecompression(b *testing.B) {
zstd := testCompress(b, data, Zstd)
b.Run(fmt.Sprintf("size=%dMiB", sizeInMiB), func(b *testing.B) {
b.Run("gzip", func(b *testing.B) {
testDecompress(b, gz)
})
original := gzipPath
defer func() {
gzipPath = original
}()
b.Run("zstd", func(b *testing.B) {
testDecompress(b, zstd)
})
if unpigzPath != "" {
original := unpigzPath
unpigzPath = ""
b.Run("gzipPureGo", func(b *testing.B) {
gzipPath = ""
b.Run("gzipPureGo", func(b *testing.B) {
testDecompress(b, gz)
})
gzipPath, err = exec.LookPath("igzip")
if err == nil {
b.Run("igzip", func(b *testing.B) {
testDecompress(b, gz)
})
}
gzipPath, err = exec.LookPath("unpigz")
if err == nil {
b.Run("unpigz", func(b *testing.B) {
testDecompress(b, gz)
})
unpigzPath = original
}
})
}

36
archive/compression/compression.go
View File

@ -47,11 +47,14 @@ const (
Zstd
)
const disablePigzEnv = "CONTAINERD_DISABLE_PIGZ"
const (
disablePigzEnv = "CONTAINERD_DISABLE_PIGZ"
disableIgzipEnv = "CONTAINERD_DISABLE_IGZIP"
)
var (
initPigz sync.Once
unpigzPath string
initGzip sync.Once
gzipPath string
)
var (
@ -259,17 +262,20 @@ func (compression *Compression) Extension() string {
}
func gzipDecompress(ctx context.Context, buf io.Reader) (io.ReadCloser, error) {
initPigz.Do(func() {
if unpigzPath = detectPigz(); unpigzPath != "" {
log.L.Debug("using pigz for decompression")
initGzip.Do(func() {
if gzipPath = detectCommand("igzip", disableIgzipEnv); gzipPath != "" {
log.L.Debug("using igzip for decompression")
return
}
if gzipPath = detectCommand("unpigz", disablePigzEnv); gzipPath != "" {
log.L.Debug("using unpigz for decompression")
}
})
if unpigzPath == "" {
if gzipPath == "" {
return gzip.NewReader(buf)
}
return cmdStream(exec.CommandContext(ctx, unpigzPath, "-d", "-c"), buf)
return cmdStream(exec.CommandContext(ctx, gzipPath, "-d", "-c"), buf)
}
func cmdStream(cmd *exec.Cmd, in io.Reader) (io.ReadCloser, error) {
@ -296,22 +302,22 @@ func cmdStream(cmd *exec.Cmd, in io.Reader) (io.ReadCloser, error) {
return reader, nil
}
func detectPigz() string {
path, err := exec.LookPath("unpigz")
func detectCommand(path, disableEnvName string) string {
path, err := exec.LookPath(path)
if err != nil {
log.L.WithError(err).Debug("unpigz not found, falling back to go gzip")
log.L.WithError(err).Debugf("%s not found", path)
return ""
}
// Check if pigz disabled via CONTAINERD_DISABLE_PIGZ env variable
value := os.Getenv(disablePigzEnv)
// Check if this command is disabled via the env variable
value := os.Getenv(disableEnvName)
if value == "" {
return path
}
disable, err := strconv.ParseBool(value)
if err != nil {
log.L.WithError(err).Warnf("could not parse %s: %s", disablePigzEnv, value)
log.L.WithError(err).Warnf("could not parse %s: %s", disableEnvName, value)
return path
}

10
archive/compression/compression_test.go
View File

@ -102,9 +102,9 @@ func testCompressDecompress(t testing.TB, size int, compression Compression) Dec
}
func TestCompressDecompressGzip(t *testing.T) {
oldUnpigzPath := unpigzPath
unpigzPath = ""
defer func() { unpigzPath = oldUnpigzPath }()
oldUnpigzPath := gzipPath
gzipPath = ""
defer func() { gzipPath = oldUnpigzPath }()
decompressor := testCompressDecompress(t, 1024*1024, Gzip)
wrapper := decompressor.(*readCloserWrapper)
@ -148,7 +148,7 @@ func TestDetectPigz(t *testing.T) {
t.Setenv("PATH", tempPath)
if pigzPath := detectPigz(); pigzPath == "" {
if pigzPath := detectCommand("unpigz", disablePigzEnv); pigzPath == "" {
t.Fatal("failed to detect pigz path")
} else if pigzPath != fullPath {
t.Fatalf("wrong pigz found: %s != %s", pigzPath, fullPath)
@ -156,7 +156,7 @@ func TestDetectPigz(t *testing.T) {
t.Setenv(disablePigzEnv, "1")
if pigzPath := detectPigz(); pigzPath != "" {
if pigzPath := detectCommand("unpigz", disablePigzEnv); pigzPath != "" {
t.Fatalf("disable via %s doesn't work", disablePigzEnv)
}
}