Use Intel ISA-L's igzip if available

Intel ISA-L is Intel's open source (BSD) library that outperforms both
gzip and pigz. This commit checks and uses igzip if available.

Signed-off-by: Kazuyoshi Kato <kaz@fly.io>

archive/compression/benchmark_test.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"os/exec"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -49,19 +50,30 @@ func BenchmarkDecompression(b *testing.B) {
 		zstd := testCompress(b, data, Zstd)
 
 		b.Run(fmt.Sprintf("size=%dMiB", sizeInMiB), func(b *testing.B) {
-			b.Run("gzip", func(b *testing.B) {
+			original := gzipPath
-				testDecompress(b, gz)
+			defer func() {
-			})
+				gzipPath = original
+			}()
+
 			b.Run("zstd", func(b *testing.B) {
 				testDecompress(b, zstd)
 			})
-			if unpigzPath != "" {
+
-				original := unpigzPath
+			gzipPath = ""
-				unpigzPath = ""
+			b.Run("gzipPureGo", func(b *testing.B) {
-				b.Run("gzipPureGo", func(b *testing.B) {
+				testDecompress(b, gz)
+			})
+			gzipPath, err = exec.LookPath("igzip")
+			if err == nil {
+				b.Run("igzip", func(b *testing.B) {
+					testDecompress(b, gz)
+				})
+			}
+			gzipPath, err = exec.LookPath("unpigz")
+			if err == nil {
+				b.Run("unpigz", func(b *testing.B) {
 					testDecompress(b, gz)
 				})
-				unpigzPath = original
 			}
 		})
 	}

archive/compression/compression.go

@@ -47,11 +47,14 @@ const (
 	Zstd
 )
 
-const disablePigzEnv = "CONTAINERD_DISABLE_PIGZ"
+const (
+	disablePigzEnv  = "CONTAINERD_DISABLE_PIGZ"
+	disableIgzipEnv = "CONTAINERD_DISABLE_IGZIP"
+)
 
 var (
-	initPigz   sync.Once
+	initGzip sync.Once
-	unpigzPath string
+	gzipPath string
 )
 
 var (
@@ -259,17 +262,20 @@ func (compression *Compression) Extension() string {
 }
 
 func gzipDecompress(ctx context.Context, buf io.Reader) (io.ReadCloser, error) {
-	initPigz.Do(func() {
+	initGzip.Do(func() {
-		if unpigzPath = detectPigz(); unpigzPath != "" {
+		if gzipPath = detectCommand("igzip", disableIgzipEnv); gzipPath != "" {
-			log.L.Debug("using pigz for decompression")
+			log.L.Debug("using igzip for decompression")
+			return
+		}
+		if gzipPath = detectCommand("unpigz", disablePigzEnv); gzipPath != "" {
+			log.L.Debug("using unpigz for decompression")
 		}
 	})
 
-	if unpigzPath == "" {
+	if gzipPath == "" {
 		return gzip.NewReader(buf)
 	}
-
+	return cmdStream(exec.CommandContext(ctx, gzipPath, "-d", "-c"), buf)
-	return cmdStream(exec.CommandContext(ctx, unpigzPath, "-d", "-c"), buf)
 }
 
 func cmdStream(cmd *exec.Cmd, in io.Reader) (io.ReadCloser, error) {
@@ -296,22 +302,22 @@ func cmdStream(cmd *exec.Cmd, in io.Reader) (io.ReadCloser, error) {
 	return reader, nil
 }
 
-func detectPigz() string {
+func detectCommand(path, disableEnvName string) string {
-	path, err := exec.LookPath("unpigz")
+	path, err := exec.LookPath(path)
 	if err != nil {
-		log.L.WithError(err).Debug("unpigz not found, falling back to go gzip")
+		log.L.WithError(err).Debugf("%s not found", path)
 		return ""
 	}
 
-	// Check if pigz disabled via CONTAINERD_DISABLE_PIGZ env variable
+	// Check if this command is disabled via the env variable
-	value := os.Getenv(disablePigzEnv)
+	value := os.Getenv(disableEnvName)
 	if value == "" {
 		return path
 	}
 
 	disable, err := strconv.ParseBool(value)
 	if err != nil {
-		log.L.WithError(err).Warnf("could not parse %s: %s", disablePigzEnv, value)
+		log.L.WithError(err).Warnf("could not parse %s: %s", disableEnvName, value)
 		return path
 	}
 

archive/compression/compression_test.go

@@ -102,9 +102,9 @@ func testCompressDecompress(t testing.TB, size int, compression Compression) Dec
 }
 
 func TestCompressDecompressGzip(t *testing.T) {
-	oldUnpigzPath := unpigzPath
+	oldUnpigzPath := gzipPath
-	unpigzPath = ""
+	gzipPath = ""
-	defer func() { unpigzPath = oldUnpigzPath }()
+	defer func() { gzipPath = oldUnpigzPath }()
 
 	decompressor := testCompressDecompress(t, 1024*1024, Gzip)
 	wrapper := decompressor.(*readCloserWrapper)
@@ -148,7 +148,7 @@ func TestDetectPigz(t *testing.T) {
 
 	t.Setenv("PATH", tempPath)
 
-	if pigzPath := detectPigz(); pigzPath == "" {
+	if pigzPath := detectCommand("unpigz", disablePigzEnv); pigzPath == "" {
 		t.Fatal("failed to detect pigz path")
 	} else if pigzPath != fullPath {
 		t.Fatalf("wrong pigz found: %s != %s", pigzPath, fullPath)
@@ -156,7 +156,7 @@ func TestDetectPigz(t *testing.T) {
 
 	t.Setenv(disablePigzEnv, "1")
 
-	if pigzPath := detectPigz(); pigzPath != "" {
+	if pigzPath := detectCommand("unpigz", disablePigzEnv); pigzPath != "" {
 		t.Fatalf("disable via %s doesn't work", disablePigzEnv)
 	}
 }