Add pprof to runc-shim

Signed-off-by: Henry Wang <henwang@amazon.com>
2024-05-03 01:05:13 +00:00
parent 8b5c218e5a
commit 243b803a19
10 changed files with 406 additions and 105 deletions
--- a/cmd/ctr/commands/pprof/pprof.go
+++ b/cmd/ctr/commands/pprof/pprof.go
@@ -65,16 +65,7 @@ var pprofGoroutinesCommand = &cli.Command{
 		},
 	},
 	Action: func(cliContext *cli.Context) error {
-		client := getPProfClient(cliContext)
-
-		debug := cliContext.Uint("debug")
-		output, err := httpGetRequest(client, fmt.Sprintf("/debug/pprof/goroutine?debug=%d", debug))
-		if err != nil {
-			return err
-		}
-		defer output.Close()
-		_, err = io.Copy(os.Stdout, output)
-		return err
+		return GoroutineProfile(cliContext, getPProfClient)
 	},
 }

@@ -89,16 +80,7 @@ var pprofHeapCommand = &cli.Command{
 		},
 	},
 	Action: func(cliContext *cli.Context) error {
-		client := getPProfClient(cliContext)
-
-		debug := cliContext.Uint("debug")
-		output, err := httpGetRequest(client, fmt.Sprintf("/debug/pprof/heap?debug=%d", debug))
-		if err != nil {
-			return err
-		}
-		defer output.Close()
-		_, err = io.Copy(os.Stdout, output)
-		return err
+		return HeapProfile(cliContext, getPProfClient)
 	},
 }

@@ -119,17 +101,7 @@ var pprofProfileCommand = &cli.Command{
 		},
 	},
 	Action: func(cliContext *cli.Context) error {
-		client := getPProfClient(cliContext)
-
-		seconds := cliContext.Duration("seconds").Seconds()
-		debug := cliContext.Uint("debug")
-		output, err := httpGetRequest(client, fmt.Sprintf("/debug/pprof/profile?seconds=%v&debug=%d", seconds, debug))
-		if err != nil {
-			return err
-		}
-		defer output.Close()
-		_, err = io.Copy(os.Stdout, output)
-		return err
+		return CPUProfile(cliContext, getPProfClient)
 	},
 }

@@ -150,18 +122,7 @@ var pprofTraceCommand = &cli.Command{
 		},
 	},
 	Action: func(cliContext *cli.Context) error {
-		client := getPProfClient(cliContext)
-
-		seconds := cliContext.Duration("seconds").Seconds()
-		debug := cliContext.Uint("debug")
-		uri := fmt.Sprintf("/debug/pprof/trace?seconds=%v&debug=%d", seconds, debug)
-		output, err := httpGetRequest(client, uri)
-		if err != nil {
-			return err
-		}
-		defer output.Close()
-		_, err = io.Copy(os.Stdout, output)
-		return err
+		return TraceProfile(cliContext, getPProfClient)
 	},
 }

@@ -176,16 +137,7 @@ var pprofBlockCommand = &cli.Command{
 		},
 	},
 	Action: func(cliContext *cli.Context) error {
-		client := getPProfClient(cliContext)
-
-		debug := cliContext.Uint("debug")
-		output, err := httpGetRequest(client, fmt.Sprintf("/debug/pprof/block?debug=%d", debug))
-		if err != nil {
-			return err
-		}
-		defer output.Close()
-		_, err = io.Copy(os.Stdout, output)
-		return err
+		return BlockProfile(cliContext, getPProfClient)
 	},
 }

@@ -200,27 +152,120 @@ var pprofThreadcreateCommand = &cli.Command{
 		},
 	},
 	Action: func(cliContext *cli.Context) error {
-		client := getPProfClient(cliContext)
-
-		debug := cliContext.Uint("debug")
-		output, err := httpGetRequest(client, fmt.Sprintf("/debug/pprof/threadcreate?debug=%d", debug))
-		if err != nil {
-			return err
-		}
-		defer output.Close()
-		_, err = io.Copy(os.Stdout, output)
-		return err
+		return ThreadcreateProfile(cliContext, getPProfClient)
 	},
 }

-func getPProfClient(cliContext *cli.Context) *http.Client {
+// Client is a func that returns a http client for a pprof server
+type Client func(cliContext *cli.Context) (*http.Client, error)
+
+// GoroutineProfile dumps goroutine stack dump
+func GoroutineProfile(cliContext *cli.Context, clientFunc Client) error {
+	client, err := clientFunc(cliContext)
+	if err != nil {
+		return err
+	}
+	debug := cliContext.Uint("debug")
+	output, err := httpGetRequest(client, fmt.Sprintf("/debug/pprof/goroutine?debug=%d", debug))
+	if err != nil {
+		return err
+	}
+	defer output.Close()
+	_, err = io.Copy(os.Stdout, output)
+	return err
+}
+
+// HeapProfile dumps the heap profile
+func HeapProfile(cliContext *cli.Context, clientFunc Client) error {
+	client, err := clientFunc(cliContext)
+	if err != nil {
+		return err
+	}
+	debug := cliContext.Uint("debug")
+	output, err := httpGetRequest(client, fmt.Sprintf("/debug/pprof/heap?debug=%d", debug))
+	if err != nil {
+		return err
+	}
+	defer output.Close()
+	_, err = io.Copy(os.Stdout, output)
+	return err
+}
+
+// CPUProfile dumps CPU profile
+func CPUProfile(cliContext *cli.Context, clientFunc Client) error {
+	client, err := clientFunc(cliContext)
+	if err != nil {
+		return err
+	}
+	seconds := cliContext.Duration("seconds").Seconds()
+	debug := cliContext.Uint("debug")
+	output, err := httpGetRequest(client, fmt.Sprintf("/debug/pprof/profile?seconds=%v&debug=%d", seconds, debug))
+	if err != nil {
+		return err
+	}
+	defer output.Close()
+	_, err = io.Copy(os.Stdout, output)
+	return err
+}
+
+// TraceProfile collects execution trace
+func TraceProfile(cliContext *cli.Context, clientFunc Client) error {
+	client, err := clientFunc(cliContext)
+	if err != nil {
+		return err
+	}
+	seconds := cliContext.Duration("seconds").Seconds()
+	debug := cliContext.Uint("debug")
+	uri := fmt.Sprintf("/debug/pprof/trace?seconds=%v&debug=%d", seconds, debug)
+	output, err := httpGetRequest(client, uri)
+	if err != nil {
+		return err
+	}
+	defer output.Close()
+	_, err = io.Copy(os.Stdout, output)
+	return err
+}
+
+// BlockProfile collects goroutine blocking profile
+func BlockProfile(cliContext *cli.Context, clientFunc Client) error {
+	client, err := clientFunc(cliContext)
+	if err != nil {
+		return err
+	}
+	debug := cliContext.Uint("debug")
+	output, err := httpGetRequest(client, fmt.Sprintf("/debug/pprof/block?debug=%d", debug))
+	if err != nil {
+		return err
+	}
+	defer output.Close()
+	_, err = io.Copy(os.Stdout, output)
+	return err
+}
+
+// ThreadcreateProfile collects goroutine thread creating profile
+func ThreadcreateProfile(cliContext *cli.Context, clientFunc Client) error {
+	client, err := clientFunc(cliContext)
+	if err != nil {
+		return err
+	}
+	debug := cliContext.Uint("debug")
+	output, err := httpGetRequest(client, fmt.Sprintf("/debug/pprof/threadcreate?debug=%d", debug))
+	if err != nil {
+		return err
+	}
+	defer output.Close()
+	_, err = io.Copy(os.Stdout, output)
+	return err
+}
+
+func getPProfClient(cliContext *cli.Context) (*http.Client, error) {
 	dialer := getPProfDialer(cliContext.String("debug-socket"))

 	tr := &http.Transport{
 		Dial: dialer.pprofDial,
 	}
 	client := &http.Client{Transport: tr}
-	return client
+	return client, nil
 }

 func httpGetRequest(client *http.Client, request string) (io.ReadCloser, error) {
--- a/cmd/ctr/commands/shim/pprof.go
+++ b/cmd/ctr/commands/shim/pprof.go
@@ -0,0 +1,165 @@
+//go:build !windows
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package shim
+
+import (
+	"context"
+	"errors"
+	"net"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/containerd/containerd/v2/cmd/ctr/commands/pprof"
+	"github.com/containerd/containerd/v2/pkg/namespaces"
+	"github.com/containerd/containerd/v2/pkg/shim"
+	"github.com/urfave/cli/v2"
+)
+
+var pprofCommand = &cli.Command{
+	Name:  "pprof",
+	Usage: "Provide golang pprof outputs for containerd-shim",
+	Subcommands: []*cli.Command{
+		pprofBlockCommand,
+		pprofGoroutinesCommand,
+		pprofHeapCommand,
+		pprofProfileCommand,
+		pprofThreadcreateCommand,
+		pprofTraceCommand,
+	},
+}
+
+var pprofGoroutinesCommand = &cli.Command{
+	Name:  "goroutines",
+	Usage: "Print goroutine stack dump",
+	Flags: []cli.Flag{
+		&cli.UintFlag{
+			Name:  "debug",
+			Usage: "Output format, value = 0: binary, value > 0: plaintext",
+			Value: 2,
+		},
+	},
+	Action: func(cliContext *cli.Context) error {
+		return pprof.GoroutineProfile(cliContext, getPProfClient)
+	},
+}
+
+var pprofHeapCommand = &cli.Command{
+	Name:  "heap",
+	Usage: "Dump heap profile",
+	Flags: []cli.Flag{
+		&cli.UintFlag{
+			Name:  "debug",
+			Usage: "Output format, value = 0: binary, value > 0: plaintext",
+			Value: 0,
+		},
+	},
+	Action: func(cliContext *cli.Context) error {
+		return pprof.HeapProfile(cliContext, getPProfClient)
+	},
+}
+
+var pprofProfileCommand = &cli.Command{
+	Name:  "profile",
+	Usage: "CPU profile",
+	Flags: []cli.Flag{
+		&cli.DurationFlag{
+			Name:    "seconds",
+			Aliases: []string{"s"},
+			Usage:   "Duration for collection (seconds)",
+			Value:   30 * time.Second,
+		},
+		&cli.UintFlag{
+			Name:  "debug",
+			Usage: "Output format, value = 0: binary, value > 0: plaintext",
+			Value: 0,
+		},
+	},
+	Action: func(cliContext *cli.Context) error {
+		return pprof.CPUProfile(cliContext, getPProfClient)
+	},
+}
+
+var pprofTraceCommand = &cli.Command{
+	Name:  "trace",
+	Usage: "Collect execution trace",
+	Flags: []cli.Flag{
+		&cli.DurationFlag{
+			Name:    "seconds",
+			Aliases: []string{"s"},
+			Usage:   "Trace time (seconds)",
+			Value:   5 * time.Second,
+		},
+		&cli.UintFlag{
+			Name:  "debug",
+			Usage: "Output format, value = 0: binary, value > 0: plaintext",
+			Value: 0,
+		},
+	},
+	Action: func(cliContext *cli.Context) error {
+		return pprof.TraceProfile(cliContext, getPProfClient)
+	},
+}
+
+var pprofBlockCommand = &cli.Command{
+	Name:  "block",
+	Usage: "Goroutine blocking profile",
+	Flags: []cli.Flag{
+		&cli.UintFlag{
+			Name:  "debug",
+			Usage: "Output format, value = 0: binary, value > 0: plaintext",
+			Value: 0,
+		},
+	},
+	Action: func(cliContext *cli.Context) error {
+		return pprof.BlockProfile(cliContext, getPProfClient)
+	},
+}
+
+var pprofThreadcreateCommand = &cli.Command{
+	Name:  "threadcreate",
+	Usage: "Goroutine thread creating profile",
+	Flags: []cli.Flag{
+		&cli.UintFlag{
+			Name:  "debug",
+			Usage: "Output format, value = 0: binary, value > 0: plaintext",
+			Value: 0,
+		},
+	},
+	Action: func(cliContext *cli.Context) error {
+		return pprof.ThreadcreateProfile(cliContext, getPProfClient)
+	},
+}
+
+func getPProfClient(cliContext *cli.Context) (*http.Client, error) {
+	id := cliContext.String("id")
+	if id == "" {
+		return nil, errors.New("container id must be provided")
+	}
+	tr := &http.Transport{
+		Dial: func(_, _ string) (net.Conn, error) {
+			ns := cliContext.String("namespace")
+			ctx := namespaces.WithNamespace(context.Background(), ns)
+			s, _ := shim.SocketAddress(ctx, cliContext.String("address"), id, true)
+			s = strings.TrimPrefix(s, "unix://")
+			return net.Dial("unix", s)
+		},
+	}
+	return &http.Client{Transport: tr}, nil
+}
--- a/cmd/ctr/commands/shim/shim.go
+++ b/cmd/ctr/commands/shim/shim.go
@@ -75,6 +75,7 @@ var Command = &cli.Command{
 		execCommand,
 		startCommand,
 		stateCommand,
+		pprofCommand,
 	},
 }

@@ -244,7 +245,7 @@ func getTaskService(cliContext *cli.Context) (task.TTRPCTaskService, error) {
 	s1 := filepath.Join(string(filepath.Separator), "containerd-shim", ns, id, "shim.sock")
 	// this should not error, ctr always get a default ns
 	ctx := namespaces.WithNamespace(context.Background(), ns)
-	s2, _ := shim.SocketAddress(ctx, cliContext.String("address"), id)
+	s2, _ := shim.SocketAddress(ctx, cliContext.String("address"), id, false)
 	s2 = strings.TrimPrefix(s2, "unix://")

 	for _, socket := range []string{s2, "\x00" + s1} {