github/containerd
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

linux/shim: reduce memory overhead by using ttrpc

Browse Source 
By replacing grpc with ttrpc, we can reduce total memory runtime
requirements and binary size. With minimal code changes, the shim can
now be controlled by the much lightweight protocol, reducing the total
memory required per container.

When reviewing this change, take particular notice of the generated shim
code.

Signed-off-by: Stephen J Day <stephen.day@docker.com>
This commit is contained in:
Stephen J Day
2017-11-15 18:06:47 -08:00
parent 8e09b565a7
commit e8f52c35ce
10 changed files with 348 additions and 726 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
cmd/containerd-shim/main_unix.go
View File

@@ -25,8 +25,8 @@ import (
"github.com/containerd/typeurl"
ptypes "github.com/gogo/protobuf/types"
"github.com/sirupsen/logrus"
"github.com/stevvooe/ttrpc"
"golang.org/x/sys/unix"
"google.golang.org/grpc"
)
var (
@@ -93,7 +93,8 @@ func executeShim() error {
return err
}
logrus.Debug("registering grpc server")
shimapi.RegisterShimServer(server, sv)
shimapi.RegisterShimService(server, sv)
socket := socketFlag
if err := serve(server, socket); err != nil {
return err
@@ -108,7 +109,7 @@ func executeShim() error {
// serve serves the grpc API over a unix socket at the provided path
// this function does not block
func serve(server *grpc.Server, path string) error {
func serve(server *ttrpc.Server, path string) error {
var (
l net.Listener
err error
@@ -133,7 +134,7 @@ func serve(server *grpc.Server, path string) error {
return nil
}
func handleSignals(logger *logrus.Entry, signals chan os.Signal, server *grpc.Server, sv *shim.Service) error {
func handleSignals(logger *logrus.Entry, signals chan os.Signal, server *ttrpc.Server, sv *shim.Service) error {
var (
termOnce sync.Once
done = make(chan struct{})
@@ -151,9 +152,12 @@ func handleSignals(logger *logrus.Entry, signals chan os.Signal, server *grpc.Se
}
case unix.SIGTERM, unix.SIGINT:
go termOnce.Do(func() {
server.Stop()
ctx := context.TODO()
if err := server.Shutdown(ctx); err != nil {
logger.WithError(err).Error("failed to shutdown server")
}
// Ensure our child is dead if any
sv.Kill(context.Background(), &shimapi.KillRequest{
sv.Kill(ctx, &shimapi.KillRequest{
Signal: uint32(syscall.SIGKILL),
All: true,
})

71
cmd/containerd-shim/shim_linux.go
View File

@@ -1,20 +1,13 @@
package main
import (
"net"
"os"
"os/signal"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
"golang.org/x/net/context"
"golang.org/x/sys/unix"
"github.com/containerd/containerd/reaper"
"github.com/containerd/containerd/sys"
runc "github.com/containerd/go-runc"
"github.com/pkg/errors"
"github.com/stevvooe/ttrpc"
)
// setupSignals creates a new signal handler for all signals and sets the shim as a
@@ -32,64 +25,6 @@ func setupSignals() (chan os.Signal, error) {
return signals, nil
}
func newServer() *grpc.Server {
return grpc.NewServer(grpc.Creds(NewUnixSocketCredentials(0, 0)))
}
type unixSocketCredentials struct {
uid int
gid int
serverName string
}
// NewUnixSocketCredentials returns TransportCredentials for a local unix socket
func NewUnixSocketCredentials(uid, gid int) credentials.TransportCredentials {
return &unixSocketCredentials{uid, gid, "locahost"}
}
func (u *unixSocketCredentials) ClientHandshake(ctx context.Context, addr string, rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) {
return nil, nil, errors.New("ClientHandshake is not supported by unixSocketCredentials")
}
func (u *unixSocketCredentials) ServerHandshake(c net.Conn) (net.Conn, credentials.AuthInfo, error) {
uc, ok := c.(*net.UnixConn)
if !ok {
return nil, nil, errors.New("unixSocketCredentials only supports unix socket")
}
f, err := uc.File()
if err != nil {
return nil, nil, errors.Wrap(err, "unixSocketCredentials: failed to retrieve connection underlying fd")
}
pcred, err := unix.GetsockoptUcred(int(f.Fd()), unix.SOL_SOCKET, unix.SO_PEERCRED)
if err != nil {
return nil, nil, errors.Wrap(err, "unixSocketCredentials: failed to retrieve socket peer credentials")
}
if (u.uid != -1 && uint32(u.uid) != pcred.Uid) || (u.gid != -1 && uint32(u.gid) != pcred.Gid) {
return nil, nil, errors.New("unixSocketCredentials: invalid credentials")
}
return c, u, nil
}
func (u *unixSocketCredentials) Info() credentials.ProtocolInfo {
return credentials.ProtocolInfo{
SecurityProtocol: "unix-socket-peer-creds",
SecurityVersion: "1.0",
ServerName: u.serverName,
}
}
func (u *unixSocketCredentials) Clone() credentials.TransportCredentials {
return &unixSocketCredentials{u.uid, u.gid, u.serverName}
}
func (u *unixSocketCredentials) OverrideServerName(serverName string) error {
u.serverName = serverName
return nil
}
func (u *unixSocketCredentials) AuthType() string {
return "unix-socket-peer-creds"
func newServer() *ttrpc.Server {
return ttrpc.NewServer()
}

7
cmd/containerd-shim/shim_unix.go
View File

@@ -6,10 +6,9 @@ import (
"os"
"os/signal"
"google.golang.org/grpc"
"github.com/containerd/containerd/reaper"
runc "github.com/containerd/go-runc"
"github.com/stevvooe/ttrpc"
)
// setupSignals creates a new signal handler for all signals and sets the shim as a
@@ -23,6 +22,6 @@ func setupSignals() (chan os.Signal, error) {
return signals, nil
}
func newServer() *grpc.Server {
return grpc.NewServer()
func newServer() *ttrpc.Server {
return ttrpc.NewServer()
}

22
cmd/ctr/commands/shim/shim.go
View File

@@ -6,12 +6,9 @@ import (
"fmt"
"io/ioutil"
"net"
"time"
gocontext "context"
"google.golang.org/grpc"
"github.com/containerd/console"
"github.com/containerd/containerd/cmd/ctr/commands"
shim "github.com/containerd/containerd/linux/shim/v1"
@@ -20,6 +17,7 @@ import (
"github.com/opencontainers/runtime-spec/specs-go"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/stevvooe/ttrpc"
"github.com/urfave/cli"
)
@@ -212,21 +210,21 @@ var execCommand = cli.Command{
},
}
func getShimService(context *cli.Context) (shim.ShimClient, error) {
func getShimService(context *cli.Context) (shim.ShimService, error) {
bindSocket := context.GlobalString("socket")
if bindSocket == "" {
return nil, errors.New("socket path must be specified")
}
dialOpts := []grpc.DialOption{grpc.WithInsecure(), grpc.WithTimeout(100 * time.Second)}
dialOpts = append(dialOpts,
grpc.WithDialer(func(addr string, timeout time.Duration) (net.Conn, error) {
return net.DialTimeout("unix", "\x00"+bindSocket, timeout)
},
))
conn, err := grpc.Dial(fmt.Sprintf("unix://%s", bindSocket), dialOpts...)
conn, err := net.Dial("unix", "\x00"+bindSocket)
if err != nil {
return nil, err
}
return shim.NewShimClient(conn), nil
client := ttrpc.NewClient(conn)
// TODO(stevvooe): This actually leaks the connection. We were leaking it
// before, so may not be a huge deal.
return shim.NewShimClient(client), nil
}

1
cmd/protoc-gen-gogoctrd/main.go
View File

@@ -5,6 +5,7 @@ import (
"github.com/gogo/protobuf/protoc-gen-gogo/descriptor"
"github.com/gogo/protobuf/vanity"
"github.com/gogo/protobuf/vanity/command"
_ "github.com/stevvooe/ttrpc/plugin"
)
func main() {