From b3f891b09fe39e724b081bc8610ca86f82983be3 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Thu, 25 May 2017 13:52:42 -0700 Subject: [PATCH] Add integration tests for running containers Add travis support for running integration tests with the client package and go test framework Signed-off-by: Michael Crosby --- .travis.yml | 8 ++ Makefile | 2 +- client.go | 25 +++++++ client_test.go | 87 +++++++++++++++++++++- container_test.go | 145 +++++++++++++++++++++++++++++++++++- io.go | 184 ++++++++++++++++++++++++++++++++++++++++++++++ task.go | 150 ------------------------------------- 7 files changed, 446 insertions(+), 155 deletions(-) create mode 100644 io.go diff --git a/.travis.yml b/.travis.yml index ccdb860b1..30268c0fb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,8 @@ dist: trusty sudo: required +# setup travis so that we can run containers for integration tests +services: + - docker language: go @@ -13,6 +16,8 @@ addons: apt: packages: - btrfs-tools + - libseccomp-dev + - libapparmor-dev env: - TRAVIS_GOOS=windows TRAVIS_CGO_ENABLED=1 @@ -25,6 +30,7 @@ install: - unzip -o -d /tmp/protobuf /tmp/protoc-3.1.0-linux-x86_64.zip - export PATH=$PATH:/tmp/protobuf/bin/ - go get -u github.com/vbatts/git-validation + - sudo wget https://github.com/crosbymichael/runc/releases/download/ctd-1/runc -O /bin/runc; sudo chmod +x /bin/runc script: - export GOOS=$TRAVIS_GOOS @@ -33,8 +39,10 @@ script: - make fmt - make vet - make binaries + - if [ "$GOOS" = "linux" ]; then sudo make install ; fi - if [ "$GOOS" = "linux" ]; then make coverage ; fi - if [ "$GOOS" = "linux" ]; then sudo PATH=$PATH GOPATH=$GOPATH make root-coverage ; fi + - if [ "$GOOS" = "linux" ]; then sudo PATH=$PATH GOPATH=$GOPATH make integration ; fi after_success: - bash <(curl -s https://codecov.io/bash) diff --git a/Makefile b/Makefile index e33c6c58d..8a1de7bed 100644 --- a/Makefile +++ b/Makefile @@ -125,7 +125,7 @@ root-test: ## run tests, except integration tests integration: ## run integration tests @echo "$(WHALE) $@" - @go test ${TESTFLAGS} ${INTEGRATION_PACKAGE} + @go test ${TESTFLAGS} FORCE: diff --git a/client.go b/client.go index f9160ab82..9cef5467a 100644 --- a/client.go +++ b/client.go @@ -33,6 +33,7 @@ import ( "github.com/pkg/errors" "google.golang.org/grpc" "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/health/grpc_health_v1" ) func init() { @@ -82,6 +83,14 @@ type Client struct { namespace string } +func (c *Client) IsServing(ctx context.Context) (bool, error) { + r, err := c.HealthService().Check(ctx, &grpc_health_v1.HealthCheckRequest{}) + if err != nil { + return false, err + } + return r.Status == grpc_health_v1.HealthCheckResponse_SERVING, nil +} + // Containers returns all containers created in containerd func (c *Client) Containers(ctx context.Context) ([]Container, error) { r, err := c.ContainerService().List(ctx, &containers.ListContainersRequest{}) @@ -309,6 +318,18 @@ func (c *Client) Pull(ctx context.Context, ref string, opts ...PullOpts) (Image, }, nil } +// GetImage returns an existing image +func (c *Client) GetImage(ctx context.Context, ref string) (Image, error) { + i, err := c.ImageService().Get(ctx, ref) + if err != nil { + return nil, err + } + return &image{ + client: c, + i: i, + }, nil +} + // Close closes the clients connection to containerd func (c *Client) Close() error { return c.conn.Close() @@ -337,3 +358,7 @@ func (c *Client) ImageService() images.Store { func (c *Client) DiffService() diff.DiffService { return diffservice.NewDiffServiceFromClient(diffapi.NewDiffClient(c.conn)) } + +func (c *Client) HealthService() grpc_health_v1.HealthClient { + return grpc_health_v1.NewHealthClient(c.conn) +} diff --git a/client_test.go b/client_test.go index 871b3167f..62c872e88 100644 --- a/client_test.go +++ b/client_test.go @@ -1,17 +1,99 @@ package containerd import ( + "bytes" "context" "flag" + "fmt" + "os" + "os/exec" + "syscall" "testing" + "time" ) +const ( + defaultRoot = "/var/lib/containerd-test" + defaultState = "/run/containerd-test" + testImage = "docker.io/library/alpine:latest" +) + +var address string + func init() { flag.StringVar(&address, "address", "/run/containerd/containerd.sock", "The address to the containerd socket for use in the tests") flag.Parse() } -var address string +func TestMain(m *testing.M) { + if testing.Short() { + os.Exit(m.Run()) + } + // setup a new containerd daemon if !testing.Short + cmd := exec.Command("containerd", + "--root", defaultRoot, + "--state", defaultState, + ) + buf := bytes.NewBuffer(nil) + cmd.Stderr = buf + if err := cmd.Start(); err != nil { + fmt.Println(err) + os.Exit(1) + } + client, err := New(address) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if err := waitForDaemonStart(client); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + // pull a seed image + if _, err = client.Pull(context.Background(), testImage, WithPullUnpack); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + + } + if err := client.Close(); err != nil { + fmt.Fprintln(os.Stderr, err) + } + + // run the test + status := m.Run() + + // tear down the daemon and resources created + if err := cmd.Process.Signal(syscall.SIGTERM); err != nil { + fmt.Fprintln(os.Stderr, err) + } + if _, err := cmd.Process.Wait(); err != nil { + fmt.Fprintln(os.Stderr, err) + } + if err := os.RemoveAll(defaultRoot); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + // only print containerd logs if the test failed + if status != 0 { + fmt.Fprintln(os.Stderr, buf.String()) + } + os.Exit(status) +} + +func waitForDaemonStart(client *Client) error { + var ( + serving bool + err error + ) + for i := 0; i < 20; i++ { + serving, err = client.IsServing(context.Background()) + if serving { + return nil + } + time.Sleep(100 * time.Millisecond) + } + return fmt.Errorf("containerd did not start within 2s: %v", err) +} func TestNewClient(t *testing.T) { if testing.Short() { @@ -39,8 +121,7 @@ func TestImagePull(t *testing.T) { } defer client.Close() - const ref = "docker.io/library/alpine:latest" - _, err = client.Pull(context.Background(), ref) + _, err = client.Pull(context.Background(), testImage) if err != nil { t.Error(err) return diff --git a/container_test.go b/container_test.go index 7f55a94c1..db79aa0bd 100644 --- a/container_test.go +++ b/container_test.go @@ -1,6 +1,7 @@ package containerd import ( + "bytes" "context" "testing" ) @@ -29,13 +30,13 @@ func TestNewContainer(t *testing.T) { if testing.Short() { t.Skip() } + id := "NewContainer" client, err := New(address) if err != nil { t.Fatal(err) } defer client.Close() - id := "test" spec, err := GenerateSpec() if err != nil { t.Error(err) @@ -46,6 +47,7 @@ func TestNewContainer(t *testing.T) { t.Error(err) return } + defer container.Delete(context.Background()) if container.ID() != id { t.Errorf("expected container id %q but received %q", id, container.ID()) } @@ -58,3 +60,144 @@ func TestNewContainer(t *testing.T) { return } } + +func TestContainerStart(t *testing.T) { + if testing.Short() { + t.Skip() + return + } + client, err := New(address) + if err != nil { + t.Fatal(err) + } + defer client.Close() + + var ( + ctx = context.Background() + id = "ContainerStart" + ) + image, err := client.GetImage(ctx, testImage) + if err != nil { + t.Error(err) + return + } + spec, err := GenerateSpec(WithImageConfig(ctx, image), WithProcessArgs("sh", "-c", "exit 7")) + if err != nil { + t.Error(err) + return + } + container, err := client.NewContainer(ctx, id, spec, WithImage(image), WithNewRootFS(id, image)) + if err != nil { + t.Error(err) + return + } + defer container.Delete(ctx) + + task, err := container.NewTask(ctx, Stdio) + if err != nil { + t.Error(err) + return + } + defer task.Delete(ctx) + + statusC := make(chan uint32, 1) + go func() { + status, err := task.Wait(ctx) + if err != nil { + t.Error(err) + } + statusC <- status + }() + + if pid := task.Pid(); pid <= 0 { + t.Errorf("invalid task pid %d", pid) + } + if err := task.Start(ctx); err != nil { + t.Error(err) + task.Delete(ctx) + return + } + status := <-statusC + if status != 7 { + t.Errorf("expected status 7 from wait but received %d", status) + } + if status, err = task.Delete(ctx); err != nil { + t.Error(err) + return + } + if status != 7 { + t.Errorf("expected status 7 from delete but received %d", status) + } +} + +func TestContainerOutput(t *testing.T) { + if testing.Short() { + t.Skip() + return + } + client, err := New(address) + if err != nil { + t.Fatal(err) + } + defer client.Close() + + var ( + ctx = context.Background() + id = "ContainerOutput" + expected = "kingkoye" + ) + image, err := client.GetImage(ctx, testImage) + if err != nil { + t.Error(err) + return + } + spec, err := GenerateSpec(WithImageConfig(ctx, image), WithProcessArgs("echo", expected)) + if err != nil { + t.Error(err) + return + } + container, err := client.NewContainer(ctx, id, spec, WithImage(image), WithNewRootFS(id, image)) + if err != nil { + t.Error(err) + return + } + defer container.Delete(ctx) + + stdout := bytes.NewBuffer(nil) + task, err := container.NewTask(ctx, BufferedIO(bytes.NewBuffer(nil), stdout, bytes.NewBuffer(nil))) + if err != nil { + t.Error(err) + return + } + defer task.Delete(ctx) + + statusC := make(chan uint32, 1) + go func() { + status, err := task.Wait(ctx) + if err != nil { + t.Error(err) + } + statusC <- status + }() + + if err := task.Start(ctx); err != nil { + t.Error(err) + return + } + + status := <-statusC + if status != 0 { + t.Errorf("expected status 0 but received %d", status) + } + if _, err := task.Delete(ctx); err != nil { + t.Error(err) + return + } + + actual := stdout.String() + // echo adds a new line + expected = expected + "\n" + if actual != expected { + t.Errorf("expected output %q but received %q", expected, actual) + } +} diff --git a/io.go b/io.go new file mode 100644 index 000000000..10d040516 --- /dev/null +++ b/io.go @@ -0,0 +1,184 @@ +package containerd + +import ( + "bytes" + "context" + "io" + "io/ioutil" + "os" + "path/filepath" + "sync" + "syscall" + + "github.com/containerd/fifo" +) + +type IO struct { + Terminal bool + Stdin string + Stdout string + Stderr string + + closer io.Closer +} + +func (i *IO) Close() error { + if i.closer == nil { + return nil + } + return i.closer.Close() +} + +type IOCreation func() (*IO, error) + +// BufferedIO returns IO that will be logged to an in memory buffer +func BufferedIO(stdin, stdout, stderr *bytes.Buffer) IOCreation { + return func() (*IO, error) { + paths, err := fifoPaths() + if err != nil { + return nil, err + } + i := &IO{ + Terminal: false, + Stdout: paths.out, + Stderr: paths.err, + Stdin: paths.in, + } + set := &ioSet{ + in: stdin, + out: stdout, + err: stderr, + } + closer, err := copyIO(paths, set, false) + if err != nil { + return nil, err + } + i.closer = closer + return i, nil + } +} + +// Stdio returns an IO implementation to be used for a task +// that outputs the container's IO as the current processes Stdio +func Stdio() (*IO, error) { + paths, err := fifoPaths() + if err != nil { + return nil, err + } + set := &ioSet{ + in: os.Stdin, + out: os.Stdout, + err: os.Stderr, + } + closer, err := copyIO(paths, set, false) + if err != nil { + return nil, err + } + return &IO{ + Terminal: false, + Stdin: paths.in, + Stdout: paths.out, + Stderr: paths.err, + closer: closer, + }, nil +} + +func fifoPaths() (*fifoSet, error) { + root := filepath.Join(os.TempDir(), "containerd") + if err := os.MkdirAll(root, 0700); err != nil { + return nil, err + } + dir, err := ioutil.TempDir(root, "") + if err != nil { + return nil, err + } + return &fifoSet{ + dir: dir, + in: filepath.Join(dir, "stdin"), + out: filepath.Join(dir, "stdout"), + err: filepath.Join(dir, "stderr"), + }, nil +} + +type fifoSet struct { + // dir is the directory holding the task fifos + dir string + in, out, err string +} + +type ioSet struct { + in io.Reader + out, err io.Writer +} + +func copyIO(fifos *fifoSet, ioset *ioSet, tty bool) (closer io.Closer, err error) { + var ( + f io.ReadWriteCloser + ctx = context.Background() + wg = &sync.WaitGroup{} + ) + + if f, err = fifo.OpenFifo(ctx, fifos.in, syscall.O_WRONLY|syscall.O_CREAT|syscall.O_NONBLOCK, 0700); err != nil { + return nil, err + } + defer func(c io.Closer) { + if err != nil { + c.Close() + } + }(f) + go func(w io.WriteCloser) { + io.Copy(w, ioset.in) + w.Close() + }(f) + + if f, err = fifo.OpenFifo(ctx, fifos.out, syscall.O_RDONLY|syscall.O_CREAT|syscall.O_NONBLOCK, 0700); err != nil { + return nil, err + } + defer func(c io.Closer) { + if err != nil { + c.Close() + } + }(f) + wg.Add(1) + go func(r io.ReadCloser) { + io.Copy(ioset.out, r) + r.Close() + wg.Done() + }(f) + + if f, err = fifo.OpenFifo(ctx, fifos.err, syscall.O_RDONLY|syscall.O_CREAT|syscall.O_NONBLOCK, 0700); err != nil { + return nil, err + } + defer func(c io.Closer) { + if err != nil { + c.Close() + } + }(f) + + if !tty { + wg.Add(1) + go func(r io.ReadCloser) { + io.Copy(ioset.err, r) + r.Close() + wg.Done() + }(f) + } + + return &wgCloser{ + wg: wg, + dir: fifos.dir, + }, nil +} + +type wgCloser struct { + wg *sync.WaitGroup + dir string +} + +func (g *wgCloser) Close() error { + g.wg.Wait() + if g.dir != "" { + return os.RemoveAll(g.dir) + } + return nil +} diff --git a/task.go b/task.go index 9b28281bd..ab37ba4ca 100644 --- a/task.go +++ b/task.go @@ -2,164 +2,14 @@ package containerd import ( "context" - "io" - "io/ioutil" - "os" - "path/filepath" - "sync" "syscall" "github.com/containerd/containerd/api/services/execution" taskapi "github.com/containerd/containerd/api/types/task" - "github.com/containerd/fifo" ) const UnknownExitStatus = 255 -type IO struct { - Terminal bool - Stdin string - Stdout string - Stderr string - - closer io.Closer -} - -func (i *IO) Close() error { - if i.closer == nil { - return nil - } - return i.closer.Close() -} - -type IOCreation func() (*IO, error) - -// Stdio returns an IO implementation to be used for a task -// that outputs the container's IO as the current processes Stdio -func Stdio() (*IO, error) { - paths, err := fifoPaths() - if err != nil { - return nil, err - } - set := &ioSet{ - in: os.Stdin, - out: os.Stdout, - err: os.Stderr, - } - closer, err := copyIO(paths, set, false) - if err != nil { - return nil, err - } - return &IO{ - Terminal: false, - Stdin: paths.in, - Stdout: paths.out, - Stderr: paths.err, - closer: closer, - }, nil -} - -func fifoPaths() (*fifoSet, error) { - root := filepath.Join(os.TempDir(), "containerd") - if err := os.MkdirAll(root, 0700); err != nil { - return nil, err - } - dir, err := ioutil.TempDir(root, "") - if err != nil { - return nil, err - } - return &fifoSet{ - dir: dir, - in: filepath.Join(dir, "stdin"), - out: filepath.Join(dir, "stdout"), - err: filepath.Join(dir, "stderr"), - }, nil -} - -type fifoSet struct { - // dir is the directory holding the task fifos - dir string - in, out, err string -} - -type ioSet struct { - in io.Reader - out, err io.Writer -} - -func copyIO(fifos *fifoSet, ioset *ioSet, tty bool) (closer io.Closer, err error) { - var ( - ctx = context.Background() - wg = &sync.WaitGroup{} - ) - - f, err := fifo.OpenFifo(ctx, fifos.in, syscall.O_WRONLY|syscall.O_CREAT|syscall.O_NONBLOCK, 0700) - if err != nil { - return nil, err - } - defer func(c io.Closer) { - if err != nil { - c.Close() - } - }(f) - go func(w io.WriteCloser) { - io.Copy(w, ioset.in) - w.Close() - }(f) - - f, err = fifo.OpenFifo(ctx, fifos.out, syscall.O_RDONLY|syscall.O_CREAT|syscall.O_NONBLOCK, 0700) - if err != nil { - return nil, err - } - defer func(c io.Closer) { - if err != nil { - c.Close() - } - }(f) - wg.Add(1) - go func(r io.ReadCloser) { - io.Copy(ioset.out, r) - r.Close() - wg.Done() - }(f) - - f, err = fifo.OpenFifo(ctx, fifos.err, syscall.O_RDONLY|syscall.O_CREAT|syscall.O_NONBLOCK, 0700) - if err != nil { - return nil, err - } - defer func(c io.Closer) { - if err != nil { - c.Close() - } - }(f) - - if !tty { - wg.Add(1) - go func(r io.ReadCloser) { - io.Copy(ioset.err, r) - r.Close() - wg.Done() - }(f) - } - return &wgCloser{ - wg: wg, - dir: fifos.dir, - }, nil -} - -type wgCloser struct { - wg *sync.WaitGroup - dir string -} - -func (g *wgCloser) Close() error { - g.wg.Wait() - if g.dir != "" { - return os.RemoveAll(g.dir) - } - return nil -} - type TaskStatus string const (