containerd/.github/workflows/ci.yml

name: CI
on:
  # When added to a merge queue.
  # See https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/configuring-pull-request-merges/managing-a-merge-queue#triggering-merge-group-checks-with-github-actions
  merge_group:
  pull_request:
    branches: ['main', 'release/**']

env:
  # Go version we currently use to build containerd across all CI.
  # Note: don't forget to update `Binaries` step, as it contains the matrix of all supported Go versions.
  GO_VERSION: "1.21.5"

permissions: # added using https://github.com/step-security/secure-workflows
  contents: read

jobs:
  #
  # golangci-lint
  #
  linters:
    permissions:
      contents: read # for actions/checkout to fetch code
      pull-requests: read # for golangci/golangci-lint-action to fetch pull requests
    name: Linters
    runs-on: ${{ matrix.os }}
    timeout-minutes: 10

    strategy:
      matrix:
        os: [ubuntu-22.04, macos-12, windows-2019]

    steps:
      - uses: actions/setup-go@v4
        with:
          go-version: ${{ env.GO_VERSION }}
          cache: false # see actions/setup-go#368

      - uses: actions/checkout@v3
      - uses: golangci/golangci-lint-action@v3
        with:
          version: v1.55.2
          skip-cache: true
          args: --timeout=8m

  #
  # Project checks
  #
  project:
    name: Project Checks
    if: github.repository == 'containerd/containerd'
    runs-on: ubuntu-22.04
    timeout-minutes: 5

    steps:
      - uses: actions/setup-go@v4
        with:
          go-version: ${{ env.GO_VERSION }}
          cache: false # see actions/setup-go#368

      - uses: actions/checkout@v3
        with:
          path: src/github.com/containerd/containerd
          fetch-depth: 100

      - uses: containerd/project-checks@v1.1.0
        with:
          working-directory: src/github.com/containerd/containerd
          repo-access-token: ${{ secrets.GITHUB_TOKEN }}

      - name: verify go modules and vendor directory
        run: |
          sudo apt-get install -y jq
          make verify-vendor
        working-directory: src/github.com/containerd/containerd

  #
  # Protobuf checks
  #
  protos:
    name: Protobuf
    runs-on: ubuntu-22.04
    timeout-minutes: 5

    defaults:
      run:
        working-directory: src/github.com/containerd/containerd

    steps:
      - uses: actions/setup-go@v4
        with:
          go-version: ${{ env.GO_VERSION }}
          cache: false # see actions/setup-go#368

      - uses: actions/checkout@v3
        with:
          path: src/github.com/containerd/containerd

      - name: Set env
        shell: bash
        run: |
          echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
          echo "${{ github.workspace }}/bin" >> $GITHUB_PATH

      - name: Install protobuf
        run: |
          sudo -E PATH=$PATH script/setup/install-protobuf
          sudo chmod +x /usr/local/bin/protoc
          sudo chmod og+rx /usr/local/include/google /usr/local/include/google/protobuf /usr/local/include/google/protobuf/compiler
          sudo chmod -R og+r /usr/local/include/google/protobuf/
          protoc --version

      - run: script/setup/install-dev-tools
      - run: make proto-fmt
      - run: make check-protos check-api-descriptors

  man:
    name: Manpages
    runs-on: ubuntu-22.04
    timeout-minutes: 5

    steps:
      - uses: actions/setup-go@v4
        with:
          go-version: ${{ env.GO_VERSION }}
          cache: false # see actions/setup-go#368
      - uses: actions/checkout@v3
      - run: go install github.com/cpuguy83/go-md2man/v2@v2.0.2
      - run: make man

  # Make sure binaries compile with other minor platforms.
  # Well-known architectures are covered in release.yml.
  crossbuild:
    name: Crossbuild Binaries
    needs: [project, linters, protos, man]
    runs-on: ubuntu-22.04
    timeout-minutes: 10
    strategy:
      fail-fast: false
      matrix:
        include:
          - goos: linux
            goarch: arm
            goarm: "7"
          - goos: linux
            goarch: arm
            goarm: "5"
          - goos: freebsd
            goarch: amd64
          - goos: freebsd
            goarch: arm64
          - goos: windows
            goarch: arm
            goarm: "7"

    steps:
      - uses: actions/setup-go@v4
        with:
          go-version: ${{ env.GO_VERSION }}
          cache: false # see actions/setup-go#368
      - uses: actions/checkout@v3
      - run: |
          set -e -x

          packages=""
          platform="${{matrix.goos}}/${{matrix.goarch}}"
          if [ -n "${{matrix.goarm}}" ]; then
            platform+="/v${{matrix.goarm}}"
          fi

          case "${platform}" in
          linux/arm/v5)
            packages+=" crossbuild-essential-armel"
            echo "CGO_ENABLED=1" >> $GITHUB_ENV
            echo "CC=arm-linux-gnueabi-gcc" >> $GITHUB_ENV
            ;;
          linux/arm/v7)
            packages+=" crossbuild-essential-armhf"
            echo "CGO_ENABLED=1" >> $GITHUB_ENV
            echo "CC=arm-linux-gnueabihf-gcc" >> $GITHUB_ENV
            ;;
          windows/arm/v7)
            echo "CGO_ENABLED=0" >> $GITHUB_ENV
            ;;
          esac

          if [ -n "${packages}" ]; then
            sudo apt-get update && sudo apt-get install -y ${packages}
          fi
        name: Install deps
      - name: Build
        env:
          GOOS: ${{matrix.goos}}
          GOARCH: ${{matrix.goarch}}
          GOARM: ${{matrix.goarm}}
        run: |
          make build
          make binaries

  #
  # Build containerd binaries
  #
  binaries:
    name: Binaries
    runs-on: ${{ matrix.os }}
    timeout-minutes: 20
    needs: [project, linters, protos, man]

    strategy:
      matrix:
        os: [ubuntu-22.04, macos-12, windows-2019, windows-2022]
        go-version: ["1.20.12", "1.21.5"]
    steps:
      - uses: actions/setup-go@v4
        with:
          go-version: ${{ matrix.go-version }}
          cache: false # see actions/setup-go#368

      - uses: actions/checkout@v3

      # NOTE(aznashwan): starting with Golang 1.21, the windows-2019 GitHub runner's
      # builtin MinGW version leads to DLL loading errors during runtime.
      - name: Upgrade MinGW on Windows 2019
        if: matrix.os == 'windows-2019'
        run: |
          choco upgrade mingw --version=12.2.0.3042023

      - name: Make
        run: |
          make build
          make binaries

  #
  # Integration and CRI tests
  #
  integration-windows:
    name: Windows Integration
    runs-on: ${{ matrix.os }}
    timeout-minutes: 90
    needs: [project, linters, protos, man]
    env:
      GOTEST: gotestsum --

    strategy:
      fail-fast: false
      matrix:
        os: [windows-2022] # windows-2019 is temporarily removed until mingw issue fixed

    defaults:
      run:
        shell: bash
        working-directory: src/github.com/containerd/containerd

    steps:
      - uses: actions/setup-go@v4
        with:
          go-version: ${{ env.GO_VERSION }}
          cache: false # see actions/setup-go#368

      - uses: actions/checkout@v3
        with:
          path: src/github.com/containerd/containerd

      - uses: actions/checkout@v3
        with:
          repository: kubernetes-sigs/cri-tools
          path: src/github.com/kubernetes-sigs/cri-tools

      - name: Set env
        run: |
          echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
          echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
          echo "${{ github.workspace }}/src/github.com/containerd/containerd/bin" >> $GITHUB_PATH
          echo "${{ github.workspace }}/src/github.com/kubernetes-sigs/cri-tools/build/bin/windows/amd64" >> $GITHUB_PATH

      - run: script/setup/install-dev-tools

      # NOTE(aznashwan): starting with Golang 1.21, the windows-2019 GitHub runner's
      # builtin MinGW version leads to DLL loading errors during runtime.
      - name: Upgrade MinGW on Windows 2019
        if: matrix.os == 'windows-2019'
        run: |
          choco upgrade mingw --version=12.2.0.3042023

      - name: Binaries
        env:
          CGO_ENABLED: 1
        run: |
          set -o xtrace
          mingw32-make.exe binaries
          cd ../../kubernetes-sigs/cri-tools
          make critest

      - run: script/setup/install-cni-windows

      - env:
          TEST_IMAGE_LIST: ${{github.workspace}}/repolist.toml
          CRI_TEST_IMAGES: ${{github.workspace}}/cri-test-images.yaml
          BUSYBOX_TESTING_IMAGE_REF: "registry.k8s.io/e2e-test-images/busybox:1.29-2"
          RESOURCE_CONSUMER_TESTING_IMAGE_REF: "registry.k8s.io/e2e-test-images/resource-consumer:1.10"
          WEBSERVER_TESTING_IMAGE_REF: "registry.k8s.io/e2e-test-images/nginx:1.14-2"
        run: |
          cat > "${{ env.TEST_IMAGE_LIST }}" << EOF
          busybox = "${{ env.BUSYBOX_TESTING_IMAGE_REF }}"
          ResourceConsumer = "${{ env.RESOURCE_CONSUMER_TESTING_IMAGE_REF }}"
          EOF
          cat > "${{ env.CRI_TEST_IMAGES }}" << EOF
          defaultTestContainerImage: ${{ env.BUSYBOX_TESTING_IMAGE_REF }}
          webServerTestImage: ${{ env.WEBSERVER_TESTING_IMAGE_REF }}
          EOF

      - name: Get crictl tool
        shell: powershell
        run: |
          # Get critctl tool. Used for cri-integration tests
          $CRICTL_DOWNLOAD_URL="https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.26.0/crictl-v1.26.0-windows-amd64.tar.gz"
          curl.exe -L $CRICTL_DOWNLOAD_URL -o c:\crictl.tar.gz
          tar -xvf c:\crictl.tar.gz
          mv crictl.exe "${{ github.workspace }}/bin/crictl.exe" # Move crictl somewhere in path

      - run: script/setup/install-gotestsum
      - run: script/setup/install-teststat
      - name: Tests
        env:
          CGO_ENABLED: 1
          GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-unit-root.xml
          GOTESTSUM_JSONFILE: ${{github.workspace}}/test-unit-root-gotest.json
        run: mingw32-make.exe test root-test
      - run: if [ -f *-gotest.json ]; then echo '# Root Test' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi
        if: always()
      - run: script/test/test2annotation.sh ${TESTFILE}
        env:
          TESTFILE: ${{github.workspace}}/test-unit-root-gotest.json
        if: always()

      - name: Integration 1
        env:
          CGO_ENABLED: 1
          GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-serial-junit.xml
          GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-serial-gotest.json
          EXTRA_TESTFLAGS: "-timeout=20m"
        run: mingw32-make.exe integration
      - run: if [ -f *-gotest.json ]; then echo '# Integration 1' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi
        if: always()
      - run: script/test/test2annotation.sh ${TESTFILE}
        env:
          TESTFILE: ${{github.workspace}}/test-integration-serial-gotest.json
        if: always()

      # Run the integration suite a second time. See discussion in github.com/containerd/containerd/pull/1759
      - name: Integration 2
        env:
          TESTFLAGS_PARALLEL: 1
          EXTRA_TESTFLAGS: "-short"
          CGO_ENABLED: 1
          GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-parallel-junit.xml
          GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-parallel-gotest.json
        run: mingw32-make.exe integration
      - run: if [ -f *-gotest.json ]; then echo '# Integration 2' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi
        if: always()
      - run: script/test/test2annotation.sh ${TESTFILE}
        env:
          TESTFILE: ${{github.workspace}}/test-integration-parallel-gotest.json
        if: always()

      - name: CRI Integration Test
        env:
          TEST_IMAGE_LIST: ${{github.workspace}}/repolist.toml
        run: |
          make cri-integration

      - name: cri-tools critest
        env:
          CRI_TEST_IMAGES: ${{github.workspace}}/cri-test-images.yaml
        shell: powershell
        run: |
          Start-Process -FilePath containerd.exe -NoNewWindow -RedirectStandardError true -PassThru
          get-process | sls containerd
          start-sleep 5
          # This test is exceedingly flaky only on ws2022 so skip for now to keep CI happy.
          # Info: https://github.com/containerd/containerd/issues/6652
          if( '${{ matrix.os }}' -eq 'windows-2022' )
          {
            $skip = "-ginkgo.skip=runtime should support exec with tty=true and stdin=true"
          }
          critest.exe --runtime-endpoint=npipe://.//pipe//containerd-containerd --test-images-file='${{env.CRI_TEST_IMAGES}}' --report-dir='${{github.workspace}}/critestreport' $skip

      - uses: actions/upload-artifact@v3
        if: always()
        with:
          name: TestResults ${{ matrix.os }}
          path: |
            ${{github.workspace}}/*-junit.xml
            ${{github.workspace}}/*-gotest.json
            ${{github.workspace}}/report/*.log

  integration-linux:
    name: Linux Integration
    runs-on: ubuntu-22.04
    timeout-minutes: 40
    needs: [project, linters, protos, man]

    strategy:
      fail-fast: false
      matrix:
        runtime:
          - io.containerd.runc.v2
        runc: [runc, crun]

    env:
      GOTEST: gotestsum --
    steps:
      - uses: actions/setup-go@v4
        with:
          go-version: ${{ env.GO_VERSION }}
          cache: false # see actions/setup-go#368

      - uses: actions/checkout@v3

      - name: Install containerd dependencies
        env:
          RUNC_FLAVOR: ${{ matrix.runc }}
        run: |
          sudo apt-get update
          sudo apt-get install -y gperf
          script/setup/install-seccomp
          script/setup/install-runc
          script/setup/install-cni $(grep containernetworking/plugins go.mod | awk '{print $2}')
          script/setup/install-critools
          script/setup/install-failpoint-binaries

      - name: Install criu
        run: |
          sudo add-apt-repository -y ppa:criu/ppa
          sudo apt-get update
          sudo apt-get install -y criu

      - name: Install containerd
        env:
          CGO_ENABLED: 1
        run: |
          make binaries GO_BUILD_FLAGS="-mod=vendor"
          sudo -E PATH=$PATH make install

      - run: script/setup/install-gotestsum
      - run: script/setup/install-teststat
      - name: Tests
        env:
          GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-unit-root-junit.xml
          GOTESTSUM_JSONFILE: ${{github.workspace}}/test-unit-root-gotest.json
        run: |
          make test
          sudo -E PATH=$PATH make root-test
      - run: if [ -f *-gotest.json ]; then echo '# Root Test' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi
        if: always()
      - run: script/test/test2annotation.sh *-gotest.json
        if: always()

      - name: Integration 1
        env:
          TEST_RUNTIME: ${{ matrix.runtime }}
          RUNC_FLAVOR: ${{ matrix.runc }}
          GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-serial-junit.xml
          GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-serial-gotest.json
        run: |
          extraflags=""
          [ "${RUNC_FLAVOR}" == "crun" ] && {
                  extraflags="EXTRA_TESTFLAGS=-no-criu";
          }
          sudo -E PATH=$PATH make integration ${extraflags} TESTFLAGS_RACE=-race
      - run: if [ -f *-gotest.json ]; then echo '# Integration 1' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi
        if: always()
      - run: script/test/test2annotation.sh *-gotest.json
        if: always()

      # Run the integration suite a second time. See discussion in github.com/containerd/containerd/pull/1759
      - name: Integration 2
        env:
          TEST_RUNTIME: ${{ matrix.runtime }}
          RUNC_FLAVOR: ${{ matrix.runc }}
          GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-parallel-junit.xml
          GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-parallel-gotest.json
        run: |
          extraflags=""
          [ "${RUNC_FLAVOR}" == "crun" ] && {
                  extraflags="EXTRA_TESTFLAGS=-no-criu";
          }
          sudo -E PATH=$PATH TESTFLAGS_PARALLEL=1 make integration ${extraflags}
      - run: if [ -f *-gotest.json ]; then echo '# Integration 2' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi
        if: always()
      - run: script/test/test2annotation.sh *-gotest.json
        if: always()

      - name: CRI Integration Test
        env:
          TEST_RUNTIME: ${{ matrix.runtime }}
        run: |
          CONTAINERD_RUNTIME=$TEST_RUNTIME make cri-integration

      - name: cri-tools critest
        env:
          TEST_RUNTIME: ${{ matrix.runtime }}
        run: |
          sudo -E PATH=$PATH ./script/critest.sh "${{github.workspace}}/report"

      # Log the status of this VM to investigate issues like
      # https://github.com/containerd/containerd/issues/4969
      - name: Host Status
        if: always()
        run: |
          set -x
          mount
          df
          losetup -l
      - name: Kernel Message
        if: failure()
        run: sudo dmesg -T -f kern

      - uses: actions/upload-artifact@v3
        if: always()
        with:
          name: TestResults ${{ matrix.runtime }} ${{matrix.runc}}
          path: |
            *-junit.xml
            *-gotest.json
            ${{github.workspace}}/report/*.xml
            ${{github.workspace}}/report/*.log

  integration-vagrant:
    name: Vagrant integration
    # "Larger" runner is needed for nested virtualization
    # https://github.com/organizations/containerd/settings/actions/runners
    runs-on: ubuntu-latest-4-cores
    timeout-minutes: 60
    needs: [project, linters, protos, man]

    strategy:
      fail-fast: false
      matrix:
        box:
          - fedora/39-cloud-base
          - rockylinux/8@8.0.0
    env:
      BOX: ${{ matrix.box }}

    steps:
      - name: Show the host info
        run: |
          set -x
          uname -a
          cat /etc/os-release
          cat /proc/cpuinfo
          free -mt
      - uses: actions/checkout@v3
      - uses: actions/cache@v3
        with:
          path: /root/.vagrant.d
          key: vagrant-${{ matrix.box }}
      - name: Set up vagrant
        run: |
          # Canonical's Vagrant 2.2.19 dpkg cannot download Fedora 38 image: https://bugs.launchpad.net/vagrant/+bug/2017828
          # So we have to install Vagrant >= 2.3.1 from the upstream: https://github.com/opencontainers/runc/blob/v1.1.8/.cirrus.yml#L41-L49
          curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
          echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
          sudo sed -i 's/^# deb-src/deb-src/' /etc/apt/sources.list
          sudo apt-get update
          sudo apt-get install -y libvirt-daemon libvirt-daemon-system vagrant
          sudo systemctl enable --now libvirtd
          sudo apt-get build-dep -y vagrant ruby-libvirt
          sudo apt-get install -y --no-install-recommends libxslt-dev libxml2-dev libvirt-dev ruby-bundler ruby-dev zlib1g-dev
          sudo vagrant plugin install vagrant-libvirt
      - name: Boot VM
        run: sudo BOX=$BOX vagrant up --no-tty
      - name: test-integration
        run: sudo BOX=$BOX vagrant up --provision-with=selinux,install-runc,install-gotestsum,test-integration
      - name: test-cri-integration
        run: sudo BOX=$BOX vagrant up --provision-with=selinux,install-runc,install-gotestsum,test-cri-integration
      - name: test-cri
        run: sudo BOX=$BOX vagrant up --provision-with=selinux,install-runc,install-gotestsum,test-cri

  tests-cri-in-userns:
    name: "CRI-in-UserNS"

    runs-on: ubuntu-22.04
    timeout-minutes: 40
    needs: [project, linters, protos, man]

    steps:
      - uses: actions/checkout@v3
      - name: Set up cgroup v2 delegation
        run: |
          sudo mkdir -p /etc/systemd/system/user@.service.d
          cat <<EOF | sudo tee /etc/systemd/system/user@.service.d/delegate.conf
          [Service]
          Delegate=cpu cpuset io memory pids
          EOF
          sudo systemctl daemon-reload
      - name: Build cri-in-userns image
        run: podman build --target cri-in-userns -t cri-in-userns -f ./contrib/Dockerfile.test .
      - name: Run cri-in-userns image
        # Rootless Podman is used for testing CRI-in-UserNS
        # (We could use rootless Docker or rootless nerdctl, but we are using Podman here because it is preinstalled)
        run: podman run --rm --privileged cri-in-userns

  tests-mac-os:
    name: MacOS unit tests
    runs-on: macos-12
    timeout-minutes: 10
    needs: [project, linters, protos, man]
    env:
      GOTEST: gotestsum --

    steps:
      - uses: actions/setup-go@v4
        with:
          go-version: ${{ env.GO_VERSION }}
          cache: false # see actions/setup-go#368
      - uses: actions/checkout@v3
      - run: script/setup/install-gotestsum
      - run: script/setup/install-teststat
      - name: Tests
        env:
          GOTESTSUM_JUNITFILE: "${{ github.workspace }}/macos-test-junit.xml"
          GOTESTSUM_JSONFILE: "${{ github.workspace }}/macos-test-gotest.json"
        run: make test
      - run: if [ -f *-gotest.json ]; then echo '# Unit Tests' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi
        if: always()
      - run: script/test/test2annotation.sh *-gotest.json
        if: always()
      - uses: actions/upload-artifact@v3
        if: always()
        with:
          name: TestResults MacOS
          path: |
            *-junit.xml
            *-gotest.json