containerd/.github/workflows/windows-periodic.yml

# Workflow intended to run containerd integration tests on Windows.

name: Windows Integration Tests

on:
  workflow_dispatch:
  workflow_call:
    secrets:
      AZURE_SUB_ID:
        required: true
      AZURE_CREDS:
        required: true
      GCP_SERVICE_ACCOUNT:
        required: true
      GCP_WORKLOAD_IDENTITY_PROVIDER:
        required: true

env:
  AZURE_DEFAULT_LOCATION: westeurope
  AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUB_ID }}
  AZURE_DEFAULT_VM_SIZE: Standard_D2s_v3
  PASSWORD: Passw0rdAdmin # temp for testing, will be generated
  DEFAULT_ADMIN_USERNAME: azureuser
  SSH_OPTS: "-o ServerAliveInterval=20 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
  REMOTE_VM_BIN_PATH: "c:\\containerd\\bin"
  BUSYBOX_TESTING_IMAGE_REF: "k8s.gcr.io/e2e-test-images/busybox:1.29-2"
  RESOURCE_CONSUMER_TESTING_IMAGE_REF: "k8s.gcr.io/e2e-test-images/resource-consumer:1.10"
  WEBSERVER_TESTING_IMAGE_REF: "k8s.gcr.io/e2e-test-images/nginx:1.14-2"


jobs:
  winIntegration:
    # NOTE: the following permissions are required by `google-github-actions/auth`:
    permissions:
      contents: 'read'
      id-token: 'write'
    strategy:
      matrix:
        win_ver: [ltsc2019, ltsc2022]
        include:
        - win_ver: ltsc2019
          AZURE_IMG: "MicrosoftWindowsServer:WindowsServer:2019-Datacenter-with-Containers-smalldisk:17763.1935.2105080716"
          AZURE_RESOURCE_GROUP: ctrd-integration-ltsc2019-${{ github.run_id }}
          GOOGLE_BUCKET: "gs://containerd-integration/logs/windows-ltsc2019/"
        - win_ver: ltsc2022
          AZURE_IMG: "MicrosoftWindowsServer:WindowsServer:2022-datacenter-smalldisk-g2:20348.169.2108120020"
          AZURE_RESOURCE_GROUP: ctrd-integration-ltsc2022-${{ github.run_id }}
          GOOGLE_BUCKET: "gs://containerd-integration/logs/windows-ltsc2022/"
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2

      - name: Install required packages
        run: |
          sudo apt-get install xmlstarlet -y

      - name: PrepareArtifacts
        run: |
          STARTED_TIME=$(date +%s)
          LOGS_DIR=$HOME/$STARTED_TIME
          echo "STARTED_TIME=$STARTED_TIME" >> $GITHUB_ENV
          echo "LOGS_DIR=$LOGS_DIR" >> $GITHUB_ENV
          mkdir -p $LOGS_DIR/artifacts

          jq -n --arg node temp --arg timestamp $STARTED_TIME '$timestamp|tonumber|{timestamp:.,$node}' > $LOGS_DIR/started.json

      - name: Generate ssh key pair
        run: |
          mkdir -p $HOME/.ssh/
          ssh-keygen -t rsa -b 4096 -C "ci@containerd.com" -f $HOME/.ssh/id_rsa -q -N ""
          echo "SSH_PUB_KEY=$(cat ~/.ssh/id_rsa.pub)" >> $GITHUB_ENV

      - name: AZLogin
        uses: azure/login@v1
        with:
          creds: ${{ secrets.AZURE_CREDS }}

      - name: AZResourceGroupCreate
        uses: azure/CLI@v1
        with:
          inlinescript: |
            az group create -n ${{ matrix.AZURE_RESOURCE_GROUP }} -l ${{ env.AZURE_DEFAULT_LOCATION }} --tags creationTimestamp=$(date -u '+%Y-%m-%dT%H:%M:%SZ')

      - name: AZTestVMCreate
        uses: azure/CLI@v1
        with:
          inlinescript: |
            az vm create -n winTestVM --admin-username ${{ env.DEFAULT_ADMIN_USERNAME }} --admin-password ${{ env.PASSWORD }} --image ${{ matrix.AZURE_IMG }} -g ${{ matrix.AZURE_RESOURCE_GROUP }} --nsg-rule SSH --size ${{ env.AZURE_DEFAULT_VM_SIZE }}

      - name: GetAZVMPublicIP
        uses: azure/CLI@v1
        with:
          inlinescript: |
            echo "VM_PUB_IP=$(az network public-ip list -g ${{ matrix.AZURE_RESOURCE_GROUP }} | jq '.[0]["ipAddress"]' | tr -d '\"')" >> $GITHUB_ENV

      - name: EnableAZVMSSH
        uses: azure/CLI@v1
        with:
          inlinescript: |
            az vm run-command invoke  --command-id RunPowerShellScript -n winTestVM -g ${{ matrix.AZURE_RESOURCE_GROUP }} --scripts @$GITHUB_WORKSPACE/script/setup/enable_ssh_windows.ps1 --parameters 'SSHPublicKey=${{ env.SSH_PUB_KEY }}'

      - name: TestSSHConnection
        run: |
          if ! ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} azureuser@${{ env.VM_PUB_IP }}  "hostname";
          then
            exit 1
          fi

      - name: InstallContainerFeatureWS2022
        if: ${{ matrix.win_ver == 'ltsc2022' }}
        run: |
          ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} azureuser@${{ env.VM_PUB_IP }} "powershell.exe -command { Install-WindowsFeature -Name 'Containers' -Restart }"

      - name: WaitForVMToRestart
        if: ${{ matrix.win_ver == 'ltsc2022' }}
        timeout-minutes: 5
        run: |
          # give the vm 30 seconds to actually stop. SSH server might actually respond while server is shutting down.
          sleep 30
          while [ ! $( ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} azureuser@${{ env.VM_PUB_IP }}  "hostname") ];
          do
             echo "Unable to connect to azurevm"
          done
          echo "Connection reestablished. VM restarted succesfully."

      - name: CreateNatNetworkWS2022
        if: ${{ matrix.win_ver == 'ltsc2022' }}
        run: |
          ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} azureuser@${{ env.VM_PUB_IP }} "powershell.exe -command { curl.exe -L 'https://raw.githubusercontent.com/microsoft/SDN/master/Kubernetes/windows/hns.psm1' -o hns.psm1 }"
          ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} azureuser@${{ env.VM_PUB_IP }} "powershell.exe -command { Import-Module .\hns.psm1 ; New-HnsNetwork -Type NAT -Name nat -AddressPrefix 172.19.208.0/20 -Gateway 172.19.208.1 }"

      - name: PrepareTestingEnv
        run: |
          scp -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }}  $GITHUB_WORKSPACE/script/setup/prepare_env_windows.ps1  azureuser@${{ env.VM_PUB_IP }}:/prepare_env_windows.ps1
          ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }}  azureuser@${{ env.VM_PUB_IP }} "c:\\prepare_env_windows.ps1"

      - name: MakeContainerDBins
        run: |
          ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} azureuser@${{ env.VM_PUB_IP }} "git clone http://github.com/containerd/containerd c:\\containerd "
          ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} azureuser@${{ env.VM_PUB_IP }} "cd c:\containerd ; make binaries"
          ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} azureuser@${{ env.VM_PUB_IP }} "git clone http://github.com/Microsoft/hcsshim c:\containerd\hcsshim "

          # Get shim commit from containerd local repo
          SHIM_COMMIT=$(grep 'Microsoft/hcsshim' go.mod | awk '{ print $2 }');

          ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} azureuser@${{ env.VM_PUB_IP }} "cd c:\containerd\hcsshim; git fetch --tags origin $SHIM_COMMIT ; \
                            git checkout $SHIM_COMMIT ; go build -mod=vendor -o ${{ env.REMOTE_VM_BIN_PATH }}\containerd-shim-runhcs-v1.exe .\cmd\containerd-shim-runhcs-v1"

      - name: RunIntegrationTests
        run: |
          ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} azureuser@${{ env.VM_PUB_IP }} "sh.exe -c 'cd /c/containerd && (make integration | tee /c/Logs/integration.log)'"
          ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} azureuser@${{ env.VM_PUB_IP }} "sh.exe -c 'cat /c/Logs/integration.log | go-junit-report.exe > /c/Logs/junit_00.xml'"

      - name: PrepareRepoList
        run: |
           ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} azureuser@${{ env.VM_PUB_IP }} "sh -c 'cat > c:/repolist.toml'" <<'EOF'
               busybox = ${{ env.BUSYBOX_TESTING_IMAGE_REF }}
               ResourceConsumer = ${{ env.RESOURCE_CONSUMER_TESTING_IMAGE_REF }}
           EOF
           ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} azureuser@${{ env.VM_PUB_IP }} "sh -c 'cat > c:/cri-test-images.yaml'" <<'EOF'
               defaultTestContainerImage: ${{ env.BUSYBOX_TESTING_IMAGE_REF }}
               webServerTestImage: ${{ env.WEBSERVER_TESTING_IMAGE_REF }}
           EOF

      - name: RunCRIIntegrationTests
        run: |
           ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} azureuser@${{ env.VM_PUB_IP }} "sh.exe -s" <<EOF
             cd c:/containerd
             ./script/setup/install-cni-windows
             mkdir /c/tmp
             export TEST_IMAGE_LIST=c:/repolist.toml
             make cri-integration | tee c:/Logs/cri-integration.log
           EOF
           ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} azureuser@${{ env.VM_PUB_IP }} "sh.exe -c 'cat /c/Logs/cri-integration.log | go-junit-report.exe > c:/Logs/junit_01.xml' "

      - name: GetCritestRepo
        run: |
          ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} azureuser@${{ env.VM_PUB_IP }} "git clone https://github.com/kubernetes-sigs/cri-tools c:/cri-tools"

      - name: BuildCritest
        run: |
          ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} azureuser@${{ env.VM_PUB_IP }} "sh.exe -c 'cd /c/cri-tools && make critest'"

      - name: RunCritest
        run: |
          ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} azureuser@${{ env.VM_PUB_IP }} "powershell.exe -command { Start-Process -FilePath C:\containerd\bin\containerd.exe -NoNewWindow -RedirectStandardError true -PassThru ; get-process | sls containerd ; start-sleep 5 ; c:\cri-tools\build\bin\critest.exe --runtime-endpoint=\"npipe:\\\\.\\pipe\\containerd-containerd\" --test-images-file='c:\cri-test-images.yaml' --report-dir='c:\Logs' }"

      - name: PullLogsFromWinNode
        run: |
          scp -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} azureuser@${{ env.VM_PUB_IP }}:c:/Logs/*.xml ${{ env.LOGS_DIR }}/artifacts/
          for f in $(ls ${{ env.LOGS_DIR }}/artifacts/*.xml); do
              xmlstarlet ed -d "/testsuites/testsuite/properties" $f > ${{ env.LOGS_DIR }}/$(basename $f)
              mv ${{ env.LOGS_DIR }}/$(basename $f) $f
          done

      - name: FinishJob
        run: |
          jq -n --arg result SUCCESS --arg timestamp $(date +%s) '$timestamp|tonumber|{timestamp:.,$result}' > ${{ env.LOGS_DIR }}/finished.json
          echo "${{ env.STARTED_TIME }}" > $HOME/latest-build.txt

      - name: AssignGcpCreds
        id: AssignGcpCreds
        run: |
          echo '::set-output name=GCP_SERVICE_ACCOUNT::${{ secrets.GCP_SERVICE_ACCOUNT }}'
          echo '::set-output name=GCP_WORKLOAD_IDENTITY_PROVIDER::${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}'

      - name: AuthGcp
        uses: google-github-actions/auth@v0
        if: steps.AssignGcpCreds.outputs.GCP_SERVICE_ACCOUNT && steps.AssignGcpCreds.outputs.GCP_WORKLOAD_IDENTITY_PROVIDER
        with:
          service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
          workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}

      - name: SetUpGcpSdk
        uses: google-github-actions/setup-gcloud@master

      - name: UploadArtifacts
        if: steps.AssignGcpCreds.outputs.GCP_PROJECT_ID && steps.AssignGcpCreds.outputs.GCP_SA_KEY
        run: |
          gsutil cp -r ${{ env.LOGS_DIR }} ${{ matrix.GOOGLE_BUCKET }}
          gsutil cp $HOME/latest-build.txt ${{ matrix.GOOGLE_BUCKET }}

      - name: ResourceCleanup
        uses: azure/CLI@v1
        with:
          inlinescript: |
            az group delete -g ${{ matrix.AZURE_RESOURCE_GROUP }} --yes