From 7e96aaab090c98bef6fb18ec3e0f13b8c2925b7f Mon Sep 17 00:00:00 2001
From: Lantao Liu <lantaol@google.com>
Date: Mon, 9 Sep 2019 16:17:04 -0700
Subject: [PATCH] Add CRI Validation test.

Signed-off-by: Lantao Liu <lantaol@google.com>
---
 .travis.yml                                |  1 -
 Makefile                                   | 18 +++-
 hack/install/install-cni.sh                |  5 +-
 hack/install/install-containerd.sh         | 11 +--
 hack/install/install-critools.sh           | 16 ++--
 hack/install/install-runc.sh               |  5 +-
 hack/install/windows/install-cni-config.sh | 86 +++++++++++++++++++
 hack/install/windows/install-cni.sh        | 37 ++++++++
 hack/install/windows/install-deps.sh       | 47 +++++++++++
 hack/install/windows/install-hcsshim.sh    | 37 ++++++++
 hack/test-cri.sh                           |  3 +-
 hack/utils.sh                              |  2 +-
 test/windows/runner.sh                     | 98 ++++++++++++++++++++++
 test/windows/setup-ssh.ps1                 | 32 +++++++
 test/windows/setup-vm.ps1                  | 23 +++++
 test/windows/test.sh                       | 45 ++++++++++
 16 files changed, 440 insertions(+), 26 deletions(-)
 create mode 100755 hack/install/windows/install-cni-config.sh
 create mode 100755 hack/install/windows/install-cni.sh
 create mode 100755 hack/install/windows/install-deps.sh
 create mode 100755 hack/install/windows/install-hcsshim.sh
 create mode 100755 test/windows/runner.sh
 create mode 100644 test/windows/setup-ssh.ps1
 create mode 100644 test/windows/setup-vm.ps1
 create mode 100755 test/windows/test.sh

diff --git a/.travis.yml b/.travis.yml
index c3edb9e7d..c1f9539bc 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -37,7 +37,6 @@ before_script:
 script:
   - make .install.gitvalidation
   - make .gitvalidation
-  - make binaries
   - make install.deps
   - make containerd
   - sudo PATH=$PATH GOPATH=$GOPATH make install-containerd
diff --git a/Makefile b/Makefile
index 9140a14b4..b3cbed693 100644
--- a/Makefile
+++ b/Makefile
@@ -129,14 +129,14 @@ containerd: $(BUILD_DIR)/containerd ## build a customized containerd with CRI pl
 
 install-containerd: containerd ## installs customized containerd to system location
 	@echo "$(WHALE) $@"
-	@install -D -m 755 $(BUILD_DIR)/containerd $(BINDIR)/containerd
+	@install -D -m 755 $(BUILD_DIR)/containerd "$(BINDIR)/containerd"
 
 install: install-containerd ## installs customized containerd to system location
 	@echo "$(WHALE) $@"
 
 uninstall: ## remove containerd from system location
 	@echo "$(WHALE) $@"
-	@rm -f $(BINDIR)/containerd
+	@rm -f "$(BINDIR)/containerd"
 
 $(BUILD_DIR)/$(TARBALL): static-binaries vendor.conf
 	@BUILD_DIR=$(BUILD_DIR) TARBALL=$(TARBALL) VERSION=$(VERSION) ./hack/release.sh
@@ -152,12 +152,22 @@ proto: ## update protobuf of the cri plugin api
 	@API_PATH=pkg/api/v1 hack/update-proto.sh
 	@API_PATH=pkg/api/runtimeoptions/v1 hack/update-proto.sh
 
-.PHONY: install.deps
+.PHONY: install.deps .install.deps.linux .install.deps.windows
 
-install.deps: ## install dependencies of cri (default 'seccomp apparmor' BUILDTAGS for runc build)
+ifeq ($(GOOS),windows)
+install.deps: .install.deps.windows ## install windows deps on windows
+else
+install.deps: .install.deps.linux ## install windows deps on linux
+endif
+
+.install.deps.linux: ## install dependencies of cri (default 'seccomp apparmor' BUILDTAGS for runc build)
 	@echo "$(WHALE) $@"
 	@./hack/install/install-deps.sh
 
+.install.deps.windows: ## install dependencies of cri on windows
+	@echo "$(WHALE) $@"
+	@./hack/install/windows/install-deps.sh
+
 .PHONY: .gitvalidation
 # When this is running in travis, it will only check the travis commit range.
 # When running outside travis, it will check from $(EPOCH_TEST_COMMIT)..HEAD.
diff --git a/hack/install/install-cni.sh b/hack/install/install-cni.sh
index 0e3dd649e..e7d120146 100755
--- a/hack/install/install-cni.sh
+++ b/hack/install/install-cni.sh
@@ -23,8 +23,7 @@ CNI_DIR=${DESTDIR}/opt/cni
 CNI_PKG=github.com/containernetworking/plugins
 
 # Create a temporary GOPATH for cni installation.
-TMPGOPATH=$(mktemp -d /tmp/cri-install-cni.XXXX)
-GOPATH=${TMPGOPATH}
+GOPATH=$(mktemp -d /tmp/cri-install-cni.XXXX)
 
 # Install cni
 from-vendor CNI github.com/containernetworking/plugins
@@ -35,4 +34,4 @@ ${SUDO} mkdir -p ${CNI_DIR}
 ${SUDO} cp -r ./bin ${CNI_DIR}
 
 # Clean the tmp GOPATH dir.
-rm -rf ${TMPGOPATH}
+rm -rf ${GOPATH}
diff --git a/hack/install/install-containerd.sh b/hack/install/install-containerd.sh
index 84c8c6e4c..e36a8bd8f 100755
--- a/hack/install/install-containerd.sh
+++ b/hack/install/install-containerd.sh
@@ -19,7 +19,7 @@ set -o nounset
 set -o pipefail
 
 source $(dirname "${BASH_SOURCE[0]}")/utils.sh
-CONTAINERD_DIR=${DESTDIR}/usr/local
+CONTAINERD_DIR=${CONTAINERD_DIR:-"${DESTDIR}/usr/local"}
 CONTAINERD_PKG=github.com/containerd/containerd
 
 # CHECKOUT_CONTAINERD indicates whether to checkout containerd repo.
@@ -29,8 +29,7 @@ CHECKOUT_CONTAINERD=${CHECKOUT_CONTAINERD:-true}
 
 if ${CHECKOUT_CONTAINERD}; then
   # Create a temporary GOPATH for containerd installation.
-  TMPGOPATH=$(mktemp -d /tmp/cri-install-containerd.XXXX)
-  GOPATH=${TMPGOPATH}
+  GOPATH=$(mktemp -d /tmp/cri-install-containerd.XXXX)
   from-vendor CONTAINERD github.com/containerd/containerd
   checkout_repo ${CONTAINERD_PKG} ${CONTAINERD_VERSION} ${CONTAINERD_REPO}
 fi
@@ -40,9 +39,11 @@ cd ${GOPATH}/src/${CONTAINERD_PKG}
 make BUILDTAGS="${BUILDTAGS}"
 # containerd make install requires `go` to work. Explicitly
 # set PATH to make sure it can find `go` even with `sudo`.
-${SUDO} sh -c "PATH=${PATH} make install -e DESTDIR=${CONTAINERD_DIR}"
+# The single quote is required because containerd Makefile
+# can't handle spaces in the path.
+${SUDO} make install -e DESTDIR="'${CONTAINERD_DIR}'"
 
 # Clean the tmp GOPATH dir.
 if ${CHECKOUT_CONTAINERD}; then
-  rm -rf ${TMPGOPATH}
+  rm -rf ${GOPATH}
 fi
diff --git a/hack/install/install-critools.sh b/hack/install/install-critools.sh
index 0194ca23a..35bb6d5f1 100755
--- a/hack/install/install-critools.sh
+++ b/hack/install/install-critools.sh
@@ -19,22 +19,22 @@ set -o nounset
 set -o pipefail
 
 source $(dirname "${BASH_SOURCE[0]}")/utils.sh
-CRITOOL_DIR=${DESTDIR}/usr/local/bin
-CRICTL_CONFIG_DIR=${DESTDIR}/etc
+CRITOOL_DIR="${CRITOOL_DIR:-${DESTDIR}/usr/local/bin}"
+CRICTL_CONFIG_DIR="${CRICTL_CONFIG_DIR:-"${DESTDIR}/etc"}"
+CRICTL_RUNTIME_ENDPOINT=${CRICTL_RUNTIME_ENDPOINT:-unix:///run/containerd/containerd.sock}
 
 # Create a temporary GOPATH for crictl installation.
-TMPGOPATH=$(mktemp -d /tmp/cri-install-crictl.XXXX)
-GOPATH=${TMPGOPATH}
+GOPATH=$(mktemp -d /tmp/cri-install-crictl.XXXX)
 
 #Install crictl
 checkout_repo ${CRITOOL_PKG} ${CRITOOL_VERSION} ${CRITOOL_REPO}
 cd ${GOPATH}/src/${CRITOOL_PKG}
 make VERSION=${CRITOOL_VERSION}
-${SUDO} make install -e BINDIR=${CRITOOL_DIR} GOPATH=${GOPATH}
+${SUDO} make install -e BINDIR="\"${CRITOOL_DIR}\"" GOPATH=${GOPATH}
 ${SUDO} mkdir -p ${CRICTL_CONFIG_DIR}
-${SUDO} bash -c 'cat >'${CRICTL_CONFIG_DIR}'/crictl.yaml <<EOF
-runtime-endpoint: unix:///run/containerd/containerd.sock
+${SUDO} bash -c 'cat >"'"${CRICTL_CONFIG_DIR}"'"/crictl.yaml <<EOF
+runtime-endpoint: '${CRICTL_RUNTIME_ENDPOINT}'
 EOF'
 
 # Clean the tmp GOPATH dir.
-rm -rf ${TMPGOPATH}
+rm -rf ${GOPATH}
diff --git a/hack/install/install-runc.sh b/hack/install/install-runc.sh
index 22427e26a..cc4a67b08 100755
--- a/hack/install/install-runc.sh
+++ b/hack/install/install-runc.sh
@@ -23,8 +23,7 @@ RUNC_DIR=${DESTDIR}
 RUNC_PKG=github.com/opencontainers/runc
 
 # Create a temporary GOPATH for runc installation.
-TMPGOPATH=$(mktemp -d /tmp/cri-install-runc.XXXX)
-GOPATH=${TMPGOPATH}
+GOPATH=$(mktemp -d /tmp/cri-install-runc.XXXX)
 
 # Install runc
 from-vendor RUNC github.com/opencontainers/runc
@@ -35,4 +34,4 @@ ${SUDO} make install -e DESTDIR=${RUNC_DIR}
 
 # Clean the tmp GOPATH dir. Use sudo because runc build generates
 # some privileged files.
-${SUDO} rm -rf ${TMPGOPATH}
+${SUDO} rm -rf ${GOPATH}
diff --git a/hack/install/windows/install-cni-config.sh b/hack/install/windows/install-cni-config.sh
new file mode 100755
index 000000000..395292026
--- /dev/null
+++ b/hack/install/windows/install-cni-config.sh
@@ -0,0 +1,86 @@
+#!/bin/bash
+
+# Copyright The containerd Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+CNI_CONFIG_DIR="${CNI_CONFIG_DIR:-"C:\\Program Files\\containerd\\cni\\conf"}"
+mkdir -p "${CNI_CONFIG_DIR}"
+
+# split_ip splits ip into a 4-element array.
+split_ip() {
+  local -r varname="$1"
+  local -r ip="$2"
+  for i in {0..3}; do
+    eval "$varname"[$i]=$( echo "$ip" | cut -d '.' -f $((i + 1)) )
+  done
+}
+
+# subnet gets subnet for a gateway, e.g. 192.168.100.0/24.
+calculate_subnet() {
+  local -r gateway="$1"
+  local -r prefix_len="$2"
+  split_ip gateway_array "$gateway"
+  local len=$prefix_len
+  for i in {0..3}; do
+    if (( len >= 8 )); then
+      mask=255
+    elif (( len > 0 )); then
+      mask=$(( 256 - 2 ** ( 8 - len ) ))
+    else
+      mask=0
+    fi
+    (( len -= 8 ))
+    result_array[i]=$(( gateway_array[i] & mask ))
+  done
+  result="$(printf ".%s" "${result_array[@]}")"
+  result="${result:1}"
+  echo "$result/$((32 - prefix_len))"
+}
+
+# nat already exists on the Windows VM, the subnet and gateway
+# we specify should match that.
+gateway="$(powershell -c "(Get-NetIPAddress -InterfaceAlias 'vEthernet (nat)' -AddressFamily IPv4).IPAddress")"
+prefix_len="$(powershell -c "(Get-NetIPAddress -InterfaceAlias 'vEthernet (nat)' -AddressFamily IPv4).PrefixLength")"
+
+subnet="$(calculate_subnet "$gateway" "$prefix_len")"
+
+# The "name" field in the config is used as the underlying
+# network type right now (see
+# https://github.com/microsoft/windows-container-networking/pull/45),
+# so it must match a network type in:
+# https://docs.microsoft.com/en-us/windows-server/networking/technologies/hcn/hcn-json-document-schemas
+bash -c 'cat >"'"${CNI_CONFIG_DIR}"'"/0-containerd-nat.conf <<EOF
+{
+    "cniVersion": "0.2.0",
+    "name": "nat",
+    "type": "nat",
+    "master": "Ethernet",
+    "ipam": {
+        "subnet": "'$subnet'",
+        "routes": [
+            {
+                "gateway": "'$gateway'"
+            }
+        ]
+    },
+    "capabilities": {
+        "portMappings": true,
+        "dns": true
+    }
+}
+EOF'
diff --git a/hack/install/windows/install-cni.sh b/hack/install/windows/install-cni.sh
new file mode 100755
index 000000000..ea0479956
--- /dev/null
+++ b/hack/install/windows/install-cni.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# Copyright The containerd Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+source $(dirname "${BASH_SOURCE[0]}")/../utils.sh
+# WINCNI_BIN_DIR is the cni plugin directory
+WINCNI_BIN_DIR="${WINCNI_BIN_DIR:-"C:\\Program Files\\containerd\\cni\\bin"}"
+WINCNI_PKG=github.com/Microsoft/windows-container-networking
+WINCNI_VERSION=33bc4764ea3ad7c6ec58c5716370d329f5eb1266
+
+# Create a temporary GOPATH for cni installation.
+GOPATH="$(mktemp -d /tmp/cri-install-cni.XXXX)"
+
+# Install cni
+checkout_repo "${WINCNI_PKG}" "${WINCNI_VERSION}" "${WINCNI_PKG}"
+cd "${GOPATH}/src/${WINCNI_PKG}"
+go build "${WINCNI_PKG}/plugins/nat"
+install -D -m 755 "nat.exe" "${WINCNI_BIN_DIR}/nat.exe"
+
+# Clean the tmp GOPATH dir.
+rm -rf "${GOPATH}"
diff --git a/hack/install/windows/install-deps.sh b/hack/install/windows/install-deps.sh
new file mode 100755
index 000000000..1b72ec97b
--- /dev/null
+++ b/hack/install/windows/install-deps.sh
@@ -0,0 +1,47 @@
+#!/bin/bash
+
+# Copyright The containerd Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+cd $(dirname "${BASH_SOURCE[0]}")
+
+# Install hcsshim
+./install-hcsshim.sh
+
+# Install cni
+./install-cni.sh
+
+# Install cni config
+./install-cni-config.sh
+
+# Install containerd
+NOSUDO=true \
+  BUILDTAGS="" \
+  CONTAINERD_DIR='C:\Program Files\Containerd' \
+  ../install-containerd.sh
+# Containerd makefile always installs into a "bin" directory.
+# Use slash instead of bach slash so that `*` can work.
+mv C:/'Program Files'/Containerd/bin/* 'C:\Program Files\Containerd\'
+rm -rf 'C:\Program Files\Containerd\bin'
+
+#Install critools
+NOSUDO=true \
+  CRITOOL_DIR='C:\Program Files\Containerd' \
+  CRICTL_RUNTIME_ENDPOINT="npipe:////./pipe/containerd-containerd" \
+  CRICTL_CONFIG_DIR="C:\\Users\\$(id -u -n)\\.crictl" \
+  ../install-critools.sh
diff --git a/hack/install/windows/install-hcsshim.sh b/hack/install/windows/install-hcsshim.sh
new file mode 100755
index 000000000..b3d2b04c8
--- /dev/null
+++ b/hack/install/windows/install-hcsshim.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# Copyright 2018 The containerd Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+source $(dirname "${BASH_SOURCE[0]}")/../utils.sh
+HCSSHIM_DIR="${HCSSHIM_DIR:-"C:\\Program Files\\Containerd"}"
+HCSSHIM_PKG=github.com/Microsoft/hcsshim
+
+# Create a temporary GOPATH for hcsshim installation.
+GOPATH="$(mktemp -d /tmp/cri-install-hcsshim.XXXX)"
+
+# Install hcsshim
+from-vendor HCSSHIM "${HCSSHIM_PKG}"
+checkout_repo "${HCSSHIM_PKG}" "${HCSSHIM_VERSION}" "${HCSSHIM_REPO}"
+cd "${GOPATH}/src/${HCSSHIM_PKG}"
+go build "${HCSSHIM_PKG}/cmd/containerd-shim-runhcs-v1"
+install -D -m 755 containerd-shim-runhcs-v1 "${HCSSHIM_DIR}"/containerd-shim-runhcs-v1
+
+# Clean the tmp GOPATH dir. Use sudo because runc build generates
+# some privileged files.
+rm -rf ${GOPATH}
diff --git a/hack/test-cri.sh b/hack/test-cri.sh
index c464afd6e..24c514549 100755
--- a/hack/test-cri.sh
+++ b/hack/test-cri.sh
@@ -52,7 +52,8 @@ if [ ! -x "$(command -v ${CRITEST})" ]; then
   cd ${GOPATH}/src/${CRITOOL_PKG}
   git fetch --all
   git checkout ${CRITOOL_VERSION}
-  make
+  make critest
+  make install-critest -e BINDIR="${GOPATH}/bin"
 fi
 which ${CRITEST}
 
diff --git a/hack/utils.sh b/hack/utils.sh
index 958d5c2e9..06f865b5e 100755
--- a/hack/utils.sh
+++ b/hack/utils.sh
@@ -17,7 +17,7 @@
 ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"/..
 
 # Not from vendor.conf.
-CRITOOL_VERSION=427262054f59f3b849391310856a19474acb7e83
+CRITOOL_VERSION=84b8540c69e82671bfc244a0b0fe28a6a98381ce
 CRITOOL_PKG=github.com/kubernetes-sigs/cri-tools
 CRITOOL_REPO=github.com/kubernetes-sigs/cri-tools
 
diff --git a/test/windows/runner.sh b/test/windows/runner.sh
new file mode 100755
index 000000000..dfa88ac18
--- /dev/null
+++ b/test/windows/runner.sh
@@ -0,0 +1,98 @@
+#!/bin/bash
+
+# Copyright The containerd Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+set -o xtrace
+
+GCE_PROJECT="${GCE_PROJECT:-"cri-containerd-node-e2e"}"
+GCE_IMAGE="${GCE_IMAGE:-"windows-server-1809-dc-core-for-containers-v20190827"}"
+GCE_IMAGE_PROJECT="${GCE_IMAGE_PROJECT:-"windows-cloud"}"
+ZONE="${ZONE:-"us-west1-b"}"
+ARTIFACTS="${ARTIFACTS:-"/tmp/test-cri-windows/_artifacts"}"
+CLEANUP="${CLEANUP:-"true"}"
+
+root="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"/../..
+script_path="${root}/test/windows"
+node_name="windows-cri-$(cat /proc/sys/kernel/random/uuid)"
+
+# log logs the test logs.
+function log() {
+  echo "$(date) $1"
+}
+
+function cleanup() {
+  if [[ "$CLEANUP" == "true" ]]; then
+    log "Delete the test instance"
+    gcloud compute instances delete -q "${node_name}"
+  fi
+}
+
+function retry() {
+  local -r MAX_ATTEMPTS=$1
+  local attempts=1
+  shift
+  until "$@" || (( attempts == MAX_ATTEMPTS ))
+  do
+    log "$* failed, retry in 1 second..."
+    (( attempts++ ))
+    sleep 1
+  done
+}
+
+gcloud config set compute/zone "${ZONE}"
+gcloud config set project "${GCE_PROJECT}"
+
+log "Create the test instance"
+gcloud compute instances create "${node_name}" --machine-type=n1-standard-2 \
+  --image="${GCE_IMAGE}" --image-project="${GCE_IMAGE_PROJECT}" \
+  --metadata-from-file=windows-startup-script-ps1="${script_path}/setup-ssh.ps1"
+trap cleanup EXIT
+
+log "Wait for ssh to be ready"
+retry 180 gcloud compute ssh "${node_name}" --command="echo ssh ready"
+
+log "Setup test environment in the test instance"
+retry 5 gcloud compute scp "${script_path}/setup-vm.ps1" "${node_name}":"C:/setup-vm.ps1"
+gcloud compute ssh "${node_name}" --command="powershell /c C:/setup-vm.ps1"
+
+log "Reboot the test instance to refresh environment variables"
+gcloud compute ssh "${node_name}" --command="powershell /c Restart-Computer"
+
+log "Wait for ssh to be ready"
+retry 180 gcloud compute ssh "${node_name}" --command="echo ssh ready"
+
+log "Run test on the test instance"
+cri_tar="/tmp/cri.tar.gz"
+tar -zcf "${cri_tar}" -C "${root}" . --owner=0 --group=0
+retry 5 gcloud compute scp "${script_path}/test.sh" "${node_name}":"C:/test.sh"
+retry 5 gcloud compute scp "${cri_tar}" "${node_name}":"C:/cri.tar.gz"
+rm "${cri_tar}"
+# git-bash doesn't return test exit code, the command should
+# succeed. We'll collect test exit code from _artifacts/.
+gcloud compute ssh "${node_name}" --command='powershell /c "Start-Process -FilePath \"C:\Program Files\Git\git-bash.exe\" -ArgumentList \"-elc\",\"`\"/c/test.sh &> /c/test.log`\"\" -Wait"'
+
+log "Collect test logs"
+mkdir -p "${ARTIFACTS}"
+retry 5 gcloud compute scp "${node_name}":"C:/test.log" "${ARTIFACTS}"
+retry 5 gcloud compute scp --recurse "${node_name}":"C:/_artifacts/*" "${ARTIFACTS}"
+
+log "Test output:"
+cat "${ARTIFACTS}/test.log"
+
+exit_code="$(cat "${ARTIFACTS}/exitcode")"
+exit "${exit_code}"
diff --git a/test/windows/setup-ssh.ps1 b/test/windows/setup-ssh.ps1
new file mode 100644
index 000000000..31565666c
--- /dev/null
+++ b/test/windows/setup-ssh.ps1
@@ -0,0 +1,32 @@
+#   Copyright The containerd Authors.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+$ProgressPreference = 'SilentlyContinue'
+$k8sversion = ("v1.16.0-beta.2")
+
+$url = ("https://raw.githubusercontent.com/kubernetes/kubernetes/$k8sversion/cluster/gce/windows/testonly/user-profile.psm1")
+Invoke-WebRequest $url -OutFile C:\user-profile.psm1
+
+$url = ("https://raw.githubusercontent.com/kubernetes/kubernetes/$k8sversion/cluster/gce/windows/common.psm1")
+Invoke-WebRequest $url -OutFile C:\common.psm1
+
+$url = ("https://raw.githubusercontent.com/kubernetes/kubernetes/$k8sversion/cluster/gce/windows/testonly/install-ssh.psm1")
+Invoke-WebRequest $url -OutFile C:\install-ssh.psm1
+
+Import-Module -Force C:\install-ssh.psm1
+
+InstallAndStart-OpenSsh
+
+StartProcess-WriteSshKeys
diff --git a/test/windows/setup-vm.ps1 b/test/windows/setup-vm.ps1
new file mode 100644
index 000000000..c3be5a2c3
--- /dev/null
+++ b/test/windows/setup-vm.ps1
@@ -0,0 +1,23 @@
+#   Copyright The containerd Authors.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+Set-ExecutionPolicy Bypass -Scope Process -Force; iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))
+
+choco install -y --no-progress git
+
+choco install -y --no-progress golang
+
+choco install -y --no-progress make
+
+choco install -y --no-progress mingw
diff --git a/test/windows/test.sh b/test/windows/test.sh
new file mode 100755
index 000000000..9e2c2bb26
--- /dev/null
+++ b/test/windows/test.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+
+# Copyright The containerd Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+export PATH="/c/Program Files/Containerd:$PATH"
+REPO_TAR="${REPO_TAR:-"/c/cri.tar.gz"}"
+FOCUS="${FOCUS:-"Conformance"}"
+SKIP="${SKIP:-"portforward"}"
+REPORT_DIR="${REPORT_DIR:-"/c/_artifacts"}"
+
+repo="$GOPATH/src/github.com/containerd/cri"
+mkdir -p "${repo}"
+cd "${repo}"
+tar -xzf "${REPO_TAR}"
+
+make install.deps
+make install -e BINDIR="/c/Program Files/Containerd"
+
+mkdir -p "${REPORT_DIR}"
+containerd -log-level debug &> "${REPORT_DIR}/containerd.log" &
+pid=$!
+ctr version
+
+set +o errexit
+critest --runtime-endpoint=npipe:////./pipe/containerd-containerd --ginkgo.focus="${FOCUS}" --ginkgo.skip="${SKIP}" --report-dir="${REPORT_DIR}" --report-prefix="windows"
+TEST_RC=$?
+set -o errexit
+kill -9 $pid
+echo -n "${TEST_RC}" > "${REPORT_DIR}/exitcode"