Update to v1.28.2 and go v1.20.8 (#8364)

* Update to v1.28.2

Signed-off-by: Johnatas <johnatasr@hotmail.com>

* Bump containerd and stargz versions

Signed-off-by: Brad Davidson <brad.davidson@rancher.com>

* Print message on upgrade fail

Signed-off-by: Brad Davidson <brad.davidson@rancher.com>

* Send Bad Gateway instead of Service Unavailable when tunnel dial fails

Works around new handling for Service Unavailable by apiserver aggregation added in kubernetes/kubernetes#119870

Signed-off-by: Brad Davidson <brad.davidson@rancher.com>

* Add 60 seconds to server upgrade wait to account for delays in apiserver readiness

Also change cleanup helper to ensure upgrade test doesn't pollute the
images for the rest of the tests.

Signed-off-by: Brad Davidson <brad.davidson@rancher.com>

---------

Signed-off-by: Johnatas <johnatasr@hotmail.com>
Signed-off-by: Brad Davidson <brad.davidson@rancher.com>
Co-authored-by: Brad Davidson <brad.davidson@rancher.com>

scripts/build

@@ -9,7 +9,6 @@ GO=${GO-go}
 
 PKG="github.com/k3s-io/k3s"
 PKG_CONTAINERD="github.com/containerd/containerd"
-PKG_K3S_CONTAINERD="github.com/k3s-io/containerd"
 PKG_CRICTL="github.com/kubernetes-sigs/cri-tools/pkg"
 PKG_K8S_BASE="k8s.io/component-base"
 PKG_K8S_CLIENT="k8s.io/client-go/pkg"
@@ -37,7 +36,7 @@ VERSIONFLAGS="
     -X ${PKG_CRICTL}/version.Version=${VERSION_CRICTL}
 
     -X ${PKG_CONTAINERD}/version.Version=${VERSION_CONTAINERD}
-    -X ${PKG_CONTAINERD}/version.Package=${PKG_K3S_CONTAINERD}
+    -X ${PKG_CONTAINERD}/version.Package=${PKG_CONTAINERD_K3S}
 
     -X ${PKG_CNI_PLUGINS}/pkg/utils/buildversion.BuildVersion=${VERSION_CNIPLUGINS}
     -X ${PKG_CNI_PLUGINS}/plugins/meta/flannel.Program=flannel

scripts/download

@@ -24,7 +24,7 @@ curl --compressed -sfL https://github.com/k3s-io/k3s-root/releases/download/${VE
 
 git clone --single-branch --branch=${VERSION_RUNC} --depth=1 https://github.com/opencontainers/runc ${RUNC_DIR}
 
-git clone --single-branch --branch=${VERSION_CONTAINERD} --depth=1 https://github.com/k3s-io/containerd ${CONTAINERD_DIR}
+git clone --single-branch --branch=${VERSION_CONTAINERD} --depth=1 https://${PKG_CONTAINERD_K3S} ${CONTAINERD_DIR}
 
 for CHART_FILE in $(grep -rlF HelmChart manifests/ | xargs yq eval --no-doc .spec.chart | xargs -n1 basename); do
   CHART_NAME=$(echo $CHART_FILE | grep -oE '^(-*[a-z])+')

scripts/test-helpers

@@ -534,8 +534,8 @@ cleanup-test-env(){
       export SERVER_ARGS=''
       export WAIT_SERVICES="${all_services[*]}"
 
-      unset AGENT_1_ARGS AGENT_2_ARGS AGENT_3_ARGS AGENT_DOCKER_ARGS
-      unset SERVER_1_ARGS SERVER_2_ARGS SERVER_3_ARGS SERVER_DOCKER_ARGS
+      unset AGENT_1_ARGS AGENT_2_ARGS AGENT_3_ARGS AGENT_DOCKER_ARGS K3S_IMAGE_AGENT
+      unset SERVER_1_ARGS SERVER_2_ARGS SERVER_3_ARGS SERVER_DOCKER_ARGS K3S_IMAGE_SERVER
 
       unset -f server-pre-hook server-post-hook agent-pre-hook agent-post-hook cluster-pre-hook cluster-post-hook test-post-hook test-cleanup-hook
 }

scripts/test-run-upgrade

@@ -25,6 +25,7 @@ server-pre-hook(){
     local testID=$(basename $TEST_DIR)
     export SERVER_DOCKER_ARGS="\
         --mount type=volume,src=k3s-server-$1-${testID,,}-rancher,dst=/var/lib/rancher/k3s \
+        --mount type=volume,src=k3s-server-$1-${testID,,}-log,dst=/var/log \
         --mount type=volume,src=k3s-server-$1-${testID,,}-etc,dst=/etc/rancher"
 }
 export -f server-pre-hook
@@ -33,6 +34,7 @@ agent-pre-hook(){
     local testID=$(basename $TEST_DIR)
     export AGENT_DOCKER_ARGS="\
         --mount type=volume,src=k3s-agent-$1-${testID,,}-rancher,dst=/var/lib/rancher/k3s \
+        --mount type=volume,src=k3s-agent-$1-${testID,,}-log,dst=/var/log \
         --mount type=volume,src=k3s-agent-$1-${testID,,}-etc,dst=/etc/rancher"
 }
 export -f agent-pre-hook
@@ -42,9 +44,11 @@ start-test() {
     kubectl get node -o wide
     kubectl create -f scripts/airgap/volume-test.yaml
 
-    # Add post-hook sleeps to give the kubelet time to update the version after startup
+    # Add post-hook sleeps to give the kubelet time to update the version after startup.
+    # Server gets an extra 60 seconds to handle the metrics-server service being unavailable:
+    # https://github.com/kubernetes/kubernetes/issues/120739
     server-post-hook(){
-        sleep 15
+        sleep 75
     }
     export -f server-post-hook
     agent-post-hook(){
@@ -69,8 +73,15 @@ start-test() {
 
     # Confirm that the nodes are running the current build and that the pod we created earlier is still there
     . ./scripts/version.sh || true
-    kubectl get node -o wide | grep -F $VERSION
+
+    verify-valid-versions $(cat $TEST_DIR/servers/1/metadata/name)
+
     kubectl get pod -n kube-system volume-test -o wide
+
+    if ! kubectl get node -o wide | grep -qF $VERSION; then
+      echo "Expected version $VERSION not found in node list"
+      return 1
+    fi
 }
 export -f start-test
 

scripts/version.sh

@@ -26,6 +26,11 @@ get-module-version(){
   go list -m -f '{{if .Replace}}{{.Replace.Version}}{{else}}{{.Version}}{{end}}' $1
 }
 
+get-module-path(){
+  go list -m -f '{{if .Replace}}{{.Replace.Path}}{{else}}{{.Path}}{{end}}' $1
+}
+
+PKG_CONTAINERD_K3S=$(get-module-path github.com/containerd/containerd)
 VERSION_CONTAINERD=$(get-module-version github.com/containerd/containerd)
 if [ -z "$VERSION_CONTAINERD" ]; then
     VERSION_CONTAINERD="v0.0.0"