public/k3s
1
0
Fork 0
mirror of https://git.vectorsigma.ru/public/k3s.git synced 2026-07-28 20:59:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Run Docker test on Github Actions arm64 runners (#11705)

Browse Source 
* Build and Run Arm64 Docker Tests
* Compile go tests for both amd64 and arm64
* Use version.sh to determine channel, not git
* Consolidate docker tests into 2D matrix
* Remove old Docker test from Drone and Github Actions

Signed-off-by: Derek Nola <derek.nola@suse.com>
This commit is contained in:
Derek Nola
2025-02-04 08:29:47 -08:00
committed by GitHub
parent f78769dd92
commit 1666b5cfff
5 changed files with 74 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
.github/workflows/build-k3s.yaml vendored
View File

@@ -3,6 +3,10 @@ name: Build K3s
on:
workflow_call:
inputs:
arch:
type: string
description: 'Architecture to build'
default: 'ubuntu-latest'
upload-repo:
type: boolean
required: false
@@ -18,7 +22,7 @@ permissions:
jobs:
build:
name: Build
runs-on: ubuntu-latest
runs-on: ${{ inputs.arch }} # defaults to ubuntu-latest, for arm64 use ubuntu-24.04-arm
timeout-minutes: 20
steps:
- name: Checkout K3s
@@ -44,9 +48,15 @@ jobs:
- name: "Save K3s image"
if: inputs.upload-image == true
run: docker image save rancher/k3s -o ./dist/artifacts/k3s-image.tar
- name: "Upload K3s binary"
if: inputs.upload-repo == false
- name: "Upload K3s Artifacts"
if: inputs.upload-repo == false && inputs.arch == 'ubuntu-latest'
uses: actions/upload-artifact@v4
with:
name: k3s
path: dist/artifacts/k3s*
- name: "Upload K3s arm64 Artifacts"
if: contains(inputs.arch, 'arm')
uses: actions/upload-artifact@v4
with:
name: k3s-arm64
path: dist/artifacts/k3s*

112
.github/workflows/e2e.yaml vendored
View File

@@ -32,6 +32,11 @@ jobs:
uses: ./.github/workflows/build-k3s.yaml
with:
upload-image: true
build-arm64:
uses: ./.github/workflows/build-k3s.yaml
with:
arch: ubuntu-24.04-arm
upload-image: true
e2e:
name: "E2E Tests"
needs: build
@@ -100,40 +105,15 @@ jobs:
files: tests/e2e/${{ matrix.etest }}/coverage.out
flags: e2etests # optional
verbose: true # optional (default = false)
docker:
needs: build
name: Docker Tests
runs-on: ubuntu-latest
timeout-minutes: 20
strategy:
fail-fast: false
matrix:
dtest: [basics, bootstraptoken, cacerts, compat, lazypull, upgrade]
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 1
- name: "Download k3s image"
uses: actions/download-artifact@v4
with:
name: k3s
path: ./dist/artifacts
- name: Load k3s image
run: docker image load -i ./dist/artifacts/k3s-image.tar
- name: Run ${{ matrix.dtest }} Test
run: |
chmod +x ./dist/artifacts/k3s
. ./scripts/version.sh
. ./tests/docker/test-helpers
. ./tests/docker/test-run-${{ matrix.dtest }}
echo "Did test-run-${{ matrix.dtest }} pass $?"
build-go-tests:
name: "Build Go Tests"
runs-on: ubuntu-latest
strategy:
matrix:
arch: [amd64, arm64]
outputs:
branch_name: ${{ steps.branch_step.outputs.BRANCH_NAME }}
channel: ${{ steps.channel_step.outputs.channel }}
steps:
- name: Checkout
uses: actions/checkout@v4
@@ -142,70 +122,62 @@ jobs:
- name: Build Go Tests
run: |
mkdir -p ./dist/artifacts
go test -c -ldflags="-w -s" -o ./dist/artifacts ./tests/docker/...
GOOS=linux GOARCH=${{ matrix.arch }} go test -c -ldflags="-w -s" -o ./dist/artifacts ./tests/docker/...
- name: Upload Go Tests
uses: actions/upload-artifact@v4
with:
name: docker-go-tests
name: docker-go-tests-${{ matrix.arch }}
path: ./dist/artifacts/*.test
compression-level: 9
retention-days: 1
# For upgrade and skew tests, we need to know the branch name this run is based off.
# Since this is predetermined, we can run this step before the docker-go job, saving time.
# For PRs we can use the base_ref (ie the target branch of the PR).
# For pushes to k3s-io/k3s from dependabot or updatecli, use master
# All other pushes should be a valid ref, master or release-1.XX.
# For pushes to a fork, we need to determine the branch name by finding the parent branch from git show-branch history.
- name: Determine branch name
id: branch_step
# For upgrade and skew tests, we need to know the channel this run is based off.
# Since this is predetermined, we can run this step before the actual test job, saving time.
- name: Determine channel
id: channel_step
run: |
if [ ${{ github.repository }} = "k3s-io/k3s" ]; then
BRANCH_NAME=$(echo ${{ github.base_ref || github.ref_name }})
if [[ $BRANCH_NAME =~ ^(dependabot|updatecli) ]]; then
BRANCH_NAME=master
fi
elif [ -z "${{ github.base_ref }}" ]; then
# We are in a fork, and need some git history to determine the branch name
# For some reason, the first fetch doesn't always get the full history, so we sleep and fetch again
git fetch origin --depth=100 +refs/heads/*:refs/remotes/origin/*
sleep 5
git fetch origin --depth=100 +refs/heads/*:refs/remotes/origin/*
BRANCH_NAME=$(git show-branch -a 2> /dev/null | grep '\*' | grep -v `git rev-parse --abbrev-ref HEAD` | head -n1 | sed 's/.*\[\(.*\/\)\(.*\)\].*/\2/' | sed 's/[\^~].*//')
else
BRANCH_NAME=${{ github.base_ref }}
fi
echo "Branch Name is $BRANCH_NAME"
echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_OUTPUT
# branch name should be either master or release-1.XX
- name: Fail if branch name does not match pattern
. ./scripts/version.sh
MINOR_VER=$(echo $VERSION_TAG | cut -d'.' -f1,2)
echo "CHANNEL=$MINOR_VER" >> $GITHUB_OUTPUT
# channel name should be v1.XX or latest
- name: Fail if channel name does not match pattern
run: |
if [[ ! ${{ steps.branch_step.outputs.branch_name }} =~ ^(master|release-[0-9]+\.[0-9]+)$ ]]; then
echo "Branch name ${{ steps.branch_step.outputs.branch_name }} does not match pattern"
echo "If this is a PR/fork, ensure you have recently rebased off master/release-1.XX branch"
if [[ ! ${{ steps.channel_step.outputs.channel }} =~ ^v1\.[0-9]+$|latest$ ]]; then
echo "Channel name ${{ steps.channel_step.outputs.channel }} does not match pattern"
exit 1
fi
docker-go:
needs: [build, build-go-tests]
name: Docker Tests In GO
runs-on: ubuntu-latest
timeout-minutes: 20
needs: [build, build-arm64, build-go-tests]
name: Docker
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
dtest: [basics, bootstraptoken, cacerts, etcd, lazypull, skew, snapshotrestore, upgrade]
arch: [amd64, arm64]
runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }}
env:
BRANCH_NAME: ${{ needs.build-go-tests.outputs.branch_name }}
CHANNEL: ${{ needs.build-go-tests.outputs.channel }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: "Download K3s image"
- name: "Download K3s image (amd64)"
if: ${{ matrix.arch == 'amd64' }}
uses: actions/download-artifact@v4
with:
name: k3s
path: ./dist/artifacts
- name: "Download K3s image (arm64)"
if: ${{ matrix.arch == 'arm64' }}
uses: actions/download-artifact@v4
with:
name: k3s-arm64
path: ./dist/artifacts
- name: Load and set K3s image
run: |
if [ ${{ matrix.arch }} = "arm64" ]; then
mv ./dist/artifacts/k3s-arm64 ./dist/artifacts/k3s
fi
chmod +x ./dist/artifacts/k3s
docker image load -i ./dist/artifacts/k3s-image.tar
IMAGE_TAG=$(docker image ls --format '{{.Repository}}:{{.Tag}}' | grep 'rancher/k3s')
@@ -213,7 +185,7 @@ jobs:
- name: Download Go Tests
uses: actions/download-artifact@v4
with:
name: docker-go-tests
name: docker-go-tests-${{ matrix.arch }}
path: ./dist/artifacts
- name: Run ${{ matrix.dtest }} Test
# Put the compiled test binary back in the same place as the test source
@@ -222,7 +194,7 @@ jobs:
mv ./dist/artifacts/${{ matrix.dtest }}.test ./tests/docker/${{ matrix.dtest }}/
cd ./tests/docker/${{ matrix.dtest }}
if [ ${{ matrix.dtest }} = "upgrade" ] || [ ${{ matrix.dtest }} = "skew" ]; then
./${{ matrix.dtest }}.test -k3sImage=$K3S_IMAGE -branch=$BRANCH_NAME
./${{ matrix.dtest }}.test -k3sImage=$K3S_IMAGE -channel=$CHANNEL
elif [ ${{ matrix.dtest }} = "snapshotrestore" ]; then
./${{ matrix.dtest }}.test -ci
else

10
scripts/test
View File

@@ -20,8 +20,9 @@ mkdir -p $artifacts
docker ps
# ---
# Only run basic tests on non amd64 archs, we use GitHub Actions for amd64
if [ "$ARCH" != 'amd64' ]; then
# Only run PR tests on arm arch, we use GitHub Actions for amd64 and arm64
# Run all tests on tag events, as we want test failures to block the release
if [ "$ARCH" == 'arm' ] || [ "$DRONE_BUILD_EVENT" = 'tag' ]; then
export K3S_IMAGE="rancher/k3s:${VERSION_TAG}${SUFFIX}"
go test ./tests/docker/basics/basics_test.go -k3sImage="$K3S_IMAGE"
@@ -47,13 +48,10 @@ if [ "$ARCH" != 'amd64' ]; then
fi
#TODO convert this to new go test framework
. ./tests/docker/test-run-hardened
echo "Did test-run-hardened $?"
. ./tests/docker/test-run-etcd
echo "Did test-run-etcd $?"
# ---
[ "$ARCH" != 'amd64' ] && \

14
tests/docker/skew/skew_test.go
View File

@@ -15,7 +15,7 @@ import (
// Using these two flags, we upgrade from the latest release of <branch> to
// the current commit build of K3s defined by <k3sImage>
var k3sImage = flag.String("k3sImage", "", "The current commit build of K3s")
var branch = flag.String("branch", "master", "The release branch to test")
var channel = flag.String("channel", "latest", "The release channel to test")
var config *tester.TestConfig
func Test_DockerSkew(t *testing.T) {
@@ -30,19 +30,17 @@ var _ = BeforeSuite(func() {
// For master and unreleased branches, we want the latest stable release
var upgradeChannel string
var err error
if *branch == "master" || *branch == "release-1.32" {
if *channel == "latest" || *channel == "v1.32" {
// disabled: AuthorizeNodeWithSelectors is now on by default, which breaks compat with agents < v1.32.
// This can be ren-enabled once the previous branch is v1.32 or higher, or when RBAC changes have been backported.
// ref: https://github.com/kubernetes/kubernetes/pull/128168
Skip("Skipping version skew tests for " + *branch + " due to AuthorizeNodeWithSelectors")
Skip("Skipping version skew tests for " + *channel + " due to AuthorizeNodeWithSelectors")
upgradeChannel = "stable"
} else {
upgradeChannel = strings.Replace(*branch, "release-", "v", 1)
// now that it is in v1.1 format, we want to substract one from the minor version
// to get the previous release
sV, err := semver.ParseTolerant(upgradeChannel)
Expect(err).NotTo(HaveOccurred(), "failed to parse version from "+upgradeChannel)
// We want to substract one from the minor version to get the previous release
sV, err := semver.ParseTolerant(*channel)
Expect(err).NotTo(HaveOccurred(), "failed to parse version from "+*channel)
sV.Minor--
upgradeChannel = fmt.Sprintf("v%d.%d", sV.Major, sV.Minor)
}

24
tests/docker/upgrade/upgrade_test.go
View File

@@ -17,7 +17,7 @@ import (
// Using these two flags, we upgrade from the latest release of <branch> to
// the current commit build of K3s defined by <k3sImage>
var k3sImage = flag.String("k3sImage", "", "The current commit build of K3s")
var branch = flag.String("branch", "master", "The release branch to test")
var channel = flag.String("channel", "latest", "The release channel to test")
var config *tester.TestConfig
var numServers = 1
@@ -34,22 +34,15 @@ var _ = Describe("Upgrade Tests", Ordered, func() {
Context("Setup Cluster with Lastest Release", func() {
var latestVersion string
It("should determine latest branch version", func() {
var upgradeChannel string
var err error
if *branch == "master" {
upgradeChannel = "latest"
} else {
upgradeChannel = strings.Replace(*branch, "release-", "v", 1)
url := fmt.Sprintf("https://update.k3s.io/v1-release/channels/%s", upgradeChannel)
resp, err := http.Head(url)
// Cover the case where the branch does not exist yet,
// such as a new unreleased minor version
if err != nil || resp.StatusCode != http.StatusOK {
upgradeChannel = "latest"
}
url := fmt.Sprintf("https://update.k3s.io/v1-release/channels/%s", *channel)
resp, err := http.Head(url)
// Cover the case where the branch does not exist yet,
// such as a new unreleased minor version
if err != nil || resp.StatusCode != http.StatusOK {
*channel = "latest"
}
latestVersion, err = tester.GetVersionFromChannel(upgradeChannel)
latestVersion, err = tester.GetVersionFromChannel(*channel)
Expect(err).NotTo(HaveOccurred())
Expect(latestVersion).To(ContainSubstring("v1."))
fmt.Println("Using latest version: ", latestVersion)
@@ -131,6 +124,7 @@ var _ = Describe("Upgrade Tests", Ordered, func() {
Expect(err).NotTo(HaveOccurred())
cVersion := strings.Split(*k3sImage, ":")[1]
cVersion = strings.Replace(cVersion, "-amd64", "", 1)
cVersion = strings.Replace(cVersion, "-arm64", "", 1)
cVersion = strings.Replace(cVersion, "-", "+", 1)
Expect(out).To(ContainSubstring(cVersion))
}