[K3s][Windows Port] Build script, multi-call binary, and Flannel (#7259)

* initial windows port. Signed-off-by: Sean Yen <seanyen@microsoft.com> Signed-off-by: Derek Nola <derek.nola@suse.com> Co-authored-by: Derek Nola <derek.nola@suse.com> Co-authored-by: Wei Ran <weiran@microsoft.com>
2026-07-28 20:59:30 +00:00 · 2023-10-16 11:53:09 -07:00
parent aaf8409096
commit 0c9bf36fe0
21 changed files with 459 additions and 105 deletions
--- a/scripts/build
+++ b/scripts/build
@@ -66,6 +66,10 @@ TAGS="ctrd apparmor seccomp netcgo osusergo providerless urfave_cli_no_docs"
 RUNC_TAGS="apparmor seccomp"
 RUNC_STATIC="static"

+if [ ${OS} = windows ]; then
+    TAGS="ctrd netcgo osusergo providerless"
+fi
+
 if [ "$SELINUX" = "true" ]; then
    TAGS="$TAGS selinux"
    RUNC_TAGS="$RUNC_TAGS selinux"
@@ -97,21 +101,41 @@ if [ ${ARCH} = s390x ]; then
    export GOARCH="s390x"
 fi

-rm -f \
-    bin/k3s-agent \
-    bin/k3s-server \
-    bin/k3s-token \
-    bin/k3s-etcd-snapshot \
-    bin/k3s-secrets-encrypt \
-    bin/k3s-certificate \
-    bin/k3s-completion \
-    bin/kubectl \
-    bin/crictl \
-    bin/ctr \
-    bin/containerd \
-    bin/containerd-shim \
-    bin/containerd-shim-runc-v2 \
-    bin/runc
+k3s_binaries=(
+    "bin/k3s-agent"
+    "bin/k3s-server"
+    "bin/k3s-token"
+    "bin/k3s-etcd-snapshot"
+    "bin/k3s-secrets-encrypt"
+    "bin/k3s-certificate"
+    "bin/k3s-completion"
+    "bin/kubectl"
+    "bin/containerd"
+    "bin/crictl"
+    "bin/ctr"
+)
+
+containerd_binaries=(
+    "bin/containerd-shim"
+    "bin/containerd-shim-runc-v2"
+    "bin/runc"
+    "bin/containerd-shim-runhcs-v1"
+    "bin/runhcs"
+)
+
+for i in "${k3s_binaries[@]}"; do
+    if [ -f "$i${BINARY_POSTFIX}" ]; then
+        echo "Removing $i${BINARY_POSTFIX}"
+        rm -f "$i${BINARY_POSTFIX}"
+    fi
+done
+
+for i in "${containerd_binaries[@]}"; do
+    if [ -f "$i${BINARY_POSTFIX}" ]; then
+        echo "Removing $i${BINARY_POSTFIX}"
+        rm -f "$i${BINARY_POSTFIX}"
+    fi
+done

 cleanup() {
    exit_status=$?
@@ -120,7 +144,7 @@ cleanup() {
 }

 INSTALLBIN=$(pwd)/bin
-if [ ! -x ${INSTALLBIN}/cni ]; then
+if [ ! -x ${INSTALLBIN}/cni${BINARY_POSTFIX} ]; then
 (
    echo Building cni
    TMPDIR=$(mktemp -d)
@@ -128,36 +152,46 @@ if [ ! -x ${INSTALLBIN}/cni ]; then
    WORKDIR=$TMPDIR/src/github.com/containernetworking/plugins
    git clone -b $VERSION_CNIPLUGINS https://github.com/rancher/plugins.git $WORKDIR
    cd $WORKDIR
-    GO111MODULE=off GOPATH=$TMPDIR CGO_ENABLED=0 "${GO}" build -tags "$TAGS" -gcflags="all=${GCFLAGS}" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o $INSTALLBIN/cni
+    GO111MODULE=off GOPATH=$TMPDIR CGO_ENABLED=0 "${GO}" build -tags "$TAGS" -gcflags="all=${GCFLAGS}" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o $INSTALLBIN/cni${BINARY_POSTFIX}
 )
 fi

 echo Building k3s
-CGO_ENABLED=1 "${GO}" build $BLDFLAGS -tags "$TAGS" -buildvcs=false -gcflags="all=${GCFLAGS}" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/k3s ./cmd/server
-ln -s k3s ./bin/containerd
-ln -s k3s ./bin/crictl
-ln -s k3s ./bin/ctr
-ln -s k3s ./bin/k3s-agent
-ln -s k3s ./bin/k3s-certificate
-ln -s k3s ./bin/k3s-completion
-ln -s k3s ./bin/k3s-etcd-snapshot
-ln -s k3s ./bin/k3s-secrets-encrypt
-ln -s k3s ./bin/k3s-server
-ln -s k3s ./bin/k3s-token
-ln -s k3s ./bin/kubectl
+CGO_ENABLED=1 "${GO}" build $BLDFLAGS -tags "$TAGS" -buildvcs=false -gcflags="all=${GCFLAGS}" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/k3s${BINARY_POSTFIX} ./cmd/server
+
+for i in "${k3s_binaries[@]}"; do
+    ln -s "k3s${BINARY_POSTFIX}" "$i${BINARY_POSTFIX}"
+done

 export GOPATH=$(pwd)/build

-echo Building containerd
-pushd ./build/src/github.com/containerd/containerd
-TAGS="${TAGS/netcgo/netgo}"
-CGO_ENABLED=1 "${GO}" build -tags "$TAGS" -gcflags="all=${GCFLAGS}" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/containerd-shim-runc-v2 ./cmd/containerd-shim-runc-v2
-popd
-cp -vf ./build/src/github.com/containerd/containerd/bin/* ./bin/
+case ${OS} in
+  linux)
+    echo Building containerd-shim
+    pushd ./build/src/github.com/containerd/containerd
+    TAGS="${TAGS/netcgo/netgo}"
+    CGO_ENABLED=1 "${GO}" build -tags "$TAGS" -gcflags="all=${GCFLAGS}" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/containerd-shim-runc-v2 ./cmd/containerd-shim-runc-v2
+    popd
+    cp -vf ./build/src/github.com/containerd/containerd/bin/* ./bin/

-echo Building runc
-pushd ./build/src/github.com/opencontainers/runc
-rm -f runc
-make EXTRA_FLAGS="-gcflags=\"all=${GCFLAGS}\"" EXTRA_LDFLAGS="$LDFLAGS" BUILDTAGS="$RUNC_TAGS" $RUNC_STATIC
-popd
-cp -vf ./build/src/github.com/opencontainers/runc/runc ./bin/
+    echo Building runc
+    pushd ./build/src/github.com/opencontainers/runc
+    rm -f runc
+    make EXTRA_FLAGS="-gcflags=\"all=${GCFLAGS}\"" EXTRA_LDFLAGS="$LDFLAGS" BUILDTAGS="$RUNC_TAGS" $RUNC_STATIC
+    popd
+    cp -vf ./build/src/github.com/opencontainers/runc/runc ./bin/
+    ;;
+  windows)
+    echo Building containerd-shim-runhcs-v1
+    pushd ./build/src/github.com/microsoft/hcsshim
+    TAGS="${TAGS/netcgo/netgo}"
+    CGO_ENABLED=0 "${GO}" build -tags "$TAGS" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/containerd-shim-runhcs-v1${BINARY_POSTFIX} ./cmd/containerd-shim-runhcs-v1
+    CGO_ENABLED=0 "${GO}" build -tags "$TAGS" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/runhcs${BINARY_POSTFIX} ./cmd/runhcs
+    popd
+    cp -vf ./build/src/github.com/microsoft/hcsshim/bin/*${BINARY_POSTFIX} ./bin/
+    ;;
+  *)
+    echo "[ERROR] unrecognized opertaing system: ${OS}"
+    exit 1
+    ;;
+esac
--- a/scripts/download
+++ b/scripts/download
@@ -10,6 +10,7 @@ CHARTS_URL=https://k3s.io/k3s-charts/assets
 CHARTS_DIR=build/static/charts
 RUNC_DIR=build/src/github.com/opencontainers/runc
 CONTAINERD_DIR=build/src/github.com/containerd/containerd
+HCSSHIM_DIR=build/src/github.com/microsoft/hcsshim
 DATA_DIR=build/data
 export TZ=UTC

@@ -17,12 +18,24 @@ umask 022
 rm -rf ${CHARTS_DIR}
 rm -rf ${RUNC_DIR}
 rm -rf ${CONTAINERD_DIR}
+rm -rf ${HCSSHIM_DIR}
 mkdir -p ${CHARTS_DIR}
 mkdir -p ${DATA_DIR}

-curl --compressed -sfL https://github.com/k3s-io/k3s-root/releases/download/${VERSION_ROOT}/k3s-root-${ARCH}.tar | tar xf -
-
-git clone --single-branch --branch=${VERSION_RUNC} --depth=1 https://github.com/opencontainers/runc ${RUNC_DIR}
+case ${OS} in
+  linux)
+    git clone --single-branch --branch=${VERSION_RUNC} --depth=1 https://github.com/opencontainers/runc ${RUNC_DIR}
+    curl --compressed -sfL https://github.com/k3s-io/k3s-root/releases/download/${VERSION_ROOT}/k3s-root-${ARCH}.tar | tar xf -
+    cp scripts/wg-add.sh bin/aux
+    ;;
+  windows)
+    git clone --single-branch --branch=${VERSION_HCSSHIM} --depth=1 https://github.com/microsoft/hcsshim ${HCSSHIM_DIR}
+    ;;
+  *)
+    echo "[ERROR] unrecognized opertaing system: ${OS}"
+    exit 1
+    ;;
+esac

 git clone --single-branch --branch=${VERSION_CONTAINERD} --depth=1 https://${PKG_CONTAINERD_K3S} ${CONTAINERD_DIR}

@@ -30,5 +43,3 @@ for CHART_FILE in $(grep -rlF HelmChart manifests/ | xargs yq eval --no-doc .spe
  CHART_NAME=$(echo $CHART_FILE | grep -oE '^(-*[a-z])+')
  curl -sfL ${CHARTS_URL}/${CHART_NAME}/${CHART_FILE} -o ${CHARTS_DIR}/${CHART_FILE}
 done
-
-cp scripts/wg-add.sh bin/aux
--- a/scripts/package-cli
+++ b/scripts/package-cli
@@ -8,13 +8,31 @@ cd $(dirname $0)/..
 GO=${GO-go}

 for i in containerd crictl kubectl k3s-agent k3s-server k3s-token k3s-etcd-snapshot k3s-secrets-encrypt k3s-certificate k3s-completion; do
-    rm -f bin/$i
-    ln -s k3s bin/$i
+    rm -f bin/$i${BINARY_POSTFIX}
+    ln -s k3s${BINARY_POSTFIX} bin/$i${BINARY_POSTFIX}
 done

-for i in bandwidth bridge firewall flannel host-local loopback portmap; do
-    rm -f bin/$i
-    ln -s cni bin/$i
+cni_binaries=(
+    "bandwidth"
+    "bridge"
+    "firewall"
+    "flannel"
+    "host-local"
+    "loopback"
+    "portmap"
+)
+
+if [ ${OS} = windows ]; then
+    cni_binaries=(
+        "win-overlay"
+        "flannel"
+        "host-local"
+    )
+fi
+
+for i in "${cni_binaries[@]}"; do
+    rm -f bin/$i${BINARY_POSTFIX}
+    ln -s cni${BINARY_POSTFIX} bin/$i${BINARY_POSTFIX}
 done

 cp contrib/util/check-config.sh bin/check-config
@@ -22,6 +40,7 @@ cp contrib/util/check-config.sh bin/check-config
 rm -rf build/data
 mkdir -p build/data build/out
 mkdir -p dist/artifacts
+mkdir -p ./etc

 (
    set +x
@@ -50,9 +69,9 @@ elif [ ${ARCH} = s390x ]; then
    BIN_SUFFIX="-s390x"
 fi

-CMD_NAME=dist/artifacts/k3s${BIN_SUFFIX}
+CMD_NAME=dist/artifacts/k3s${BIN_SUFFIX}${BINARY_POSTFIX}

-"${GO}" generate
+GOOS=linux CC=gcc CXX=g++ "${GO}" generate
 LDFLAGS="
    -X github.com/k3s-io/k3s/pkg/version.Version=$VERSION
    -X github.com/k3s-io/k3s/pkg/version.GitCommit=${COMMIT:0:8}
@@ -60,7 +79,7 @@ LDFLAGS="
 "
 TAGS="urfave_cli_no_docs"
 STATIC="-extldflags '-static'"
-CGO_ENABLED=0 "${GO}" build -tags "$TAGS" -ldflags "$LDFLAGS $STATIC" -o ${CMD_NAME} ./cmd/k3s/main.go
+CGO_ENABLED=0 "${GO}" build -tags "$TAGS" -buildvcs=false -ldflags "$LDFLAGS $STATIC" -o ${CMD_NAME} ./cmd/k3s

 stat ${CMD_NAME}

--- a/scripts/validate
+++ b/scripts/validate
@@ -15,7 +15,7 @@ echo Running: go mod tidy
 go mod tidy

 echo Running: go generate
-go generate
+GOOS=linux CC=gcc CXX=g++ go generate

 echo Running validation

--- a/scripts/version.sh
+++ b/scripts/version.sh
@@ -2,6 +2,7 @@

 GO=${GO-go}
 ARCH=${ARCH:-$("${GO}" env GOARCH)}
+OS=${OS:-$("${GO}" env GOOS)}
 SUFFIX="-${ARCH}"
 GIT_TAG=$DRONE_TAG
 TREE_STATE=clean
@@ -52,6 +53,11 @@ if [ -z "$VERSION_RUNC" ]; then
    VERSION_RUNC="v0.0.0"
 fi

+VERSION_HCSSHIM=$(get-module-version github.com/Microsoft/hcsshim)
+if [ -z "$VERSION_HCSSHIM" ]; then
+    VERSION_HCSSHIM="v0.0.0"
+fi
+
 VERSION_FLANNEL=$(get-module-version github.com/flannel-io/flannel)
 if [ -z "$VERSION_FLANNEL" ]; then
  VERSION_FLANNEL="v0.0.0"
@@ -81,3 +87,8 @@ else
    VERSION="$VERSION_K8S+k3s-${COMMIT:0:8}$DIRTY"
 fi
 VERSION_TAG="$(sed -e 's/+/-/g' <<< "$VERSION")"
+
+BINARY_POSTFIX=
+if [ ${OS} = windows ]; then
+    BINARY_POSTFIX=.exe
+fi