diff --git a/.drone.yml b/.drone.yml index fc7c9791ea..06412183ec 100644 --- a/.drone.yml +++ b/.drone.yml @@ -9,12 +9,14 @@ platform: steps: - name: build image: rancher/dapper:v0.5.0 - secrets: [ gcloud_auth, unprivileged_github_token ] + secrets: [ AWS_SECRET_ACCESS_KEY-artifact-uploader, AWS_ACCESS_KEY_ID-artifact-uploader, unprivileged_github_token ] environment: GITHUB_TOKEN: from_secret: unprivileged_github_token - GCLOUD_AUTH: - from_secret: gcloud_auth + AWS_SECRET_ACCESS_KEY: + from_secret: AWS_SECRET_ACCESS_KEY-artifact-uploader + AWS_ACCESS_KEY_ID: + from_secret: AWS_ACCESS_KEY_ID-artifact-uploader commands: - dapper ci - echo "${DRONE_TAG}-amd64" | sed -e 's/+/-/g' >.tags @@ -88,11 +90,13 @@ steps: - name: test image: rancher/dapper:v0.5.0 - secrets: [ gcloud_auth ] + secrets: [ AWS_SECRET_ACCESS_KEY-artifact-uploader, AWS_ACCESS_KEY_ID-artifact-uploader ] environment: ENABLE_REGISTRY: 'true' - GCLOUD_AUTH: - from_secret: gcloud_auth + AWS_SECRET_ACCESS_KEY: + from_secret: AWS_SECRET_ACCESS_KEY-artifact-uploader + AWS_ACCESS_KEY_ID: + from_secret: AWS_ACCESS_KEY_ID-artifact-uploader commands: - dapper -f Dockerfile.test.dapper volumes: @@ -132,10 +136,12 @@ platform: steps: - name: build image: rancher/dapper:v0.5.0 - secrets: [ gcloud_auth ] + secrets: [ AWS_SECRET_ACCESS_KEY-artifact-uploader, AWS_ACCESS_KEY_ID-artifact-uploader ] environment: - GCLOUD_AUTH: - from_secret: gcloud_auth + AWS_SECRET_ACCESS_KEY: + from_secret: AWS_SECRET_ACCESS_KEY-artifact-uploader + AWS_ACCESS_KEY_ID: + from_secret: AWS_ACCESS_KEY_ID-artifact-uploader commands: - dapper ci - echo "${DRONE_TAG}-arm64" | sed -e 's/+/-/g' >.tags @@ -184,11 +190,13 @@ steps: - name: test image: rancher/dapper:v0.5.0 - secrets: [ gcloud_auth ] + secrets: [ AWS_SECRET_ACCESS_KEY-artifact-uploader, AWS_ACCESS_KEY_ID-artifact-uploader ] environment: ENABLE_REGISTRY: 'true' - GCLOUD_AUTH: - from_secret: gcloud_auth + AWS_SECRET_ACCESS_KEY: + from_secret: AWS_SECRET_ACCESS_KEY-artifact-uploader + AWS_ACCESS_KEY_ID: + from_secret: AWS_ACCESS_KEY_ID-artifact-uploader commands: - dapper -f Dockerfile.test.dapper volumes: @@ -211,10 +219,12 @@ platform: steps: - name: build image: rancher/dapper:v0.5.0 - secrets: [ gcloud_auth ] + secrets: [ AWS_SECRET_ACCESS_KEY-artifact-uploader, AWS_ACCESS_KEY_ID-artifact-uploader ] environment: - GCLOUD_AUTH: - from_secret: gcloud_auth + AWS_SECRET_ACCESS_KEY: + from_secret: AWS_SECRET_ACCESS_KEY-artifact-uploader + AWS_ACCESS_KEY_ID: + from_secret: AWS_ACCESS_KEY_ID-artifact-uploader commands: - dapper ci - echo "${DRONE_TAG}-arm" | sed -e 's/+/-/g' >.tags @@ -263,11 +273,13 @@ steps: - name: test image: rancher/dapper:v0.5.0 - secrets: [ gcloud_auth ] + secrets: [ AWS_SECRET_ACCESS_KEY-artifact-uploader, AWS_ACCESS_KEY_ID-artifact-uploader ] environment: ENABLE_REGISTRY: 'true' - GCLOUD_AUTH: - from_secret: gcloud_auth + AWS_SECRET_ACCESS_KEY: + from_secret: AWS_SECRET_ACCESS_KEY-artifact-uploader + AWS_ACCESS_KEY_ID: + from_secret: AWS_ACCESS_KEY_ID-artifact-uploader commands: - dapper -f Dockerfile.test.dapper volumes: @@ -304,10 +316,12 @@ steps: - name: build image: rancher/dapper:v0.5.8 - secrets: [ gcloud_auth ] + secrets: [ AWS_SECRET_ACCESS_KEY-artifact-uploader, AWS_ACCESS_KEY_ID-artifact-uploader ] environment: - GCLOUD_AUTH: - from_secret: gcloud_auth + AWS_SECRET_ACCESS_KEY: + from_secret: AWS_SECRET_ACCESS_KEY-artifact-uploader + AWS_ACCESS_KEY_ID: + from_secret: AWS_ACCESS_KEY_ID-artifact-uploader commands: - dapper ci - echo "${DRONE_TAG}-s390x" | sed -e 's/+/-/g' >.tags @@ -359,11 +373,13 @@ steps: - name: test image: rancher/dapper:v0.5.8 - secrets: [ gcloud_auth ] + secrets: [ AWS_SECRET_ACCESS_KEY-artifact-uploader, AWS_ACCESS_KEY_ID-artifact-uploader ] environment: ENABLE_REGISTRY: 'true' - GCLOUD_AUTH: - from_secret: gcloud_auth + AWS_SECRET_ACCESS_KEY: + from_secret: AWS_SECRET_ACCESS_KEY-artifact-uploader + AWS_ACCESS_KEY_ID: + from_secret: AWS_ACCESS_KEY_ID-artifact-uploader commands: - dapper -f Dockerfile.test.dapper volumes: diff --git a/Dockerfile.dapper b/Dockerfile.dapper index 3c16bb1936..4cfe5ec13c 100644 --- a/Dockerfile.dapper +++ b/Dockerfile.dapper @@ -9,7 +9,7 @@ ENV https_proxy=$https_proxy ENV no_proxy=$no_proxy RUN apk -U --no-cache add bash git gcc musl-dev docker vim less file curl wget ca-certificates jq linux-headers \ - zlib-dev tar zip squashfs-tools npm coreutils python3 openssl-dev libffi-dev libseccomp libseccomp-dev \ + zlib-dev tar zip squashfs-tools npm coreutils python3 py3-pip openssl-dev libffi-dev libseccomp libseccomp-dev \ libseccomp-static make libuv-static sqlite-dev sqlite-static libselinux libselinux-dev zlib-dev zlib-static \ zstd pigz alpine-sdk binutils-gold btrfs-progs-dev btrfs-progs-static gawk \ && \ @@ -17,6 +17,8 @@ RUN apk -U --no-cache add bash git gcc musl-dev docker vim less file curl wget c apk -U --no-cache add mingw-w64-gcc; \ fi +RUN python3 -m pip install awscli + RUN if [ "$(go env GOARCH)" = "arm64" ]; then \ wget https://github.com/aquasecurity/trivy/releases/download/v0.25.3/trivy_0.25.3_Linux-ARM64.tar.gz && \ tar -zxvf trivy_0.25.3_Linux-ARM64.tar.gz && \ @@ -45,7 +47,7 @@ ARG SELINUX=true ENV SELINUX $SELINUX ENV DAPPER_RUN_ARGS --privileged -v k3s-cache:/go/src/github.com/k3s-io/k3s/.cache -v trivy-cache:/root/.cache/trivy -ENV DAPPER_ENV REPO TAG DRONE_TAG IMAGE_NAME SKIP_VALIDATE SKIP_AIRGAP GCLOUD_AUTH GITHUB_TOKEN GOLANG DEBUG +ENV DAPPER_ENV REPO TAG DRONE_TAG IMAGE_NAME SKIP_VALIDATE SKIP_AIRGAP AWS_SECRET_ACCESS_KEY AWS_ACCESS_KEY_ID GITHUB_TOKEN GOLANG DEBUG ENV DAPPER_SOURCE /go/src/github.com/k3s-io/k3s/ ENV DAPPER_OUTPUT ./bin ./dist ./build/out ./build/static ./pkg/static ./pkg/deploy diff --git a/Dockerfile.test.dapper b/Dockerfile.test.dapper index dd624a5bb8..6c58db6ff6 100644 --- a/Dockerfile.test.dapper +++ b/Dockerfile.test.dapper @@ -19,7 +19,7 @@ RUN OS=linux; \ ENV TEST_CLEANUP true ENV DAPPER_RUN_ARGS --privileged --network host -v /tmp:/tmp -ENV DAPPER_ENV REPO TAG DRONE_TAG DRONE_BUILD_EVENT IMAGE_NAME GCLOUD_AUTH SONOBUOY_VERSION ENABLE_REGISTRY +ENV DAPPER_ENV REPO TAG DRONE_TAG DRONE_BUILD_EVENT IMAGE_NAME AWS_SECRET_ACCESS_KEY AWS_ACCESS_KEY_ID SONOBUOY_VERSION ENABLE_REGISTRY ENV DAPPER_SOURCE /go/src/github.com/k3s-io/k3s/ ENV DAPPER_OUTPUT ./dist ENV DAPPER_DOCKER_SOCKET true diff --git a/install.sh b/install.sh index 2d92682aac..7ea2e20b37 100755 --- a/install.sh +++ b/install.sh @@ -92,7 +92,7 @@ set -o noglob # Defaults to 'stable'. GITHUB_URL=https://github.com/k3s-io/k3s/releases -STORAGE_URL=https://storage.googleapis.com/k3s-ci-builds +STORAGE_URL=https://k3s-ci-builds.s3.amazonaws.com DOWNLOADER= # --- helper functions for logs --- diff --git a/package/rpm/install.sh b/package/rpm/install.sh index c393dce19c..432090ba03 100755 --- a/package/rpm/install.sh +++ b/package/rpm/install.sh @@ -94,7 +94,7 @@ if [ "$INSTALL_K3S_DEBUG" = "true" ]; then fi GITHUB_URL=https://github.com/k3s-io/k3s/releases -STORAGE_URL=https://storage.googleapis.com/k3s-ci-builds +STORAGE_URL=https://k3s-ci-builds.s3.amazonaws.com DOWNLOADER= # --- helper functions for logs --- diff --git a/scripts/build-upload b/scripts/build-upload index 95f1238a2a..a5d5cc00ee 100755 --- a/scripts/build-upload +++ b/scripts/build-upload @@ -1,8 +1,13 @@ #!/bin/bash -[ -n "$GCLOUD_AUTH" ] || { +# Check for AWS Credentials +[ -n "$AWS_SECRET_ACCESS_KEY" ] || { exit 0 } +[ -n "$AWS_ACCESS_KEY_ID" ] || { + exit 0 +} + [ -x "$1" ] || { echo "First argument should be an executable" >&2 exit 1 @@ -23,35 +28,10 @@ cleanup() { } trap cleanup EXIT INT -GCLOUD_JSON=${TMPDIR}/.gcloud.json -[ -z "${GCLOUD_AUTH}" ] || echo "${GCLOUD_AUTH}" >${GCLOUD_JSON} -[ -s "${GCLOUD_JSON}" ] || { - echo "gcloud auth not defined" >&2 - exit 1 -} - -BOTO_CONF=${TMPDIR}/.boto -[ -s "${BOTO_CONF}" ] || cat >${BOTO_CONF} <${TMPDIR}/${BUILD_NAME}.sha256sum cp $1 ${TMPDIR}/${BUILD_NAME} -[ -d "${TMPDIR}/gsutil" ] || curl -sfL https://storage.googleapis.com/pub/gsutil.tar.gz | tar xz -C ${TMPDIR} - -HOME=${TMPDIR} -PATH=${PATH}:${HOME}/gsutil - -gsutil cp ${TMPDIR}/${BUILD_NAME}* gs://k3s-ci-builds || exit 1 - +aws s3 cp ${TMPDIR}/${BUILD_NAME}* s3://k3s-ci-builds || exit 1 echo "Build uploaded" >&2 -echo "https://storage.googleapis.com/k3s-ci-builds/${BUILD_NAME}" +echo "https://k3s-ci-builds.s3.amazonaws.com/${BUILD_NAME}" diff --git a/scripts/log-upload b/scripts/log-upload index 8db501e3f6..fd946c362f 100755 --- a/scripts/log-upload +++ b/scripts/log-upload @@ -1,5 +1,13 @@ #!/bin/bash +# Check for AWS Credentials +[ -n "$AWS_SECRET_ACCESS_KEY" ] || { + exit 0 +} +[ -n "$AWS_ACCESS_KEY_ID" ] || { + exit 0 +} + [ -d "$1" ] || { echo "First argument should be a directory" >&2 exit 1 @@ -18,33 +26,10 @@ cleanup() { } trap cleanup EXIT INT -GCLOUD_JSON=${TMPDIR}/.gcloud.json -[ -z "${GCLOUD_AUTH}" ] || echo "${GCLOUD_AUTH}" >${GCLOUD_JSON} -[ -s "${GCLOUD_JSON}" ] || { - echo "gcloud auth not defined" >&2 - exit 1 -} - -BOTO_CONF=${TMPDIR}/.boto -[ -s "${BOTO_CONF}" ] || cat >${BOTO_CONF} <&2 -echo "https://storage.googleapis.com/k3s-ci-logs/${LOG_TGZ}" +echo "https://k3s-ci-logs.s3.amazonaws.com/${LOG_TGZ}" + diff --git a/tests/e2e/scripts/latest_commit.sh b/tests/e2e/scripts/latest_commit.sh index 5bdfe4245a..d60ff3aab2 100755 --- a/tests/e2e/scripts/latest_commit.sh +++ b/tests/e2e/scripts/latest_commit.sh @@ -3,17 +3,17 @@ iterations=0 curl -s -H 'Accept: application/vnd.github.v3+json' "https://api.github.com/repos/k3s-io/k3s/commits?per_page=5&sha=$1" | jq -r '.[] | .sha' &> $2 # The VMs take time on startup to hit googleapis.com, wait loop until we can -while ! curl -s --fail https://storage.googleapis.com/k3s-ci-builds > /dev/null; do +while ! curl -s --fail https://k3s-ci-builds.s3.amazonaws.com > /dev/null; do ((iterations++)) if [ "$iterations" -ge 30 ]; then - echo "Unable to hit googleapis.com/k3s-ci-builds" + echo "Unable to hit https://k3s-ci-builds.s3.amazonaws.com" exit 1 fi sleep 1 done iterations=0 -curl -s --fail https://storage.googleapis.com/k3s-ci-builds/k3s-$(head -n 1 $2).sha256sum +curl -s --fail https://k3s-ci-builds.s3.amazonaws.com/k3s-$(head -n 1 $2).sha256sum while [ $? -ne 0 ]; do ((iterations++)) if [ "$iterations" -ge 6 ]; then @@ -22,5 +22,5 @@ while [ $? -ne 0 ]; do fi sed -i 1d "$2" sleep 1 - curl -s --fail https://storage.googleapis.com/k3s-ci-builds/k3s-$(head -n 1 $2).sha256sum + curl -s --fail https://k3s-ci-builds.s3.amazonaws.com/k3s-ci-builds/k3s-$(head -n 1 $2).sha256sum done \ No newline at end of file