diff --git a/tests/e2e/Dockerfile.build b/tests/e2e/Dockerfile.build deleted file mode 100644 index 281b427d9e..0000000000 --- a/tests/e2e/Dockerfile.build +++ /dev/null @@ -1,24 +0,0 @@ -FROM golang:alpine - -ARG TERRAFORM_VERSION=0.12.10 -ENV TERRAFORM_VERSION=$TERRAFORM_VERSION - - -RUN apk update && \ - apk upgrade --update-cache --available && \ - apk add curl git jq bash openssh unzip gcc g++ make ca-certificates && \ - curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh - -RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && \ - chmod +x ./kubectl && \ - mv ./kubectl /usr/local/bin -RUN mkdir tmp && \ - curl "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip" -o tmp/terraform.zip && \ - unzip tmp/terraform.zip -d /usr/local/bin && \ - chmod +x /usr/local/bin/terraform && \ - rm -rf tmp - -WORKDIR $GOPATH/src/github.com/k3s-io/k3s-io/k3s/tests/e2e - -COPY . . - diff --git a/tests/e2e/amd64_resource_files/clusterip.yaml b/tests/e2e/amd64_resource_files/clusterip.yaml index f1c501e8fa..e972f32d19 100644 --- a/tests/e2e/amd64_resource_files/clusterip.yaml +++ b/tests/e2e/amd64_resource_files/clusterip.yaml @@ -24,7 +24,7 @@ metadata: labels: k8s-app: nginx-app-clusterip name: nginx-clusterip-svc - namespace: default + namespace: default spec: type: ClusterIP ports: diff --git a/tests/e2e/arm_resource_files/ingress.yaml b/tests/e2e/arm_resource_files/ingress.yaml deleted file mode 100644 index 67adff28e9..0000000000 --- a/tests/e2e/arm_resource_files/ingress.yaml +++ /dev/null @@ -1,48 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: Ingress -metadata: - name: ingress -spec: - rules: - - host: foo1.bar.com - http: - paths: - - path: /name.html - backend: - serviceName: nginx-ingress-svc - servicePort: 80 ---- -apiVersion: v1 -kind: Service -metadata: - name: nginx-ingress-svc - labels: - k8s-app: nginx-app-ingress -spec: - ports: - - port: 80 - targetPort: 80 - protocol: TCP - name: http - selector: - k8s-app: nginx-app-ingress ---- -apiVersion: v1 -kind: ReplicationController -metadata: - name: test-ingress -spec: - replicas: 2 - selector: - k8s-app: nginx-app-ingress - template: - metadata: - labels: - k8s-app: nginx-app-ingress - spec: - terminationGracePeriodSeconds: 60 - containers: - - name: testcontainer - image: shylajarancher19/shylajaarm64:v1.0 - ports: - - containerPort: 80 diff --git a/tests/e2e/arm_resource_files/clusterip.yaml b/tests/e2e/cis_amd64_resource_files/clusterip.yaml similarity index 79% rename from tests/e2e/arm_resource_files/clusterip.yaml rename to tests/e2e/cis_amd64_resource_files/clusterip.yaml index ae5efe6a1e..24f3d9ac71 100644 --- a/tests/e2e/arm_resource_files/clusterip.yaml +++ b/tests/e2e/cis_amd64_resource_files/clusterip.yaml @@ -14,9 +14,9 @@ spec: spec: containers: - name: nginx - image: shylajarancher19/shylajaarm64:v1.0 + image: ranchertest/mytestcontainer:unprivileged ports: - - containerPort: 80 + - containerPort: 8080 --- apiVersion: v1 kind: Service @@ -24,10 +24,10 @@ metadata: labels: k8s-app: nginx-app-clusterip name: nginx-clusterip-svc - namespace: default + namespace: default spec: type: ClusterIP ports: - - port: 80 + - port: 8080 selector: k8s-app: nginx-app-clusterip diff --git a/tests/e2e/arm_resource_files/daemonset.yaml b/tests/e2e/cis_amd64_resource_files/daemonset.yaml similarity index 100% rename from tests/e2e/arm_resource_files/daemonset.yaml rename to tests/e2e/cis_amd64_resource_files/daemonset.yaml diff --git a/tests/e2e/cis_amd64_resource_files/dnsutils.yaml b/tests/e2e/cis_amd64_resource_files/dnsutils.yaml new file mode 100644 index 0000000000..9192bf3502 --- /dev/null +++ b/tests/e2e/cis_amd64_resource_files/dnsutils.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Pod +metadata: + name: dnsutils + namespace: default +spec: + containers: + - name: dnsutils + image: gcr.io/kubernetes-e2e-test-images/dnsutils:1.3 + command: + - sleep + - "3600" + imagePullPolicy: IfNotPresent + restartPolicy: Always diff --git a/tests/e2e/cis_amd64_resource_files/ingress.yaml b/tests/e2e/cis_amd64_resource_files/ingress.yaml new file mode 100644 index 0000000000..48366f268c --- /dev/null +++ b/tests/e2e/cis_amd64_resource_files/ingress.yaml @@ -0,0 +1,28 @@ +apiVersion: v1 +kind: Service +metadata: + name: nginx-ingress-svc + labels: + k8s-app: nginx-app-ingress +spec: + ports: + - port: 8080 + targetPort: 8080 + protocol: TCP + name: http + selector: + k8s-app: nginx-app-ingress +--- +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: test-ingress +spec: + rules: + - host: foo1.bar.com + http: + paths: + - backend: + serviceName: nginx-ingress-svc + servicePort: 8080 + pathType: ImplementationSpecific diff --git a/tests/e2e/arm_resource_files/loadbalancer.yaml b/tests/e2e/cis_amd64_resource_files/loadbalancer.yaml similarity index 51% rename from tests/e2e/arm_resource_files/loadbalancer.yaml rename to tests/e2e/cis_amd64_resource_files/loadbalancer.yaml index 6cc02bd898..9bf77f3b3b 100644 --- a/tests/e2e/arm_resource_files/loadbalancer.yaml +++ b/tests/e2e/cis_amd64_resource_files/loadbalancer.yaml @@ -1,4 +1,3 @@ ---- apiVersion: apps/v1 kind: Deployment metadata: @@ -15,22 +14,23 @@ spec: spec: containers: - name: nginx - image: shylajarancher19/shylajaarm64:v1.0 + image: ranchertest/mytestcontainer:unprivileged ports: - - containerPort: 80 + - containerPort: 8080 --- apiVersion: v1 kind: Service metadata: - name: nginx-loadbalancer-svc - labels: - k8s-app: nginx-app-loadbalancer + labels: + k8s-app: nginx-app-loadbalancer + name: nginx-loadbalancer-svc + namespace: default spec: - type: LoadBalancer - ports: - - port: 81 - targetPort: 80 - protocol: TCP - name: http - selector: - k8s-app: nginx-app-loadbalancer + type: LoadBalancer + ports: + - port: 81 + targetPort: 8080 + protocol: TCP + name: http + selector: + k8s-app: nginx-app-loadbalancer diff --git a/tests/e2e/arm_resource_files/local-path-provisioner.yaml b/tests/e2e/cis_amd64_resource_files/local-path-provisioner.yaml similarity index 100% rename from tests/e2e/arm_resource_files/local-path-provisioner.yaml rename to tests/e2e/cis_amd64_resource_files/local-path-provisioner.yaml diff --git a/tests/e2e/arm_resource_files/nodeport.yaml b/tests/e2e/cis_amd64_resource_files/nodeport.yaml similarity index 83% rename from tests/e2e/arm_resource_files/nodeport.yaml rename to tests/e2e/cis_amd64_resource_files/nodeport.yaml index 8978efaebb..a7f41eff1b 100644 --- a/tests/e2e/arm_resource_files/nodeport.yaml +++ b/tests/e2e/cis_amd64_resource_files/nodeport.yaml @@ -14,9 +14,9 @@ spec: spec: containers: - name: nginx - image: shylajarancher19/shylajaarm64:v1.0 + image: ranchertest/mytestcontainer:unprivileged ports: - - containerPort: 80 + - containerPort: 8080 --- apiVersion: v1 kind: Service @@ -28,7 +28,7 @@ metadata: spec: type: NodePort ports: - - port: 80 + - port: 8080 nodePort: 30096 name: http selector: diff --git a/tests/e2e/cis_amd64_resource_files/secrets.yaml b/tests/e2e/cis_amd64_resource_files/secrets.yaml new file mode 100644 index 0000000000..3f8d060685 --- /dev/null +++ b/tests/e2e/cis_amd64_resource_files/secrets.yaml @@ -0,0 +1,49 @@ +apiVersion: v1 +kind: Secret +metadata: + name: e2e-secret1 +type: Opaque +stringData: + config.yaml: | + key: "hello" + val: "world" +--- +apiVersion: v1 +kind: Secret +metadata: + name: e2e-secret2 +type: Opaque +stringData: + config.yaml: | + key: "good" + val: "day" +--- +apiVersion: v1 +kind: Secret +metadata: + name: e2e-secret3 +type: Opaque +stringData: + config.yaml: | + key: "top-secret" + val: "information" +--- +apiVersion: v1 +kind: Secret +metadata: + name: e2e-secret4 +type: Opaque +stringData: + config.yaml: | + key: "lock" + val: "key" +--- +apiVersion: v1 +kind: Secret +metadata: + name: e2e-secret5 +type: Opaque +stringData: + config.yaml: | + key: "last" + val: "call" \ No newline at end of file diff --git a/tests/e2e/dualstack/dualstack_test.go b/tests/e2e/dualstack/dualstack_test.go index a69688c3d6..50c6945cb4 100644 --- a/tests/e2e/dualstack/dualstack_test.go +++ b/tests/e2e/dualstack/dualstack_test.go @@ -16,6 +16,7 @@ import ( var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system") var serverCount = flag.Int("serverCount", 3, "number of server nodes") var agentCount = flag.Int("agentCount", 0, "number of agent nodes") +var hardened = flag.Bool("hardened", false, "true or false") // Environment Variables Info: // E2E_RELEASE_VERSION=v1.23.1+k3s1 or nil for latest commit from master @@ -129,7 +130,7 @@ var _ = Describe("Verify DualStack Configuration", func() { }) It("Verifies ClusterIP Service", func() { - _, err := e2e.DeployWorkload("dualstack_clusterip.yaml", kubeConfigFile, false) + _, err := e2e.DeployWorkload("dualstack_clusterip.yaml", kubeConfigFile, *hardened) Expect(err).NotTo(HaveOccurred()) Eventually(func() (string, error) { cmd := "kubectl get pods -o=name -l k8s-app=nginx-app-clusterip --field-selector=status.phase=Running --kubeconfig=" + kubeConfigFile @@ -157,7 +158,7 @@ var _ = Describe("Verify DualStack Configuration", func() { } }) It("Verifies Ingress", func() { - _, err := e2e.DeployWorkload("dualstack_ingress.yaml", kubeConfigFile, false) + _, err := e2e.DeployWorkload("dualstack_ingress.yaml", kubeConfigFile, *hardened) Expect(err).NotTo(HaveOccurred(), "Ingress manifest not deployed") cmd := "kubectl get ingress ds-ingress --kubeconfig=" + kubeConfigFile + " -o jsonpath=\"{.spec.rules[*].host}\"" hostName, err := e2e.RunCommand(cmd) @@ -177,7 +178,7 @@ var _ = Describe("Verify DualStack Configuration", func() { }) It("Verifies NodePort Service", func() { - _, err := e2e.DeployWorkload("dualstack_nodeport.yaml", kubeConfigFile, false) + _, err := e2e.DeployWorkload("dualstack_nodeport.yaml", kubeConfigFile, *hardened) Expect(err).NotTo(HaveOccurred()) cmd := "kubectl get service ds-nodeport-svc --kubeconfig=" + kubeConfigFile + " --output jsonpath=\"{.spec.ports[0].nodePort}\"" nodeport, err := e2e.RunCommand(cmd) diff --git a/tests/e2e/e2e_test_playbook.yaml b/tests/e2e/e2e_test_playbook.yaml new file mode 100644 index 0000000000..098d270dae --- /dev/null +++ b/tests/e2e/e2e_test_playbook.yaml @@ -0,0 +1,34 @@ +--- +- name: Run Cluster validation tests + gather_facts: false + hosts: node + vars: + user: "{{ user }}" + key: "{{ key }}" + ip: "{{ ip }}" + dir: scripts + tasks: + - name : Print var + debug: + msg: | + user: {{ user }} + key: {{ key }} + ip: {{ ip }} + db: {{ db }} + nodeOS: {{ nodeOS }} + serverCount: {{ serverCount }} + agentCount: {{ agentCount }} + + - name: build container image + command: chdir=k3s/tests/e2e /usr/local/bin/docker build -t k3s_nightly_build -f {{ dir }}/Dockerfile . + + - name: Delete docker container + command: "/usr/local/bin/docker rm createcluster -f" + ignore_errors: True + + - name: create docker container + command: "/usr/local/bin/docker run -d -it -v ~/config:/config --cap-add=NET_ADMIN --device /dev/net/tun --sysctl net.ipv6.conf.all.disable_ipv6=0 --name createcluster --privileged k3s_nightly_build" + + - name: execute command in docker + shell: | + /usr/local/bin/docker exec -it createcluster /bin/bash ./run_tests.sh {{ key }} {{ user }} {{ ip }} {{ db }} {{ nodeOS }} {{ serverCount }} {{ agentCount }} diff --git a/tests/e2e/scripts/Dockerfile b/tests/e2e/scripts/Dockerfile new file mode 100644 index 0000000000..deb2dc4132 --- /dev/null +++ b/tests/e2e/scripts/Dockerfile @@ -0,0 +1,19 @@ +FROM ubuntu:20.04 +ARG EXTERNAL_ENCODED_VPN +ARG VPN_ENCODED_LOGIN + +RUN apt-get update && \ + apt-get install -y curl bridge-utils iputils-ping openvpn openssh-client && \ + mkdir -p /dev/net && \ + mknod /dev/net/tun c 10 200 && \ + chmod 600 /dev/net/tun + +RUN if [[ -z "$EXTERNAL_ENCODED_VPN" ]] ; then echo "no vpn provided" ; \ + else echo -n $EXTERNAL_ENCODED_VPN | base64 -di > external.ovpn && \ + if [[ -z "$VPN_ENCODED_LOGIN" ]]; then echo "no passcode provided" ; \ + else echo -n $VPN_ENCODED_LOGIN | base64 -di > authfile && \ + sed -i 's/auth-user-pass/auth-user-pass authfile/g' external.ovpn; fi ; fi + +WORKDIR . +COPY scripts/run_tests.sh . +COPY scripts/init.sh . diff --git a/tests/e2e/scripts/Jenkinsfile b/tests/e2e/scripts/Jenkinsfile new file mode 100644 index 0000000000..06d22a3e0a --- /dev/null +++ b/tests/e2e/scripts/Jenkinsfile @@ -0,0 +1,53 @@ +pipeline { + agent any + environment { + AWS_SSH_PEM_KEY = credentials('AWS_SSH_PEM_KEY') + } + + stages { + stage('Git Checkout') { + steps { + git branch: 'master', url: 'https://github.com/k3s-io/k3s.git' + script { + dir("${WORKSPACE}/tests/e2e") { + if (env.AWS_SSH_PEM_KEY && env.AWS_SSH_KEY_NAME) { + def decoded = new String(AWS_SSH_PEM_KEY.decodeBase64()) + writeFile file: AWS_SSH_KEY_NAME, text: decoded + } + } + } + } + } + stage('Build Cluster'){ + steps { + script { + try { + sh """ + cd ${WORKSPACE}/tests/e2e/ + /usr/bin/docker build -f ./scripts/Dockerfile --build-arg EXTERNAL_ENCODED_VPN="$EXTERNAL_ENCODED_VPN" \ + --build-arg VPN_ENCODED_LOGIN="$VPN_ENCODED_LOGIN" -t k3s_create_cluster . + /usr/bin/docker run -d -it --cap-add=NET_ADMIN --device /dev/net/tun --sysctl net.ipv6.conf.all.disable_ipv6=0 \ + --name createcluster_${BUILD_NUMBER} --privileged k3s_create_cluster + /usr/bin/docker cp ${AWS_SSH_KEY_NAME} createcluster_${BUILD_NUMBER}:/ + /usr/bin/docker exec createcluster_${BUILD_NUMBER} ./run_tests.sh ${AWS_SSH_KEY_NAME} ${USER} ${NODEIP} \ + ${NODEOS} ${SERVERCOUNT} ${AGENTCOUNT} ${DB} ${HARDENED} + """ + } catch(error) { + echo "First build failed, let's retry if accepted" + retry(2) { + build 'k3s_nightly_automation' + } + } + } + } + } + stage('Test Report') { + steps { + sh """ + /usr/bin/docker rm -f createcluster_${BUILD_NUMBER} + /usr/bin/docker rmi -f k3s_create_cluster + """ + } + } + } +} diff --git a/tests/e2e/scripts/hosts b/tests/e2e/scripts/hosts new file mode 100644 index 0000000000..e8042a4f27 --- /dev/null +++ b/tests/e2e/scripts/hosts @@ -0,0 +1,2 @@ +[node] +localhost ansible_python_interpreter=/usr/local/opt/python@3.9/bin/python3.9 \ No newline at end of file diff --git a/tests/e2e/scripts/init.sh b/tests/e2e/scripts/init.sh new file mode 100755 index 0000000000..683a487f66 --- /dev/null +++ b/tests/e2e/scripts/init.sh @@ -0,0 +1,39 @@ +#!/bin/sh +# Script to install pre-requisites on vsphere node to run e2e test + +export PATH=$PATH:/usr/local/go/bin/:/usr/local/bin +echo 'Installing Kubectl' +curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" && \ +sudo mv kubectl /usr/local/bin/ && \ +chmod a+x /usr/local/bin/kubectl + +echo 'Installing jq' +sudo apt-get -y install jq + +echo 'Installing Go' +curl -L https://dl.google.com/go/go1.16.10.linux-amd64.tar.gz | tar xz +sudo mv go /usr/local +/usr/local/go/bin/go version +go version + +echo 'Installing Virtualbox' +sudo apt-get -y update && sudo apt-get -y upgrade +sudo apt-get -y install virtualbox +vboxmanage --version + +# Virtualbox >= 6.1.28 require `/etc/vbox/network.conf +ls /etc/vbox 2>/dev/null || sudo mkdir -p /etc/vbox +echo "* 10.0.0.0/8 192.168.0.0/16">>/etc/vbox/networks.conf + +echo 'Installing vagrant' +sudo apt-get -y install -f unzip +curl -O https://releases.hashicorp.com/vagrant/2.2.19/vagrant_2.2.19_linux_amd64.zip +unzip vagrant_2.2.19_linux_amd64.zip +sudo cp vagrant /usr/local/bin/ +vagrant --version +sudo apt-get -y install libarchive-tools +vagrant plugin install vagrant-k3s +vagrant plugin install vagrant-reload + +echo 'Cloning repo' +ls k3s 2>/dev/null || git clone https://github.com/k3s-io/k3s.git diff --git a/tests/e2e/scripts/run_tests.sh b/tests/e2e/scripts/run_tests.sh new file mode 100755 index 0000000000..d01a2579d8 --- /dev/null +++ b/tests/e2e/scripts/run_tests.sh @@ -0,0 +1,32 @@ +#!/bin/bash +servercount=${5:-3} +agentcount=${6:-2} +db=${7:-"etcd"} +k3s_version=${k3s_version} +k3s_channel=${k3s_channel:-"commit"} +hardened=${8:-""} + +E2E_EXTERNAL_DB=$db && export E2E_EXTERNAL_DB + +eval openvpn --daemon --config external.ovpn &>/dev/null & +sleep 10 + +ssh -i "$1" -o 'StrictHostKeyChecking no' $2@$3 'cd k3s && git pull --rebase origin master' +ssh -i "$1" -o 'StrictHostKeyChecking no' $2@$3 '/usr/local/go/bin/go get github.com/onsi/ginkgo/v2' +ssh -i "$1" -o 'StrictHostKeyChecking no' $2@$3 '/usr/local/go/bin/go get github.com/onsi/gomega' +ssh -i "$1" -o 'StrictHostKeyChecking no' $2@$3 '/usr/local/go/bin/go get github.com/k3s-io/k3s/tests/e2e' + +echo 'RUNNING CLUSTER VALIDATION TEST' +ssh -i "$1" -o 'StrictHostKeyChecking no' $2@$3 'cd k3s/tests/e2e/validatecluster && vagrant destroy -f' +ssh -i "$1" -o 'StrictHostKeyChecking no' $2@$3 "cd k3s/tests/e2e && E2E_HARDENED=$hardened /usr/local/go/bin/go test -v validatecluster/validatecluster_test.go -nodeOS="$4" -serverCount=$((servercount)) -agentCount=$((agentcount)) -timeout=1h" + +echo 'RUNNING SECRETS ENCRYPTION TEST' +#ssh -i "$1" -o 'StrictHostKeyChecking no' $2@$3 'cd k3s/tests/e2e/secretsencryption && vagrant destroy -f' +#ssh -i "$1" -o 'StrictHostKeyChecking no' $2@$3 "cd k3s/tests/e2e && /usr/local/go/bin/go test -v secretsencryption/secretsencryption_test.go -nodeOS="$4" -serverCount=$((servercount)) -timeout=1h" + +E2E_RELEASE_VERSION=$k3s_version && export E2E_RELEASE_VERSION +E2E_RELEASE_CHANNEL=$k3s_channel && export E2E_RELEASE_CHANNEL + +echo 'RUNNING CLUSTER UPGRADE TEST' +#ssh -i "$1" -o 'StrictHostKeyChecking no' $2@$3 'cd k3s/tests/e2e/upgradecluster && vagrant destroy -f' +#ssh -i "$1" -o 'StrictHostKeyChecking no' $2@$3 "cd k3s/tests/e2e && /usr/local/go/bin/go test -v upgradecluster/upgradecluster_test.go -nodeOS="$4" -serverCount=$((servercount)) -agentCount=$((agentcount)) -timeout=1h" diff --git a/tests/e2e/secretsencryption/Vagrantfile b/tests/e2e/secretsencryption/Vagrantfile index acc43a4ad2..b2b33c5713 100644 --- a/tests/e2e/secretsencryption/Vagrantfile +++ b/tests/e2e/secretsencryption/Vagrantfile @@ -1,12 +1,12 @@ ENV['VAGRANT_NO_PARALLEL'] = 'no' -NODE_ROLES = (ENV['NODE_ROLES'] || +NODE_ROLES = (ENV['E2E_NODE_ROLES'] || ["server-0", "server-1", "server-2"]) -NODE_BOXES = (ENV['NODE_BOXES'] || +NODE_BOXES = (ENV['E2E_NODE_BOXES'] || ['generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004']) -GITHUB_BRANCH = (ENV['GITHUB_BRANCH'] || "master") -RELEASE_VERSION = (ENV['RELEASE_VERSION'] || "") -NODE_CPUS = (ENV['NODE_CPUS'] || 2).to_i -NODE_MEMORY = (ENV['NODE_MEMORY'] || 1024).to_i +GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master") +RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "") +NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i +NODE_MEMORY = (ENV['E2E_NODE_MEMORY'] || 1024).to_i # Virtualbox >= 6.1.28 require `/etc/vbox/network.conf` for expanded private networks NETWORK_PREFIX = "10.10.10" install_type = "" diff --git a/tests/e2e/secretsencryption/secretsencryption_test.go b/tests/e2e/secretsencryption/secretsencryption_test.go index dd507e7184..74eaacd28c 100644 --- a/tests/e2e/secretsencryption/secretsencryption_test.go +++ b/tests/e2e/secretsencryption/secretsencryption_test.go @@ -6,7 +6,6 @@ import ( "os" "strings" "testing" - "time" "github.com/k3s-io/k3s/tests/e2e" . "github.com/onsi/ginkgo/v2" @@ -16,14 +15,15 @@ import ( // Valid nodeOS: generic/ubuntu2004, opensuse/Leap-15.3.x86_64, dweomer/microos.amd64 var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system") var serverCount = flag.Int("serverCount", 3, "number of server nodes") +var hardened = flag.Bool("hardened", false, "true or false") // Environment Variables Info: // E2E_RELEASE_VERSION=v1.23.1+k3s2 or nil for latest commit from master -func Test_E2EClusterValidation(t *testing.T) { +func Test_E2ESecretsEncryption(t *testing.T) { RegisterFailHandler(Fail) flag.Parse() - RunSpecs(t, "Create Cluster Test Suite") + RunSpecs(t, "Secrets Encryption Test Suite") } var ( @@ -71,7 +71,7 @@ var _ = Describe("Verify Secrets Encryption Rotation", func() { }) It("Deploys several secrets", func() { - _, err := e2e.DeployWorkload("secrets.yaml", kubeConfigFile, false) + _, err := e2e.DeployWorkload("secrets.yaml", kubeConfigFile, *hardened) Expect(err).NotTo(HaveOccurred(), "Secrets not deployed") }) @@ -127,16 +127,19 @@ var _ = Describe("Verify Secrets Encryption Rotation", func() { } } }, "420s", "5s").Should(Succeed()) + _, _ = e2e.ParseNodes(kubeConfigFile, true) }) It("Verifies encryption prepare stage", func() { cmd := "sudo k3s secrets-encrypt status" for _, nodeName := range serverNodeNames { - res, err := e2e.RunCmdOnNode(cmd, nodeName) - Expect(err).NotTo(HaveOccurred()) - Expect(res).Should(ContainSubstring("Encryption Status: Enabled")) - Expect(res).Should(ContainSubstring("Current Rotation Stage: prepare")) - Expect(res).Should(ContainSubstring("Server Encryption Hashes: All hashes match")) + Eventually(func(g Gomega) { + res, err := e2e.RunCmdOnNode(cmd, nodeName) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(res).Should(ContainSubstring("Encryption Status: Enabled")) + g.Expect(res).Should(ContainSubstring("Current Rotation Stage: prepare")) + g.Expect(res).Should(ContainSubstring("Server Encryption Hashes: All hashes match")) + }, "420s", "2s").Should(Succeed()) } }) @@ -145,31 +148,34 @@ var _ = Describe("Verify Secrets Encryption Rotation", func() { res, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0]) Expect(err).NotTo(HaveOccurred(), res) for i, nodeName := range serverNodeNames { - cmd := "sudo k3s secrets-encrypt status" - res, err := e2e.RunCmdOnNode(cmd, nodeName) - Expect(err).NotTo(HaveOccurred(), res) - Expect(res).Should(ContainSubstring("Server Encryption Hashes: hash does not match")) - if i == 0 { - Expect(res).Should(ContainSubstring("Current Rotation Stage: rotate")) - } else { - Expect(res).Should(ContainSubstring("Current Rotation Stage: prepare")) - } + Eventually(func(g Gomega) { + cmd := "sudo k3s secrets-encrypt status" + res, err := e2e.RunCmdOnNode(cmd, nodeName) + g.Expect(err).NotTo(HaveOccurred(), res) + g.Expect(res).Should(ContainSubstring("Server Encryption Hashes: hash does not match")) + if i == 0 { + g.Expect(res).Should(ContainSubstring("Current Rotation Stage: rotate")) + } else { + g.Expect(res).Should(ContainSubstring("Current Rotation Stage: prepare")) + } + }, "420s", "2s").Should(Succeed()) } }) It("Restarts K3s servers", func() { Expect(e2e.RestartCluster(serverNodeNames)).To(Succeed()) - time.Sleep(20 * time.Second) }) It("Verifies encryption rotate stage", func() { cmd := "sudo k3s secrets-encrypt status" for _, nodeName := range serverNodeNames { - res, err := e2e.RunCmdOnNode(cmd, nodeName) - Expect(err).NotTo(HaveOccurred()) - Expect(res).Should(ContainSubstring("Encryption Status: Enabled")) - Expect(res).Should(ContainSubstring("Current Rotation Stage: rotate")) - Expect(res).Should(ContainSubstring("Server Encryption Hashes: All hashes match")) + Eventually(func(g Gomega) { + res, err := e2e.RunCmdOnNode(cmd, nodeName) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(res).Should(ContainSubstring("Encryption Status: Enabled")) + g.Expect(res).Should(ContainSubstring("Current Rotation Stage: rotate")) + g.Expect(res).Should(ContainSubstring("Server Encryption Hashes: All hashes match")) + }, "420s", "2s").Should(Succeed()) } }) @@ -181,7 +187,7 @@ var _ = Describe("Verify Secrets Encryption Rotation", func() { cmd = "sudo k3s secrets-encrypt status" Eventually(func() (string, error) { return e2e.RunCmdOnNode(cmd, serverNodeNames[0]) - }, "30s", "5s").Should(ContainSubstring("Current Rotation Stage: reencrypt_finished")) + }, "180s", "5s").Should(ContainSubstring("Current Rotation Stage: reencrypt_finished")) for _, nodeName := range serverNodeNames[1:] { res, err := e2e.RunCmdOnNode(cmd, nodeName) @@ -193,17 +199,18 @@ var _ = Describe("Verify Secrets Encryption Rotation", func() { It("Restarts K3s Servers", func() { Expect(e2e.RestartCluster(serverNodeNames)).To(Succeed()) - time.Sleep(20 * time.Second) }) It("Verifies Encryption Reencrypt Stage", func() { cmd := "sudo k3s secrets-encrypt status" for _, nodeName := range serverNodeNames { - res, err := e2e.RunCmdOnNode(cmd, nodeName) - Expect(err).NotTo(HaveOccurred()) - Expect(res).Should(ContainSubstring("Encryption Status: Enabled")) - Expect(res).Should(ContainSubstring("Current Rotation Stage: reencrypt_finished")) - Expect(res).Should(ContainSubstring("Server Encryption Hashes: All hashes match")) + Eventually(func(g Gomega) { + res, err := e2e.RunCmdOnNode(cmd, nodeName) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(res).Should(ContainSubstring("Encryption Status: Enabled")) + g.Expect(res).Should(ContainSubstring("Current Rotation Stage: reencrypt_finished")) + g.Expect(res).Should(ContainSubstring("Server Encryption Hashes: All hashes match")) + }, "420s", "2s").Should(Succeed()) } }) }) @@ -220,28 +227,31 @@ var _ = Describe("Verify Secrets Encryption Rotation", func() { cmd = "sudo k3s secrets-encrypt status" Eventually(func() (string, error) { return e2e.RunCmdOnNode(cmd, serverNodeNames[0]) - }, "30s", "5s").Should(ContainSubstring("Current Rotation Stage: reencrypt_finished")) + }, "180s", "5s").Should(ContainSubstring("Current Rotation Stage: reencrypt_finished")) for i, nodeName := range serverNodeNames { - res, err := e2e.RunCmdOnNode(cmd, nodeName) - Expect(err).NotTo(HaveOccurred(), res) - if i == 0 { - Expect(res).Should(ContainSubstring("Encryption Status: Disabled")) - } else { - Expect(res).Should(ContainSubstring("Encryption Status: Enabled")) - } + Eventually(func(g Gomega) { + res, err := e2e.RunCmdOnNode(cmd, nodeName) + g.Expect(err).NotTo(HaveOccurred(), res) + if i == 0 { + g.Expect(res).Should(ContainSubstring("Encryption Status: Disabled")) + } else { + g.Expect(res).Should(ContainSubstring("Encryption Status: Enabled")) + } + }, "420s", "2s").Should(Succeed()) } }) It("Restarts K3s servers", func() { Expect(e2e.RestartCluster(serverNodeNames)).To(Succeed()) - time.Sleep(20 * time.Second) }) It("Verifies encryption disabled on all nodes", func() { cmd := "sudo k3s secrets-encrypt status" for _, nodeName := range serverNodeNames { - Expect(e2e.RunCmdOnNode(cmd, nodeName)).Should(ContainSubstring("Encryption Status: Disabled")) + Eventually(func(g Gomega) { + g.Expect(e2e.RunCmdOnNode(cmd, nodeName)).Should(ContainSubstring("Encryption Status: Disabled")) + }, "420s", "2s").Should(Succeed()) } }) @@ -257,19 +267,21 @@ var _ = Describe("Verify Secrets Encryption Rotation", func() { cmd = "sudo k3s secrets-encrypt status" Eventually(func() (string, error) { return e2e.RunCmdOnNode(cmd, serverNodeNames[0]) - }, "30s", "5s").Should(ContainSubstring("Current Rotation Stage: reencrypt_finished")) + }, "180s", "5s").Should(ContainSubstring("Current Rotation Stage: reencrypt_finished")) }) It("Restarts K3s servers", func() { Expect(e2e.RestartCluster(serverNodeNames)).To(Succeed()) - time.Sleep(20 * time.Second) }) It("Verifies encryption enabled on all nodes", func() { cmd := "sudo k3s secrets-encrypt status" for _, nodeName := range serverNodeNames { - Expect(e2e.RunCmdOnNode(cmd, nodeName)).Should(ContainSubstring("Encryption Status: Enabled")) + Eventually(func(g Gomega) { + g.Expect(e2e.RunCmdOnNode(cmd, nodeName)).Should(ContainSubstring("Encryption Status: Enabled")) + }, "420s", "2s").Should(Succeed()) } + }) }) diff --git a/tests/e2e/splitserver/splitserver_test.go b/tests/e2e/splitserver/splitserver_test.go index 612b0faf5a..983d639b1d 100644 --- a/tests/e2e/splitserver/splitserver_test.go +++ b/tests/e2e/splitserver/splitserver_test.go @@ -18,6 +18,7 @@ var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system") var etcdCount = flag.Int("etcdCount", 1, "number of server nodes only deploying etcd") var controlPlaneCount = flag.Int("controlPlaneCount", 1, "number of server nodes acting as control plane") var agentCount = flag.Int("agentCount", 1, "number of agent nodes") +var hardened = flag.Bool("hardened", false, "true or false") // Environment Variables Info: // E2E_RELEASE_VERSION=v1.23.1+k3s2 or nil for latest commit from master @@ -111,7 +112,7 @@ var _ = Describe("Verify Create", func() { }) It("Verifies ClusterIP Service", func() { - _, err := e2e.DeployWorkload("clusterip.yaml", kubeConfigFile, false) + _, err := e2e.DeployWorkload("clusterip.yaml", kubeConfigFile, *hardened) Expect(err).NotTo(HaveOccurred(), "Cluster IP manifest not deployed") cmd := "kubectl get pods -o=name -l k8s-app=nginx-app-clusterip --field-selector=status.phase=Running --kubeconfig=" + kubeConfigFile @@ -129,7 +130,7 @@ var _ = Describe("Verify Create", func() { }) It("Verifies NodePort Service", func() { - _, err := e2e.DeployWorkload("nodeport.yaml", kubeConfigFile, false) + _, err := e2e.DeployWorkload("nodeport.yaml", kubeConfigFile, *hardened) Expect(err).NotTo(HaveOccurred(), "NodePort manifest not deployed") for _, nodeName := range cpNodeNames { @@ -151,7 +152,7 @@ var _ = Describe("Verify Create", func() { }) It("Verifies LoadBalancer Service", func() { - _, err := e2e.DeployWorkload("loadbalancer.yaml", kubeConfigFile, false) + _, err := e2e.DeployWorkload("loadbalancer.yaml", kubeConfigFile, *hardened) Expect(err).NotTo(HaveOccurred(), "Loadbalancer manifest not deployed") for _, nodeName := range cpNodeNames { @@ -174,7 +175,7 @@ var _ = Describe("Verify Create", func() { }) It("Verifies Ingress", func() { - _, err := e2e.DeployWorkload("ingress.yaml", kubeConfigFile, false) + _, err := e2e.DeployWorkload("ingress.yaml", kubeConfigFile, *hardened) Expect(err).NotTo(HaveOccurred(), "Ingress manifest not deployed") for _, nodeName := range cpNodeNames { @@ -187,7 +188,7 @@ var _ = Describe("Verify Create", func() { }) It("Verifies Daemonset", func() { - _, err := e2e.DeployWorkload("daemonset.yaml", kubeConfigFile, false) + _, err := e2e.DeployWorkload("daemonset.yaml", kubeConfigFile, *hardened) Expect(err).NotTo(HaveOccurred(), "Daemonset manifest not deployed") Eventually(func(g Gomega) { @@ -202,7 +203,7 @@ var _ = Describe("Verify Create", func() { }) It("Verifies dns access", func() { - _, err := e2e.DeployWorkload("dnsutils.yaml", kubeConfigFile, false) + _, err := e2e.DeployWorkload("dnsutils.yaml", kubeConfigFile, *hardened) Expect(err).NotTo(HaveOccurred(), "dnsutils manifest not deployed") cmd := "kubectl get pods dnsutils --kubeconfig=" + kubeConfigFile diff --git a/tests/e2e/testutils.go b/tests/e2e/testutils.go index e3e1d65152..b279632645 100644 --- a/tests/e2e/testutils.go +++ b/tests/e2e/testutils.go @@ -69,10 +69,10 @@ func CreateCluster(nodeOS string, serverCount, agentCount int) ([]string, []stri return serverNodeNames, agentNodeNames, nil } -func DeployWorkload(workload, kubeconfig string, arch bool) (string, error) { +func DeployWorkload(workload, kubeconfig string, hardened bool) (string, error) { resourceDir := "../amd64_resource_files" - if arch { - resourceDir = "../arm64_resource_files" + if hardened { + resourceDir = "../cis_amd64_resource_files" } files, err := ioutil.ReadDir(resourceDir) if err != nil { diff --git a/tests/e2e/upgradecluster/upgradecluster_test.go b/tests/e2e/upgradecluster/upgradecluster_test.go index a4dddfa5ed..4603f3d780 100644 --- a/tests/e2e/upgradecluster/upgradecluster_test.go +++ b/tests/e2e/upgradecluster/upgradecluster_test.go @@ -16,6 +16,7 @@ import ( var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system") var serverCount = flag.Int("serverCount", 3, "number of server nodes") var agentCount = flag.Int("agentCount", 2, "number of agent nodes") +var hardened = flag.Bool("hardened", false, "true or false") // Environment Variables Info: // E2E_RELEASE_VERSION=v1.23.3+k3s1 @@ -75,7 +76,7 @@ var _ = Describe("Verify Upgrade", func() { }) It("Verifies ClusterIP Service", func() { - _, err := e2e.DeployWorkload("clusterip.yaml", kubeConfigFile, false) + _, err := e2e.DeployWorkload("clusterip.yaml", kubeConfigFile, *hardened) Expect(err).NotTo(HaveOccurred(), "Cluster IP manifest not deployed") @@ -94,7 +95,7 @@ var _ = Describe("Verify Upgrade", func() { }) It("Verifies NodePort Service", func() { - _, err := e2e.DeployWorkload("nodeport.yaml", kubeConfigFile, false) + _, err := e2e.DeployWorkload("nodeport.yaml", kubeConfigFile, *hardened) Expect(err).NotTo(HaveOccurred(), "NodePort manifest not deployed") for _, nodeName := range serverNodeNames { @@ -117,7 +118,7 @@ var _ = Describe("Verify Upgrade", func() { }) It("Verifies LoadBalancer Service", func() { - _, err := e2e.DeployWorkload("loadbalancer.yaml", kubeConfigFile, false) + _, err := e2e.DeployWorkload("loadbalancer.yaml", kubeConfigFile, *hardened) Expect(err).NotTo(HaveOccurred(), "Loadbalancer manifest not deployed") for _, nodeName := range serverNodeNames { ip, _ := e2e.FetchNodeExternalIP(nodeName) @@ -138,7 +139,7 @@ var _ = Describe("Verify Upgrade", func() { }) It("Verifies Ingress", func() { - _, err := e2e.DeployWorkload("ingress.yaml", kubeConfigFile, false) + _, err := e2e.DeployWorkload("ingress.yaml", kubeConfigFile, *hardened) Expect(err).NotTo(HaveOccurred(), "Ingress manifest not deployed") for _, nodeName := range serverNodeNames { @@ -151,7 +152,7 @@ var _ = Describe("Verify Upgrade", func() { }) It("Verifies Daemonset", func() { - _, err := e2e.DeployWorkload("daemonset.yaml", kubeConfigFile, false) + _, err := e2e.DeployWorkload("daemonset.yaml", kubeConfigFile, *hardened) Expect(err).NotTo(HaveOccurred(), "Daemonset manifest not deployed") nodes, _ := e2e.ParseNodes(kubeConfigFile, false) //nodes := @@ -168,7 +169,7 @@ var _ = Describe("Verify Upgrade", func() { }) It("Verifies dns access", func() { - _, err := e2e.DeployWorkload("dnsutils.yaml", kubeConfigFile, false) + _, err := e2e.DeployWorkload("dnsutils.yaml", kubeConfigFile, *hardened) Expect(err).NotTo(HaveOccurred(), "dnsutils manifest not deployed") Eventually(func() (string, error) { @@ -183,7 +184,7 @@ var _ = Describe("Verify Upgrade", func() { }) It("Verifies Local Path Provisioner storage ", func() { - _, err := e2e.DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, false) + _, err := e2e.DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, *hardened) Expect(err).NotTo(HaveOccurred(), "local-path-provisioner manifest not deployed") Eventually(func(g Gomega) { cmd := "kubectl get pvc local-path-pvc --kubeconfig=" + kubeConfigFile @@ -214,7 +215,7 @@ var _ = Describe("Verify Upgrade", func() { Expect(err).NotTo(HaveOccurred()) fmt.Println(res) - _, err = e2e.DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, false) + _, err = e2e.DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, *hardened) Expect(err).NotTo(HaveOccurred(), "local-path-provisioner manifest not deployed") Eventually(func() (string, error) { diff --git a/tests/e2e/validatecluster/validatecluster_test.go b/tests/e2e/validatecluster/validatecluster_test.go index daa8c75cab..a65a580625 100644 --- a/tests/e2e/validatecluster/validatecluster_test.go +++ b/tests/e2e/validatecluster/validatecluster_test.go @@ -16,6 +16,7 @@ import ( var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system") var serverCount = flag.Int("serverCount", 3, "number of server nodes") var agentCount = flag.Int("agentCount", 2, "number of agent nodes") +var hardened = flag.Bool("hardened", false, "true or false") // Environment Variables Info: // E2E_EXTERNAL_DB: mysql or postgres, nil for embedded etcd @@ -74,7 +75,7 @@ var _ = Describe("Verify Create", func() { }) It("Verifies ClusterIP Service", func() { - _, err := e2e.DeployWorkload("clusterip.yaml", kubeConfigFile, false) + _, err := e2e.DeployWorkload("clusterip.yaml", kubeConfigFile, *hardened) Expect(err).NotTo(HaveOccurred(), "Cluster IP manifest not deployed") Eventually(func(g Gomega) { @@ -98,7 +99,7 @@ var _ = Describe("Verify Create", func() { }) It("Verifies NodePort Service", func() { - _, err := e2e.DeployWorkload("nodeport.yaml", kubeConfigFile, false) + _, err := e2e.DeployWorkload("nodeport.yaml", kubeConfigFile, *hardened) Expect(err).NotTo(HaveOccurred(), "NodePort manifest not deployed") for _, nodeName := range serverNodeNames { @@ -126,7 +127,7 @@ var _ = Describe("Verify Create", func() { }) It("Verifies LoadBalancer Service", func() { - _, err := e2e.DeployWorkload("loadbalancer.yaml", kubeConfigFile, false) + _, err := e2e.DeployWorkload("loadbalancer.yaml", kubeConfigFile, *hardened) Expect(err).NotTo(HaveOccurred(), "Loadbalancer manifest not deployed") for _, nodeName := range serverNodeNames { @@ -155,7 +156,7 @@ var _ = Describe("Verify Create", func() { }) It("Verifies Ingress", func() { - _, err := e2e.DeployWorkload("ingress.yaml", kubeConfigFile, false) + _, err := e2e.DeployWorkload("ingress.yaml", kubeConfigFile, *hardened) Expect(err).NotTo(HaveOccurred(), "Ingress manifest not deployed") for _, nodeName := range serverNodeNames { @@ -173,7 +174,7 @@ var _ = Describe("Verify Create", func() { }) It("Verifies Daemonset", func() { - _, err := e2e.DeployWorkload("daemonset.yaml", kubeConfigFile, false) + _, err := e2e.DeployWorkload("daemonset.yaml", kubeConfigFile, *hardened) Expect(err).NotTo(HaveOccurred(), "Daemonset manifest not deployed") nodes, _ := e2e.ParseNodes(kubeConfigFile, false) @@ -190,7 +191,7 @@ var _ = Describe("Verify Create", func() { }) It("Verifies dns access", func() { - _, err := e2e.DeployWorkload("dnsutils.yaml", kubeConfigFile, false) + _, err := e2e.DeployWorkload("dnsutils.yaml", kubeConfigFile, *hardened) Expect(err).NotTo(HaveOccurred(), "dnsutils manifest not deployed") Eventually(func(g Gomega) { @@ -210,7 +211,7 @@ var _ = Describe("Verify Create", func() { }) It("Verifies Local Path Provisioner storage ", func() { - _, err := e2e.DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, false) + _, err := e2e.DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, *hardened) Expect(err).NotTo(HaveOccurred(), "local-path-provisioner manifest not deployed") Eventually(func(g Gomega) { @@ -241,7 +242,7 @@ var _ = Describe("Verify Create", func() { Expect(err).NotTo(HaveOccurred()) fmt.Println(res) - _, err = e2e.DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, false) + _, err = e2e.DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, *hardened) Expect(err).NotTo(HaveOccurred(), "local-path-provisioner manifest not deployed") Eventually(func(g Gomega) {