From bd63aeccc254276d3e073ef0c2bb7ec8b81dfc12 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Tue, 14 Jun 2022 15:17:41 -0700 Subject: [PATCH] [Release-1.23] Integration and E2E test improvements (#5686) * Integration Test: Startup (#5630) * New startup integration test * Add testing section to PR template * Move helper functions to direct k8s client calls Signed-off-by: Derek Nola * E2E Improvements and groundwork for test-pad tool (#5593) * Add rancher install sript, taints to cp/etcd roles * Revert back to generic/ubuntu2004, libvirt networking is unreliable on opensuse * Added support for alpine * Rancher deployment script * Refactor installType into function * Cleanup splitserver test Signed-off-by: Derek Nola * E2E: Dualstack test (#5617) * E2E dualstack test * Improve testing documentation Signed-off-by: Derek Nola --- .github/PULL_REQUEST_TEMPLATE.md | 5 + tests/e2e/README.md | 8 +- .../dualstack_clusterip.yaml | 36 +++ .../dualstack_ingress.yaml | 16 ++ .../dualstack_nodeport.yaml | 36 +++ tests/e2e/dualstack/Vagrantfile | 111 +++++++++ tests/e2e/dualstack/dualstack_test.go | 213 ++++++++++++++++++ tests/e2e/scripts/ipv6.sh | 16 ++ tests/e2e/scripts/rancher.sh | 63 ++++++ tests/e2e/secretsencryption/Vagrantfile | 10 +- tests/e2e/splitserver/Vagrantfile | 38 +++- tests/e2e/splitserver/splitserver_test.go | 110 ++++----- tests/e2e/testutils.go | 11 +- tests/e2e/upgradecluster/Vagrantfile | 2 +- .../e2e/upgradecluster/upgradecluster_test.go | 4 +- tests/e2e/vagrantdefaults.rb | 34 ++- tests/e2e/validatecluster/Vagrantfile | 29 ++- .../validatecluster/validatecluster_test.go | 2 +- .../certrotation/certrotation_int_test.go | 12 +- .../dualstack/dualstack_int_test.go | 6 +- .../etcdrestore/etcd_restore_int_test.go | 12 +- .../etcdsnapshot/etcdsnapshot_int_test.go | 6 +- tests/integration/integration.go | 91 +++++++- .../localstorage/localstorage_int_test.go | 6 +- .../secretsencryption_int_test.go | 24 +- tests/integration/startup/startup_int_test.go | 123 ++++++++++ 26 files changed, 866 insertions(+), 158 deletions(-) create mode 100644 tests/e2e/amd64_resource_files/dualstack_clusterip.yaml create mode 100644 tests/e2e/amd64_resource_files/dualstack_ingress.yaml create mode 100644 tests/e2e/amd64_resource_files/dualstack_nodeport.yaml create mode 100644 tests/e2e/dualstack/Vagrantfile create mode 100644 tests/e2e/dualstack/dualstack_test.go create mode 100644 tests/e2e/scripts/ipv6.sh create mode 100644 tests/e2e/scripts/rancher.sh create mode 100644 tests/integration/startup/startup_int_test.go diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 882fa40bce..85acea2a50 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -13,6 +13,11 @@ +#### Testing #### + + + + #### Linked Issues #### diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 0f7f7895bd..b55679e14d 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -42,4 +42,10 @@ To run the all E2E tests and generate JUnit testing reports: ginkgo --junit-report=result.xml ./tests/e2e/... ``` -Note: The `go test` default timeout is 10 minutes, thus the `-timeout` flag should be used. The `ginkgo` default timeout is 1 hour, no timeout flag is needed. \ No newline at end of file +Note: The `go test` default timeout is 10 minutes, thus the `-timeout` flag should be used. The `ginkgo` default timeout is 1 hour, no timeout flag is needed. + +# Debugging +In the event of a test failure, the cluster and VMs are retained in their broken state. Startup logs are retained in `vagrant.log`. +To see a list of nodes: `vagrant status` +To ssh into a node: `vagrant ssh ` +Once you are done/ready to restart the test, use `vagrant destroy -f` to remove the broken cluster. \ No newline at end of file diff --git a/tests/e2e/amd64_resource_files/dualstack_clusterip.yaml b/tests/e2e/amd64_resource_files/dualstack_clusterip.yaml new file mode 100644 index 0000000000..c9d4476403 --- /dev/null +++ b/tests/e2e/amd64_resource_files/dualstack_clusterip.yaml @@ -0,0 +1,36 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ds-clusterip-pod +spec: + selector: + matchLabels: + k8s-app: nginx-app-clusterip + replicas: 2 + template: + metadata: + labels: + k8s-app: nginx-app-clusterip + spec: + containers: + - name: nginx + image: ranchertest/mytestcontainer + ports: + - containerPort: 80 +--- +apiVersion: v1 +kind: Service +metadata: + labels: + k8s-app: nginx-app-clusterip + name: ds-clusterip-svc + namespace: default +spec: + type: ClusterIP + ipFamilyPolicy: PreferDualStack + ports: + - protocol: TCP + port: 80 + targetPort: 80 + selector: + k8s-app: nginx-app-clusterip diff --git a/tests/e2e/amd64_resource_files/dualstack_ingress.yaml b/tests/e2e/amd64_resource_files/dualstack_ingress.yaml new file mode 100644 index 0000000000..ee9110086c --- /dev/null +++ b/tests/e2e/amd64_resource_files/dualstack_ingress.yaml @@ -0,0 +1,16 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ds-ingress +spec: + rules: + - host: testds.com + http: + paths: + - backend: + service: + # Reliant on dualstack_clusterip.yaml + name: ds-clusterip-svc + port: + number: 80 + pathType: ImplementationSpecific \ No newline at end of file diff --git a/tests/e2e/amd64_resource_files/dualstack_nodeport.yaml b/tests/e2e/amd64_resource_files/dualstack_nodeport.yaml new file mode 100644 index 0000000000..2a5b38f1c4 --- /dev/null +++ b/tests/e2e/amd64_resource_files/dualstack_nodeport.yaml @@ -0,0 +1,36 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ds-nodeport-pod +spec: + selector: + matchLabels: + k8s-app: nginx-app-nodeport + replicas: 2 + template: + metadata: + labels: + k8s-app: nginx-app-nodeport + spec: + containers: + - name: nginx + image: ranchertest/mytestcontainer + ports: + - containerPort: 80 +--- +apiVersion: v1 +kind: Service +metadata: + labels: + k8s-app: nginx-app-nodeport + name: ds-nodeport-svc + namespace: default +spec: + type: NodePort + ipFamilyPolicy: PreferDualStack + ports: + - port: 80 + nodePort: 30096 + name: http + selector: + k8s-app: nginx-app-nodeport diff --git a/tests/e2e/dualstack/Vagrantfile b/tests/e2e/dualstack/Vagrantfile new file mode 100644 index 0000000000..84342ef7ae --- /dev/null +++ b/tests/e2e/dualstack/Vagrantfile @@ -0,0 +1,111 @@ +ENV['VAGRANT_NO_PARALLEL'] = 'no' +NODE_ROLES = (ENV['E2E_NODE_ROLES'] || + ["server-0", "server-1", "server-2", "agent-0" ]) +NODE_BOXES = (ENV['E2E_NODE_BOXES'] || + ['generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004']) +GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master") +RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "") +NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i +NODE_MEMORY = (ENV['E2E_NODE_MEMORY'] || 2048).to_i +NETWORK4_PREFIX = "10.10.10" +NETWORK6_PREFIX = "a11:decf:c0ff:ee" +install_type = "" + +def provision(vm, roles, role_num, node_num) + vm.box = NODE_BOXES[node_num] + vm.hostname = "#{roles[0]}-#{role_num}" + node_ip4 = "#{NETWORK4_PREFIX}.#{100+node_num}" + node_ip6 = "#{NETWORK6_PREFIX}::#{10+node_num}" + # Only works with libvirt, which allows IPv4 + IPv6 on a single network/interface + vm.network "private_network", + :ip => node_ip4, + :netmask => "255.255.255.0", + :libvirt__dhcp_enabled => false, + :libvirt__forward_mode => "none", + :libvirt__guest_ipv6 => "yes", + :libvirt__ipv6_address => "#{NETWORK6_PREFIX}::1", + :libvirt__ipv6_prefix => "64" + + vagrant_defaults = '../vagrantdefaults.rb' + load vagrant_defaults if File.exists?(vagrant_defaults) + + defaultOSConfigure(vm) + + vm.provision "IPv6 Setup", type: "shell", path: "../scripts/ipv6.sh", args: [node_ip4, node_ip6, vm.box] + install_type = getInstallType(vm, RELEASE_VERSION, GITHUB_BRANCH) + + vm.provision "Ping Check", type: "shell", inline: "ping -c 2 k3s.io" + + if roles.include?("server") && role_num == 0 + vm.provision :k3s, run: 'once' do |k3s| + k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 + k3s.args = "server " + k3s.config = <<~YAML + node-external-ip: #{node_ip4},#{node_ip6} + node-ip: #{node_ip4},#{node_ip6} + cluster-init: true + token: vagrant + cluster-cidr: 10.42.0.0/16,2001:cafe:42:0::/56 + service-cidr: 10.43.0.0/16,2001:cafe:42:1::/112 + bind-address: #{NETWORK4_PREFIX}.100 + flannel-iface: eth1 + YAML + k3s.env = ["K3S_KUBECONFIG_MODE=0644", install_type] + end + elsif roles.include?("server") && role_num != 0 + vm.provision :k3s, run: 'once' do |k3s| + k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 + k3s.args = "server " + k3s.config = <<~YAML + node-external-ip: #{node_ip4},#{node_ip6} + node-ip: #{node_ip4},#{node_ip6} + server: https://#{NETWORK4_PREFIX}.100:6443 + token: vagrant + cluster-cidr: 10.42.0.0/16,2001:cafe:42:0::/56 + service-cidr: 10.43.0.0/16,2001:cafe:42:1::/112 + flannel-iface: eth1 + YAML + k3s.env = ["K3S_KUBECONFIG_MODE=0644", install_type] + end + end + if roles.include?("agent") + vm.provision :k3s, run: 'once' do |k3s| + k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 + k3s.args = "agent " + k3s.config = <<~YAML + node-external-ip: #{node_ip4},#{node_ip6} + node-ip: #{node_ip4},#{node_ip6} + server: https://#{NETWORK4_PREFIX}.100:6443 + token: vagrant + flannel-iface: eth1 + YAML + k3s.env = ["K3S_KUBECONFIG_MODE=0644", install_type] + end + end +end + +Vagrant.configure("2") do |config| + config.vagrant.plugins = ["vagrant-k3s", "vagrant-reload", "vagrant-libvirt"] + config.vm.provider "libvirt" do |v| + v.cpus = NODE_CPUS + v.memory = NODE_MEMORY + end + + if NODE_ROLES.kind_of?(String) + NODE_ROLES = NODE_ROLES.split(" ", -1) + end + if NODE_BOXES.kind_of?(String) + NODE_BOXES = NODE_BOXES.split(" ", -1) + end + + # Must iterate on the index, vagrant does not understand iterating + # over the node roles themselves + NODE_ROLES.length.times do |i| + name = NODE_ROLES[i] + config.vm.define name do |node| + roles = name.split("-", -1) + role_num = roles.pop.to_i + provision(node.vm, roles, role_num, i) + end + end +end diff --git a/tests/e2e/dualstack/dualstack_test.go b/tests/e2e/dualstack/dualstack_test.go new file mode 100644 index 0000000000..a69688c3d6 --- /dev/null +++ b/tests/e2e/dualstack/dualstack_test.go @@ -0,0 +1,213 @@ +package validatecluster + +import ( + "flag" + "fmt" + "os" + "strings" + "testing" + + "github.com/k3s-io/k3s/tests/e2e" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// Valid nodeOS: generic/ubuntu2004, opensuse/Leap-15.3.x86_64 +var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system") +var serverCount = flag.Int("serverCount", 3, "number of server nodes") +var agentCount = flag.Int("agentCount", 0, "number of agent nodes") + +// Environment Variables Info: +// E2E_RELEASE_VERSION=v1.23.1+k3s1 or nil for latest commit from master + +type objIP struct { + name string + ipv4 string + ipv6 string +} + +func getPodIPs(kubeConfigFile string) ([]objIP, error) { + cmd := `kubectl get pods -A -o=jsonpath='{range .items[*]}{.metadata.name}{" "}{.status.podIPs[*].ip}{"\n"}{end}' --kubeconfig=` + kubeConfigFile + return getObjIPs(cmd) +} +func getNodeIPs(kubeConfigFile string) ([]objIP, error) { + cmd := `kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{" "}{.status.addresses[?(@.type == "InternalIP")].address}{"\n"}{end}' --kubeconfig=` + kubeConfigFile + return getObjIPs(cmd) +} + +func getObjIPs(cmd string) ([]objIP, error) { + var objIPs []objIP + res, err := e2e.RunCommand(cmd) + if err != nil { + return nil, err + } + objs := strings.Split(res, "\n") + objs = objs[:len(objs)-1] + + for _, obj := range objs { + fields := strings.Fields(obj) + if len(fields) > 2 { + objIPs = append(objIPs, objIP{name: fields[0], ipv4: fields[1], ipv6: fields[2]}) + } else if len(fields) > 1 { + objIPs = append(objIPs, objIP{name: fields[0], ipv4: fields[1]}) + } else { + objIPs = append(objIPs, objIP{name: fields[0]}) + } + } + return objIPs, nil +} + +func Test_E2EDualStack(t *testing.T) { + flag.Parse() + RegisterFailHandler(Fail) + RunSpecs(t, "Validate DualStack Suite") +} + +var ( + kubeConfigFile string + serverNodeNames []string + agentNodeNames []string +) + +var _ = Describe("Verify DualStack Configuration", func() { + + It("Starts up with no issues", func() { + var err error + serverNodeNames, agentNodeNames, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount) + Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog()) + fmt.Println("CLUSTER CONFIG") + fmt.Println("OS:", *nodeOS) + fmt.Println("Server Nodes:", serverNodeNames) + fmt.Println("Agent Nodes:", agentNodeNames) + kubeConfigFile, err = e2e.GenKubeConfigFile(serverNodeNames[0]) + Expect(err).NotTo(HaveOccurred()) + }) + + It("Checks Node Status", func() { + Eventually(func(g Gomega) { + nodes, err := e2e.ParseNodes(kubeConfigFile, false) + g.Expect(err).NotTo(HaveOccurred()) + for _, node := range nodes { + g.Expect(node.Status).Should(Equal("Ready")) + } + }, "420s", "5s").Should(Succeed()) + _, err := e2e.ParseNodes(kubeConfigFile, true) + Expect(err).NotTo(HaveOccurred()) + }) + + It("Checks Pod Status", func() { + Eventually(func(g Gomega) { + pods, err := e2e.ParsePods(kubeConfigFile, false) + g.Expect(err).NotTo(HaveOccurred()) + for _, pod := range pods { + if strings.Contains(pod.Name, "helm-install") { + g.Expect(pod.Status).Should(Equal("Completed"), pod.Name) + } else { + g.Expect(pod.Status).Should(Equal("Running"), pod.Name) + } + } + }, "420s", "5s").Should(Succeed()) + _, err := e2e.ParsePods(kubeConfigFile, true) + Expect(err).NotTo(HaveOccurred()) + }) + + It("Verifies that each node has IPv4 and IPv6", func() { + nodeIPs, err := getNodeIPs(kubeConfigFile) + Expect(err).NotTo(HaveOccurred()) + for _, node := range nodeIPs { + Expect(node.ipv4).Should(ContainSubstring("10.10.10")) + Expect(node.ipv6).Should(ContainSubstring("a11:decf:c0ff")) + } + }) + It("Verifies that each pod has IPv4 and IPv6", func() { + podIPs, err := getPodIPs(kubeConfigFile) + Expect(err).NotTo(HaveOccurred()) + for _, pod := range podIPs { + Expect(pod.ipv4).Should(Or(ContainSubstring("10.10.10"), ContainSubstring("10.42.")), pod.name) + Expect(pod.ipv6).Should(Or(ContainSubstring("a11:decf:c0ff"), ContainSubstring("2001:cafe:42")), pod.name) + } + }) + + It("Verifies ClusterIP Service", func() { + _, err := e2e.DeployWorkload("dualstack_clusterip.yaml", kubeConfigFile, false) + Expect(err).NotTo(HaveOccurred()) + Eventually(func() (string, error) { + cmd := "kubectl get pods -o=name -l k8s-app=nginx-app-clusterip --field-selector=status.phase=Running --kubeconfig=" + kubeConfigFile + return e2e.RunCommand(cmd) + }, "120s", "5s").Should(ContainSubstring("ds-clusterip-pod")) + + // Checks both IPv4 and IPv6 + clusterips, err := e2e.FetchClusterIP(kubeConfigFile, "ds-clusterip-svc", true) + Expect(err).NotTo(HaveOccurred()) + for _, ip := range strings.Split(clusterips, ",") { + if strings.Contains(ip, "::") { + ip = "[" + ip + "]" + } + pods, err := e2e.ParsePods(kubeConfigFile, false) + Expect(err).NotTo(HaveOccurred()) + for _, pod := range pods { + if !strings.HasPrefix(pod.Name, "ds-clusterip-pod") { + continue + } + cmd := fmt.Sprintf("curl -L --insecure http://%s", ip) + Eventually(func() (string, error) { + return e2e.RunCmdOnNode(cmd, serverNodeNames[0]) + }, "60s", "5s").Should(ContainSubstring("Welcome to nginx!"), "failed cmd: "+cmd) + } + } + }) + It("Verifies Ingress", func() { + _, err := e2e.DeployWorkload("dualstack_ingress.yaml", kubeConfigFile, false) + Expect(err).NotTo(HaveOccurred(), "Ingress manifest not deployed") + cmd := "kubectl get ingress ds-ingress --kubeconfig=" + kubeConfigFile + " -o jsonpath=\"{.spec.rules[*].host}\"" + hostName, err := e2e.RunCommand(cmd) + Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd) + nodeIPs, err := getNodeIPs(kubeConfigFile) + Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd) + for _, node := range nodeIPs { + cmd := fmt.Sprintf("curl --header host:%s http://%s/name.html", hostName, node.ipv4) + Eventually(func() (string, error) { + return e2e.RunCommand(cmd) + }, "10s", "2s").Should(ContainSubstring("ds-clusterip-pod"), "failed cmd: "+cmd) + cmd = fmt.Sprintf("curl --header host:%s http://[%s]/name.html", hostName, node.ipv6) + Eventually(func() (string, error) { + return e2e.RunCommand(cmd) + }, "5s", "1s").Should(ContainSubstring("ds-clusterip-pod"), "failed cmd: "+cmd) + } + }) + + It("Verifies NodePort Service", func() { + _, err := e2e.DeployWorkload("dualstack_nodeport.yaml", kubeConfigFile, false) + Expect(err).NotTo(HaveOccurred()) + cmd := "kubectl get service ds-nodeport-svc --kubeconfig=" + kubeConfigFile + " --output jsonpath=\"{.spec.ports[0].nodePort}\"" + nodeport, err := e2e.RunCommand(cmd) + Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd) + nodeIPs, err := getNodeIPs(kubeConfigFile) + Expect(err).NotTo(HaveOccurred()) + for _, node := range nodeIPs { + cmd = "curl -L --insecure http://" + node.ipv4 + ":" + nodeport + "/name.html" + Eventually(func() (string, error) { + return e2e.RunCommand(cmd) + }, "10s", "1s").Should(ContainSubstring("ds-nodeport-pod"), "failed cmd: "+cmd) + cmd = "curl -L --insecure http://[" + node.ipv6 + "]:" + nodeport + "/name.html" + Eventually(func() (string, error) { + return e2e.RunCommand(cmd) + }, "10s", "1s").Should(ContainSubstring("ds-nodeport-pod"), "failed cmd: "+cmd) + } + }) + +}) + +var failed bool +var _ = AfterEach(func() { + failed = failed || CurrentGinkgoTestDescription().Failed +}) + +var _ = AfterSuite(func() { + if failed { + fmt.Println("FAILED!") + } else { + Expect(e2e.DestroyCluster()).To(Succeed()) + Expect(os.Remove(kubeConfigFile)).To(Succeed()) + } +}) diff --git a/tests/e2e/scripts/ipv6.sh b/tests/e2e/scripts/ipv6.sh new file mode 100644 index 0000000000..af7f823a93 --- /dev/null +++ b/tests/e2e/scripts/ipv6.sh @@ -0,0 +1,16 @@ +#!/bin/bash +ip4_addr=$1 +ip6_addr=$2 +os=$3 + +sysctl -w net.ipv6.conf.all.disable_ipv6=0 +sysctl -w net.ipv6.conf.eth1.accept_dad=0 + +if [ -z "${os##*ubuntu*}" ]; then + netplan set ethernets.eth1.accept-ra=false + netplan set ethernets.eth1.addresses=["$ip4_addr"/24,"$ip6_addr"/64] + netplan apply +else + ip -6 addr add "$ip6_addr"/64 dev eth1 +fi +ip addr show dev eth1 \ No newline at end of file diff --git a/tests/e2e/scripts/rancher.sh b/tests/e2e/scripts/rancher.sh new file mode 100644 index 0000000000..4c7d8efa16 --- /dev/null +++ b/tests/e2e/scripts/rancher.sh @@ -0,0 +1,63 @@ +#!/bin/bash +node_ip=$1 + +echo "Give K3s time to startup" +sleep 10 +kubectl -n kube-system rollout status deploy/coredns +kubectl -n kube-system rollout status deploy/local-path-provisioner + +cat << EOF > /var/lib/rancher/k3s/server/manifests/rancher.yaml +--- +apiVersion: v1 +kind: Namespace +metadata: + name: cert-manager +--- +apiVersion: v1 +kind: Namespace +metadata: + name: cattle-system +--- +apiVersion: helm.cattle.io/v1 +kind: HelmChart +metadata: + namespace: kube-system + name: cert-manager +spec: + targetNamespace: cert-manager + version: v1.6.1 + chart: cert-manager + repo: https://charts.jetstack.io + set: + installCRDs: "true" +--- +apiVersion: helm.cattle.io/v1 +kind: HelmChart +metadata: + namespace: kube-system + name: rancher +spec: + targetNamespace: cattle-system + version: 2.6.5 + chart: rancher + repo: https://releases.rancher.com/server-charts/latest + set: + ingress.tls.source: "rancher" + hostname: "$node_ip.nip.io" + replicas: 1 +EOF + + +echo "Give Rancher time to startup" +sleep 20 +kubectl -n cert-manager rollout status deploy/cert-manager +while ! kubectl get secret --namespace cattle-system bootstrap-secret -o go-template='{{.data.bootstrapPassword|base64decode}}' &> /dev/null; do + ((iterations++)) + if [ "$iterations" -ge 8 ]; then + echo "Unable to find bootstrap-secret" + exit 1 + fi + echo "waiting for bootstrap-secret..." + sleep 20 +done +echo https://"$node_ip".nip.io/dashboard/?setup=$(kubectl get secret --namespace cattle-system bootstrap-secret -o go-template='{{.data.bootstrapPassword|base64decode}}') \ No newline at end of file diff --git a/tests/e2e/secretsencryption/Vagrantfile b/tests/e2e/secretsencryption/Vagrantfile index e6fe9f0479..acc43a4ad2 100644 --- a/tests/e2e/secretsencryption/Vagrantfile +++ b/tests/e2e/secretsencryption/Vagrantfile @@ -21,14 +21,8 @@ def provision(vm, role, role_num, node_num) load vagrant_defaults if File.exists?(vagrant_defaults) defaultOSConfigure(vm) + install_type = getInstallType(vm, RELEASE_VERSION, GITHUB_BRANCH) - if !RELEASE_VERSION.empty? - install_type = "INSTALL_K3S_VERSION=#{RELEASE_VERSION}" - else - # Grabs the last 5 commit SHA's from the given branch, then purges any commits that do not have a passing CI build - vm.provision "shell", path: "../scripts/latest_commit.sh", args: [GITHUB_BRANCH, "/tmp/k3s_commits"] - install_type = "INSTALL_K3S_COMMIT=$(head\ -n\ 1\ /tmp/k3s_commits)" - end vm.provision "shell", inline: "ping -c 2 k3s.io" if role.include?("server") && role_num == 0 @@ -44,7 +38,7 @@ def provision(vm, role, role_num, node_num) k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 end end - if vm.box.include?("microos") + if vm.box.to_s.include?("microos") vm.provision 'k3s-reload', type: 'reload', run: 'once' end end diff --git a/tests/e2e/splitserver/Vagrantfile b/tests/e2e/splitserver/Vagrantfile index c244751060..803b186186 100644 --- a/tests/e2e/splitserver/Vagrantfile +++ b/tests/e2e/splitserver/Vagrantfile @@ -9,7 +9,6 @@ NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i NODE_MEMORY = (ENV['E2E_NODE_MEMORY'] || 1024).to_i # Virtualbox >= 6.1.28 require `/etc/vbox/network.conf` for expanded private networks NETWORK_PREFIX = "10.10.10" -install_type = "" def provision(vm, role, role_num, node_num) vm.box = NODE_BOXES[node_num] @@ -17,20 +16,13 @@ def provision(vm, role, role_num, node_num) # An expanded netmask is required to allow VM<-->VM communication, virtualbox defaults to /32 vm.network "private_network", ip: "#{NETWORK_PREFIX}.#{100+node_num}", netmask: "255.255.255.0" - scripts_location = Dir.exists?("./scripts") ? "./scripts" : "../scripts" vagrant_defaults = File.exists?("./vagrantdefaults.rb") ? "./vagrantdefaults.rb" : "../vagrantdefaults.rb" load vagrant_defaults defaultOSConfigure(vm) - if !RELEASE_VERSION.empty? - install_type = "INSTALL_K3S_VERSION=#{RELEASE_VERSION}" - else - # Grabs the last 5 commit SHA's from the given branch, then purges any commits that do not have a passing CI build - # MicroOS requires it not be in a /tmp/ or other root system folder - vm.provision "Acquire latest commit", type: "shell", path: scripts_location + "/latest_commit.sh", args: [GITHUB_BRANCH, "/home/vagrant/k3s_commits"] - install_type = "INSTALL_K3S_COMMIT=$(head\ -n\ 1\ /home/vagrant/k3s_commits)" - end + install_type = getInstallType(vm, RELEASE_VERSION, GITHUB_BRANCH) + vm.provision "ping k3s.io", type: "shell", inline: "ping -c 2 k3s.io" if node_num == 0 && !role.include?("server") && !role.include?("etcd") @@ -46,6 +38,8 @@ def provision(vm, role, role_num, node_num) disable-apiserver: true disable-controller-manager: true disable-scheduler: true + node-taint: + - node-role.kubernetes.io/etcd:NoExecute YAML k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant #{install_type}] k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 @@ -59,13 +53,33 @@ def provision(vm, role, role_num, node_num) disable-apiserver: true disable-controller-manager: true disable-scheduler: true + node-taint: + - node-role.kubernetes.io/etcd:NoExecute + YAML k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant #{install_type}] k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 end elsif role.include?("server") && role.include?("cp") vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s| - k3s.args = "server --disable-etcd --server https://#{NETWORK_PREFIX}.100:6443 --flannel-iface=eth1" + k3s.args = "server" + k3s.config = <<~YAML + server: https://#{NETWORK_PREFIX}.100:6443 + flannel-iface: eth1 + disable-etcd: true + node-taint: + - node-role.kubernetes.io/control-plane:NoSchedule + YAML + k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant #{install_type}] + k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 + end + elsif role.include?("server") && role.include?("all") + vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s| + k3s.args = "server" + k3s.config = <<~YAML + server: https://#{NETWORK_PREFIX}.100:6443 + flannel-iface: eth1 + YAML k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant #{install_type}] k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 end @@ -77,7 +91,7 @@ def provision(vm, role, role_num, node_num) k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 end end - if vm.box.include?("microos") + if vm.box.to_s.include?("microos") vm.provision 'k3s-reload', type: 'reload', run: 'once' end end diff --git a/tests/e2e/splitserver/splitserver_test.go b/tests/e2e/splitserver/splitserver_test.go index a9de6a69b1..612b0faf5a 100644 --- a/tests/e2e/splitserver/splitserver_test.go +++ b/tests/e2e/splitserver/splitserver_test.go @@ -114,23 +114,17 @@ var _ = Describe("Verify Create", func() { _, err := e2e.DeployWorkload("clusterip.yaml", kubeConfigFile, false) Expect(err).NotTo(HaveOccurred(), "Cluster IP manifest not deployed") - Eventually(func(g Gomega) { - cmd := "kubectl get pods -o=name -l k8s-app=nginx-app-clusterip --field-selector=status.phase=Running --kubeconfig=" + kubeConfigFile - res, err := e2e.RunCommand(cmd) - Expect(err).NotTo(HaveOccurred()) - g.Expect(res).Should((ContainSubstring("test-clusterip"))) - }, "240s", "5s").Should(Succeed()) + cmd := "kubectl get pods -o=name -l k8s-app=nginx-app-clusterip --field-selector=status.phase=Running --kubeconfig=" + kubeConfigFile + Eventually(func() (string, error) { + return e2e.RunCommand(cmd) + }, "240s", "5s").Should(ContainSubstring("test-clusterip"), "failed cmd: "+cmd) - clusterip, _ := e2e.FetchClusterIP(kubeConfigFile, "nginx-clusterip-svc") - cmd := "curl -L --insecure http://" + clusterip + "/name.html" - fmt.Println(cmd) + clusterip, _ := e2e.FetchClusterIP(kubeConfigFile, "nginx-clusterip-svc", false) + cmd = "curl -L --insecure http://" + clusterip + "/name.html" for _, nodeName := range cpNodeNames { - Eventually(func(g Gomega) { - res, err := e2e.RunCmdOnNode(cmd, nodeName) - g.Expect(err).NotTo(HaveOccurred()) - fmt.Println(res) - Expect(res).Should(ContainSubstring("test-clusterip")) - }, "120s", "10s").Should(Succeed()) + Eventually(func() (string, error) { + return e2e.RunCmdOnNode(cmd, nodeName) + }, "120s", "10s").Should(ContainSubstring("test-clusterip"), "failed cmd: "+cmd) } }) @@ -144,21 +138,15 @@ var _ = Describe("Verify Create", func() { nodeport, err := e2e.RunCommand(cmd) Expect(err).NotTo(HaveOccurred()) - Eventually(func(g Gomega) { - cmd := "kubectl get pods -o=name -l k8s-app=nginx-app-nodeport --field-selector=status.phase=Running --kubeconfig=" + kubeConfigFile - res, err := e2e.RunCommand(cmd) - Expect(err).NotTo(HaveOccurred()) - g.Expect(res).Should(ContainSubstring("test-nodeport"), "nodeport pod was not created") - }, "240s", "5s").Should(Succeed()) + cmd = "kubectl get pods -o=name -l k8s-app=nginx-app-nodeport --field-selector=status.phase=Running --kubeconfig=" + kubeConfigFile + Eventually(func() (string, error) { + return e2e.RunCommand(cmd) + }, "240s", "5s").Should(ContainSubstring("test-nodeport"), "nodeport pod was not created") cmd = "curl -L --insecure http://" + nodeExternalIP + ":" + nodeport + "/name.html" - fmt.Println(cmd) - Eventually(func(g Gomega) { - res, err := e2e.RunCommand(cmd) - Expect(err).NotTo(HaveOccurred()) - fmt.Println(res) - g.Expect(res).Should(ContainSubstring("test-nodeport")) - }, "240s", "5s").Should(Succeed()) + Eventually(func() (string, error) { + return e2e.RunCommand(cmd) + }, "240s", "5s").Should(ContainSubstring("test-nodeport"), "failed cmd: "+cmd) } }) @@ -173,21 +161,15 @@ var _ = Describe("Verify Create", func() { port, err := e2e.RunCommand(cmd) Expect(err).NotTo(HaveOccurred()) - Eventually(func(g Gomega) { - cmd := "kubectl get pods -o=name -l k8s-app=nginx-app-loadbalancer --field-selector=status.phase=Running --kubeconfig=" + kubeConfigFile - res, err := e2e.RunCommand(cmd) - Expect(err).NotTo(HaveOccurred()) - g.Expect(res).Should(ContainSubstring("test-loadbalancer")) - }, "240s", "5s").Should(Succeed()) + cmd = "kubectl get pods -o=name -l k8s-app=nginx-app-loadbalancer --field-selector=status.phase=Running --kubeconfig=" + kubeConfigFile + Eventually(func() (string, error) { + return e2e.RunCommand(cmd) + }, "240s", "5s").Should(ContainSubstring("test-loadbalancer"), "failed cmd: "+cmd) - Eventually(func(g Gomega) { - cmd = "curl -L --insecure http://" + ip + ":" + port + "/name.html" - fmt.Println(cmd) - res, err := e2e.RunCommand(cmd) - Expect(err).NotTo(HaveOccurred()) - fmt.Println(res) - g.Expect(res).Should(ContainSubstring("test-loadbalancer")) - }, "240s", "5s").Should(Succeed()) + cmd = "curl -L --insecure http://" + ip + ":" + port + "/name.html" + Eventually(func() (string, error) { + return e2e.RunCommand(cmd) + }, "240s", "5s").Should(ContainSubstring("test-loadbalancer"), "failed cmd: "+cmd) } }) @@ -198,14 +180,9 @@ var _ = Describe("Verify Create", func() { for _, nodeName := range cpNodeNames { ip, _ := e2e.FetchNodeExternalIP(nodeName) cmd := "curl --header host:foo1.bar.com" + " http://" + ip + "/name.html" - fmt.Println(cmd) - - Eventually(func(g Gomega) { - res, err := e2e.RunCommand(cmd) - g.Expect(err).NotTo(HaveOccurred()) - fmt.Println(res) - g.Expect(res).Should(ContainSubstring("test-ingress")) - }, "240s", "5s").Should(Succeed()) + Eventually(func() (string, error) { + return e2e.RunCommand(cmd) + }, "240s", "5s").Should(ContainSubstring("test-ingress"), "failed cmd: "+cmd) } }) @@ -213,37 +190,30 @@ var _ = Describe("Verify Create", func() { _, err := e2e.DeployWorkload("daemonset.yaml", kubeConfigFile, false) Expect(err).NotTo(HaveOccurred(), "Daemonset manifest not deployed") - nodes, _ := e2e.ParseNodes(kubeConfigFile, false) - pods, _ := e2e.ParsePods(kubeConfigFile, false) - Eventually(func(g Gomega) { + pods, _ := e2e.ParsePods(kubeConfigFile, false) count := e2e.CountOfStringInSlice("test-daemonset", pods) fmt.Println("POD COUNT") fmt.Println(count) - fmt.Println("NODE COUNT") - fmt.Println(len(nodes)) - g.Expect(len(nodes)).Should((Equal(count)), "Daemonset pod count does not match node count") - }, "420s", "10s").Should(Succeed()) + fmt.Println("CP COUNT") + fmt.Println(len(cpNodeNames)) + g.Expect(len(cpNodeNames)).Should((Equal(count)), "Daemonset pod count does not match cp node count") + }, "240s", "10s").Should(Succeed()) }) It("Verifies dns access", func() { _, err := e2e.DeployWorkload("dnsutils.yaml", kubeConfigFile, false) Expect(err).NotTo(HaveOccurred(), "dnsutils manifest not deployed") - Eventually(func(g Gomega) { - cmd := "kubectl get pods dnsutils --kubeconfig=" + kubeConfigFile - res, _ := e2e.RunCommand(cmd) - fmt.Println(res) - g.Expect(res).Should(ContainSubstring("dnsutils")) - }, "420s", "2s").Should(Succeed()) + cmd := "kubectl get pods dnsutils --kubeconfig=" + kubeConfigFile + Eventually(func() (string, error) { + return e2e.RunCommand(cmd) + }, "420s", "2s").Should(ContainSubstring("dnsutils"), "failed cmd: "+cmd) - Eventually(func(g Gomega) { - cmd := "kubectl --kubeconfig=" + kubeConfigFile + " exec -i -t dnsutils -- nslookup kubernetes.default" - fmt.Println(cmd) - res, _ := e2e.RunCommand(cmd) - fmt.Println(res) - g.Expect(res).Should(ContainSubstring("kubernetes.default.svc.cluster.local")) - }, "420s", "2s").Should(Succeed()) + cmd = "kubectl --kubeconfig=" + kubeConfigFile + " exec -i -t dnsutils -- nslookup kubernetes.default" + Eventually(func() (string, error) { + return e2e.RunCommand(cmd) + }, "420s", "2s").Should(ContainSubstring("kubernetes.default.svc.cluster.local"), "failed cmd: "+cmd) }) }) }) diff --git a/tests/e2e/testutils.go b/tests/e2e/testutils.go index c955a35037..e3e1d65152 100644 --- a/tests/e2e/testutils.go +++ b/tests/e2e/testutils.go @@ -97,7 +97,16 @@ func DestroyCluster() error { return os.Remove("vagrant.log") } -func FetchClusterIP(kubeconfig string, servicename string) (string, error) { +func FetchClusterIP(kubeconfig string, servicename string, dualStack bool) (string, error) { + if dualStack { + cmd := "kubectl get svc " + servicename + " -o jsonpath='{.spec.clusterIPs}' --kubeconfig=" + kubeconfig + res, err := RunCommand(cmd) + if err != nil { + return res, err + } + res = strings.ReplaceAll(res, "\"", "") + return strings.Trim(res, "[]"), nil + } cmd := "kubectl get svc " + servicename + " -o jsonpath='{.spec.clusterIP}' --kubeconfig=" + kubeconfig return RunCommand(cmd) } diff --git a/tests/e2e/upgradecluster/Vagrantfile b/tests/e2e/upgradecluster/Vagrantfile index 3602f96422..ae0120f703 100644 --- a/tests/e2e/upgradecluster/Vagrantfile +++ b/tests/e2e/upgradecluster/Vagrantfile @@ -55,7 +55,7 @@ def provision(vm, role, role_num, node_num) k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 end end - if vm.box.include?("microos") + if vm.box.to_s.include?("microos") vm.provision 'k3s-reload', type: 'reload', run: 'once' end end diff --git a/tests/e2e/upgradecluster/upgradecluster_test.go b/tests/e2e/upgradecluster/upgradecluster_test.go index 882dbca33b..a4dddfa5ed 100644 --- a/tests/e2e/upgradecluster/upgradecluster_test.go +++ b/tests/e2e/upgradecluster/upgradecluster_test.go @@ -84,7 +84,7 @@ var _ = Describe("Verify Upgrade", func() { return e2e.RunCommand(cmd) }, "240s", "5s").Should(ContainSubstring("test-clusterip"), "failed cmd: "+cmd) - clusterip, _ := e2e.FetchClusterIP(kubeConfigFile, "nginx-clusterip-svc") + clusterip, _ := e2e.FetchClusterIP(kubeConfigFile, "nginx-clusterip-svc", false) cmd = "curl -L --insecure http://" + clusterip + "/name.html" for _, nodeName := range serverNodeNames { Eventually(func() (string, error) { @@ -280,7 +280,7 @@ var _ = Describe("Verify Upgrade", func() { return e2e.RunCommand(cmd) }, "420s", "5s").Should(ContainSubstring("test-clusterip")) - clusterip, _ := e2e.FetchClusterIP(kubeConfigFile, "nginx-clusterip-svc") + clusterip, _ := e2e.FetchClusterIP(kubeConfigFile, "nginx-clusterip-svc", false) cmd := "curl -L --insecure http://" + clusterip + "/name.html" fmt.Println(cmd) for _, nodeName := range serverNodeNames { diff --git a/tests/e2e/vagrantdefaults.rb b/tests/e2e/vagrantdefaults.rb index 2dda456015..9bbc91192e 100644 --- a/tests/e2e/vagrantdefaults.rb +++ b/tests/e2e/vagrantdefaults.rb @@ -1,11 +1,27 @@ def defaultOSConfigure(vm) - if vm.box.include?("generic/ubuntu") - vm.provision "Set DNS", type: "shell", inline: "systemd-resolve --set-dns=8.8.8.8 --interface=eth0" - vm.provision "Install jq", type: "shell", inline: "apt install -y jq" - elsif vm.box.include?("Leap") - vm.provision "Install jq", type: "shell", inline: "zypper install -y jq" - elsif vm.box.include?("microos") - vm.provision "Install jq", type: "shell", inline: "transactional-update pkg install -y jq" - vm.provision 'reload', run: 'once' - end + box = vm.box.to_s + if box.include?("generic/ubuntu") + vm.provision "Set DNS", type: "shell", inline: "netplan set ethernets.eth0.nameservers.addresses=[8.8.8.8,1.1.1.1]; netplan apply", run: 'once' + vm.provision "Install jq", type: "shell", inline: "apt install -y jq" + elsif box.include?("Leap") || box.include?("Tumbleweed") + vm.provision "Install jq", type: "shell", inline: "zypper install -y jq" + elsif box.include?("alpine") + vm.provision "Install tools", type: "shell", inline: "apk add jq coreutils" + elsif box.include?("microos") + vm.provision "Install jq", type: "shell", inline: "transactional-update pkg install -y jq" + vm.provision 'reload', run: 'once' + end +end + +def getInstallType(vm, release_version, branch) + if release_version == "skip" + install_type = "INSTALL_K3S_SKIP_DOWNLOAD=true" + elsif !release_version.empty? + return "INSTALL_K3S_VERSION=#{release_version}" + else + # Grabs the last 5 commit SHA's from the given branch, then purges any commits that do not have a passing CI build + # MicroOS requires it not be in a /tmp/ or other root system folder + vm.provision "Get latest commit", type: "shell", path: "../scripts/latest_commit.sh", args: [branch, "/tmp/k3s_commits"] + return "INSTALL_K3S_COMMIT=$(head\ -n\ 1\ /tmp/k3s_commits)" + end end \ No newline at end of file diff --git a/tests/e2e/validatecluster/Vagrantfile b/tests/e2e/validatecluster/Vagrantfile index 082329ff7c..4129be20c2 100644 --- a/tests/e2e/validatecluster/Vagrantfile +++ b/tests/e2e/validatecluster/Vagrantfile @@ -2,11 +2,12 @@ ENV['VAGRANT_NO_PARALLEL'] = 'no' NODE_ROLES = (ENV['E2E_NODE_ROLES'] || ["server-0", "server-1", "server-2", "agent-0", "agent-1"]) NODE_BOXES = (ENV['E2E_NODE_BOXES'] || - ['opensuse/Leap-15.3.x86_64', 'opensuse/Leap-15.3.x86_64', 'opensuse/Leap-15.3.x86_64', 'opensuse/Leap-15.3.x86_64', 'opensuse/Leap-15.3.x86_64']) + ['generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004']) GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master") RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "") EXTERNAL_DB = (ENV['E2E_EXTERNAL_DB'] || "etcd") HARDENED = (ENV['E2E_HARDENED'] || "") +RANCHER = (ENV['E2E_RANCHER'] || "") NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i NODE_MEMORY = (ENV['E2E_NODE_MEMORY'] || 1024).to_i # Virtualbox >= 6.1.28 require `/etc/vbox/network.conf` for expanded private networks @@ -26,15 +27,8 @@ def provision(vm, role, role_num, node_num) load vagrant_defaults defaultOSConfigure(vm) + install_type = getInstallType(vm, RELEASE_VERSION, GITHUB_BRANCH) - if !RELEASE_VERSION.empty? - install_type = "INSTALL_K3S_VERSION=#{RELEASE_VERSION}" - else - # Grabs the last 5 commit SHA's from the given branch, then purges any commits that do not have a passing CI build - # MicroOS requires it not be in a /tmp/ or other root system folder - vm.provision "Get latest commit", type: "shell", path: scripts_location + "/latest_commit.sh", args: [GITHUB_BRANCH, "/home/vagrant/k3s_commits"] - install_type = "INSTALL_K3S_COMMIT=$(head\ -n\ 1\ /home/vagrant/k3s_commits)" - end vm.provision "shell", inline: "ping -c 2 k3s.io" db_type = getDBType(role, role_num, vm) @@ -51,6 +45,7 @@ def provision(vm, role, role_num, node_num) token: vagrant node-external-ip: #{NETWORK_PREFIX}.100 flannel-iface: eth1 + tls-san: #{NETWORK_PREFIX}.100.nip.io #{db_type} #{hardened_arg} YAML @@ -89,12 +84,16 @@ def provision(vm, role, role_num, node_num) k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 end end - if vm.box.include?("microos") + if vm.box.to_s.include?("microos") vm.provision 'k3s-reload', type: 'reload', run: 'once' if !EXTERNAL_DB.empty? vm.provision "shell", inline: "docker start #{EXTERNAL_DB}" end end + # This step does not run by default and is designed to be called by higher level tools + if !RANCHER.empty? + vm.provision "Install Rancher", type: "shell", run: "never", path: scripts_location + "/rancher.sh", args: node_ip + end end def getDBType(role, role_num, vm) @@ -139,14 +138,14 @@ def dockerInstall(vm) vm.provider "virtualbox" do |v| v.memory = NODE_MEMORY + 1024 end - if vm.box.include?("ubuntu") + if vm.box.to_s.include?("ubuntu") vm.provision "shell", inline: "apt install -y docker.io" end - if vm.box.include?("Leap") - vm.provision "shell", inline: "zypper install -y docker" + if vm.box.to_s.include?("Leap") + vm.provision "shell", inline: "zypper install -y docker apparmor-parser" end - if vm.box.include?("microos") - vm.provision "shell", inline: "transactional-update pkg install -y docker" + if vm.box.to_s.include?("microos") + vm.provision "shell", inline: "transactional-update pkg install -y docker apparmor-parser" vm.provision 'docker-reload', type: 'reload', run: 'once' vm.provision "shell", inline: "systemctl enable --now docker" end diff --git a/tests/e2e/validatecluster/validatecluster_test.go b/tests/e2e/validatecluster/validatecluster_test.go index f4230a3e2d..daa8c75cab 100644 --- a/tests/e2e/validatecluster/validatecluster_test.go +++ b/tests/e2e/validatecluster/validatecluster_test.go @@ -84,7 +84,7 @@ var _ = Describe("Verify Create", func() { g.Expect(res).Should((ContainSubstring("test-clusterip"))) }, "240s", "5s").Should(Succeed()) - clusterip, _ := e2e.FetchClusterIP(kubeConfigFile, "nginx-clusterip-svc") + clusterip, _ := e2e.FetchClusterIP(kubeConfigFile, "nginx-clusterip-svc", false) cmd := "curl -L --insecure http://" + clusterip + "/name.html" fmt.Println(cmd) for _, nodeName := range serverNodeNames { diff --git a/tests/integration/certrotation/certrotation_int_test.go b/tests/integration/certrotation/certrotation_int_test.go index c562f87d5c..a54f73dcb0 100644 --- a/tests/integration/certrotation/certrotation_int_test.go +++ b/tests/integration/certrotation/certrotation_int_test.go @@ -34,9 +34,9 @@ var _ = Describe("certificate rotation", func() { }) When("a new server is created", func() { It("starts up with no problems", func() { - Eventually(func() (string, error) { - return testutil.K3sCmd("kubectl", "get pods -A") - }, "180s", "5s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running")) + Eventually(func() error { + return testutil.K3sDefaultDeployments() + }, "180s", "5s").Should(Succeed()) }) It("get certificate hash", func() { // get md5sum of the CA certs @@ -60,9 +60,9 @@ var _ = Describe("certificate rotation", func() { Expect(err).ToNot(HaveOccurred()) }) It("starts up with no problems", func() { - Eventually(func() (string, error) { - return testutil.K3sCmd("kubectl", "get", "pods", "-A") - }, "360s", "5s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running")) + Eventually(func() error { + return testutil.K3sDefaultDeployments() + }, "360s", "5s").Should(Succeed()) }) It("get certificate hash", func() { // get md5sum of the CA certs diff --git a/tests/integration/dualstack/dualstack_int_test.go b/tests/integration/dualstack/dualstack_int_test.go index e4ba57038f..dc433eb7a2 100644 --- a/tests/integration/dualstack/dualstack_int_test.go +++ b/tests/integration/dualstack/dualstack_int_test.go @@ -39,9 +39,9 @@ var _ = Describe("dual stack", func() { }) When("a ipv4 and ipv6 cidr is present", func() { It("starts up with no problems", func() { - Eventually(func() (string, error) { - return testutil.K3sCmd("kubectl", "get", "pods", "-A") - }, "180s", "5s").Should(MatchRegexp("kube-system.+traefik.+1\\/1.+Running")) + Eventually(func() error { + return testutil.K3sDefaultDeployments() + }, "180s", "10s").Should(Succeed()) }) It("creates pods with two IPs", func() { podname, err := testutil.K3sCmd("kubectl", "get", "pods", "-n", "kube-system", "-o", "jsonpath={.items[?(@.metadata.labels.app\\.kubernetes\\.io/name==\"traefik\")].metadata.name}") diff --git a/tests/integration/etcdrestore/etcd_restore_int_test.go b/tests/integration/etcdrestore/etcd_restore_int_test.go index c106096266..f39e6f14f0 100644 --- a/tests/integration/etcdrestore/etcd_restore_int_test.go +++ b/tests/integration/etcdrestore/etcd_restore_int_test.go @@ -32,9 +32,9 @@ var _ = Describe("etcd snapshot restore", func() { }) When("a snapshot is restored on existing node", func() { It("etcd starts up with no problems", func() { - Eventually(func() (string, error) { - return testutil.K3sCmd("kubectl", "get", "pods", "-A") - }, "360s", "5s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running")) + Eventually(func() error { + return testutil.K3sDefaultDeployments() + }, "180s", "5s").Should(Succeed()) }) It("create a workload", func() { result, err := testutil.K3sCmd("kubectl", "create", "-f", "./testdata/temp_depl.yaml") @@ -79,9 +79,9 @@ var _ = Describe("etcd snapshot restore", func() { Expect(err).ToNot(HaveOccurred()) }) It("starts up with no problems", func() { - Eventually(func() (string, error) { - return testutil.K3sCmd("kubectl", "get", "pods", "-A") - }, "360s", "5s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running")) + Eventually(func() error { + return testutil.K3sDefaultDeployments() + }, "360s", "5s").Should(Succeed()) }) It("Make sure Workload 1 exists", func() { Eventually(func() (string, error) { diff --git a/tests/integration/etcdsnapshot/etcdsnapshot_int_test.go b/tests/integration/etcdsnapshot/etcdsnapshot_int_test.go index 3956032636..a6f0ec57d4 100644 --- a/tests/integration/etcdsnapshot/etcdsnapshot_int_test.go +++ b/tests/integration/etcdsnapshot/etcdsnapshot_int_test.go @@ -33,9 +33,9 @@ var _ = Describe("etcd snapshots", func() { }) When("a new etcd is created", func() { It("starts up with no problems", func() { - Eventually(func() (string, error) { - return testutil.K3sCmd("kubectl", "get pods -A") - }, "180s", "5s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running")) + Eventually(func() error { + return testutil.K3sDefaultDeployments() + }, "180s", "10s").Should(Succeed()) }) It("saves an etcd snapshot", func() { Expect(testutil.K3sCmd("etcd-snapshot", "save")). diff --git a/tests/integration/integration.go b/tests/integration/integration.go index 030c628925..2989a0c4b0 100644 --- a/tests/integration/integration.go +++ b/tests/integration/integration.go @@ -3,6 +3,7 @@ package integration import ( "bufio" "bytes" + "context" "encoding/json" "fmt" "os" @@ -15,6 +16,10 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" ) // Compile-time variable @@ -120,6 +125,67 @@ func K3sServerArgs() []string { return args } +// K3sDefaultDeployments checks if the default deployments for K3s are ready, otherwise returns an error +func K3sDefaultDeployments() error { + return CheckDeployments([]string{"coredns", "local-path-provisioner", "metrics-server", "traefik"}) +} + +// CheckDeployments checks if the provided list of deployments are ready, otherwise returns an error +func CheckDeployments(deployments []string) error { + + deploymentSet := make(map[string]bool) + for _, d := range deployments { + deploymentSet[d] = false + } + + client, err := k8sClient() + if err != nil { + return err + } + deploymentList, err := client.AppsV1().Deployments("").List(context.Background(), metav1.ListOptions{}) + if err != nil { + return err + } + for _, deployment := range deploymentList.Items { + if _, ok := deploymentSet[deployment.Name]; ok && deployment.Status.ReadyReplicas == deployment.Status.Replicas { + deploymentSet[deployment.Name] = true + } + } + for d, found := range deploymentSet { + if !found { + return fmt.Errorf("failed to deploy %s", d) + } + } + + return nil +} + +func ParsePods() ([]corev1.Pod, error) { + clientSet, err := k8sClient() + if err != nil { + return nil, err + } + pods, err := clientSet.CoreV1().Pods("").List(context.Background(), metav1.ListOptions{}) + if err != nil { + return nil, err + } + + return pods.Items, nil +} + +func ParseNodes() ([]corev1.Node, error) { + clientSet, err := k8sClient() + if err != nil { + return nil, err + } + nodes, err := clientSet.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{}) + if err != nil { + return nil, err + } + + return nodes.Items, nil +} + func FindStringInCmdAsync(scanner *bufio.Scanner, target string) bool { for scanner.Scan() { if strings.Contains(scanner.Text(), target) { @@ -158,7 +224,6 @@ func K3sStartServer(inputArgs ...string) (*K3sServer, error) { } // K3sKillServer terminates the running K3s server and its children -// and unlocks the file for other tests func K3sKillServer(server *K3sServer) error { pgid, err := syscall.Getpgid(server.cmd.Process.Pid) if err != nil { @@ -180,9 +245,10 @@ func K3sKillServer(server *K3sServer) error { return nil } -// K3sCleanup attempts to cleanup networking and files leftover from an integration test -// this is similar to the k3s-killall.sh script, but we dynamically generate that on -// install, so we don't have access to it in testing. +// K3sCleanup unlocks the test-lock and +// attempts to cleanup networking and files leftover from an integration test. +// This is similar to the k3s-killall.sh script, but we dynamically generate that on +// install, so we don't have access to it during testing. func K3sCleanup(k3sTestLock int, dataDir string) error { if cni0Link, err := netlink.LinkByName("cni0"); err == nil { links, _ := netlink.LinkList() @@ -206,7 +272,10 @@ func K3sCleanup(k3sTestLock int, dataDir string) error { if err := os.RemoveAll(dataDir); err != nil { return err } - return flock.Release(k3sTestLock) + if k3sTestLock != -1 { + return flock.Release(k3sTestLock) + } + return nil } // RunCommand Runs command on the host @@ -220,3 +289,15 @@ func RunCommand(cmd string) (string, error) { } return out.String(), nil } + +func k8sClient() (*kubernetes.Clientset, error) { + config, err := clientcmd.BuildConfigFromFlags("", "/etc/rancher/k3s/k3s.yaml") + if err != nil { + return nil, err + } + clientSet, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, err + } + return clientSet, nil +} diff --git a/tests/integration/localstorage/localstorage_int_test.go b/tests/integration/localstorage/localstorage_int_test.go index b541c1836a..9504a6a334 100644 --- a/tests/integration/localstorage/localstorage_int_test.go +++ b/tests/integration/localstorage/localstorage_int_test.go @@ -34,9 +34,9 @@ var _ = Describe("local storage", func() { }) When("a new local storage is created", func() { It("starts up with no problems", func() { - Eventually(func() (string, error) { - return testutil.K3sCmd("kubectl get pods -A") - }, "90s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running")) + Eventually(func() error { + return testutil.K3sDefaultDeployments() + }, "120s", "5s").Should(Succeed()) }) It("creates a new pvc", func() { result, err := testutil.K3sCmd("kubectl create -f ./testdata/localstorage_pvc.yaml") diff --git a/tests/integration/secretsencryption/secretsencryption_int_test.go b/tests/integration/secretsencryption/secretsencryption_int_test.go index 92aa101be2..18e8d38912 100644 --- a/tests/integration/secretsencryption/secretsencryption_int_test.go +++ b/tests/integration/secretsencryption/secretsencryption_int_test.go @@ -34,9 +34,9 @@ var _ = Describe("secrets encryption rotation", func() { }) When("A server starts with secrets encryption", func() { It("starts up with no problems", func() { - Eventually(func() (string, error) { - return testutil.K3sCmd("kubectl get pods -A") - }, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running")) + Eventually(func() error { + return testutil.K3sDefaultDeployments() + }, "180s", "5s").Should(Succeed()) }) It("it creates a encryption key", func() { result, err := testutil.K3sCmd("secrets-encrypt status -d", secretsEncryptionDataDir) @@ -65,9 +65,9 @@ var _ = Describe("secrets encryption rotation", func() { Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed()) secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...) Expect(err).ToNot(HaveOccurred()) - Eventually(func() (string, error) { - return testutil.K3sCmd("kubectl get pods -A") - }, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running")) + Eventually(func() error { + return testutil.K3sDefaultDeployments() + }, "180s", "5s").Should(Succeed()) }) It("rotates the keys", func() { Eventually(func() (string, error) { @@ -89,9 +89,9 @@ var _ = Describe("secrets encryption rotation", func() { Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed()) secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...) Expect(err).ToNot(HaveOccurred()) - Eventually(func() (string, error) { - return testutil.K3sCmd("kubectl get pods -A") - }, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running")) + Eventually(func() error { + return testutil.K3sDefaultDeployments() + }, "180s", "5s").Should(Succeed()) time.Sleep(10 * time.Second) }) It("reencrypts the keys", func() { @@ -123,9 +123,9 @@ var _ = Describe("secrets encryption rotation", func() { Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed()) secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...) Expect(err).ToNot(HaveOccurred()) - Eventually(func() (string, error) { - return testutil.K3sCmd("kubectl get pods -A") - }, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running")) + Eventually(func() error { + return testutil.K3sDefaultDeployments() + }, "180s", "5s").Should(Succeed()) time.Sleep(10 * time.Second) }) It("reencrypts the keys", func() { diff --git a/tests/integration/startup/startup_int_test.go b/tests/integration/startup/startup_int_test.go new file mode 100644 index 0000000000..64ca170bae --- /dev/null +++ b/tests/integration/startup/startup_int_test.go @@ -0,0 +1,123 @@ +package integration + +import ( + "testing" + + testutil "github.com/k3s-io/k3s/tests/integration" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" +) + +var startupServer *testutil.K3sServer +var startupServerArgs = []string{} +var testLock int + +var _ = BeforeSuite(func() { + if testutil.IsExistingServer() { + Skip("Test does not support running on existing k3s servers") + } + var err error + testLock, err = testutil.K3sTestLock() + Expect(err).ToNot(HaveOccurred()) +}) + +var _ = Describe("startup tests", func() { + + When("a default server is created", func() { + It("is created with no arguments", func() { + var err error + startupServer, err = testutil.K3sStartServer(startupServerArgs...) + Expect(err).ToNot(HaveOccurred()) + }) + It("has the default pods deployed", func() { + Eventually(func() error { + return testutil.K3sDefaultDeployments() + }, "90s", "5s").Should(Succeed()) + }) + It("dies cleanly", func() { + Expect(testutil.K3sKillServer(startupServer)).To(Succeed()) + }) + }) + When("a etcd backed server is created", func() { + It("is created with cluster-init arguments", func() { + var err error + startupServerArgs = []string{"--cluster-init"} + startupServer, err = testutil.K3sStartServer(startupServerArgs...) + Expect(err).ToNot(HaveOccurred()) + }) + It("has the default pods deployed", func() { + Eventually(func() error { + return testutil.K3sDefaultDeployments() + }, "90s", "5s").Should(Succeed()) + }) + It("dies cleanly", func() { + Expect(testutil.K3sKillServer(startupServer)).To(Succeed()) + }) + }) + When("a server without traefik is created", func() { + It("is created with disable arguments", func() { + var err error + startupServerArgs = []string{"--disable", "traefik"} + startupServer, err = testutil.K3sStartServer(startupServerArgs...) + Expect(err).ToNot(HaveOccurred()) + }) + It("has the default pods without traefik deployed", func() { + Eventually(func() error { + return testutil.CheckDeployments([]string{"coredns", "local-path-provisioner", "metrics-server"}) + }, "90s", "10s").Should(Succeed()) + }) + It("dies cleanly", func() { + Expect(testutil.K3sKillServer(startupServer)).To(Succeed()) + Expect(testutil.K3sCleanup(-1, "")).To(Succeed()) + }) + }) + When("a server with different IPs is created", func() { + It("creates dummy interfaces", func() { + Expect(testutil.RunCommand("ip link add dummy2 type dummy")).To(Equal("")) + Expect(testutil.RunCommand("ip link add dummy3 type dummy")).To(Equal("")) + Expect(testutil.RunCommand("ip addr add 11.22.33.44/24 dev dummy2")).To(Equal("")) + Expect(testutil.RunCommand("ip addr add 55.66.77.88/24 dev dummy3")).To(Equal("")) + }) + It("is created with node-ip arguments", func() { + var err error + startupServerArgs = []string{"--node-ip", "11.22.33.44", "--node-external-ip", "55.66.77.88"} + startupServer, err = testutil.K3sStartServer(startupServerArgs...) + Expect(err).ToNot(HaveOccurred()) + }) + It("has the node deployed with correct IPs", func() { + Eventually(func() error { + return testutil.K3sDefaultDeployments() + }, "90s", "10s").Should(Succeed()) + + nodes, err := testutil.ParseNodes() + Expect(err).NotTo(HaveOccurred()) + Expect(nodes).To(HaveLen(1)) + Expect(nodes[0].Status.Addresses).To(ContainElements([]v1.NodeAddress{ + { + Type: "InternalIP", + Address: "11.22.33.44", + }, + { + Type: "ExternalIP", + Address: "55.66.77.88", + }})) + }) + It("dies cleanly", func() { + Expect(testutil.K3sKillServer(startupServer)).To(Succeed()) + Expect(testutil.RunCommand("ip link del dummy2")).To(Equal("")) + Expect(testutil.RunCommand("ip link del dummy3")).To(Equal("")) + }) + }) +}) + +var _ = AfterSuite(func() { + if !testutil.IsExistingServer() { + Expect(testutil.K3sCleanup(testLock, "")).To(Succeed()) + } +}) + +func Test_IntegrationStartup(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Startup Suite") +}