diff --git a/tests/e2e/scripts/latest_commit.sh b/tests/e2e/scripts/latest_commit.sh index db63eb6d70..5bdfe4245a 100755 --- a/tests/e2e/scripts/latest_commit.sh +++ b/tests/e2e/scripts/latest_commit.sh @@ -2,6 +2,17 @@ # Grabs the last 5 commit SHA's from the given branch, then purges any commits that do not have a passing CI build iterations=0 curl -s -H 'Accept: application/vnd.github.v3+json' "https://api.github.com/repos/k3s-io/k3s/commits?per_page=5&sha=$1" | jq -r '.[] | .sha' &> $2 +# The VMs take time on startup to hit googleapis.com, wait loop until we can +while ! curl -s --fail https://storage.googleapis.com/k3s-ci-builds > /dev/null; do + ((iterations++)) + if [ "$iterations" -ge 30 ]; then + echo "Unable to hit googleapis.com/k3s-ci-builds" + exit 1 + fi + sleep 1 +done + +iterations=0 curl -s --fail https://storage.googleapis.com/k3s-ci-builds/k3s-$(head -n 1 $2).sha256sum while [ $? -ne 0 ]; do ((iterations++)) diff --git a/tests/e2e/secretsencryption/secretsencryption_test.go b/tests/e2e/secretsencryption/secretsencryption_test.go index a9715c43f1..9b5bc61d96 100644 --- a/tests/e2e/secretsencryption/secretsencryption_test.go +++ b/tests/e2e/secretsencryption/secretsencryption_test.go @@ -28,19 +28,19 @@ func Test_E2EClusterValidation(t *testing.T) { var ( kubeConfigFile string - serverNodenames []string + serverNodeNames []string ) var _ = Describe("Verify Secrets Encryption Rotation", func() { Context("Cluster :", func() { It("Starts up with no issues", func() { var err error - serverNodenames, _, err = e2e.CreateCluster(*nodeOS, *serverCount, 0, *installType) + serverNodeNames, _, err = e2e.CreateCluster(*nodeOS, *serverCount, 0, *installType) Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog) fmt.Println("CLUSTER CONFIG") fmt.Println("OS:", *nodeOS) - fmt.Println("Server Nodes:", serverNodenames) - kubeConfigFile, err = e2e.GenKubeConfigFile(serverNodenames[0]) + fmt.Println("Server Nodes:", serverNodeNames) + kubeConfigFile, err = e2e.GenKubeConfigFile(serverNodeNames[0]) Expect(err).NotTo(HaveOccurred()) }) @@ -77,7 +77,7 @@ var _ = Describe("Verify Secrets Encryption Rotation", func() { It("Verifies encryption start stage", func() { cmd := "sudo k3s secrets-encrypt status" - for _, nodeName := range serverNodenames { + for _, nodeName := range serverNodeNames { res, err := e2e.RunCmdOnNode(cmd, nodeName) Expect(err).NotTo(HaveOccurred()) Expect(res).Should(ContainSubstring("Encryption Status: Enabled")) @@ -88,9 +88,9 @@ var _ = Describe("Verify Secrets Encryption Rotation", func() { It("Prepares for Secrets-Encryption Rotation", func() { cmd := "sudo k3s secrets-encrypt prepare" - res, err := e2e.RunCmdOnNode(cmd, serverNodenames[0]) + res, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0]) Expect(err).NotTo(HaveOccurred(), res) - for i, nodeName := range serverNodenames { + for i, nodeName := range serverNodeNames { cmd := "sudo k3s secrets-encrypt status" res, err := e2e.RunCmdOnNode(cmd, nodeName) Expect(err).NotTo(HaveOccurred(), res) @@ -104,7 +104,7 @@ var _ = Describe("Verify Secrets Encryption Rotation", func() { }) It("Restarts K3s servers", func() { - Expect(e2e.RestartCluster(serverNodenames)).To(Succeed()) + Expect(e2e.RestartCluster(serverNodeNames)).To(Succeed()) }) It("Checks node and pod status", func() { @@ -131,7 +131,7 @@ var _ = Describe("Verify Secrets Encryption Rotation", func() { It("Verifies encryption prepare stage", func() { cmd := "sudo k3s secrets-encrypt status" - for _, nodeName := range serverNodenames { + for _, nodeName := range serverNodeNames { res, err := e2e.RunCmdOnNode(cmd, nodeName) Expect(err).NotTo(HaveOccurred()) Expect(res).Should(ContainSubstring("Encryption Status: Enabled")) @@ -142,9 +142,9 @@ var _ = Describe("Verify Secrets Encryption Rotation", func() { It("Rotates the Secrets-Encryption Keys", func() { cmd := "sudo k3s secrets-encrypt rotate" - res, err := e2e.RunCmdOnNode(cmd, serverNodenames[0]) + res, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0]) Expect(err).NotTo(HaveOccurred(), res) - for i, nodeName := range serverNodenames { + for i, nodeName := range serverNodeNames { cmd := "sudo k3s secrets-encrypt status" res, err := e2e.RunCmdOnNode(cmd, nodeName) Expect(err).NotTo(HaveOccurred(), res) @@ -158,13 +158,13 @@ var _ = Describe("Verify Secrets Encryption Rotation", func() { }) It("Restarts K3s servers", func() { - Expect(e2e.RestartCluster(serverNodenames)).To(Succeed()) + Expect(e2e.RestartCluster(serverNodeNames)).To(Succeed()) time.Sleep(20 * time.Second) }) It("Verifies encryption rotate stage", func() { cmd := "sudo k3s secrets-encrypt status" - for _, nodeName := range serverNodenames { + for _, nodeName := range serverNodeNames { res, err := e2e.RunCmdOnNode(cmd, nodeName) Expect(err).NotTo(HaveOccurred()) Expect(res).Should(ContainSubstring("Encryption Status: Enabled")) @@ -175,15 +175,15 @@ var _ = Describe("Verify Secrets Encryption Rotation", func() { It("Reencrypts the Secrets-Encryption Keys", func() { cmd := "sudo k3s secrets-encrypt reencrypt" - res, err := e2e.RunCmdOnNode(cmd, serverNodenames[0]) + res, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0]) Expect(err).NotTo(HaveOccurred(), res) cmd = "sudo k3s secrets-encrypt status" Eventually(func() (string, error) { - return e2e.RunCmdOnNode(cmd, serverNodenames[0]) + return e2e.RunCmdOnNode(cmd, serverNodeNames[0]) }, "30s", "5s").Should(ContainSubstring("Current Rotation Stage: reencrypt_finished")) - for _, nodeName := range serverNodenames[1:] { + for _, nodeName := range serverNodeNames[1:] { res, err := e2e.RunCmdOnNode(cmd, nodeName) Expect(err).NotTo(HaveOccurred(), res) Expect(res).Should(ContainSubstring("Server Encryption Hashes: hash does not match")) @@ -192,13 +192,13 @@ var _ = Describe("Verify Secrets Encryption Rotation", func() { }) It("Restarts K3s Servers", func() { - Expect(e2e.RestartCluster(serverNodenames)).To(Succeed()) + Expect(e2e.RestartCluster(serverNodeNames)).To(Succeed()) time.Sleep(20 * time.Second) }) It("Verifies Encryption Reencrypt Stage", func() { cmd := "sudo k3s secrets-encrypt status" - for _, nodeName := range serverNodenames { + for _, nodeName := range serverNodeNames { res, err := e2e.RunCmdOnNode(cmd, nodeName) Expect(err).NotTo(HaveOccurred()) Expect(res).Should(ContainSubstring("Encryption Status: Enabled")) @@ -210,19 +210,19 @@ var _ = Describe("Verify Secrets Encryption Rotation", func() { It("Disables encryption", func() { cmd := "sudo k3s secrets-encrypt disable" - res, err := e2e.RunCmdOnNode(cmd, serverNodenames[0]) + res, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0]) Expect(err).NotTo(HaveOccurred(), res) cmd = "sudo k3s secrets-encrypt reencrypt -f --skip" - res, err = e2e.RunCmdOnNode(cmd, serverNodenames[0]) + res, err = e2e.RunCmdOnNode(cmd, serverNodeNames[0]) Expect(err).NotTo(HaveOccurred(), res) cmd = "sudo k3s secrets-encrypt status" Eventually(func() (string, error) { - return e2e.RunCmdOnNode(cmd, serverNodenames[0]) + return e2e.RunCmdOnNode(cmd, serverNodeNames[0]) }, "30s", "5s").Should(ContainSubstring("Current Rotation Stage: reencrypt_finished")) - for i, nodeName := range serverNodenames { + for i, nodeName := range serverNodeNames { res, err := e2e.RunCmdOnNode(cmd, nodeName) Expect(err).NotTo(HaveOccurred(), res) if i == 0 { @@ -234,40 +234,40 @@ var _ = Describe("Verify Secrets Encryption Rotation", func() { }) It("Restarts K3s servers", func() { - Expect(e2e.RestartCluster(serverNodenames)).To(Succeed()) + Expect(e2e.RestartCluster(serverNodeNames)).To(Succeed()) time.Sleep(20 * time.Second) }) It("Verifies encryption disabled on all nodes", func() { cmd := "sudo k3s secrets-encrypt status" - for _, nodeName := range serverNodenames { + for _, nodeName := range serverNodeNames { Expect(e2e.RunCmdOnNode(cmd, nodeName)).Should(ContainSubstring("Encryption Status: Disabled")) } }) It("Enables encryption", func() { cmd := "sudo k3s secrets-encrypt enable" - res, err := e2e.RunCmdOnNode(cmd, serverNodenames[0]) + res, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0]) Expect(err).NotTo(HaveOccurred(), res) cmd = "sudo k3s secrets-encrypt reencrypt -f --skip" - res, err = e2e.RunCmdOnNode(cmd, serverNodenames[0]) + res, err = e2e.RunCmdOnNode(cmd, serverNodeNames[0]) Expect(err).NotTo(HaveOccurred(), res) cmd = "sudo k3s secrets-encrypt status" Eventually(func() (string, error) { - return e2e.RunCmdOnNode(cmd, serverNodenames[0]) + return e2e.RunCmdOnNode(cmd, serverNodeNames[0]) }, "30s", "5s").Should(ContainSubstring("Current Rotation Stage: reencrypt_finished")) }) It("Restarts K3s servers", func() { - Expect(e2e.RestartCluster(serverNodenames)).To(Succeed()) + Expect(e2e.RestartCluster(serverNodeNames)).To(Succeed()) time.Sleep(20 * time.Second) }) It("Verifies encryption enabled on all nodes", func() { cmd := "sudo k3s secrets-encrypt status" - for _, nodeName := range serverNodenames { + for _, nodeName := range serverNodeNames { Expect(e2e.RunCmdOnNode(cmd, nodeName)).Should(ContainSubstring("Encryption Status: Enabled")) } }) diff --git a/tests/e2e/testutils.go b/tests/e2e/testutils.go index 2672b499b1..a9575ad262 100644 --- a/tests/e2e/testutils.go +++ b/tests/e2e/testutils.go @@ -38,27 +38,35 @@ func CountOfStringInSlice(str string, pods []Pod) int { return count } -func CreateCluster(nodeOS string, serverCount int, agentCount int, installType string) ([]string, []string, error) { - serverNodenames := make([]string, serverCount) +func CreateCluster(nodeOS string, serverCount, agentCount int) ([]string, []string, error) { + serverNodeNames := make([]string, serverCount) for i := 0; i < serverCount; i++ { - serverNodenames[i] = "server-" + strconv.Itoa(i) + serverNodeNames[i] = "server-" + strconv.Itoa(i) } - agentNodenames := make([]string, agentCount) + agentNodeNames := make([]string, agentCount) for i := 0; i < agentCount; i++ { - agentNodenames[i] = "agent-" + strconv.Itoa(i) + agentNodeNames[i] = "agent-" + strconv.Itoa(i) } - nodeRoles := strings.Join(serverNodenames, " ") + " " + strings.Join(agentNodenames, " ") + nodeRoles := strings.Join(serverNodeNames, " ") + " " + strings.Join(agentNodeNames, " ") nodeRoles = strings.TrimSpace(nodeRoles) nodeBoxes := strings.Repeat(nodeOS+" ", serverCount+agentCount) nodeBoxes = strings.TrimSpace(nodeBoxes) - cmd := fmt.Sprintf("NODE_ROLES=\"%s\" NODE_BOXES=\"%s\" %s vagrant up &> vagrant.log", nodeRoles, nodeBoxes, installType) + + var testOptions string + for _, env := range os.Environ() { + if strings.HasPrefix(env, "E2E_") { + testOptions += " " + env + } + } + + cmd := fmt.Sprintf(`E2E_NODE_ROLES="%s" E2E_NODE_BOXES="%s" %s vagrant up &> vagrant.log`, nodeRoles, nodeBoxes, testOptions) fmt.Println(cmd) if _, err := RunCommand(cmd); err != nil { fmt.Println("Error Creating Cluster", err) return nil, nil, err } - return serverNodenames, agentNodenames, nil + return serverNodeNames, agentNodeNames, nil } func DeployWorkload(workload, kubeconfig string, arch bool) (string, error) { @@ -231,8 +239,8 @@ func RunCommand(cmd string) (string, error) { return string(out), err } -func UpgradeCluster(serverNodenames []string, agentNodenames []string) error { - for _, nodeName := range serverNodenames { +func UpgradeCluster(serverNodeNames []string, agentNodeNames []string) error { + for _, nodeName := range serverNodeNames { cmd := "RELEASE_CHANNEL=commit vagrant provision " + nodeName fmt.Println(cmd) if out, err := RunCommand(cmd); err != nil { @@ -240,7 +248,7 @@ func UpgradeCluster(serverNodenames []string, agentNodenames []string) error { return err } } - for _, nodeName := range agentNodenames { + for _, nodeName := range agentNodeNames { cmd := "RELEASE_CHANNEL=commit vagrant provision " + nodeName if _, err := RunCommand(cmd); err != nil { fmt.Println("Error Upgrading Cluster", err) diff --git a/tests/e2e/upgradecluster/upgradecluster_test.go b/tests/e2e/upgradecluster/upgradecluster_test.go index a0878d64f7..28ae64a62b 100644 --- a/tests/e2e/upgradecluster/upgradecluster_test.go +++ b/tests/e2e/upgradecluster/upgradecluster_test.go @@ -17,8 +17,8 @@ var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system") var serverCount = flag.Int("serverCount", 3, "number of server nodes") var agentCount = flag.Int("agentCount", 2, "number of agent nodes") -//valid format: RELEASE_VERSION=v1.23.1+k3s2 or nil for latest commit from master -var installType = flag.String("installType", "", "version or nil to use latest commit") +// Environment Variables Info: +// E2E_RELEASE_VERSION=v1.23.1+k3s2 or nil for latest commit from master func Test_E2EUpgradeValidation(t *testing.T) { RegisterFailHandler(Fail) @@ -28,21 +28,21 @@ func Test_E2EUpgradeValidation(t *testing.T) { var ( kubeConfigFile string - serverNodenames []string - agentNodenames []string + serverNodeNames []string + agentNodeNames []string ) var _ = Describe("Verify Upgrade", func() { Context("Cluster :", func() { It("Starts up with no issues", func() { var err error - serverNodenames, agentNodenames, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount, *installType) - Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog()) + serverNodeNames, agentNodeNames, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount) + Expect(err).NotTo(HaveOccurred()) fmt.Println("CLUSTER CONFIG") fmt.Println("OS:", *nodeOS) - fmt.Println("Server Nodes:", serverNodenames) - fmt.Println("Agent Nodes:", agentNodenames) - kubeConfigFile, err = e2e.GenKubeConfigFile(serverNodenames[0]) + fmt.Println("Server Nodes:", serverNodeNames) + fmt.Println("Agent Nodes:", agentNodeNames) + kubeConfigFile, err = e2e.GenKubeConfigFile(serverNodeNames[0]) Expect(err).NotTo(HaveOccurred()) }) @@ -84,7 +84,7 @@ var _ = Describe("Verify Upgrade", func() { clusterip, _ := e2e.FetchClusterIP(kubeConfigFile, "nginx-clusterip-svc") cmd = "curl -L --insecure http://" + clusterip + "/name.html" - for _, nodeName := range serverNodenames { + for _, nodeName := range serverNodeNames { Eventually(func() (string, error) { return e2e.RunCmdOnNode(cmd, nodeName) }, "120s", "10s").Should(ContainSubstring("test-clusterip"), "failed cmd: "+cmd) @@ -95,7 +95,7 @@ var _ = Describe("Verify Upgrade", func() { _, err := e2e.DeployWorkload("nodeport.yaml", kubeConfigFile, false) Expect(err).NotTo(HaveOccurred(), "NodePort manifest not deployed") - for _, nodeName := range serverNodenames { + for _, nodeName := range serverNodeNames { node_external_ip, _ := e2e.FetchNodeExternalIP(nodeName) cmd := "kubectl get service nginx-nodeport-svc --kubeconfig=" + kubeConfigFile + " --output jsonpath=\"{.spec.ports[0].nodePort}\"" nodeport, err := e2e.RunCommand(cmd) @@ -117,7 +117,7 @@ var _ = Describe("Verify Upgrade", func() { It("Verifies LoadBalancer Service", func() { _, err := e2e.DeployWorkload("loadbalancer.yaml", kubeConfigFile, false) Expect(err).NotTo(HaveOccurred(), "Loadbalancer manifest not deployed") - for _, nodeName := range serverNodenames { + for _, nodeName := range serverNodeNames { ip, _ := e2e.FetchNodeExternalIP(nodeName) cmd := "kubectl get service nginx-loadbalancer-svc --kubeconfig=" + kubeConfigFile + " --output jsonpath=\"{.spec.ports[0].port}\"" port, err := e2e.RunCommand(cmd) @@ -139,7 +139,7 @@ var _ = Describe("Verify Upgrade", func() { _, err := e2e.DeployWorkload("ingress.yaml", kubeConfigFile, false) Expect(err).NotTo(HaveOccurred(), "Ingress manifest not deployed") - for _, nodeName := range serverNodenames { + for _, nodeName := range serverNodeNames { ip, _ := e2e.FetchNodeExternalIP(nodeName) cmd := "curl --header host:foo1.bar.com" + " http://" + ip + "/name.html" Eventually(func() (string, error) { @@ -238,11 +238,11 @@ var _ = Describe("Verify Upgrade", func() { It("Upgrades with no issues", func() { var err error - err = e2e.UpgradeCluster(serverNodenames, agentNodenames) + err = e2e.UpgradeCluster(serverNodeNames, agentNodeNames) fmt.Println(err) Expect(err).NotTo(HaveOccurred()) fmt.Println("CLUSTER UPGRADED") - kubeConfigFile, err = e2e.GenKubeConfigFile(serverNodenames[0]) + kubeConfigFile, err = e2e.GenKubeConfigFile(serverNodeNames[0]) Expect(err).NotTo(HaveOccurred()) }) @@ -281,7 +281,7 @@ var _ = Describe("Verify Upgrade", func() { clusterip, _ := e2e.FetchClusterIP(kubeConfigFile, "nginx-clusterip-svc") cmd := "curl -L --insecure http://" + clusterip + "/name.html" fmt.Println(cmd) - for _, nodeName := range serverNodenames { + for _, nodeName := range serverNodeNames { Eventually(func() (string, error) { return e2e.RunCmdOnNode(cmd, nodeName) }, "120s", "10s").Should(ContainSubstring("test-clusterip"), "failed cmd: "+cmd) @@ -290,7 +290,7 @@ var _ = Describe("Verify Upgrade", func() { It("After upgrade verifies NodePort Service", func() { - for _, nodeName := range serverNodenames { + for _, nodeName := range serverNodeNames { node_external_ip, _ := e2e.FetchNodeExternalIP(nodeName) cmd := "kubectl get service nginx-nodeport-svc --kubeconfig=" + kubeConfigFile + " --output jsonpath=\"{.spec.ports[0].nodePort}\"" nodeport, err := e2e.RunCommand(cmd) @@ -310,7 +310,7 @@ var _ = Describe("Verify Upgrade", func() { }) It("After upgrade verifies LoadBalancer Service", func() { - for _, nodeName := range serverNodenames { + for _, nodeName := range serverNodeNames { ip, _ := e2e.FetchNodeExternalIP(nodeName) cmd := "kubectl get service nginx-loadbalancer-svc --kubeconfig=" + kubeConfigFile + " --output jsonpath=\"{.spec.ports[0].port}\"" port, err := e2e.RunCommand(cmd) @@ -328,7 +328,7 @@ var _ = Describe("Verify Upgrade", func() { }) It("After upgrade verifies Ingress", func() { - for _, nodeName := range serverNodenames { + for _, nodeName := range serverNodeNames { ip, _ := e2e.FetchNodeExternalIP(nodeName) cmd := "curl --header host:foo1.bar.com" + " http://" + ip + "/name.html" fmt.Println(cmd) diff --git a/tests/e2e/validatecluster/Vagrantfile b/tests/e2e/validatecluster/Vagrantfile index 6cf7d81022..d85b47bbd8 100644 --- a/tests/e2e/validatecluster/Vagrantfile +++ b/tests/e2e/validatecluster/Vagrantfile @@ -1,15 +1,17 @@ ENV['VAGRANT_NO_PARALLEL'] = 'no' -NODE_ROLES = (ENV['NODE_ROLES'] || +NODE_ROLES = (ENV['E2E_NODE_ROLES'] || ["server-0", "server-1", "server-2", "agent-0", "agent-1"]) -NODE_BOXES = (ENV['NODE_BOXES'] || +NODE_BOXES = (ENV['E2E_NODE_BOXES'] || ['generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004']) -GITHUB_BRANCH = (ENV['GITHUB_BRANCH'] || "master") -RELEASE_VERSION = (ENV['RELEASE_VERSION'] || "") -NODE_CPUS = (ENV['NODE_CPUS'] || 2).to_i -NODE_MEMORY = (ENV['NODE_MEMORY'] || 1024).to_i +GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master") +RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "") +EXTERNAL_DB = (ENV['E2E_EXTERNAL_DB'] || "mysql") +NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i +NODE_MEMORY = (ENV['E2E_NODE_MEMORY'] || 1024).to_i # Virtualbox >= 6.1.28 require `/etc/vbox/network.conf` for expanded private networks NETWORK_PREFIX = "10.10.10" install_type = "" +db_type = "" def provision(vm, roles, role_num, node_num) vm.box = NODE_BOXES[node_num] @@ -26,33 +28,80 @@ def provision(vm, roles, role_num, node_num) install_type = "INSTALL_K3S_VERSION=#{RELEASE_VERSION}" else # Grabs the last 5 commit SHA's from the given branch, then purges any commits that do not have a passing CI build - vm.provision "shell", path: "../scripts/latest_commit.sh", args: [GITHUB_BRANCH, "/tmp/k3s_commits"] - install_type = "INSTALL_K3S_COMMIT=$(head\ -n\ 1\ /tmp/k3s_commits)" + # MicroOS requires it not be in a /tmp/ or other root system folder + vm.provision "shell", path: "../scripts/latest_commit.sh", args: [GITHUB_BRANCH, "/home/vagrant/k3s_commits"] + install_type = "INSTALL_K3S_COMMIT=$(head\ -n\ 1\ /home/vagrant/k3s_commits)" end vm.provision "shell", inline: "ping -c 2 k3s.io" if roles.include?("server") && role_num == 0 + if EXTERNAL_DB == "mysql" + dockerInstall(vm) + vm.provision "shell", inline: "docker run -d -p 3306:3306 --name #{EXTERNAL_DB} -e MYSQL_ROOT_PASSWORD=e2e mysql:5.7" + vm.provision "shell", inline: "echo \"Wait for mysql to startup\"; sleep 10" + db_type = "--datastore-endpoint='mysql://root:e2e@tcp(#{NETWORK_PREFIX}.100:3306)/k3s'" + elsif EXTERNAL_DB == "postgres" + dockerInstall(vm) + vm.provision "shell", inline: "docker run -d -p 5432:5432 --name #{EXTERNAL_DB} -e POSTGRES_PASSWORD=e2e postgres:14-alpine" + vm.provision "shell", inline: "echo \"Wait for postgres to startup\"; sleep 10" + db_type = "--datastore-endpoint='postgres://postgres:e2e@#{NETWORK_PREFIX}.100:5432/k3s?sslmode=disable'" + elsif EXTERNAL_DB == "" || EXTERNAL_DB == "etcd" + db_type = "--cluster-init" + else + puts "Unknown EXTERNAL_DB: " + EXTERNAL_DB + abort + end + vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s| - k3s.args = %W[server --cluster-init --node-external-ip=#{NETWORK_PREFIX}.100 --flannel-iface=eth1] + k3s.args = "server #{db_type} --node-external-ip=#{NETWORK_PREFIX}.100 --flannel-iface=eth1" k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant #{install_type}] k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 end elsif roles.include?("server") && role_num != 0 + if EXTERNAL_DB == "mysql" + db_type = "--datastore-endpoint='mysql://root:e2e@tcp(#{NETWORK_PREFIX}.100:3306)/k3s'" + elsif EXTERNAL_DB == "postgres" + db_type = "--datastore-endpoint='postgres://postgres:e2e@#{NETWORK_PREFIX}.100:5432/k3s?sslmode=disable'" + end + vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s| - k3s.args = %W[server --server https://#{NETWORK_PREFIX}.100:6443 --flannel-iface=eth1] + k3s.args = "server #{db_type} --server https://#{NETWORK_PREFIX}.100:6443 --flannel-iface=eth1" k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant #{install_type}] k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 end end if roles.include?("agent") vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s| - k3s.args = %W[agent --server https://#{NETWORK_PREFIX}.100:6443 --flannel-iface=eth1] + k3s.args = %W[agent #{db_type} --server https://#{NETWORK_PREFIX}.100:6443 --flannel-iface=eth1] k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant #{install_type}] k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 end end if vm.box.include?("microos") vm.provision 'k3s-reload', type: 'reload', run: 'once' + if !EXTERNAL_DB.empty? + vm.provision "shell", inline: "docker start #{EXTERNAL_DB}" + end + end +end + +def dockerInstall(vm) + vm.provider "libvirt" do |v| + v.memory = NODE_MEMORY + 1024 + end + vm.provider "virtualbox" do |v| + v.memory = NODE_MEMORY + 1024 + end + if vm.box.include?("ubuntu2004") + vm.provision "shell", inline: "apt install -y docker.io" + end + if vm.box.include?("Leap") + vm.provision "shell", inline: "zypper install -y docker" + end + if vm.box.include?("microos") + vm.provision "shell", inline: "transactional-update pkg install -y docker" + vm.provision 'docker-reload', type: 'reload', run: 'once' + vm.provision "shell", inline: "systemctl enable --now docker" end end diff --git a/tests/e2e/validatecluster/validatecluster_test.go b/tests/e2e/validatecluster/validatecluster_test.go index e078acebe5..98000c4f3e 100644 --- a/tests/e2e/validatecluster/validatecluster_test.go +++ b/tests/e2e/validatecluster/validatecluster_test.go @@ -17,8 +17,9 @@ var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system") var serverCount = flag.Int("serverCount", 3, "number of server nodes") var agentCount = flag.Int("agentCount", 2, "number of agent nodes") -//valid format: RELEASE_VERSION=v1.23.1+k3s2 or nil for latest commit from master -var installType = flag.String("installType", "", "version or nil to use latest commit") +// Environment Variables Info: +// E2E_EXTERNAL_DB: mysql or postgres, nil for embedded etcd +// E2E_RELEASE_VERSION=v1.23.1+k3s2 or nil for latest commit from master func Test_E2EClusterValidation(t *testing.T) { RegisterFailHandler(Fail) @@ -28,21 +29,21 @@ func Test_E2EClusterValidation(t *testing.T) { var ( kubeConfigFile string - serverNodenames []string - agentNodenames []string + serverNodeNames []string + agentNodeNames []string ) var _ = Describe("Verify Create", func() { Context("Cluster :", func() { It("Starts up with no issues", func() { var err error - serverNodenames, agentNodenames, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount, *installType) - Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog) + serverNodeNames, agentNodeNames, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount) + Expect(err).NotTo(HaveOccurred()) fmt.Println("CLUSTER CONFIG") fmt.Println("OS:", *nodeOS) - fmt.Println("Server Nodes:", serverNodenames) - fmt.Println("Agent Nodes:", agentNodenames) - kubeConfigFile, err = e2e.GenKubeConfigFile(serverNodenames[0]) + fmt.Println("Server Nodes:", serverNodeNames) + fmt.Println("Agent Nodes:", agentNodeNames) + kubeConfigFile, err = e2e.GenKubeConfigFile(serverNodeNames[0]) Expect(err).NotTo(HaveOccurred()) }) @@ -86,7 +87,7 @@ var _ = Describe("Verify Create", func() { clusterip, _ := e2e.FetchClusterIP(kubeConfigFile, "nginx-clusterip-svc") cmd := "curl -L --insecure http://" + clusterip + "/name.html" fmt.Println(cmd) - for _, nodeName := range serverNodenames { + for _, nodeName := range serverNodeNames { Eventually(func(g Gomega) { res, err := e2e.RunCmdOnNode(cmd, nodeName) g.Expect(err).NotTo(HaveOccurred()) @@ -100,7 +101,7 @@ var _ = Describe("Verify Create", func() { _, err := e2e.DeployWorkload("nodeport.yaml", kubeConfigFile, false) Expect(err).NotTo(HaveOccurred(), "NodePort manifest not deployed") - for _, nodeName := range serverNodenames { + for _, nodeName := range serverNodeNames { node_external_ip, _ := e2e.FetchNodeExternalIP(nodeName) cmd := "kubectl get service nginx-nodeport-svc --kubeconfig=" + kubeConfigFile + " --output jsonpath=\"{.spec.ports[0].nodePort}\"" nodeport, err := e2e.RunCommand(cmd) @@ -128,7 +129,7 @@ var _ = Describe("Verify Create", func() { _, err := e2e.DeployWorkload("loadbalancer.yaml", kubeConfigFile, false) Expect(err).NotTo(HaveOccurred(), "Loadbalancer manifest not deployed") - for _, nodeName := range serverNodenames { + for _, nodeName := range serverNodeNames { ip, _ := e2e.FetchNodeExternalIP(nodeName) cmd := "kubectl get service nginx-loadbalancer-svc --kubeconfig=" + kubeConfigFile + " --output jsonpath=\"{.spec.ports[0].port}\"" @@ -157,7 +158,7 @@ var _ = Describe("Verify Create", func() { _, err := e2e.DeployWorkload("ingress.yaml", kubeConfigFile, false) Expect(err).NotTo(HaveOccurred(), "Ingress manifest not deployed") - for _, nodeName := range serverNodenames { + for _, nodeName := range serverNodeNames { ip, _ := e2e.FetchNodeExternalIP(nodeName) cmd := "curl --header host:foo1.bar.com" + " http://" + ip + "/name.html" fmt.Println(cmd)