From 1f7abe5dbb8440491fd0696c12df6348e39a8c97 Mon Sep 17 00:00:00 2001
From: Derek Nola <derek.nola@suse.com>
Date: Tue, 15 Mar 2022 10:29:56 -0700
Subject: [PATCH] Testing directory and documentation rework. (#5256)

* Removed vagrant folder
* Fix comments around E2E ENVs
* Eliminate testutil folder
* Convert flock integration test to unit test
* Point to other READMEs

Signed-off-by: Derek Nola <derek.nola@suse.com>
---
 .github/workflows/cgroup.yaml                 |   8 +-
 .github/workflows/install.yaml                |   8 +-
 .github/workflows/integration.yaml            |   8 +-
 .github/workflows/snapshotter.yaml            |   8 +-
 .github/workflows/unitcoverage.yaml           |   8 +-
 pkg/etcd/etcd_test.go                         |   2 +-
 pkg/flock/flock_int_test.go                   |  37 -----
 pkg/flock/flock_unix_test.go                  |  62 +++++++++
 scripts/build-tests-sonobuoy                  |   4 +-
 tests/TESTING.md                              | 129 ++++--------------
 .../cgroup/unified/fedora-34/Vagrantfile      |   6 +-
 tests/e2e/README.md                           |  45 ++++++
 tests/e2e/go.mod                              |  10 --
 tests/e2e/go.sum                              | 109 ---------------
 .../secretsencryption_test.go                 |   6 +-
 tests/e2e/testutils.go                        |   8 +-
 .../e2e/upgradecluster/upgradecluster_test.go |  12 +-
 .../validatecluster/validatecluster_test.go   |   5 +-
 .../install/centos-7/Vagrantfile              |   2 +-
 .../install/centos-8/Vagrantfile              |   2 +-
 .../install/fedora-coreos/Vagrantfile         |   2 +-
 .../install/opensuse-leap/Vagrantfile         |   2 +-
 .../install/opensuse-microos/Vagrantfile      |   2 +-
 .../install/ubuntu-focal/Vagrantfile          |   2 +-
 tests/integration/README.md                   |  48 +++++++
 .../custometcdargs/custometcdargs_int_test.go |   2 +-
 .../dualstack/dualstack_int_test.go           |   2 +-
 .../etcdrestore/etcd_restore_int_test.go      |   2 +-
 .../etcdsnapshot/etcdsnapshot_int_test.go     |   2 +-
 tests/{util => integration}/integration.go    |   2 +-
 .../localstorage/localstorage_int_test.go     |   2 +-
 .../secretsencryption_int_test.go             |   2 +-
 .../btrfs/opensuse-leap/Vagrantfile           |   2 +-
 tests/{util => }/unit.go                      |   2 +-
 tests/util/wait-for-coredns.sh                |  17 ---
 tests/vagrant/.gitignore                      |   1 -
 36 files changed, 244 insertions(+), 327 deletions(-)
 delete mode 100644 pkg/flock/flock_int_test.go
 create mode 100644 pkg/flock/flock_unix_test.go
 rename tests/{vagrant => }/cgroup/unified/fedora-34/Vagrantfile (95%)
 create mode 100644 tests/e2e/README.md
 delete mode 100644 tests/e2e/go.mod
 delete mode 100644 tests/e2e/go.sum
 rename tests/{vagrant => }/install/centos-7/Vagrantfile (98%)
 rename tests/{vagrant => }/install/centos-8/Vagrantfile (98%)
 rename tests/{vagrant => }/install/fedora-coreos/Vagrantfile (98%)
 rename tests/{vagrant => }/install/opensuse-leap/Vagrantfile (98%)
 rename tests/{vagrant => }/install/opensuse-microos/Vagrantfile (98%)
 rename tests/{vagrant => }/install/ubuntu-focal/Vagrantfile (98%)
 create mode 100644 tests/integration/README.md
 rename tests/{util => integration}/integration.go (99%)
 rename tests/{vagrant => }/snapshotter/btrfs/opensuse-leap/Vagrantfile (97%)
 rename tests/{util => }/unit.go (99%)
 delete mode 100755 tests/util/wait-for-coredns.sh
 delete mode 100644 tests/vagrant/.gitignore

diff --git a/.github/workflows/cgroup.yaml b/.github/workflows/cgroup.yaml
index 335e5839ad..fdb0903677 100644
--- a/.github/workflows/cgroup.yaml
+++ b/.github/workflows/cgroup.yaml
@@ -6,7 +6,7 @@ on:
       - "channel.yaml"
       - "install.sh"
       - "tests/**"
-      - "!tests/vagrant/cgroup/**"
+      - "!tests/cgroup/**"
       - ".github/**"
       - "!.github/workflows/cgroup.yaml"
   pull_request:
@@ -15,7 +15,7 @@ on:
       - "channel.yaml"
       - "install.sh"
       - "tests/**"
-      - "!tests/vagrant/cgroup/**"
+      - "!tests/cgroup/**"
       - ".github/**"
       - "!.github/workflows/cgroup.yaml"
   workflow_dispatch: {}
@@ -47,7 +47,7 @@ jobs:
       max-parallel: 1
     defaults:
       run:
-        working-directory: tests/vagrant/cgroup/${{ matrix.mode }}/${{ matrix.vm }}
+        working-directory: tests/cgroup/${{ matrix.mode }}/${{ matrix.vm }}
     steps:
       - name: "Checkout"
         uses: actions/checkout@v2
@@ -61,7 +61,7 @@ jobs:
           path: |
             ~/.vagrant.d/boxes
             ~/.vagrant.d/gems
-          key: cgroup-${{ hashFiles(format('tests/vagrant/cgroup/{0}/{1}/Vagrantfile', matrix.mode, matrix.vm)) }}
+          key: cgroup-${{ hashFiles(format('tests/cgroup/{0}/{1}/Vagrantfile', matrix.mode, matrix.vm)) }}
         id: vagrant-cache
         continue-on-error: true
       - name: "Vagrant Plugin(s)"
diff --git a/.github/workflows/install.yaml b/.github/workflows/install.yaml
index 20477943cf..d626f52c85 100644
--- a/.github/workflows/install.yaml
+++ b/.github/workflows/install.yaml
@@ -5,12 +5,12 @@ on:
     paths:
       - "channel.yaml"
       - "install.sh"
-      - "tests/vagrant/install/**"
+      - "tests/install/**"
   pull_request:
     branches: [main, master]
     paths:
       - "install.sh"
-      - "tests/vagrant/install/**"
+      - "tests/install/**"
   workflow_dispatch: {}
 jobs:
   test:
@@ -32,7 +32,7 @@ jobs:
       max-parallel: 2
     defaults:
       run:
-        working-directory: tests/vagrant/install/${{ matrix.vm }}
+        working-directory: tests/install/${{ matrix.vm }}
     env:
       INSTALL_K3S_CHANNEL: ${{ matrix.channel }}
     steps:
@@ -45,7 +45,7 @@ jobs:
           path: |
             ~/.vagrant.d/boxes
             ~/.vagrant.d/gems
-          key: install-${{ hashFiles(format('tests/vagrant/install/{0}/Vagrantfile', matrix.vm)) }}
+          key: install-${{ hashFiles(format('tests/install/{0}/Vagrantfile', matrix.vm)) }}
         id: vagrant-cache
         continue-on-error: true
       - name: "Vagrant Plugin(s)"
diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml
index fac2bfc6ca..85be47fac2 100644
--- a/.github/workflows/integration.yaml
+++ b/.github/workflows/integration.yaml
@@ -5,7 +5,9 @@ on:
       - "**.md"
       - "channel.yaml"
       - "install.sh"
-      - "tests/vagrant/**"
+      - "tests/snapshotter/**"
+      - "tests/install/**"
+      - "tests/cgroup/**"
       - ".github/**"
       - "!.github/workflows/integration.yaml"
   pull_request:
@@ -13,7 +15,9 @@ on:
       - "**.md"
       - "channel.yaml"
       - "install.sh"
-      - "tests/vagrant/**"
+      - "tests/snapshotter/**"
+      - "tests/install/**"
+      - "tests/cgroup/**"
       - ".github/**"
       - "!.github/workflows/integration.yaml"
   workflow_dispatch: {}
diff --git a/.github/workflows/snapshotter.yaml b/.github/workflows/snapshotter.yaml
index c875817f37..a3c0fa8351 100644
--- a/.github/workflows/snapshotter.yaml
+++ b/.github/workflows/snapshotter.yaml
@@ -6,7 +6,7 @@ on:
       - "channel.yaml"
       - "install.sh"
       - "tests/**"
-      - "!tests/vagrant/snapshotter/**"
+      - "!tests/snapshotter/**"
       - ".github/**"
       - "!.github/workflows/snapshotter.yaml"
   pull_request:
@@ -15,7 +15,7 @@ on:
       - "channel.yaml"
       - "install.sh"
       - "tests/**"
-      - "!tests/vagrant/snapshotter/**"
+      - "!tests/snapshotter/**"
       - ".github/**"
       - "!.github/workflows/snapshotter.yaml"
   workflow_dispatch: {}
@@ -47,7 +47,7 @@ jobs:
       max-parallel: 1
     defaults:
       run:
-        working-directory: tests/vagrant/snapshotter/${{ matrix.snapshotter }}/${{ matrix.vm }}
+        working-directory: tests/snapshotter/${{ matrix.snapshotter }}/${{ matrix.vm }}
     env:
       VAGRANT_EXPERIMENTAL: disks
     steps:
@@ -63,7 +63,7 @@ jobs:
           path: |
             ~/.vagrant.d/boxes
             ~/.vagrant.d/gems
-          key: snapshotter-${{ hashFiles(format('tests/vagrant/snapshotter/{0}/{1}/Vagrantfile', matrix.snapshotter, matrix.vm)) }}
+          key: snapshotter-${{ hashFiles(format('tests/snapshotter/{0}/{1}/Vagrantfile', matrix.snapshotter, matrix.vm)) }}
         id: vagrant-cache
         continue-on-error: true
       - name: "Vagrant Plugin(s)"
diff --git a/.github/workflows/unitcoverage.yaml b/.github/workflows/unitcoverage.yaml
index f10eeb0697..b56e19ee57 100644
--- a/.github/workflows/unitcoverage.yaml
+++ b/.github/workflows/unitcoverage.yaml
@@ -5,7 +5,9 @@ on:
       - "**.md"
       - "channel.yaml"
       - "install.sh"
-      - "tests/vagrant/**"
+      - "tests/snapshotter/**"
+      - "tests/install/**"
+      - "tests/cgroup/**"
       - ".github/**"
       - "!.github/workflows/unitcoverage.yaml"
   pull_request:
@@ -13,7 +15,9 @@ on:
       - "**.md"
       - "channel.yaml"
       - "install.sh"
-      - "tests/vagrant/**"
+      - "tests/snapshotter/**"
+      - "tests/install/**"
+      - "tests/cgroup/**"
       - ".github/**"
       - "!.github/workflows/unitcoverage.yaml"
   workflow_dispatch: {}
diff --git a/pkg/etcd/etcd_test.go b/pkg/etcd/etcd_test.go
index e673d1de91..cff23e2f9e 100644
--- a/pkg/etcd/etcd_test.go
+++ b/pkg/etcd/etcd_test.go
@@ -11,7 +11,7 @@ import (
 
 	"github.com/k3s-io/k3s/pkg/clientaccess"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
-	testutil "github.com/k3s-io/k3s/tests/util"
+	testutil "github.com/k3s-io/k3s/tests"
 	"github.com/robfig/cron/v3"
 	clientv3 "go.etcd.io/etcd/client/v3"
 	"go.etcd.io/etcd/server/v3/etcdserver"
diff --git a/pkg/flock/flock_int_test.go b/pkg/flock/flock_int_test.go
deleted file mode 100644
index aed7bbbe92..0000000000
--- a/pkg/flock/flock_int_test.go
+++ /dev/null
@@ -1,37 +0,0 @@
-//go:build linux || darwin || freebsd || openbsd || netbsd || dragonfly
-// +build linux darwin freebsd openbsd netbsd dragonfly
-
-package flock_test
-
-import (
-	"testing"
-
-	"github.com/k3s-io/k3s/pkg/flock"
-	. "github.com/onsi/ginkgo/v2"
-	. "github.com/onsi/gomega"
-)
-
-const lockfile = "/tmp/testlock.test"
-
-var lock int
-var _ = Describe("file locks", func() {
-	When("a new exclusive lock is created", func() {
-		It("starts up with no problems", func() {
-			var err error
-			lock, err = flock.Acquire(lockfile)
-			Expect(err).ToNot(HaveOccurred())
-		})
-		It("has a write lock on the file", func() {
-			Expect(flock.CheckLock(lockfile)).To(BeTrue())
-		})
-		It("release the lock correctly", func() {
-			Expect(flock.Release(lock)).To(Succeed())
-			Expect(flock.CheckLock(lockfile)).To(BeFalse())
-		})
-	})
-})
-
-func TestFlock(t *testing.T) {
-	RegisterFailHandler(Fail)
-	RunSpecs(t, "Flock Suite")
-}
diff --git a/pkg/flock/flock_unix_test.go b/pkg/flock/flock_unix_test.go
new file mode 100644
index 0000000000..36a0643c04
--- /dev/null
+++ b/pkg/flock/flock_unix_test.go
@@ -0,0 +1,62 @@
+//go:build linux || darwin || freebsd || openbsd || netbsd || dragonfly
+// +build linux darwin freebsd openbsd netbsd dragonfly
+
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package flock
+
+import (
+	"testing"
+)
+
+func Test_UnitFlock(t *testing.T) {
+	tests := []struct {
+		name      string
+		path      string
+		wantCheck bool
+		wantErr   bool
+	}{
+		{
+			name: "Basic Flock Test",
+			path: "/tmp/testlock.test",
+
+			wantCheck: true,
+			wantErr:   false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			lock, err := Acquire(tt.path)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Acquire() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+
+			if got := CheckLock(tt.path); got != tt.wantCheck {
+				t.Errorf("CheckLock() = %+v\nWant = %+v", got, tt.wantCheck)
+			}
+
+			if err := Release(lock); (err != nil) != tt.wantErr {
+				t.Errorf("Release() error = %v, wantErr %v", err, tt.wantErr)
+			}
+
+			if got := CheckLock(tt.path); got == tt.wantCheck {
+				t.Errorf("CheckLock() = %+v\nWant = %+v", got, !tt.wantCheck)
+			}
+		})
+	}
+}
diff --git a/scripts/build-tests-sonobuoy b/scripts/build-tests-sonobuoy
index 6312e57b7d..a016e6a4e4 100755
--- a/scripts/build-tests-sonobuoy
+++ b/scripts/build-tests-sonobuoy
@@ -14,7 +14,7 @@ PKG_TO_TEST=$(find ./pkg/ -type f -name "*_int_test.go" | sed -r 's|/[^/]+$||' |
 for i in $PKG_TO_TEST; do
     name=$(echo "${i##*/}")
     echo $name
-    go test -c -v -ldflags "-X 'github.com/k3s-io/k3s/tests/util.existingServer=True'" -o dist/artifacts/k3s-integration-$name.test $i -run Integration
+    go test -c -v -ldflags "-X 'github.com/k3s-io/k3s/tests/integration.existingServer=True'" -o dist/artifacts/k3s-integration-$name.test $i -run Integration
 done
 
 # Integration tests under /tests
@@ -22,7 +22,7 @@ PKG_TO_TEST=$(find ./tests/integration -type f -name "*_int_test.go" | sed -r 's
 for i in $PKG_TO_TEST; do
     name=$(echo "${i##*/}")
     echo $name
-    go test -c -v -ldflags "-X 'github.com/k3s-io/k3s/tests/util.existingServer=True'" -o dist/artifacts/k3s-integration-$name.test $i -run Integration
+    go test -c -v -ldflags "-X 'github.com/k3s-io/k3s/tests/integration.existingServer=True'" -o dist/artifacts/k3s-integration-$name.test $i -run Integration
 done
 docker build -f ./tests/integration/Dockerfile.test -t $REPO .
 docker save $REPO -o ./dist/artifacts/$REPO.tar
diff --git a/tests/TESTING.md b/tests/TESTING.md
index 9f447007e6..26d3dfed3b 100644
--- a/tests/TESTING.md
+++ b/tests/TESTING.md
@@ -1,9 +1,10 @@
 # Testing Standards in K3s
 
-Testing in K3s comes in 4 forms: 
+Testing in K3s comes in 5 forms: 
 - [Unit](#unit-tests)
 - [Integration](#integration-tests)
 - [Smoke](#smoke-tests)
+- [Performance](#performance)
 - [End-to-End (E2E)](#end-to-end-e2e-tests)
 
 This document will explain *when* each test should be written and *how* each test should be
@@ -53,71 +54,27 @@ ___
 Integration tests should be used to test a specific functionality of k3s that exists across multiple Go packages, either via exported function calls, or more often, CLI comands.
 Integration tests should be used for "black box" testing. 
 
-### Framework
-
-All integration tests in K3s follow a [Behavior Diven Development (BDD)](https://en.wikipedia.org/wiki/Behavior-driven_development) style. Specifically, K3s uses [Ginkgo](https://onsi.github.io/ginkgo/) and [Gomega](https://onsi.github.io/gomega/) to drive the tests.  
-To generate an initial test, the command `ginkgo bootstrap` can be used.
-
-To facilitate K3s CLI testing, see `tests/util/cmd.go` helper functions.
-
-### Format
-
-All integration tests should be placed under `tests/integration/<TEST_NAME>`.  
-All integration test files should be named: `<TEST_NAME>_int_test.go`.  
-All integration test functions should be named: `Test_Integration<TEST_NAME>`.  
-See the [local storage test](../tests/integration/localstorage/localstorage_int_test.go) as an example.
-
-### Running
-
-Integration tests can be run with no k3s cluster present, each test will spin up and kill the appropriate k3s server it needs.  
-Note: Integration tests must be run as root, prefix the commands below with `sudo -E env "PATH=$PATH"` if a sudo user.
-```bash
-go test ./tests/integration/... -run Integration
-```
-
-Additionally, to generate JUnit reporting for the tests, the Ginkgo CLI is used
-```
-ginkgo --junit-report=result.xml ./tests/integration/...
-```
-
-Integration tests can be run on an existing single-node cluster via compile time flag, tests will skip if the server is not configured correctly.
-```bash
-go test -ldflags "-X 'github.com/k3s-io/k3s/tests/util.existingServer=True'" ./tests/integration/... -run Integration
-```
-
-Integration tests can also be run via a [Sonobuoy](https://sonobuoy.io/docs/v0.53.2/) plugin on an existing single-node cluster.
-```bash
-./scripts/build-tests-sonobuoy
-sudo KUBECONFIG=/etc/rancher/k3s/k3s.yaml sonobuoy run --plugin ./dist/artifacts/k3s-int-tests.yaml
-```
-Check the sonobuoy status and retrieve results
-``` 
-sudo KUBECONFIG=/etc/rancher/k3s/k3s.yaml sonobuoy status
-sudo KUBECONFIG=/etc/rancher/k3s/k3s.yaml sonobuoy retrieve
-sudo KUBECONFIG=/etc/rancher/k3s/k3s.yaml sonobuoy results <TAR_FILE_FROM_RETRIEVE>
-```
+See [integration/README.md](./integration/README.md) for more info.
 
 ___
 
 ## Smoke Tests
 
-Smoke tests are defined under the [tests/vagrant](../tests/vagrant) path at the root of this repository.
-The sub-directories therein contain fixtures for running simple clusters to assert correct behavior for "happy path"
-scenarios. These fixtures are mostly self-contained Vagrantfiles describing single-node installations that are
-easily spun up with Vagrant for the `libvirt` and `virtualbox` providers:
+Smoke tests are a collection of tests defined under the [tests](../tests) path at the root of this repository.
+The sub-directories therein contain fixtures for running simple clusters to assert correct behavior for "happy path" scenarios. These fixtures are mostly self-contained Vagrantfiles describing single-node installations that are easily spun up with Vagrant for the `libvirt` and `virtualbox` providers:
 
-- [Install Script](../tests/vagrant/install) :arrow_right: on proposed changes to [install.sh](../install.sh) 
-  - [CentOS 7](../tests/vagrant/install/centos-7) (stand-in for RHEL 7)
-  - [CentOS 8](../tests/vagrant/install/centos-8) (stand-in for RHEL 8)
-  - [Leap 15.3](../tests/vagrant/install/opensuse-microos) (stand-in for SLES)
-  - [MicroOS](../tests/vagrant/install/opensuse-microos) (stand-in for SLE-Micro)
-  - [Ubuntu 20.04](../tests/vagrant/install/ubuntu-focal) (Focal Fossa)
-- [Control Groups](../tests/vagrant/cgroup) :arrow_right: on any code change
-  - [mode=unified](../tests/vagrant/cgroup/unified) (cgroups v2)
-    - [Fedora 34](../tests/vagrant/cgroup/unified/fedora-34) (rootfull + rootless)
-- [Snapshotter](../tests/vagrant/snapshotter/btrfs/opensuse-leap) :arrow_right: on any code change
-  - [BTRFS](../tests/vagrant/snapshotter/btrfs) ([containerd built-in](https://github.com/containerd/containerd/tree/main/snapshots/btrfs))
-    - [Leap 15.3](../tests/vagrant/snapshotter/btrfs/opensuse-leap)
+- [Install Script](../tests/install) :arrow_right: on proposed changes to [install.sh](../install.sh) 
+  - [CentOS 7](../tests/install/centos-7) (stand-in for RHEL 7)
+  - [CentOS 8](../tests/install/centos-8) (stand-in for RHEL 8)
+  - [Leap 15.3](../tests/install/opensuse-microos) (stand-in for SLES)
+  - [MicroOS](../tests/install/opensuse-microos) (stand-in for SLE-Micro)
+  - [Ubuntu 20.04](../tests/install/ubuntu-focal) (Focal Fossa)
+- [Control Groups](../tests/cgroup) :arrow_right: on any code change
+  - [mode=unified](../tests/cgroup/unified) (cgroups v2)
+    - [Fedora 34](../tests/cgroup/unified/fedora-34) (rootfull + rootless)
+- [Snapshotter](../tests/snapshotter/btrfs/opensuse-leap) :arrow_right: on any code change
+  - [BTRFS](../tests/snapshotter/btrfs) ([containerd built-in](https://github.com/containerd/containerd/tree/main/snapshots/btrfs))
+    - [Leap 15.3](../tests/snapshotter/btrfs/opensuse-leap)
 
 When adding new installer test(s) please copy the prevalent style for the `Vagrantfile`.
 Ideally, the boxes used for additional assertions will support the default `virtualbox` provider which
@@ -153,7 +110,7 @@ These can be set on the CLI or exported before invoking Vagrant:
 
 The **Install Script** tests can be run by changing to the fixture directory and invoking `vagrant up`, e.g.:
 ```shell
-cd tests/vagrant/install/centos-8
+cd tests/install/centos-8
 vagrant up
 # the following provisioners are optional. the do not run by default but are invoked
 # explicitly by github actions workflow to avoid certain timeout issues on slow runners
@@ -173,54 +130,22 @@ an idea of how they can be invoked.
 
 ___
 
+## Performance Tests
+
+Performance tests use Terraform to test large scale deployments of K3s clusters.
+
+See [perf/README.md](./perf/README.md) for more info.
+___
+
 ## End-to-End (E2E) Tests
 
 E2E tests cover multi-node K3s configuration and administration: bringup, update, teardown etc. across a wide range of operating systems. E2E tests are run nightly as part of K3s quality assurance (QA).
 
-### Framework 
-End-to-end tests utilize [Ginkgo](https://onsi.github.io/ginkgo/) and [Gomega](https://onsi.github.io/gomega/) like the integration tests, but rely on [Vagrant](https://www.vagrantup.com/) to provide the underlying cluster configuration. 
+See [e2e/README.md](./e2e/README.md) for more info.
 
-Currently tested operating systems are:
-- [Ubuntu 20.04](https://app.vagrantup.com/generic/boxes/ubuntu2004)
-- [Leap 15.3](https://app.vagrantup.com/opensuse/boxes/Leap-15.3.x86_64) (stand-in for SLE-Server)
-- [MicroOS](https://app.vagrantup.com/dweomer/boxes/microos.amd64) (stand-in for SLE-Micro)
-
-### Format
-
-All E2E tests should be placed under `tests/e2e/<TEST_NAME>`.  
-All E2E test functions should be named: `Test_E2E<TEST_NAME>`.  
-A E2E test consists of two parts:
-1. `Vagrantfile`: a vagrant file which describes and configures the VMs upon which the cluster and test will run
-2. `<TEST_NAME>.go`: A go test file which calls `vagrant up` and controls the actual testing of the cluster
-
-See the [validate cluster test](../tests/e2e/validatecluster/validatecluster_test.go) as an example.
-
-### Running
-
-Generally, E2E tests are run as a nightly Jenkins job for QA. They can still be run locally but additional setup may be required. By default, all E2E tests are designed with `libvirt` as the underlying VM provider. Instructions for installing libvirt and its associated vagrant plugin, `vagrant-libvirt` can be found [here.](https://github.com/vagrant-libvirt/vagrant-libvirt#installation) `VirtualBox` is also supported as a backup VM provider.
-
-Once setup is complete, all E2E tests can be run with:
-```bash
-go test -timeout=15m ./tests/e2e/... -run E2E
-```
-Tests can be run individually with:
-```bash
-go test -timeout=15m ./tests/e2e/validatecluster/... -run E2E
-#or
-go test -timeout=15m ./tests/e2e/... -run E2EClusterValidation
-```
-
-Additionally, to generate junit reporting for the tests, the Ginkgo CLI is used. Installation instructions can be found [here.](https://onsi.github.io/ginkgo/#getting-started)  
-
-To run the all E2E tests and generate JUnit testing reports:
-```
-ginkgo --junit-report=result.xml ./tests/e2e/...
-```
-
-Note: The `go test` default timeout is 10 minutes, thus the `-timeout` flag should be used. The `ginkgo` default timeout is 1 hour, no timeout flag is needed.
+___
 
 ## Contributing New Or Updated Tests
 
-___
 We gladly accept new and updated tests of all types. If you wish to create
 a new test or update an existing test, please submit a PR with a title that includes the words `<NAME_OF_TEST> (Created/Updated)`.
diff --git a/tests/vagrant/cgroup/unified/fedora-34/Vagrantfile b/tests/cgroup/unified/fedora-34/Vagrantfile
similarity index 95%
rename from tests/vagrant/cgroup/unified/fedora-34/Vagrantfile
rename to tests/cgroup/unified/fedora-34/Vagrantfile
index 3c10005a59..3f4ec10391 100644
--- a/tests/vagrant/cgroup/unified/fedora-34/Vagrantfile
+++ b/tests/cgroup/unified/fedora-34/Vagrantfile
@@ -3,8 +3,8 @@
 #
 # Vagrant box for testing k3s with cgroup v2.
 
-ENV['TEST_UNITFILE_ROOTFULL'] ||= '../../../../../k3s.service'
-ENV['TEST_UNITFILE_ROOTLESS'] ||= '../../../../../k3s-rootless.service'
+ENV['TEST_UNITFILE_ROOTFULL'] ||= '../../../../k3s.service'
+ENV['TEST_UNITFILE_ROOTLESS'] ||= '../../../../k3s-rootless.service'
 
 Vagrant.configure("2") do |config|
   config.vagrant.plugins = {
@@ -12,7 +12,7 @@ Vagrant.configure("2") do |config|
   }
   config.vm.box = "fedora/34-cloud-base"
   config.vm.boot_timeout = ENV['TEST_VM_BOOT_TIMEOUT'] || 600 # seconds
-  config.vm.synced_folder '../../../../../dist/artifacts', '/vagrant', type: 'rsync', disabled: false,
+  config.vm.synced_folder '../../../../dist/artifacts', '/vagrant', type: 'rsync', disabled: false,
     rsync__exclude: ENV['RSYNC_EXCLUDE'] || '*.tar.*'
 
   config.vm.define 'cgroup-unified', primary: true do |test|
diff --git a/tests/e2e/README.md b/tests/e2e/README.md
new file mode 100644
index 0000000000..0f7f7895bd
--- /dev/null
+++ b/tests/e2e/README.md
@@ -0,0 +1,45 @@
+# End-to-End (E2E) Tests
+
+E2E tests cover multi-node K3s configuration and administration: bringup, update, teardown etc. across a wide range of operating systems. E2E tests are run nightly as part of K3s quality assurance (QA).
+
+## Framework 
+End-to-end tests utilize [Ginkgo](https://onsi.github.io/ginkgo/) and [Gomega](https://onsi.github.io/gomega/) like the integration tests, but rely on [Vagrant](https://www.vagrantup.com/) to provide the underlying cluster configuration. 
+
+Currently tested operating systems are:
+- [Ubuntu 20.04](https://app.vagrantup.com/generic/boxes/ubuntu2004)
+- [Leap 15.3](https://app.vagrantup.com/opensuse/boxes/Leap-15.3.x86_64) (stand-in for SLE-Server)
+- [MicroOS](https://app.vagrantup.com/dweomer/boxes/microos.amd64) (stand-in for SLE-Micro)
+
+## Format
+
+All E2E tests should be placed under `tests/e2e/<TEST_NAME>`.  
+All E2E test functions should be named: `Test_E2E<TEST_NAME>`.  
+A E2E test consists of two parts:
+1. `Vagrantfile`: a vagrant file which describes and configures the VMs upon which the cluster and test will run
+2. `<TEST_NAME>.go`: A go test file which calls `vagrant up` and controls the actual testing of the cluster
+
+See the [validate cluster test](../tests/e2e/validatecluster/validatecluster_test.go) as an example.
+
+## Running
+
+Generally, E2E tests are run as a nightly Jenkins job for QA. They can still be run locally but additional setup may be required. By default, all E2E tests are designed with `libvirt` as the underlying VM provider. Instructions for installing libvirt and its associated vagrant plugin, `vagrant-libvirt` can be found [here.](https://github.com/vagrant-libvirt/vagrant-libvirt#installation) `VirtualBox` is also supported as a backup VM provider.
+
+Once setup is complete, all E2E tests can be run with:
+```bash
+go test -timeout=15m ./tests/e2e/... -run E2E
+```
+Tests can be run individually with:
+```bash
+go test -timeout=15m ./tests/e2e/validatecluster/... -run E2E
+#or
+go test -timeout=15m ./tests/e2e/... -run E2EClusterValidation
+```
+
+Additionally, to generate junit reporting for the tests, the Ginkgo CLI is used. Installation instructions can be found [here.](https://onsi.github.io/ginkgo/#getting-started)  
+
+To run the all E2E tests and generate JUnit testing reports:
+```
+ginkgo --junit-report=result.xml ./tests/e2e/...
+```
+
+Note: The `go test` default timeout is 10 minutes, thus the `-timeout` flag should be used. The `ginkgo` default timeout is 1 hour, no timeout flag is needed.
\ No newline at end of file
diff --git a/tests/e2e/go.mod b/tests/e2e/go.mod
deleted file mode 100644
index 589bba4b6a..0000000000
--- a/tests/e2e/go.mod
+++ /dev/null
@@ -1,10 +0,0 @@
-module github.com/k3s-io/k3s/tests/e2e
-
-go 1.15
-
-require (
-	github.com/kr/pretty v0.2.0 // indirect
-	github.com/onsi/ginkgo/v2 v2.1.1
-	github.com/onsi/gomega v1.17.0
-	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
-)
diff --git a/tests/e2e/go.sum b/tests/e2e/go.sum
deleted file mode 100644
index f9529a34fe..0000000000
--- a/tests/e2e/go.sum
+++ /dev/null
@@ -1,109 +0,0 @@
-github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
-github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
-github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
-github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
-github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
-github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
-github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
-github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
-github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
-github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
-github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
-github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
-github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
-github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
-github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
-github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
-github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
-github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
-github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
-github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
-github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
-github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
-github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc=
-github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
-github.com/onsi/ginkgo/v2 v2.1.1 h1:LCnPB85AvFNr91s0B2aDzEiiIg6MUwLYbryC1NSlWi8=
-github.com/onsi/ginkgo/v2 v2.1.1/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c=
-github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
-github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
-github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE=
-github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
-github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0=
-golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
-golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c=
-golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
-golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
-google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
-google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
-google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
-google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
-google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
-google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
-gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
-gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
diff --git a/tests/e2e/secretsencryption/secretsencryption_test.go b/tests/e2e/secretsencryption/secretsencryption_test.go
index 8ead055998..dd507e7184 100644
--- a/tests/e2e/secretsencryption/secretsencryption_test.go
+++ b/tests/e2e/secretsencryption/secretsencryption_test.go
@@ -17,8 +17,8 @@ import (
 var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system")
 var serverCount = flag.Int("serverCount", 3, "number of server nodes")
 
-//valid format: RELEASE_VERSION=v1.23.1+k3s2 or nil for latest commit from master
-var installType = flag.String("installType", "", "version or nil to use latest commit")
+// Environment Variables Info:
+// E2E_RELEASE_VERSION=v1.23.1+k3s2 or nil for latest commit from master
 
 func Test_E2EClusterValidation(t *testing.T) {
 	RegisterFailHandler(Fail)
@@ -35,7 +35,7 @@ var _ = Describe("Verify Secrets Encryption Rotation", func() {
 	Context("Cluster :", func() {
 		It("Starts up with no issues", func() {
 			var err error
-			serverNodeNames, _, err = e2e.CreateCluster(*nodeOS, *serverCount, 0, *installType)
+			serverNodeNames, _, err = e2e.CreateCluster(*nodeOS, *serverCount, 0)
 			Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog)
 			fmt.Println("CLUSTER CONFIG")
 			fmt.Println("OS:", *nodeOS)
diff --git a/tests/e2e/testutils.go b/tests/e2e/testutils.go
index a9575ad262..4f9f166b46 100644
--- a/tests/e2e/testutils.go
+++ b/tests/e2e/testutils.go
@@ -70,18 +70,18 @@ func CreateCluster(nodeOS string, serverCount, agentCount int) ([]string, []stri
 }
 
 func DeployWorkload(workload, kubeconfig string, arch bool) (string, error) {
-	resource_dir := "../amd64_resource_files"
+	resourceDir := "../amd64_resource_files"
 	if arch {
-		resource_dir = "../arm64_resource_files"
+		resourceDir = "../arm64_resource_files"
 	}
-	files, err := ioutil.ReadDir(resource_dir)
+	files, err := ioutil.ReadDir(resourceDir)
 	if err != nil {
 		err = fmt.Errorf("%s : Unable to read resource manifest file for %s", err, workload)
 		return "", err
 	}
 	fmt.Println("\nDeploying", workload)
 	for _, f := range files {
-		filename := filepath.Join(resource_dir, f.Name())
+		filename := filepath.Join(resourceDir, f.Name())
 		if strings.TrimSpace(f.Name()) == workload {
 			cmd := "kubectl apply -f " + filename + " --kubeconfig=" + kubeconfig
 			return RunCommand(cmd)
diff --git a/tests/e2e/upgradecluster/upgradecluster_test.go b/tests/e2e/upgradecluster/upgradecluster_test.go
index afee907cbf..242ed120a1 100644
--- a/tests/e2e/upgradecluster/upgradecluster_test.go
+++ b/tests/e2e/upgradecluster/upgradecluster_test.go
@@ -18,7 +18,9 @@ var serverCount = flag.Int("serverCount", 3, "number of server nodes")
 var agentCount = flag.Int("agentCount", 2, "number of agent nodes")
 
 // Environment Variables Info:
-// E2E_RELEASE_VERSION=v1.23.1+k3s2 or nil for latest commit from master
+// E2E_RELEASE_VERSION=v1.23.3+k3s1
+// OR
+// E2E_RELEASE_CHANNEL=(commit|latest|stable), commit pulls latest commit from master
 
 func Test_E2EUpgradeValidation(t *testing.T) {
 	RegisterFailHandler(Fail)
@@ -96,7 +98,7 @@ var _ = Describe("Verify Upgrade", func() {
 			Expect(err).NotTo(HaveOccurred(), "NodePort manifest not deployed")
 
 			for _, nodeName := range serverNodeNames {
-				node_external_ip, _ := e2e.FetchNodeExternalIP(nodeName)
+				nodeExternalIP, _ := e2e.FetchNodeExternalIP(nodeName)
 				cmd := "kubectl get service nginx-nodeport-svc --kubeconfig=" + kubeConfigFile + " --output jsonpath=\"{.spec.ports[0].nodePort}\""
 				nodeport, err := e2e.RunCommand(cmd)
 				Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd)
@@ -106,7 +108,7 @@ var _ = Describe("Verify Upgrade", func() {
 					return e2e.RunCommand(cmd)
 				}, "240s", "5s").Should(ContainSubstring("test-nodeport"), "nodeport pod was not created")
 
-				cmd = "curl -L --insecure http://" + node_external_ip + ":" + nodeport + "/name.html"
+				cmd = "curl -L --insecure http://" + nodeExternalIP + ":" + nodeport + "/name.html"
 				fmt.Println(cmd)
 				Eventually(func() (string, error) {
 					return e2e.RunCommand(cmd)
@@ -291,7 +293,7 @@ var _ = Describe("Verify Upgrade", func() {
 		It("After upgrade verifies NodePort Service", func() {
 
 			for _, nodeName := range serverNodeNames {
-				node_external_ip, _ := e2e.FetchNodeExternalIP(nodeName)
+				nodeExternalIP, _ := e2e.FetchNodeExternalIP(nodeName)
 				cmd := "kubectl get service nginx-nodeport-svc --kubeconfig=" + kubeConfigFile + " --output jsonpath=\"{.spec.ports[0].nodePort}\""
 				nodeport, err := e2e.RunCommand(cmd)
 				Expect(err).NotTo(HaveOccurred())
@@ -301,7 +303,7 @@ var _ = Describe("Verify Upgrade", func() {
 					return e2e.RunCommand(cmd)
 				}, "240s", "5s").Should(ContainSubstring("test-nodeport"), "nodeport pod was not created")
 
-				cmd = "curl -L --insecure http://" + node_external_ip + ":" + nodeport + "/name.html"
+				cmd = "curl -L --insecure http://" + nodeExternalIP + ":" + nodeport + "/name.html"
 				fmt.Println(cmd)
 				Eventually(func() (string, error) {
 					return e2e.RunCommand(cmd)
diff --git a/tests/e2e/validatecluster/validatecluster_test.go b/tests/e2e/validatecluster/validatecluster_test.go
index 9032242a29..0846308823 100644
--- a/tests/e2e/validatecluster/validatecluster_test.go
+++ b/tests/e2e/validatecluster/validatecluster_test.go
@@ -7,6 +7,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/k3s-io/k3s/tests/e2e"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )
@@ -101,7 +102,7 @@ var _ = Describe("Verify Create", func() {
 			Expect(err).NotTo(HaveOccurred(), "NodePort manifest not deployed")
 
 			for _, nodeName := range serverNodeNames {
-				node_external_ip, _ := e2e.FetchNodeExternalIP(nodeName)
+				nodeExternalIP, _ := e2e.FetchNodeExternalIP(nodeName)
 				cmd := "kubectl get service nginx-nodeport-svc --kubeconfig=" + kubeConfigFile + " --output jsonpath=\"{.spec.ports[0].nodePort}\""
 				nodeport, err := e2e.RunCommand(cmd)
 				Expect(err).NotTo(HaveOccurred())
@@ -113,7 +114,7 @@ var _ = Describe("Verify Create", func() {
 					g.Expect(res).Should(ContainSubstring("test-nodeport"), "nodeport pod was not created")
 				}, "240s", "5s").Should(Succeed())
 
-				cmd = "curl -L --insecure http://" + node_external_ip + ":" + nodeport + "/name.html"
+				cmd = "curl -L --insecure http://" + nodeExternalIP + ":" + nodeport + "/name.html"
 				fmt.Println(cmd)
 				Eventually(func(g Gomega) {
 					res, err := e2e.RunCommand(cmd)
diff --git a/tests/vagrant/install/centos-7/Vagrantfile b/tests/install/centos-7/Vagrantfile
similarity index 98%
rename from tests/vagrant/install/centos-7/Vagrantfile
rename to tests/install/centos-7/Vagrantfile
index 12cde07a20..b8253b60bf 100644
--- a/tests/vagrant/install/centos-7/Vagrantfile
+++ b/tests/install/centos-7/Vagrantfile
@@ -2,7 +2,7 @@
 # vi: set ft=ruby :
 #
 
-ENV['TEST_INSTALL_SH'] ||= '../../../../install.sh'
+ENV['TEST_INSTALL_SH'] ||= '../../../install.sh'
 
 Vagrant.configure("2") do |config|
   config.vagrant.plugins = {
diff --git a/tests/vagrant/install/centos-8/Vagrantfile b/tests/install/centos-8/Vagrantfile
similarity index 98%
rename from tests/vagrant/install/centos-8/Vagrantfile
rename to tests/install/centos-8/Vagrantfile
index e3bec9dad3..d746f06448 100644
--- a/tests/vagrant/install/centos-8/Vagrantfile
+++ b/tests/install/centos-8/Vagrantfile
@@ -2,7 +2,7 @@
 # vi: set ft=ruby :
 #
 
-ENV['TEST_INSTALL_SH'] ||= '../../../../install.sh'
+ENV['TEST_INSTALL_SH'] ||= '../../../install.sh'
 
 Vagrant.configure("2") do |config|
   config.vagrant.plugins = {
diff --git a/tests/vagrant/install/fedora-coreos/Vagrantfile b/tests/install/fedora-coreos/Vagrantfile
similarity index 98%
rename from tests/vagrant/install/fedora-coreos/Vagrantfile
rename to tests/install/fedora-coreos/Vagrantfile
index 66fbe9dc6b..480b145d9d 100644
--- a/tests/vagrant/install/fedora-coreos/Vagrantfile
+++ b/tests/install/fedora-coreos/Vagrantfile
@@ -2,7 +2,7 @@
 # vi: set ft=ruby :
 #
 
-ENV['TEST_INSTALL_SH'] ||= '../../../../install.sh'
+ENV['TEST_INSTALL_SH'] ||= '../../../install.sh'
 
 Vagrant.configure("2") do |config|
   config.vagrant.plugins = {
diff --git a/tests/vagrant/install/opensuse-leap/Vagrantfile b/tests/install/opensuse-leap/Vagrantfile
similarity index 98%
rename from tests/vagrant/install/opensuse-leap/Vagrantfile
rename to tests/install/opensuse-leap/Vagrantfile
index f0f8370bd4..6873c3e3fa 100644
--- a/tests/vagrant/install/opensuse-leap/Vagrantfile
+++ b/tests/install/opensuse-leap/Vagrantfile
@@ -2,7 +2,7 @@
 # vi: set ft=ruby :
 #
 
-ENV['TEST_INSTALL_SH'] ||= '../../../../install.sh'
+ENV['TEST_INSTALL_SH'] ||= '../../../install.sh'
 
 Vagrant.configure("2") do |config|
   config.vagrant.plugins = {
diff --git a/tests/vagrant/install/opensuse-microos/Vagrantfile b/tests/install/opensuse-microos/Vagrantfile
similarity index 98%
rename from tests/vagrant/install/opensuse-microos/Vagrantfile
rename to tests/install/opensuse-microos/Vagrantfile
index 30934ba045..28a6c3f4d4 100644
--- a/tests/vagrant/install/opensuse-microos/Vagrantfile
+++ b/tests/install/opensuse-microos/Vagrantfile
@@ -2,7 +2,7 @@
 # vi: set ft=ruby :
 #
 
-ENV['TEST_INSTALL_SH'] ||= '../../../../install.sh'
+ENV['TEST_INSTALL_SH'] ||= '../../../install.sh'
 
 Vagrant.configure("2") do |config|
   config.vagrant.plugins = {
diff --git a/tests/vagrant/install/ubuntu-focal/Vagrantfile b/tests/install/ubuntu-focal/Vagrantfile
similarity index 98%
rename from tests/vagrant/install/ubuntu-focal/Vagrantfile
rename to tests/install/ubuntu-focal/Vagrantfile
index eafd61d609..db6b93234c 100644
--- a/tests/vagrant/install/ubuntu-focal/Vagrantfile
+++ b/tests/install/ubuntu-focal/Vagrantfile
@@ -2,7 +2,7 @@
 # vi: set ft=ruby :
 #
 
-ENV['TEST_INSTALL_SH'] ||= '../../../../install.sh'
+ENV['TEST_INSTALL_SH'] ||= '../../../install.sh'
 
 Vagrant.configure("2") do |config|
   config.vagrant.plugins = {
diff --git a/tests/integration/README.md b/tests/integration/README.md
new file mode 100644
index 0000000000..54d89a0a02
--- /dev/null
+++ b/tests/integration/README.md
@@ -0,0 +1,48 @@
+# Integration Tests
+
+Integration tests should be used to test a specific functionality of k3s that exists across multiple Go packages, either via exported function calls, or more often, CLI comands.
+Integration tests should be used for "black box" testing. 
+
+## Framework
+
+All integration tests in K3s follow a [Behavior Diven Development (BDD)](https://en.wikipedia.org/wiki/Behavior-driven_development) style. Specifically, K3s uses [Ginkgo](https://onsi.github.io/ginkgo/) and [Gomega](https://onsi.github.io/gomega/) to drive the tests.  
+To generate an initial test, the command `ginkgo bootstrap` can be used.
+
+To facilitate K3s CLI testing, see `tests/util/cmd.go` helper functions.
+
+## Format
+
+All integration tests should be placed under `tests/integration/<TEST_NAME>`.  
+All integration test files should be named: `<TEST_NAME>_int_test.go`.  
+All integration test functions should be named: `Test_Integration<TEST_NAME>`.  
+See the [local storage test](../tests/integration/localstorage/localstorage_int_test.go) as an example.
+
+## Running
+
+Integration tests can be run with no k3s cluster present, each test will spin up and kill the appropriate k3s server it needs.  
+Note: Integration tests must be run as root, prefix the commands below with `sudo -E env "PATH=$PATH"` if a sudo user.
+```bash
+go test ./tests/integration/... -run Integration
+```
+
+Additionally, to generate JUnit reporting for the tests, the Ginkgo CLI is used
+```
+ginkgo --junit-report=result.xml ./tests/integration/...
+```
+
+Integration tests can be run on an existing single-node cluster via compile time flag, tests will skip if the server is not configured correctly.
+```bash
+go test -ldflags "-X 'github.com/k3s-io/k3s/tests/integration.existingServer=True'" ./tests/integration/... -run Integration
+```
+
+Integration tests can also be run via a [Sonobuoy](https://sonobuoy.io/docs/v0.53.2/) plugin on an existing single-node cluster.
+```bash
+./scripts/build-tests-sonobuoy
+sudo KUBECONFIG=/etc/rancher/k3s/k3s.yaml sonobuoy run --plugin ./dist/artifacts/k3s-int-tests.yaml
+```
+Check the sonobuoy status and retrieve results
+``` 
+sudo KUBECONFIG=/etc/rancher/k3s/k3s.yaml sonobuoy status
+sudo KUBECONFIG=/etc/rancher/k3s/k3s.yaml sonobuoy retrieve
+sudo KUBECONFIG=/etc/rancher/k3s/k3s.yaml sonobuoy results <TAR_FILE_FROM_RETRIEVE>
+```
diff --git a/tests/integration/custometcdargs/custometcdargs_int_test.go b/tests/integration/custometcdargs/custometcdargs_int_test.go
index c1658254f9..92b107e9a9 100644
--- a/tests/integration/custometcdargs/custometcdargs_int_test.go
+++ b/tests/integration/custometcdargs/custometcdargs_int_test.go
@@ -5,7 +5,7 @@ import (
 	"strings"
 	"testing"
 
-	testutil "github.com/k3s-io/k3s/tests/util"
+	testutil "github.com/k3s-io/k3s/tests/integration"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )
diff --git a/tests/integration/dualstack/dualstack_int_test.go b/tests/integration/dualstack/dualstack_int_test.go
index a4797d6eb9..e4ba57038f 100644
--- a/tests/integration/dualstack/dualstack_int_test.go
+++ b/tests/integration/dualstack/dualstack_int_test.go
@@ -5,7 +5,7 @@ import (
 	"strings"
 	"testing"
 
-	testutil "github.com/k3s-io/k3s/tests/util"
+	testutil "github.com/k3s-io/k3s/tests/integration"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )
diff --git a/tests/integration/etcdrestore/etcd_restore_int_test.go b/tests/integration/etcdrestore/etcd_restore_int_test.go
index dc33848740..ec6a8daa7e 100644
--- a/tests/integration/etcdrestore/etcd_restore_int_test.go
+++ b/tests/integration/etcdrestore/etcd_restore_int_test.go
@@ -4,7 +4,7 @@ import (
 	"strings"
 	"testing"
 
-	testutil "github.com/k3s-io/k3s/tests/util"
+	testutil "github.com/k3s-io/k3s/tests/integration"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )
diff --git a/tests/integration/etcdsnapshot/etcdsnapshot_int_test.go b/tests/integration/etcdsnapshot/etcdsnapshot_int_test.go
index d5cb8f755a..3956032636 100644
--- a/tests/integration/etcdsnapshot/etcdsnapshot_int_test.go
+++ b/tests/integration/etcdsnapshot/etcdsnapshot_int_test.go
@@ -6,7 +6,7 @@ import (
 	"testing"
 	"time"
 
-	testutil "github.com/k3s-io/k3s/tests/util"
+	testutil "github.com/k3s-io/k3s/tests/integration"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )
diff --git a/tests/util/integration.go b/tests/integration/integration.go
similarity index 99%
rename from tests/util/integration.go
rename to tests/integration/integration.go
index 0c40c459b1..eaaa1307c2 100644
--- a/tests/util/integration.go
+++ b/tests/integration/integration.go
@@ -1,4 +1,4 @@
-package util
+package integration
 
 import (
 	"bufio"
diff --git a/tests/integration/localstorage/localstorage_int_test.go b/tests/integration/localstorage/localstorage_int_test.go
index 6b75b7cb79..b541c1836a 100644
--- a/tests/integration/localstorage/localstorage_int_test.go
+++ b/tests/integration/localstorage/localstorage_int_test.go
@@ -7,7 +7,7 @@ import (
 	"strings"
 	"testing"
 
-	testutil "github.com/k3s-io/k3s/tests/util"
+	testutil "github.com/k3s-io/k3s/tests/integration"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )
diff --git a/tests/integration/secretsencryption/secretsencryption_int_test.go b/tests/integration/secretsencryption/secretsencryption_int_test.go
index e587c1a541..a397ece3b7 100644
--- a/tests/integration/secretsencryption/secretsencryption_int_test.go
+++ b/tests/integration/secretsencryption/secretsencryption_int_test.go
@@ -6,7 +6,7 @@ import (
 	"testing"
 	"time"
 
-	testutil "github.com/k3s-io/k3s/tests/util"
+	testutil "github.com/k3s-io/k3s/tests/integration"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )
diff --git a/tests/vagrant/snapshotter/btrfs/opensuse-leap/Vagrantfile b/tests/snapshotter/btrfs/opensuse-leap/Vagrantfile
similarity index 97%
rename from tests/vagrant/snapshotter/btrfs/opensuse-leap/Vagrantfile
rename to tests/snapshotter/btrfs/opensuse-leap/Vagrantfile
index c2f24057d4..a3ca1c91fd 100644
--- a/tests/vagrant/snapshotter/btrfs/opensuse-leap/Vagrantfile
+++ b/tests/snapshotter/btrfs/opensuse-leap/Vagrantfile
@@ -10,7 +10,7 @@ Vagrant.configure("2") do |config|
   }
   config.vm.box = "opensuse/Leap-15.3.x86_64"
   config.vm.boot_timeout = ENV['TEST_VM_BOOT_TIMEOUT'] || 600 # seconds
-  config.vm.synced_folder '../../../../../dist/artifacts', '/vagrant', type: 'rsync', disabled: false,
+  config.vm.synced_folder '../../../../dist/artifacts', '/vagrant', type: 'rsync', disabled: false,
     rsync__exclude: ENV['RSYNC_EXCLUDE'] || '*.tar.*'
 
   config.vm.define 'snapshotter-btrfs', primary: true do |test|
diff --git a/tests/util/unit.go b/tests/unit.go
similarity index 99%
rename from tests/util/unit.go
rename to tests/unit.go
index 907d33858f..9df0acd039 100644
--- a/tests/util/unit.go
+++ b/tests/unit.go
@@ -1,4 +1,4 @@
-package util
+package tests
 
 import (
 	"net"
diff --git a/tests/util/wait-for-coredns.sh b/tests/util/wait-for-coredns.sh
deleted file mode 100755
index 10141b327d..0000000000
--- a/tests/util/wait-for-coredns.sh
+++ /dev/null
@@ -1,17 +0,0 @@
-#!/bin/bash
-# Wait for CoreDNS pods to be ready.
-
-set -x
-echo "Waiting for CoreDNS pods to be ready..."
-counter=0
-# `kubectl wait` fails when the pods with the specified label are not created yet
-until kubectl wait --for=condition=ready pods --namespace=kube-system -l k8s-app=kube-dns; do
-  ((counter++))
-  if [[ $counter -eq 20 ]]; then
-    echo "CoreDNS not running?"
-    kubectl get pods -A
-    kubectl get nodes -o wide
-    exit 1
-  fi
-  sleep 10
-done
diff --git a/tests/vagrant/.gitignore b/tests/vagrant/.gitignore
deleted file mode 100644
index a977916f65..0000000000
--- a/tests/vagrant/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-.vagrant/