diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 8731e4bf91..13f51d6336 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -160,7 +160,7 @@ jobs: strategy: fail-fast: false matrix: - dtest: [basics, bootstraptoken, cacerts, etcd, lazypull, skew, snapshotrestore, token, upgrade] + dtest: [basics, bootstraptoken, cacerts, etcd, hardened, lazypull, skew, snapshotrestore, token, upgrade] arch: [amd64, arm64] runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} env: @@ -202,7 +202,7 @@ jobs: cd ./tests/docker/${{ matrix.dtest }} if [ ${{ matrix.dtest }} = "upgrade" ] || [ ${{ matrix.dtest }} = "skew" ]; then ./${{ matrix.dtest }}.test -k3sImage=$K3S_IMAGE -channel=$CHANNEL - elif [ ${{ matrix.dtest }} = "snapshotrestore" ]; then + elif [ ${{ matrix.dtest }} = "snapshotrestore" ] || [ ${{ matrix.dtest }} = "hardened" ]; then ./${{ matrix.dtest }}.test -ci else ./${{ matrix.dtest }}.test -k3sImage=$K3S_IMAGE diff --git a/tests/docker/hardened/cluster-level-pss.yaml b/tests/docker/hardened/cluster-level-pss.yaml new file mode 100644 index 0000000000..ea18a8dd31 --- /dev/null +++ b/tests/docker/hardened/cluster-level-pss.yaml @@ -0,0 +1,18 @@ +apiVersion: apiserver.config.k8s.io/v1 +kind: AdmissionConfiguration +plugins: +- name: PodSecurity + configuration: + apiVersion: pod-security.admission.config.k8s.io/v1beta1 + kind: PodSecurityConfiguration + defaults: + enforce: "privileged" + enforce-version: "latest" + audit: "baseline" + audit-version: "latest" + warn: "baseline" + warn-version: "latest" + exemptions: + usernames: [] + runtimeClasses: [] + namespaces: [kube-system] diff --git a/tests/docker/hardened/hardened_test.go b/tests/docker/hardened/hardened_test.go new file mode 100644 index 0000000000..fd10b914bb --- /dev/null +++ b/tests/docker/hardened/hardened_test.go @@ -0,0 +1,124 @@ +package main + +import ( + "flag" + "fmt" + "strings" + "testing" + + "github.com/k3s-io/k3s/tests" + "github.com/k3s-io/k3s/tests/docker" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var config *docker.TestConfig +var ci = flag.Bool("ci", false, "running on CI") + +func Test_DockerHardened(t *testing.T) { + flag.Parse() + RegisterFailHandler(Fail) + RunSpecs(t, "Hardened Docker Test Suite") +} + +var _ = Describe("Hardened Tests", Ordered, func() { + + Context("Setup Cluster", func() { + It("should provision servers and agents", func() { + var err error + config, err = docker.NewTestConfig("rancher/systemd-node") + Expect(err).NotTo(HaveOccurred()) + config.ServerYaml = ` +secrets-encryption: true +kube-controller-manager-arg: + - 'terminated-pod-gc-threshold=10' +kubelet-arg: + - 'streaming-connection-idle-timeout=5m' + - 'make-iptables-util-chains=true' + - 'event-qps=0' + - "tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305" +kube-apiserver-arg: + - 'admission-control-config-file=/tmp/cluster-level-pss.yaml' + - 'audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log' + - 'audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml' + - 'audit-log-maxage=30' + - 'audit-log-maxbackup=10' + - 'audit-log-maxsize=100' +` + config.AgentYaml = ` +kubelet-arg: + - 'streaming-connection-idle-timeout=5m' + - "tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305" +` + config.SkipStart = true + Expect(config.ProvisionServers(1)).To(Succeed()) + + for _, server := range config.Servers { + cmd := "docker cp ./cluster-level-pss.yaml " + server.Name + ":/tmp/cluster-level-pss.yaml" + Expect(docker.RunCommand(cmd)).Error().NotTo(HaveOccurred()) + + cmd = "mkdir -p /var/lib/rancher/k3s/server/logs" + Expect(server.RunCmdOnNode(cmd)).Error().NotTo(HaveOccurred()) + auditYaml := "apiVersion: audit.k8s.io/v1\nkind: Policy\nrules:\n- level: Metadata" + cmd = fmt.Sprintf("echo -e '%s' > /var/lib/rancher/k3s/server/audit.yaml", auditYaml) + Expect(server.RunCmdOnNode(cmd)).Error().NotTo(HaveOccurred()) + Expect(server.RunCmdOnNode("systemctl start k3s")).Error().NotTo(HaveOccurred()) + } + Expect(config.CopyAndModifyKubeconfig()).To(Succeed()) + config.SkipStart = false + Expect(config.ProvisionAgents(1)).To(Succeed()) + Eventually(func() error { + return tests.CheckDeployments([]string{"coredns", "local-path-provisioner", "metrics-server", "traefik"}, config.KubeconfigFile) + }, "60s", "5s").Should(Succeed()) + Eventually(func() error { + return tests.NodesReady(config.KubeconfigFile, config.GetNodeNames()) + }, "40s", "5s").Should(Succeed()) + }) + }) + + Context("Verify Network Policies", func() { + It("applies network policies", func() { + _, err := config.DeployWorkload("hardened-ingress.yaml") + Expect(err).NotTo(HaveOccurred()) + Eventually(func() (string, error) { + cmd := "kubectl get daemonset -n default example -o jsonpath='{.status.numberReady}' --kubeconfig=" + config.KubeconfigFile + return docker.RunCommand(cmd) + }, "60s", "5s").Should(Equal("2")) + _, err = config.DeployWorkload("hardened-netpool.yaml") + }) + It("checks ingress connections", func() { + for _, scheme := range []string{"http", "https"} { + for _, server := range config.Servers { + cmd := fmt.Sprintf("curl -vksf -H 'Host: example.com' %s://%s/", scheme, server.IP) + Expect(docker.RunCommand(cmd)).Error().NotTo(HaveOccurred()) + } + for _, agent := range config.Agents { + cmd := fmt.Sprintf("curl -vksf -H 'Host: example.com' %s://%s/", scheme, agent.IP) + Expect(docker.RunCommand(cmd)).Error().NotTo(HaveOccurred()) + } + } + }) + It("confirms we can make a request through the nodeport service", func() { + for _, server := range config.Servers { + cmd := "kubectl get service/example -o 'jsonpath={.spec.ports[*].nodePort}' --kubeconfig=" + config.KubeconfigFile + ports, err := docker.RunCommand(cmd) + Expect(err).NotTo(HaveOccurred()) + for _, port := range strings.Split(ports, " ") { + cmd := fmt.Sprintf("curl -vksf -H 'Host: example.com' http://%s:%s", server.IP, port) + Expect(docker.RunCommand(cmd)).Error().NotTo(HaveOccurred()) + } + } + }) + }) +}) + +var failed bool +var _ = AfterEach(func() { + failed = failed || CurrentSpecReport().Failed() +}) + +var _ = AfterSuite(func() { + if *ci || (config != nil && !failed) { + Expect(config.Cleanup()).To(Succeed()) + } +}) diff --git a/tests/docker/resources/hardened-ingress.yaml b/tests/docker/resources/hardened-ingress.yaml new file mode 100644 index 0000000000..19fc18f370 --- /dev/null +++ b/tests/docker/resources/hardened-ingress.yaml @@ -0,0 +1,128 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: example + namespace: default + labels: + app.kubernetes.io: example +spec: + selector: + matchLabels: + app.kubernetes.io/name: example + template: + metadata: + labels: + app.kubernetes.io/name: example + spec: + automountServiceAccountToken: false + securityContext: + runAsUser: 405 + runAsGroup: 100 + containers: + - name: socat + image: docker.io/alpine/socat:1.7.4.3-r1 + args: + - "TCP-LISTEN:8080,reuseaddr,fork" + - "EXEC:echo -e 'HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n$(NODE_IP) $(POD_NAMESPACE)/$(POD_NAME)\r\n'" + ports: + - containerPort: 8080 + name: http + env: + - name: NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + readinessProbe: + initialDelaySeconds: 2 + periodSeconds: 10 + httpGet: + path: / + port: 8080 +--- +apiVersion: v1 +kind: Service +metadata: + name: example + namespace: default +spec: + type: NodePort + selector: + app.kubernetes.io/name: example + ports: + - name: http + protocol: TCP + port: 80 + nodePort: 30096 + targetPort: http +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: example +spec: + rules: + - host: "example.com" + http: + paths: + - pathType: Prefix + path: "/" + backend: + service: + name: example + port: + name: http +--- +# Allow access to example backend from traefik ingress +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: ingress-to-backend-example + namespace: default +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: example + ingress: + - ports: + - port: 8080 + protocol: TCP + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + app.kubernetes.io/name: traefik + policyTypes: + - Ingress +--- +# Allow access to example backend from outside the cluster via nodeport service +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: nodeport-to-backend-example + namespace: default +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: example + ingress: + - ports: + - port: 8080 + protocol: TCP + - from: + - ipBlock: + cidr: 0.0.0.0/0 + except: + - 10.42.0.0/16 + - 10.43.0.0/16 + policyTypes: + - Ingress diff --git a/tests/docker/resources/hardened-netpol.yaml b/tests/docker/resources/hardened-netpol.yaml new file mode 100644 index 0000000000..162bdadec3 --- /dev/null +++ b/tests/docker/resources/hardened-netpol.yaml @@ -0,0 +1,120 @@ +--- +# Allow all traffic within the kube-system namespace; block all other access +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: intra-namespace + namespace: kube-system +spec: + podSelector: {} + ingress: + - from: + - namespaceSelector: + matchLabels: + name: kube-system + policyTypes: + - Ingress +--- +# Allow all traffic within the default namespace; block all other access +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: intra-namespace + namespace: default +spec: + podSelector: {} + ingress: + - from: + - namespaceSelector: + matchLabels: + name: default + policyTypes: + - Ingress +--- +# Allow traffic within the kube-public namespace; block all other access +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: intra-namespace + namespace: kube-public +spec: + podSelector: {} + ingress: + - from: + - namespaceSelector: + matchLabels: + name: kube-public + policyTypes: + - Ingress +--- +# Allow all access to metrics-server +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-all-metrics-server + namespace: kube-system +spec: + podSelector: + matchLabels: + k8s-app: metrics-server + ingress: + - {} + policyTypes: + - Ingress +--- +# Allow all access to coredns DNS ports +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-network-dns-policy + namespace: kube-system +spec: + ingress: + - ports: + - port: 53 + protocol: TCP + - port: 53 + protocol: UDP + podSelector: + matchLabels: + k8s-app: kube-dns + policyTypes: + - Ingress +--- +# Allow all access to the servicelb traefik HTTP/HTTPS ports +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-all-svclb-traefik + namespace: kube-system +spec: + podSelector: + matchLabels: + svccontroller.k3s.cattle.io/svcname: traefik + ingress: + - ports: + - port: 80 + protocol: TCP + - port: 443 + protocol: TCP + policyTypes: + - Ingress +--- +# Allow all access to traefik HTTP/HTTPS ports +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-all-traefik + namespace: kube-system +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: traefik + ingress: + - ports: + - port: 8000 + protocol: TCP + - port: 8443 + protocol: TCP + policyTypes: + - Ingress diff --git a/tests/docker/test-helpers.go b/tests/docker/test-helpers.go index 41b5b93e70..1a146f02eb 100644 --- a/tests/docker/test-helpers.go +++ b/tests/docker/test-helpers.go @@ -177,14 +177,13 @@ func (config *TestConfig) ProvisionServers(numOfServers int) error { } // Write the raw YAML directly to the config.yaml on the systemd-node container if config.ServerYaml != "" { - cmd = fmt.Sprintf("echo \"%s\" > /etc/rancher/k3s/config.yaml", config.ServerYaml) + cmd = fmt.Sprintf("echo '%s' > /etc/rancher/k3s/config.yaml", config.ServerYaml) if out, err := newServer.RunCmdOnNode(cmd); err != nil { return fmt.Errorf("failed to write server yaml: %s: %v", out, err) } } - // The pipe requires that we use sh -c with "" to run the command - cmd = fmt.Sprintf("/bin/sh -c \"curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC='%s' %s INSTALL_K3S_SKIP_DOWNLOAD=true sh -\"", + cmd = fmt.Sprintf("curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC='%s' %s INSTALL_K3S_SKIP_DOWNLOAD=true sh -", dbConnect+" "+joinServer+" "+os.Getenv(fmt.Sprintf("SERVER_%d_ARGS", i)), skipStart) if _, err := newServer.RunCmdOnNode(cmd); err != nil { // Attempt to dump the last few lines of the journalctl logs @@ -247,7 +246,7 @@ func (config *TestConfig) ProvisionServers(numOfServers int) error { } // Wait for kubeconfig to be available time.Sleep(5 * time.Second) - return CopyAndModifyKubeconfig(config) + return config.CopyAndModifyKubeconfig() } // setupDatabase will start the configured database if startDB is true, @@ -333,14 +332,13 @@ func (config *TestConfig) ProvisionAgents(numOfAgents int) error { } // Write the raw YAML directly to the config.yaml on the systemd-node container if config.AgentYaml != "" { - cmd = fmt.Sprintf("echo \"%s\" > /etc/rancher/k3s/config.yaml", config.ServerYaml) + cmd = fmt.Sprintf("echo '%s' > /etc/rancher/k3s/config.yaml", config.AgentYaml) if out, err := newAgent.RunCmdOnNode(cmd); err != nil { return fmt.Errorf("failed to write server yaml: %s: %v", out, err) } } - // The pipe requires that we use sh -c with "" to run the command - sCmd := fmt.Sprintf("/bin/sh -c \"curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC='agent %s' %s INSTALL_K3S_SKIP_DOWNLOAD=true sh -\"", + sCmd := fmt.Sprintf("curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC='agent %s' %s INSTALL_K3S_SKIP_DOWNLOAD=true sh -", os.Getenv(agentInstanceArgs), skipStart) if _, err := newAgent.RunCmdOnNode(sCmd); err != nil { // Attempt to dump the last few lines of the journalctl logs @@ -470,7 +468,7 @@ func (config *TestConfig) Cleanup() error { // CopyAndModifyKubeconfig copies out kubeconfig from first control-plane server // and updates the port to match the external port -func CopyAndModifyKubeconfig(config *TestConfig) error { +func (config *TestConfig) CopyAndModifyKubeconfig() error { if len(config.Servers) == 0 { return fmt.Errorf("no servers available to copy kubeconfig") } @@ -500,7 +498,7 @@ func CopyAndModifyKubeconfig(config *TestConfig) error { // RunCmdOnNode runs a command on a docker container func (node DockerNode) RunCmdOnNode(cmd string) (string, error) { - dCmd := fmt.Sprintf("docker exec %s %s", node.Name, cmd) + dCmd := fmt.Sprintf("docker exec %s /bin/sh -c \"%s\"", node.Name, cmd) out, err := RunCommand(dCmd) if err != nil { return out, fmt.Errorf("%v: on node %s: %s", err, node.Name, out) diff --git a/tests/docker/token/token_test.go b/tests/docker/token/token_test.go index 553dddb469..e123f4707a 100644 --- a/tests/docker/token/token_test.go +++ b/tests/docker/token/token_test.go @@ -57,8 +57,7 @@ var _ = Describe("Use the token CLI to create and join agents", Ordered, func() Expect(res).To(MatchRegexp(`perage\s+\s+`)) }) It("Joins an agent with the permanent token", func() { - // wrap in bash to handle pipe - cmd := fmt.Sprintf("/bin/bash -c \"echo 'token: %s' | tee -a /etc/rancher/k3s/config.yaml > /dev/null\"", permToken) + cmd := fmt.Sprintf("echo 'token: %s' | tee -a /etc/rancher/k3s/config.yaml > /dev/null", permToken) _, err := tc.Agents[0].RunCmdOnNode(cmd) Expect(err).NotTo(HaveOccurred()) _, err = tc.Agents[0].RunCmdOnNode("systemctl start k3s-agent") @@ -95,7 +94,7 @@ var _ = Describe("Use the token CLI to create and join agents", Ordered, func() Expect(res).To(MatchRegexp(`10mint\s+[0-9]m`)) }) It("Joins an agent with the 10m token", func() { - cmd := fmt.Sprintf("/bin/bash -c \"echo 'token: %s' | tee -a /etc/rancher/k3s/config.yaml > /dev/null\"", tempToken) + cmd := fmt.Sprintf("echo 'token: %s' | tee -a /etc/rancher/k3s/config.yaml > /dev/null", tempToken) _, err := tc.Agents[1].RunCmdOnNode(cmd) Expect(err).NotTo(HaveOccurred()) _, err = tc.Agents[1].RunCmdOnNode("systemctl start k3s-agent") @@ -117,7 +116,7 @@ var _ = Describe("Use the token CLI to create and join agents", Ordered, func() To(ContainSubstring("Token rotated, restart k3s nodes with new token")) }) It("Restarts servers with the new token", func() { - cmd := fmt.Sprintf("/bin/bash -c \"echo 'token: %s' | tee -a /etc/rancher/k3s/config.yaml > /dev/null\"", serverToken) + cmd := fmt.Sprintf("echo 'token: %s' | tee -a /etc/rancher/k3s/config.yaml > /dev/null", serverToken) for _, node := range tc.Servers { _, err := node.RunCmdOnNode(cmd) Expect(err).NotTo(HaveOccurred())