public/k3s
1
0
Fork 0
mirror of https://git.vectorsigma.ru/public/k3s.git synced 2026-07-28 23:09:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Migrate hardened docker test

Browse Source 
Signed-off-by: Derek Nola <derek.nola@suse.com>
This commit is contained in:
Derek Nola
2025-02-06 13:40:11 -08:00
parent 2724f08be8
commit 793d216d02
7 changed files with 402 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/e2e.yaml vendored
View File

@@ -160,7 +160,7 @@ jobs:
strategy:
fail-fast: false
matrix:
dtest: [basics, bootstraptoken, cacerts, etcd, lazypull, skew, snapshotrestore, token, upgrade]
dtest: [basics, bootstraptoken, cacerts, etcd, hardened, lazypull, skew, snapshotrestore, token, upgrade]
arch: [amd64, arm64]
runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }}
env:
@@ -202,7 +202,7 @@ jobs:
cd ./tests/docker/${{ matrix.dtest }}
if [ ${{ matrix.dtest }} = "upgrade" ] || [ ${{ matrix.dtest }} = "skew" ]; then
./${{ matrix.dtest }}.test -k3sImage=$K3S_IMAGE -channel=$CHANNEL
elif [ ${{ matrix.dtest }} = "snapshotrestore" ]; then
elif [ ${{ matrix.dtest }} = "snapshotrestore" ] || [ ${{ matrix.dtest }} = "hardened" ]; then
./${{ matrix.dtest }}.test -ci
else
./${{ matrix.dtest }}.test -k3sImage=$K3S_IMAGE

18
tests/docker/hardened/cluster-level-pss.yaml Normal file
View File

@@ -0,0 +1,18 @@
apiVersion: apiserver.config.k8s.io/v1
kind: AdmissionConfiguration
plugins:
- name: PodSecurity
configuration:
apiVersion: pod-security.admission.config.k8s.io/v1beta1
kind: PodSecurityConfiguration
defaults:
enforce: "privileged"
enforce-version: "latest"
audit: "baseline"
audit-version: "latest"
warn: "baseline"
warn-version: "latest"
exemptions:
usernames: []
runtimeClasses: []
namespaces: [kube-system]

124
tests/docker/hardened/hardened_test.go Normal file
View File

@@ -0,0 +1,124 @@
package main
import (
"flag"
"fmt"
"strings"
"testing"
"github.com/k3s-io/k3s/tests"
"github.com/k3s-io/k3s/tests/docker"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
var config *docker.TestConfig
var ci = flag.Bool("ci", false, "running on CI")
func Test_DockerHardened(t *testing.T) {
flag.Parse()
RegisterFailHandler(Fail)
RunSpecs(t, "Hardened Docker Test Suite")
}
var _ = Describe("Hardened Tests", Ordered, func() {
Context("Setup Cluster", func() {
It("should provision servers and agents", func() {
var err error
config, err = docker.NewTestConfig("rancher/systemd-node")
Expect(err).NotTo(HaveOccurred())
config.ServerYaml = `
secrets-encryption: true
kube-controller-manager-arg:
- 'terminated-pod-gc-threshold=10'
kubelet-arg:
- 'streaming-connection-idle-timeout=5m'
- 'make-iptables-util-chains=true'
- 'event-qps=0'
- "tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305"
kube-apiserver-arg:
- 'admission-control-config-file=/tmp/cluster-level-pss.yaml'
- 'audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log'
- 'audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml'
- 'audit-log-maxage=30'
- 'audit-log-maxbackup=10'
- 'audit-log-maxsize=100'
`
config.AgentYaml = `
kubelet-arg:
- 'streaming-connection-idle-timeout=5m'
- "tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305"
`
config.SkipStart = true
Expect(config.ProvisionServers(1)).To(Succeed())
for _, server := range config.Servers {
cmd := "docker cp ./cluster-level-pss.yaml " + server.Name + ":/tmp/cluster-level-pss.yaml"
Expect(docker.RunCommand(cmd)).Error().NotTo(HaveOccurred())
cmd = "mkdir -p /var/lib/rancher/k3s/server/logs"
Expect(server.RunCmdOnNode(cmd)).Error().NotTo(HaveOccurred())
auditYaml := "apiVersion: audit.k8s.io/v1\nkind: Policy\nrules:\n- level: Metadata"
cmd = fmt.Sprintf("echo -e '%s' > /var/lib/rancher/k3s/server/audit.yaml", auditYaml)
Expect(server.RunCmdOnNode(cmd)).Error().NotTo(HaveOccurred())
Expect(server.RunCmdOnNode("systemctl start k3s")).Error().NotTo(HaveOccurred())
}
Expect(config.CopyAndModifyKubeconfig()).To(Succeed())
config.SkipStart = false
Expect(config.ProvisionAgents(1)).To(Succeed())
Eventually(func() error {
return tests.CheckDeployments([]string{"coredns", "local-path-provisioner", "metrics-server", "traefik"}, config.KubeconfigFile)
}, "60s", "5s").Should(Succeed())
Eventually(func() error {
return tests.NodesReady(config.KubeconfigFile, config.GetNodeNames())
}, "40s", "5s").Should(Succeed())
})
})
Context("Verify Network Policies", func() {
It("applies network policies", func() {
_, err := config.DeployWorkload("hardened-ingress.yaml")
Expect(err).NotTo(HaveOccurred())
Eventually(func() (string, error) {
cmd := "kubectl get daemonset -n default example -o jsonpath='{.status.numberReady}' --kubeconfig=" + config.KubeconfigFile
return docker.RunCommand(cmd)
}, "60s", "5s").Should(Equal("2"))
_, err = config.DeployWorkload("hardened-netpool.yaml")
})
It("checks ingress connections", func() {
for _, scheme := range []string{"http", "https"} {
for _, server := range config.Servers {
cmd := fmt.Sprintf("curl -vksf -H 'Host: example.com' %s://%s/", scheme, server.IP)
Expect(docker.RunCommand(cmd)).Error().NotTo(HaveOccurred())
}
for _, agent := range config.Agents {
cmd := fmt.Sprintf("curl -vksf -H 'Host: example.com' %s://%s/", scheme, agent.IP)
Expect(docker.RunCommand(cmd)).Error().NotTo(HaveOccurred())
}
}
})
It("confirms we can make a request through the nodeport service", func() {
for _, server := range config.Servers {
cmd := "kubectl get service/example -o 'jsonpath={.spec.ports[*].nodePort}' --kubeconfig=" + config.KubeconfigFile
ports, err := docker.RunCommand(cmd)
Expect(err).NotTo(HaveOccurred())
for _, port := range strings.Split(ports, " ") {
cmd := fmt.Sprintf("curl -vksf -H 'Host: example.com' http://%s:%s", server.IP, port)
Expect(docker.RunCommand(cmd)).Error().NotTo(HaveOccurred())
}
}
})
})
})
var failed bool
var _ = AfterEach(func() {
failed = failed || CurrentSpecReport().Failed()
})
var _ = AfterSuite(func() {
if *ci || (config != nil && !failed) {
Expect(config.Cleanup()).To(Succeed())
}
})

128
tests/docker/resources/hardened-ingress.yaml Normal file
View File

@@ -0,0 +1,128 @@
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: example
namespace: default
labels:
app.kubernetes.io: example
spec:
selector:
matchLabels:
app.kubernetes.io/name: example
template:
metadata:
labels:
app.kubernetes.io/name: example
spec:
automountServiceAccountToken: false
securityContext:
runAsUser: 405
runAsGroup: 100
containers:
- name: socat
image: docker.io/alpine/socat:1.7.4.3-r1
args:
- "TCP-LISTEN:8080,reuseaddr,fork"
- "EXEC:echo -e 'HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n$(NODE_IP) $(POD_NAMESPACE)/$(POD_NAME)\r\n'"
ports:
- containerPort: 8080
name: http
env:
- name: NODE_IP
valueFrom:
fieldRef:
fieldPath: status.hostIP
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
readinessProbe:
initialDelaySeconds: 2
periodSeconds: 10
httpGet:
path: /
port: 8080
---
apiVersion: v1
kind: Service
metadata:
name: example
namespace: default
spec:
type: NodePort
selector:
app.kubernetes.io/name: example
ports:
- name: http
protocol: TCP
port: 80
nodePort: 30096
targetPort: http
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: example
spec:
rules:
- host: "example.com"
http:
paths:
- pathType: Prefix
path: "/"
backend:
service:
name: example
port:
name: http
---
# Allow access to example backend from traefik ingress
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: ingress-to-backend-example
namespace: default
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: example
ingress:
- ports:
- port: 8080
protocol: TCP
- from:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: kube-system
podSelector:
matchLabels:
app.kubernetes.io/name: traefik
policyTypes:
- Ingress
---
# Allow access to example backend from outside the cluster via nodeport service
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: nodeport-to-backend-example
namespace: default
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: example
ingress:
- ports:
- port: 8080
protocol: TCP
- from:
- ipBlock:
cidr: 0.0.0.0/0
except:
- 10.42.0.0/16
- 10.43.0.0/16
policyTypes:
- Ingress

120
tests/docker/resources/hardened-netpol.yaml Normal file
View File

@@ -0,0 +1,120 @@
---
# Allow all traffic within the kube-system namespace; block all other access
kind: NetworkPolicy
apiVersion: networking.k8s.io/v1
metadata:
name: intra-namespace
namespace: kube-system
spec:
podSelector: {}
ingress:
- from:
- namespaceSelector:
matchLabels:
name: kube-system
policyTypes:
- Ingress
---
# Allow all traffic within the default namespace; block all other access
kind: NetworkPolicy
apiVersion: networking.k8s.io/v1
metadata:
name: intra-namespace
namespace: default
spec:
podSelector: {}
ingress:
- from:
- namespaceSelector:
matchLabels:
name: default
policyTypes:
- Ingress
---
# Allow traffic within the kube-public namespace; block all other access
kind: NetworkPolicy
apiVersion: networking.k8s.io/v1
metadata:
name: intra-namespace
namespace: kube-public
spec:
podSelector: {}
ingress:
- from:
- namespaceSelector:
matchLabels:
name: kube-public
policyTypes:
- Ingress
---
# Allow all access to metrics-server
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: allow-all-metrics-server
namespace: kube-system
spec:
podSelector:
matchLabels:
k8s-app: metrics-server
ingress:
- {}
policyTypes:
- Ingress
---
# Allow all access to coredns DNS ports
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: default-network-dns-policy
namespace: kube-system
spec:
ingress:
- ports:
- port: 53
protocol: TCP
- port: 53
protocol: UDP
podSelector:
matchLabels:
k8s-app: kube-dns
policyTypes:
- Ingress
---
# Allow all access to the servicelb traefik HTTP/HTTPS ports
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: allow-all-svclb-traefik
namespace: kube-system
spec:
podSelector:
matchLabels:
svccontroller.k3s.cattle.io/svcname: traefik
ingress:
- ports:
- port: 80
protocol: TCP
- port: 443
protocol: TCP
policyTypes:
- Ingress
---
# Allow all access to traefik HTTP/HTTPS ports
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: allow-all-traefik
namespace: kube-system
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: traefik
ingress:
- ports:
- port: 8000
protocol: TCP
- port: 8443
protocol: TCP
policyTypes:
- Ingress

16
tests/docker/test-helpers.go
View File

@@ -177,14 +177,13 @@ func (config *TestConfig) ProvisionServers(numOfServers int) error {
}
// Write the raw YAML directly to the config.yaml on the systemd-node container
if config.ServerYaml != "" {
cmd = fmt.Sprintf("echo \"%s\" > /etc/rancher/k3s/config.yaml", config.ServerYaml)
cmd = fmt.Sprintf("echo '%s' > /etc/rancher/k3s/config.yaml", config.ServerYaml)
if out, err := newServer.RunCmdOnNode(cmd); err != nil {
return fmt.Errorf("failed to write server yaml: %s: %v", out, err)
}
}
// The pipe requires that we use sh -c with "" to run the command
cmd = fmt.Sprintf("/bin/sh -c \"curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC='%s' %s INSTALL_K3S_SKIP_DOWNLOAD=true sh -\"",
cmd = fmt.Sprintf("curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC='%s' %s INSTALL_K3S_SKIP_DOWNLOAD=true sh -",
dbConnect+" "+joinServer+" "+os.Getenv(fmt.Sprintf("SERVER_%d_ARGS", i)), skipStart)
if _, err := newServer.RunCmdOnNode(cmd); err != nil {
// Attempt to dump the last few lines of the journalctl logs
@@ -247,7 +246,7 @@ func (config *TestConfig) ProvisionServers(numOfServers int) error {
}
// Wait for kubeconfig to be available
time.Sleep(5 * time.Second)
return CopyAndModifyKubeconfig(config)
return config.CopyAndModifyKubeconfig()
}
// setupDatabase will start the configured database if startDB is true,
@@ -333,14 +332,13 @@ func (config *TestConfig) ProvisionAgents(numOfAgents int) error {
}
// Write the raw YAML directly to the config.yaml on the systemd-node container
if config.AgentYaml != "" {
cmd = fmt.Sprintf("echo \"%s\" > /etc/rancher/k3s/config.yaml", config.ServerYaml)
cmd = fmt.Sprintf("echo '%s' > /etc/rancher/k3s/config.yaml", config.AgentYaml)
if out, err := newAgent.RunCmdOnNode(cmd); err != nil {
return fmt.Errorf("failed to write server yaml: %s: %v", out, err)
}
}
// The pipe requires that we use sh -c with "" to run the command
sCmd := fmt.Sprintf("/bin/sh -c \"curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC='agent %s' %s INSTALL_K3S_SKIP_DOWNLOAD=true sh -\"",
sCmd := fmt.Sprintf("curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC='agent %s' %s INSTALL_K3S_SKIP_DOWNLOAD=true sh -",
os.Getenv(agentInstanceArgs), skipStart)
if _, err := newAgent.RunCmdOnNode(sCmd); err != nil {
// Attempt to dump the last few lines of the journalctl logs
@@ -470,7 +468,7 @@ func (config *TestConfig) Cleanup() error {
// CopyAndModifyKubeconfig copies out kubeconfig from first control-plane server
// and updates the port to match the external port
func CopyAndModifyKubeconfig(config *TestConfig) error {
func (config *TestConfig) CopyAndModifyKubeconfig() error {
if len(config.Servers) == 0 {
return fmt.Errorf("no servers available to copy kubeconfig")
}
@@ -500,7 +498,7 @@ func CopyAndModifyKubeconfig(config *TestConfig) error {
// RunCmdOnNode runs a command on a docker container
func (node DockerNode) RunCmdOnNode(cmd string) (string, error) {
dCmd := fmt.Sprintf("docker exec %s %s", node.Name, cmd)
dCmd := fmt.Sprintf("docker exec %s /bin/sh -c \"%s\"", node.Name, cmd)
out, err := RunCommand(dCmd)
if err != nil {
return out, fmt.Errorf("%v: on node %s: %s", err, node.Name, out)

7
tests/docker/token/token_test.go
View File

@@ -57,8 +57,7 @@ var _ = Describe("Use the token CLI to create and join agents", Ordered, func()
Expect(res).To(MatchRegexp(`perage\s+<forever>\s+<never>`))
})
It("Joins an agent with the permanent token", func() {
// wrap in bash to handle pipe
cmd := fmt.Sprintf("/bin/bash -c \"echo 'token: %s' | tee -a /etc/rancher/k3s/config.yaml > /dev/null\"", permToken)
cmd := fmt.Sprintf("echo 'token: %s' | tee -a /etc/rancher/k3s/config.yaml > /dev/null", permToken)
_, err := tc.Agents[0].RunCmdOnNode(cmd)
Expect(err).NotTo(HaveOccurred())
_, err = tc.Agents[0].RunCmdOnNode("systemctl start k3s-agent")
@@ -95,7 +94,7 @@ var _ = Describe("Use the token CLI to create and join agents", Ordered, func()
Expect(res).To(MatchRegexp(`10mint\s+[0-9]m`))
})
It("Joins an agent with the 10m token", func() {
cmd := fmt.Sprintf("/bin/bash -c \"echo 'token: %s' | tee -a /etc/rancher/k3s/config.yaml > /dev/null\"", tempToken)
cmd := fmt.Sprintf("echo 'token: %s' | tee -a /etc/rancher/k3s/config.yaml > /dev/null", tempToken)
_, err := tc.Agents[1].RunCmdOnNode(cmd)
Expect(err).NotTo(HaveOccurred())
_, err = tc.Agents[1].RunCmdOnNode("systemctl start k3s-agent")
@@ -117,7 +116,7 @@ var _ = Describe("Use the token CLI to create and join agents", Ordered, func()
To(ContainSubstring("Token rotated, restart k3s nodes with new token"))
})
It("Restarts servers with the new token", func() {
cmd := fmt.Sprintf("/bin/bash -c \"echo 'token: %s' | tee -a /etc/rancher/k3s/config.yaml > /dev/null\"", serverToken)
cmd := fmt.Sprintf("echo 'token: %s' | tee -a /etc/rancher/k3s/config.yaml > /dev/null", serverToken)
for _, node := range tc.Servers {
_, err := node.RunCmdOnNode(cmd)
Expect(err).NotTo(HaveOccurred())