From 4e8d85a2e4f598824d302235e9d3cb14607e05f7 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Mon, 14 Apr 2025 13:44:48 -0700 Subject: [PATCH] Split E2E Drone pipeline into matrix (#12086) * Split drone e2e into multiple blocks, explicit virsh cleanup * Create multiple registries once and reuse as long as they exist Signed-off-by: Derek Nola --- .drone.yml | 42 +++++++++++++---- tests/e2e/scripts/cleanup_vms.sh | 21 ++++++--- tests/e2e/scripts/drone_registries.sh | 66 +++++++++++++++++++++++++++ tests/e2e/scripts/registry.sh | 14 +++++- 4 files changed, 127 insertions(+), 16 deletions(-) create mode 100755 tests/e2e/scripts/drone_registries.sh diff --git a/.drone.yml b/.drone.yml index 1f0a5fbc07..4c54bc3733 100644 --- a/.drone.yml +++ b/.drone.yml @@ -657,7 +657,9 @@ steps: - name: docker path: /var/run/docker.sock -- name: test-e2e +- name: test-e2e-validatecluster + depends_on: + - build-e2e-image image: test-e2e pull: never resources: @@ -669,24 +671,46 @@ steps: commands: - mkdir -p dist/artifacts - cp /tmp/artifacts/* dist/artifacts/ - - docker stop registry && docker rm registry # Cleanup VMs that are older than 2h. Happens if a previous test panics or is canceled - tests/e2e/scripts/cleanup_vms.sh - - docker run -d -p 5000:5000 -e REGISTRY_PROXY_REMOTEURL=https://registry-1.docker.io --name registry registry:2 + - tests/e2e/scripts/drone_registries.sh - | cd tests/e2e/validatecluster - vagrant destroy -f + ../scripts/cleanup_vms.sh 'validatecluster_([0-9]+)_(server|agent)' go test -v -timeout=45m ./validatecluster_test.go -ci -local cp ./coverage.out /tmp/artifacts/validate-coverage.out + volumes: + - name: libvirt + path: /var/run/libvirt/ + - name: docker + path: /var/run/docker.sock + - name: cache + path: /tmp/artifacts + +- name: test-e2e-splitserver + depends_on: + - build-e2e-image + image: test-e2e + pull: never + resources: + cpu: 6000 + memory: 10Gi + environment: + E2E_REGISTRY: 'true' + E2E_GOCOVER: 'true' + commands: + - mkdir -p dist/artifacts + - cp /tmp/artifacts/* dist/artifacts/ + - tests/e2e/scripts/drone_registries.sh - | - cd ../splitserver - vagrant destroy -f + cd tests/e2e/splitserver + ../scripts/cleanup_vms.sh 'splitserver_([0-9]+)' go test -v -timeout=30m ./splitserver_test.go -ci -local cp ./coverage.out /tmp/artifacts/split-coverage.out - | if [ "$DRONE_BUILD_EVENT" = "pull_request" ]; then cd ../upgradecluster - vagrant destroy -f + ../scripts/cleanup_vms.sh 'upgradecluster_([0-9]+)_(server|agent)' # Convert release-1.XX branch to v1.XX channel if [ "$DRONE_BRANCH" = "master" ]; then UPGRADE_CHANNEL="latest" @@ -700,7 +724,6 @@ steps: E2E_RELEASE_CHANNEL=$UPGRADE_CHANNEL go test -v -timeout=45m ./upgradecluster_test.go -ci -local -ginkgo.v cp ./coverage.out /tmp/artifacts/upgrade-coverage.out fi - - docker stop registry && docker rm registry volumes: - name: libvirt @@ -711,6 +734,9 @@ steps: path: /tmp/artifacts - name: upload to codecov + depends_on: + - test-e2e-validatecluster + - test-e2e-splitserver image: robertstettner/drone-codecov settings: token: diff --git a/tests/e2e/scripts/cleanup_vms.sh b/tests/e2e/scripts/cleanup_vms.sh index f5c1b909c2..f31e3957d3 100755 --- a/tests/e2e/scripts/cleanup_vms.sh +++ b/tests/e2e/scripts/cleanup_vms.sh @@ -1,21 +1,28 @@ #!/bin/bash -# Clean up any VMS that are older than 2 hours. -# +# Usage: ./cleanup_vms.sh [regex_pattern] +# Default pattern matches timestamped VMs older than 2 hours. # We embed the time in the VM name, so we can easily filter them out. # Get the current time in seconds since the epoch current_time=$(date +%s) + +def_pattern="_([0-9]+)_(server|agent)" +if [ -n "$1" ]; then + pattern="$1" +else + pattern="$def_pattern" +fi + # Get the list of VMs vms=$(virsh list --name --all) -time_regex="_([0-9]+)_(server|agent)" # Cleanup running VMs, happens if a previous test panics for vm in $vms; do - if [[ $vm =~ $time_regex ]]; then + if [[ $vm =~ $pattern ]]; then vm_time="${BASH_REMATCH[1]}" age=$((current_time - vm_time)) - if [ $age -gt 7200 ]; then + if [ $age -gt 7200 ] || [ "$pattern" != "$def_pattern" ]; then virsh destroy $vm virsh undefine $vm --remove-all-storage fi @@ -25,10 +32,10 @@ done # Cleanup inactive domains, happens if previous test is canceled vms=$(virsh list --name --inactive) for vm in $vms; do - if [[ $vm =~ $time_regex ]]; then + if [[ $vm =~ $pattern ]]; then vm_time="${BASH_REMATCH[1]}" age=$((current_time - vm_time)) - if [ $age -gt 7200 ]; then + if [ $age -gt 7200 ] || [ "$pattern" != "$def_pattern" ]; then virsh undefine $vm --remove-all-storage fi fi diff --git a/tests/e2e/scripts/drone_registries.sh b/tests/e2e/scripts/drone_registries.sh new file mode 100755 index 0000000000..aa1527ee25 --- /dev/null +++ b/tests/e2e/scripts/drone_registries.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +# Script to set up Docker registry proxies for various public registries +# Creates proxy registries for: +# - registry-1.docker.io +# - registry.k8s.io +# - gcr.io +# - quay.io +# - ghcr.io +# +# Not persistent - containers will not survive host reboot + +declare -A registries +declare -A registry_ports +registries=( + ["dockerhub"]="registry-1.docker.io" + ["k8s_io"]="registry.k8s.io" + ["gcr_io"]="gcr.io" + ["quay_io"]="quay.io" + ["ghcr_io"]="ghcr.io" +) +registry_ports=( + ["dockerhub"]=15000 + ["k8s_io"]=15001 + ["gcr_io"]=15002 + ["quay_io"]=15003 + ["ghcr_io"]=15004 +) + +# is_registry_running checsk if a registry is already exists +is_registry_running() { + local name=$1 + docker ps --format '{{.Names}}' | grep -q "^${name}$" + return $? +} + +create_registry_proxy() { + local name=$1 + local upstream=$2 + local port=$3 + + echo "Setting up registry proxy for ${upstream} on port ${port}" + + docker run -d \ + --name "${name}" \ + -e "REGISTRY_PROXY_REMOTEURL=https://${upstream}" \ + -e "REGISTRY_HTTP_SECRET=shared-secret" \ + -e "REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY=/var/lib/registry/$name" \ + -p "${port}:5000" \ + registry:2 + + echo "Registry proxy for ${upstream} started on port ${port}" +} + + + +# Set up each registry proxy +for name in "${!registries[@]}"; do + upstream=${registries[$name]} + port=${registry_ports[$name]} + if is_registry_running "registry_${name}"; then + echo "Registry proxy for ${upstream} already running" + else + create_registry_proxy "registry_${name}" "${upstream}" "${port}" + fi +done \ No newline at end of file diff --git a/tests/e2e/scripts/registry.sh b/tests/e2e/scripts/registry.sh index ef16ad616f..6beec5cca5 100644 --- a/tests/e2e/scripts/registry.sh +++ b/tests/e2e/scripts/registry.sh @@ -8,4 +8,16 @@ mkdir -p /etc/rancher/k3s/ echo "mirrors: docker.io: endpoint: - - \"http://$ip_addr:5000\"" >> /etc/rancher/k3s/registries.yaml \ No newline at end of file + - \"http://$ip_addr:15000\" + registry.k8s.io: + endpoint: + - \"http://$ip_addr:15001\" + gcr.io: + endpoint: + - \"http://$ip_addr:15002\" + quay.io: + endpoint: + - \"http://$ip_addr:15003\" + ghcr.io: + endpoint: + - \"http://$ip_addr:15004\"" >> /etc/rancher/k3s/registries.yaml \ No newline at end of file