Split E2E Drone pipeline into matrix (#12086)

* Split drone e2e into multiple blocks, explicit virsh cleanup

* Create multiple registries once and reuse as long as they exist

Signed-off-by: Derek Nola <derek.nola@suse.com>

.drone.yml

@@ -657,7 +657,9 @@ steps:
   - name: docker
     path: /var/run/docker.sock
 
-- name: test-e2e
+- name: test-e2e-validatecluster
+  depends_on:
+    - build-e2e-image
   image: test-e2e
   pull: never
   resources:
@@ -669,24 +671,46 @@ steps:
   commands:
   - mkdir -p dist/artifacts
   - cp /tmp/artifacts/* dist/artifacts/
-  - docker stop registry && docker rm registry
   # Cleanup VMs that are older than 2h. Happens if a previous test panics or is canceled
   - tests/e2e/scripts/cleanup_vms.sh
-  - docker run -d -p 5000:5000 -e REGISTRY_PROXY_REMOTEURL=https://registry-1.docker.io --name registry registry:2
+  - tests/e2e/scripts/drone_registries.sh
   - |
     cd tests/e2e/validatecluster
-    vagrant destroy -f
+    ../scripts/cleanup_vms.sh 'validatecluster_([0-9]+)_(server|agent)'
     go test -v -timeout=45m ./validatecluster_test.go -ci -local
     cp ./coverage.out /tmp/artifacts/validate-coverage.out
+  volumes:
+  - name: libvirt
+    path: /var/run/libvirt/
+  - name: docker
+    path: /var/run/docker.sock
+  - name: cache
+    path: /tmp/artifacts
+
+- name: test-e2e-splitserver
+  depends_on:
+    - build-e2e-image
+  image: test-e2e
+  pull: never
+  resources:
+    cpu: 6000
+    memory: 10Gi
+  environment:
+    E2E_REGISTRY: 'true'
+    E2E_GOCOVER: 'true'
+  commands:
+  - mkdir -p dist/artifacts
+  - cp /tmp/artifacts/* dist/artifacts/
+  - tests/e2e/scripts/drone_registries.sh
   - |
-    cd ../splitserver
-    vagrant destroy -f
+    cd tests/e2e/splitserver
+    ../scripts/cleanup_vms.sh 'splitserver_([0-9]+)'
     go test -v -timeout=30m ./splitserver_test.go -ci -local
     cp ./coverage.out /tmp/artifacts/split-coverage.out
   - |
     if [ "$DRONE_BUILD_EVENT" = "pull_request" ]; then
       cd ../upgradecluster
-      vagrant destroy -f
+      ../scripts/cleanup_vms.sh 'upgradecluster_([0-9]+)_(server|agent)'
       # Convert release-1.XX branch to v1.XX channel
       if [ "$DRONE_BRANCH" = "master" ]; then
         UPGRADE_CHANNEL="latest"
@@ -700,7 +724,6 @@ steps:
       E2E_RELEASE_CHANNEL=$UPGRADE_CHANNEL go test -v -timeout=45m ./upgradecluster_test.go  -ci -local -ginkgo.v
       cp ./coverage.out /tmp/artifacts/upgrade-coverage.out
     fi
-  - docker stop registry && docker rm registry
   
   volumes:
   - name: libvirt
@@ -711,6 +734,9 @@ steps:
     path: /tmp/artifacts
 
 - name: upload to codecov
+  depends_on:
+    - test-e2e-validatecluster
+    - test-e2e-splitserver
   image: robertstettner/drone-codecov
   settings:
     token: 

tests/e2e/scripts/cleanup_vms.sh

@@ -1,21 +1,28 @@
 #!/bin/bash
 
-# Clean up any VMS that are older than 2 hours.
-# 
+# Usage: ./cleanup_vms.sh [regex_pattern]
+# Default pattern matches timestamped VMs older than 2 hours.
 # We embed the time in the VM name, so we can easily filter them out.
 
 # Get the current time in seconds since the epoch
 current_time=$(date +%s)
 
+
+def_pattern="_([0-9]+)_(server|agent)"
+if [ -n "$1" ]; then
+    pattern="$1"
+else
+    pattern="$def_pattern"
+fi
+
 # Get the list of VMs
 vms=$(virsh list --name --all)
-time_regex="_([0-9]+)_(server|agent)"
 # Cleanup running VMs, happens if a previous test panics
 for vm in $vms; do
-    if [[ $vm =~ $time_regex ]]; then
+    if [[ $vm =~ $pattern ]]; then
         vm_time="${BASH_REMATCH[1]}"
         age=$((current_time - vm_time))
-        if [ $age -gt 7200 ]; then
+        if [ $age -gt 7200 ] || [ "$pattern" != "$def_pattern" ]; then
             virsh destroy $vm
             virsh undefine $vm --remove-all-storage
         fi
@@ -25,10 +32,10 @@ done
 # Cleanup inactive domains, happens if previous test is canceled
 vms=$(virsh list --name --inactive)
 for vm in $vms; do
-    if [[ $vm =~ $time_regex ]]; then
+    if [[ $vm =~ $pattern ]]; then
         vm_time="${BASH_REMATCH[1]}"
         age=$((current_time - vm_time))
-        if [ $age -gt 7200 ]; then
+        if [ $age -gt 7200 ] || [ "$pattern" != "$def_pattern" ]; then
             virsh undefine $vm  --remove-all-storage
         fi
     fi

tests/e2e/scripts/drone_registries.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+
+# Script to set up Docker registry proxies for various public registries
+# Creates proxy registries for: 
+# - registry-1.docker.io
+# - registry.k8s.io
+# - gcr.io
+# - quay.io
+# - ghcr.io
+#
+# Not persistent - containers will not survive host reboot
+
+declare -A registries
+declare -A registry_ports
+registries=(
+    ["dockerhub"]="registry-1.docker.io"
+    ["k8s_io"]="registry.k8s.io"
+    ["gcr_io"]="gcr.io"
+    ["quay_io"]="quay.io"
+    ["ghcr_io"]="ghcr.io"
+)
+registry_ports=(
+    ["dockerhub"]=15000
+    ["k8s_io"]=15001
+    ["gcr_io"]=15002
+    ["quay_io"]=15003
+    ["ghcr_io"]=15004
+)
+
+# is_registry_running checsk if a registry is already exists
+is_registry_running() {
+    local name=$1
+    docker ps --format '{{.Names}}' | grep -q "^${name}$"
+    return $?
+}
+
+create_registry_proxy() {
+    local name=$1
+    local upstream=$2
+    local port=$3
+    
+    echo "Setting up registry proxy for ${upstream} on port ${port}"
+    
+    docker run -d \
+        --name "${name}" \
+        -e "REGISTRY_PROXY_REMOTEURL=https://${upstream}" \
+        -e "REGISTRY_HTTP_SECRET=shared-secret" \
+        -e "REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY=/var/lib/registry/$name" \
+        -p "${port}:5000" \
+        registry:2
+        
+    echo "Registry proxy for ${upstream} started on port ${port}"
+}
+
+
+
+# Set up each registry proxy
+for name in "${!registries[@]}"; do
+    upstream=${registries[$name]}
+    port=${registry_ports[$name]}
+    if is_registry_running "registry_${name}"; then
+        echo "Registry proxy for ${upstream} already running"
+    else
+        create_registry_proxy "registry_${name}" "${upstream}" "${port}"
+    fi
+done

tests/e2e/scripts/registry.sh

@@ -8,4 +8,16 @@ mkdir -p /etc/rancher/k3s/
 echo "mirrors:
   docker.io:
     endpoint:
-      - \"http://$ip_addr:5000\"" >> /etc/rancher/k3s/registries.yaml
+      - \"http://$ip_addr:15000\"
+  registry.k8s.io:
+    endpoint:
+      - \"http://$ip_addr:15001\"
+  gcr.io:
+    endpoint:
+      - \"http://$ip_addr:15002\"
+  quay.io:
+    endpoint:
+      - \"http://$ip_addr:15003\"
+  ghcr.io:
+    endpoint:
+      - \"http://$ip_addr:15004\"" >> /etc/rancher/k3s/registries.yaml