chore: Add test to make sure flags are up to date and sorted (#4845)

Signed-off-by: Luke Massa <lukefrederickmassa@gmail.com>
Co-authored-by: PePe Amengual <2208324+jamengual@users.noreply.github.com>

cmd/server_test.go

@@ -14,10 +14,13 @@
 package cmd
 
 import (
+	"bufio"
+	"cmp"
 	"fmt"
 	"os"
 	"path/filepath"
 	"reflect"
+	"slices"
 	"strings"
 	"testing"
 
@@ -226,21 +229,32 @@ func TestExecute_Flags(t *testing.T) {
 	}
 }
 
-func TestUserConfigAllTested(t *testing.T) {
-	t.Log("All settings in userConfig should be tested.")
-
+func getUserConfigKeysWithFlags() []string {
+	var ret []string
 	u := reflect.TypeOf(server.UserConfig{})
 
 	for i := 0; i < u.NumField(); i++ {
 
 		userConfigKey := u.Field(i).Tag.Get("mapstructure")
+		// By default, we expect all fields in UserConfig to have flags defined in server.go and tested here in server_test.go
+		// Some fields are too complicated to have flags, so are only expressible in the config yaml
+		flagKey := u.Field(i).Tag.Get("flag")
+		if flagKey == "false" {
+			continue
+		}
+		ret = append(ret, userConfigKey)
+
+	}
+	return ret
+
+}
+
+func TestUserConfigAllTested(t *testing.T) {
+	t.Log("All settings in userConfig should be tested.")
+
+	for _, userConfigKey := range getUserConfigKeysWithFlags() {
+
 		t.Run(userConfigKey, func(t *testing.T) {
-			// By default, we expect all fields in UserConfig to have flags defined in server.go and tested here in server_test.go
-			// Some fields are too complicated to have flags, so are only expressible in the config yaml
-			flagKey := u.Field(i).Tag.Get("flag")
-			if flagKey == "false" {
-				return
-			}
 			// If a setting is configured in server.UserConfig, it should be tested here. If there is no corresponding const
 			// for specifying the flag, that probably means one *also* needs to be added to server.go
 			if _, ok := testFlags[userConfigKey]; !ok {
@@ -252,6 +266,94 @@ func TestUserConfigAllTested(t *testing.T) {
 
 }
 
+func getDocumentedFlags(t *testing.T) []string {
+
+	var ret []string
+	docFile := "../runatlantis.io/docs/server-configuration.md"
+
+	file, err := os.Open(docFile)
+	Ok(t, err)
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if !strings.HasPrefix(line, "### ") {
+			continue
+		}
+		split := strings.Split(line, "`")
+		if len(split) != 3 {
+			t.Errorf("Unexpected line in %s: %s", docFile, line)
+			continue
+		}
+		flag := split[1]
+		if !strings.HasPrefix(flag, "--") {
+			t.Errorf("Unexpected line in %s: %s", docFile, line)
+			continue
+		}
+		flag = strings.TrimPrefix(flag, "--")
+		ret = append(ret, flag)
+	}
+
+	err = scanner.Err()
+	Ok(t, err)
+
+	return ret
+}
+
+func testIsSorted[S ~[]E, E cmp.Ordered](t *testing.T, x S) {
+	// TODO: This is n^2, probably a better algorithm for this
+	// Also, this works best for lists that are mostly sorted, if the whole thing is wrong, it's just
+	// going to say that every individual element is out of order
+	for i, elem := range x {
+		for j, compareTo := range x {
+			if i == j {
+				continue
+			}
+			if i > j && cmp.Less(elem, compareTo) {
+				t.Errorf("%v is out of order (should be before %v)", elem, compareTo)
+				break
+			}
+			if i < j && cmp.Less(compareTo, elem) {
+				t.Errorf("%v is out of order (should be after %v)", elem, compareTo)
+				break
+			}
+		}
+	}
+}
+
+func TestAllFlagsDocumented(t *testing.T) {
+	// This is not a unit test per se, but is a helpful way of making sure when flags are added/removed
+	// the corresponding documentation is kept up-to-date.
+	t.Log("All flags in userConfig should have documentation in server-configuration.md.")
+
+	userConfigKeys := getUserConfigKeysWithFlags()
+	documentedFlags := getDocumentedFlags(t)
+
+	testIsSorted(t, documentedFlags)
+	slices.Sort(userConfigKeys)
+	slices.Sort(documentedFlags)
+
+	for _, userConfigKey := range userConfigKeys {
+		_, found := slices.BinarySearch(documentedFlags, userConfigKey)
+		if !found {
+			t.Errorf("Found undocumented config key: %s", userConfigKey)
+		}
+	}
+
+	for _, documentedFlag := range documentedFlags {
+		// --help and --config are documented but don't have a setting on userConfig
+		if documentedFlag == "help" || documentedFlag == "config" {
+			continue
+		}
+		_, found := slices.BinarySearch(userConfigKeys, documentedFlag)
+		if !found {
+			t.Errorf("Found documentation for flag that doesn't exist: %s", documentedFlag)
+		}
+	}
+
+}
+
 func TestExecute_ConfigFile(t *testing.T) {
 	t.Log("Should use all the values from the config file.")
 	// Use yaml package to quote values that need quoting