refactor: use hc-install for TF downloads + constraints (#4494)

Co-authored-by: PePe Amengual <jose.amengual@gmail.com>

server/controllers/events/events_controller_e2e_test.go

@@ -53,12 +53,12 @@ var mockPreWorkflowHookRunner *runtimemocks.MockPreWorkflowHookRunner
 
 var mockPostWorkflowHookRunner *runtimemocks.MockPostWorkflowHookRunner
 
-func (m *NoopTFDownloader) GetFile(_, _ string) error {
+func (m *NoopTFDownloader) GetAny(_, _ string) error {
 	return nil
 }
 
-func (m *NoopTFDownloader) GetAny(_, _ string) error {
-	return nil
+func (m *NoopTFDownloader) Install(_ string, _ string, _ *version.Version) (string, error) {
+	return "", nil
 }
 
 type LocalConftestCache struct {

server/core/runtime/policy/conftest_client_test.go

@@ -33,7 +33,6 @@ func TestConfTestVersionDownloader(t *testing.T) {
 
 	t.Run("success", func(t *testing.T) {
 
-		When(mockDownloader.GetFile(Eq(destPath), Eq(fullURL))).ThenReturn(nil)
 		binPath, err := subject.downloadConfTestVersion(version, destPath)
 
 		mockDownloader.VerifyWasCalledOnce().GetAny(Eq(destPath), Eq(fullURL))

server/core/terraform/mocks/mock_downloader.go

@@ -4,6 +4,7 @@
 package mocks
 
 import (
+	go_version "github.com/hashicorp/go-version"
 	pegomock "github.com/petergtz/pegomock/v4"
 	"reflect"
 	"time"
@@ -39,19 +40,23 @@ func (mock *MockDownloader) GetAny(dst string, src string) error {
 	return ret0
 }
 
-func (mock *MockDownloader) GetFile(dst string, src string) error {
+func (mock *MockDownloader) Install(dir string, downloadURL string, v *go_version.Version) (string, error) {
 	if mock == nil {
 		panic("mock must not be nil. Use myMock := NewMockDownloader().")
 	}
-	params := []pegomock.Param{dst, src}
-	result := pegomock.GetGenericMockFrom(mock).Invoke("GetFile", params, []reflect.Type{reflect.TypeOf((*error)(nil)).Elem()})
-	var ret0 error
+	params := []pegomock.Param{dir, downloadURL, v}
+	result := pegomock.GetGenericMockFrom(mock).Invoke("Install", params, []reflect.Type{reflect.TypeOf((*string)(nil)).Elem(), reflect.TypeOf((*error)(nil)).Elem()})
+	var ret0 string
+	var ret1 error
 	if len(result) != 0 {
 		if result[0] != nil {
-			ret0 = result[0].(error)
+			ret0 = result[0].(string)
+		}
+		if result[1] != nil {
+			ret1 = result[1].(error)
 		}
 	}
-	return ret0
+	return ret0, ret1
 }
 
 func (mock *MockDownloader) VerifyWasCalledOnce() *VerifierMockDownloader {
@@ -122,23 +127,23 @@ func (c *MockDownloader_GetAny_OngoingVerification) GetAllCapturedArguments() (_
 	return
 }
 
-func (verifier *VerifierMockDownloader) GetFile(dst string, src string) *MockDownloader_GetFile_OngoingVerification {
-	params := []pegomock.Param{dst, src}
-	methodInvocations := pegomock.GetGenericMockFrom(verifier.mock).Verify(verifier.inOrderContext, verifier.invocationCountMatcher, "GetFile", params, verifier.timeout)
-	return &MockDownloader_GetFile_OngoingVerification{mock: verifier.mock, methodInvocations: methodInvocations}
+func (verifier *VerifierMockDownloader) Install(dir string, downloadURL string, v *go_version.Version) *MockDownloader_Install_OngoingVerification {
+	params := []pegomock.Param{dir, downloadURL, v}
+	methodInvocations := pegomock.GetGenericMockFrom(verifier.mock).Verify(verifier.inOrderContext, verifier.invocationCountMatcher, "Install", params, verifier.timeout)
+	return &MockDownloader_Install_OngoingVerification{mock: verifier.mock, methodInvocations: methodInvocations}
 }
 
-type MockDownloader_GetFile_OngoingVerification struct {
+type MockDownloader_Install_OngoingVerification struct {
 	mock              *MockDownloader
 	methodInvocations []pegomock.MethodInvocation
 }
 
-func (c *MockDownloader_GetFile_OngoingVerification) GetCapturedArguments() (string, string) {
-	dst, src := c.GetAllCapturedArguments()
-	return dst[len(dst)-1], src[len(src)-1]
+func (c *MockDownloader_Install_OngoingVerification) GetCapturedArguments() (string, string, *go_version.Version) {
+	dir, downloadURL, v := c.GetAllCapturedArguments()
+	return dir[len(dir)-1], downloadURL[len(downloadURL)-1], v[len(v)-1]
 }
 
-func (c *MockDownloader_GetFile_OngoingVerification) GetAllCapturedArguments() (_param0 []string, _param1 []string) {
+func (c *MockDownloader_Install_OngoingVerification) GetAllCapturedArguments() (_param0 []string, _param1 []string, _param2 []*go_version.Version) {
 	params := pegomock.GetGenericMockFrom(c.mock).GetInvocationParams(c.methodInvocations)
 	if len(params) > 0 {
 		_param0 = make([]string, len(c.methodInvocations))
@@ -149,6 +154,10 @@ func (c *MockDownloader_GetFile_OngoingVerification) GetAllCapturedArguments() (
 		for u, param := range params[1] {
 			_param1[u] = param.(string)
 		}
+		_param2 = make([]*go_version.Version, len(c.methodInvocations))
+		for u, param := range params[2] {
+			_param2[u] = param.(*go_version.Version)
+		}
 	}
 	return
 }

server/core/terraform/mocks/mock_terraform_client.go

@@ -57,25 +57,6 @@ func (mock *MockClient) EnsureVersion(log logging.SimpleLogging, v *go_version.V
 	return ret0
 }
 
-func (mock *MockClient) ListAvailableVersions(log logging.SimpleLogging) ([]string, error) {
-	if mock == nil {
-		panic("mock must not be nil. Use myMock := NewMockClient().")
-	}
-	params := []pegomock.Param{log}
-	result := pegomock.GetGenericMockFrom(mock).Invoke("ListAvailableVersions", params, []reflect.Type{reflect.TypeOf((*[]string)(nil)).Elem(), reflect.TypeOf((*error)(nil)).Elem()})
-	var ret0 []string
-	var ret1 error
-	if len(result) != 0 {
-		if result[0] != nil {
-			ret0 = result[0].([]string)
-		}
-		if result[1] != nil {
-			ret1 = result[1].(error)
-		}
-	}
-	return ret0, ret1
-}
-
 func (mock *MockClient) RunCommandWithVersion(ctx command.ProjectContext, path string, args []string, envs map[string]string, v *go_version.Version, workspace string) (string, error) {
 	if mock == nil {
 		panic("mock must not be nil. Use myMock := NewMockClient().")
@@ -194,33 +175,6 @@ func (c *MockClient_EnsureVersion_OngoingVerification) GetAllCapturedArguments()
 	return
 }
 
-func (verifier *VerifierMockClient) ListAvailableVersions(log logging.SimpleLogging) *MockClient_ListAvailableVersions_OngoingVerification {
-	params := []pegomock.Param{log}
-	methodInvocations := pegomock.GetGenericMockFrom(verifier.mock).Verify(verifier.inOrderContext, verifier.invocationCountMatcher, "ListAvailableVersions", params, verifier.timeout)
-	return &MockClient_ListAvailableVersions_OngoingVerification{mock: verifier.mock, methodInvocations: methodInvocations}
-}
-
-type MockClient_ListAvailableVersions_OngoingVerification struct {
-	mock              *MockClient
-	methodInvocations []pegomock.MethodInvocation
-}
-
-func (c *MockClient_ListAvailableVersions_OngoingVerification) GetCapturedArguments() logging.SimpleLogging {
-	log := c.GetAllCapturedArguments()
-	return log[len(log)-1]
-}
-
-func (c *MockClient_ListAvailableVersions_OngoingVerification) GetAllCapturedArguments() (_param0 []logging.SimpleLogging) {
-	params := pegomock.GetGenericMockFrom(c.mock).GetInvocationParams(c.methodInvocations)
-	if len(params) > 0 {
-		_param0 = make([]logging.SimpleLogging, len(c.methodInvocations))
-		for u, param := range params[0] {
-			_param0[u] = param.(logging.SimpleLogging)
-		}
-	}
-	return
-}
-
 func (verifier *VerifierMockClient) RunCommandWithVersion(ctx command.ProjectContext, path string, args []string, envs map[string]string, v *go_version.Version, workspace string) *MockClient_RunCommandWithVersion_OngoingVerification {
 	params := []pegomock.Param{ctx, path, args, envs, v, workspace}
 	methodInvocations := pegomock.GetGenericMockFrom(verifier.mock).Verify(verifier.inOrderContext, verifier.invocationCountMatcher, "RunCommandWithVersion", params, verifier.timeout)

server/core/terraform/terraform_client.go

@@ -19,23 +19,23 @@ package terraform
 import (
 	"context"
 	"fmt"
-	"net/http"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"regexp"
-	"runtime"
-	"sort"
 	"strings"
 	"sync"
 	"time"
 
 	"github.com/hashicorp/go-getter/v2"
 	"github.com/hashicorp/go-version"
+	install "github.com/hashicorp/hc-install"
+	"github.com/hashicorp/hc-install/product"
+	"github.com/hashicorp/hc-install/releases"
+	"github.com/hashicorp/hc-install/src"
 	"github.com/hashicorp/terraform-config-inspect/tfconfig"
 	"github.com/mitchellh/go-homedir"
 	"github.com/pkg/errors"
-	"github.com/warrensbox/terraform-switcher/lib"
 
 	"github.com/runatlantis/atlantis/server/core/runtime/models"
 	"github.com/runatlantis/atlantis/server/events/command"
@@ -57,9 +57,6 @@ type Client interface {
 	// EnsureVersion makes sure that terraform version `v` is available to use
 	EnsureVersion(log logging.SimpleLogging, v *version.Version) error
 
-	// ListAvailableVersions returns all available version of Terraform, if available; otherwise this will return an empty list.
-	ListAvailableVersions(log logging.SimpleLogging) ([]string, error)
-
 	// DetectVersion Extracts required_version from Terraform configuration in the specified project directory. Returns nil if unable to determine the version.
 	DetectVersion(log logging.SimpleLogging, projectDirectory string) *version.Version
 }
@@ -97,7 +94,7 @@ type DefaultClient struct {
 
 // Downloader is for downloading terraform versions.
 type Downloader interface {
-	GetFile(dst, src string) error
+	Install(dir string, downloadURL string, v *version.Version) (string, error)
 	GetAny(dst, src string) error
 }
 
@@ -278,99 +275,83 @@ func (c *DefaultClient) TerraformBinDir() string {
 	return c.binDir
 }
 
-// ListAvailableVersions returns all available version of Terraform. If downloads are not allowed, this will return an empty list.
-func (c *DefaultClient) ListAvailableVersions(log logging.SimpleLogging) ([]string, error) {
-	url := fmt.Sprintf("%s/terraform", c.downloadBaseURL)
-
-	if !c.downloadAllowed {
-		log.Debug("Terraform downloads disabled. Won't list Terraform versions available at %s", url)
-		return []string{}, nil
+// ExtractExactRegex attempts to extract an exact version number from the provided string as a fallback.
+// The function expects the version string to be in one of the following formats: "= x.y.z", "=x.y.z", or "x.y.z" where x, y, and z are integers.
+// If the version string matches one of these formats, the function returns a slice containing the exact version number.
+// If the version string does not match any of these formats, the function logs a debug message and returns nil.
+func (c *DefaultClient) ExtractExactRegex(log logging.SimpleLogging, version string) []string {
+	re := regexp.MustCompile(`^=?\s*([0-9.]+)\s*$`)
+	matched := re.FindStringSubmatch(version)
+	if len(matched) == 0 {
+		log.Debug("exact version regex not found in the version %q", version)
+		return nil
 	}
-
-	log.Debug("Listing Terraform versions available at: %s", url)
-
-	// terraform-switcher calls os.Exit(1) if it fails to successfully GET the configured URL.
-	// So, before calling it, test if we can connect. Then we can return an error instead if the request fails.
-	resp, err := http.Get(url) // #nosec G107 -- terraform-switch makes this same call below. Also, we don't process the response payload.
-	if err != nil {
-		return nil, fmt.Errorf("Unable to list Terraform versions: %s", err)
-	}
-	defer resp.Body.Close() // nolint: errcheck
-
-	if resp.StatusCode != http.StatusOK {
-		return nil, fmt.Errorf("Unable to list Terraform versions: response code %d from %s", resp.StatusCode, url)
-	}
-
-	versions, err := lib.GetTFList(url, true)
-	return versions, err
+	// The first element of the slice is the entire string, so we want the second element (the first capture group)
+	tfVersions := []string{matched[1]}
+	log.Debug("extracted exact version %q from version %q", tfVersions[0], version)
+	return tfVersions
 }
 
-// DetectVersion Extracts required_version from Terraform configuration in the specified project directory. Returns nil if unable to determine the version.
-// This will also try to intelligently evaluate non-exact matches by listing the available versions of Terraform and picking the best match.
+// DetectVersion extracts required_version from Terraform configuration in the specified project directory. Returns nil if unable to determine the version.
+// It will also try to evaluate non-exact matches by passing the Constraints to the hc-install Releases API, which will return a list of available versions.
+// It will then select the highest version that satisfies the constraint.
 func (c *DefaultClient) DetectVersion(log logging.SimpleLogging, projectDirectory string) *version.Version {
 	module, diags := tfconfig.LoadModule(projectDirectory)
 	if diags.HasErrors() {
-		log.Err("Trying to detect required version: %s", diags.Error())
+		log.Err("trying to detect required version: %s", diags.Error())
 	}
 
 	if len(module.RequiredCore) != 1 {
-		log.Info("Cannot determine which version to use from terraform configuration, detected %d possibilities.", len(module.RequiredCore))
+		log.Info("cannot determine which version to use from terraform configuration, detected %d possibilities.", len(module.RequiredCore))
 		return nil
 	}
 	requiredVersionSetting := module.RequiredCore[0]
 	log.Debug("Found required_version setting of %q", requiredVersionSetting)
 
-	tfVersions, err := c.ListAvailableVersions(log)
-	if err != nil {
-		log.Err("Unable to list Terraform versions, may fall back to default: %s", err)
-	}
-
-	if len(tfVersions) == 0 {
-		// Fall back to an exact required version string
-		// We allow `= x.y.z`, `=x.y.z` or `x.y.z` where `x`, `y` and `z` are integers.
-		re := regexp.MustCompile(`^=?\s*([0-9.]+)\s*$`)
-		matched := re.FindStringSubmatch(requiredVersionSetting)
+	if !c.downloadAllowed {
+		log.Debug("terraform downloads disabled.")
+		matched := c.ExtractExactRegex(log, requiredVersionSetting)
 		if len(matched) == 0 {
-			log.Debug("Did not specify exact version in terraform configuration, found %q", requiredVersionSetting)
+			log.Debug("did not specify exact version in terraform configuration, found %q", requiredVersionSetting)
 			return nil
 		}
-		tfVersions = []string{matched[1]}
-	}
 
-	constraint, _ := version.NewConstraint(requiredVersionSetting)
-	// Since terraform version 1.8.2, terraform is not a single file download anymore and
-	// Atlantis fails to download version 1.8.2 and higher. So, as a short-term fix,
-	// we need to block any version higher than 1.8.1 until proper solution is implemented.
-	// More details on the issue here - https://github.com/runatlantis/atlantis/issues/4471
-	highestSupportedConstraint, _ := version.NewConstraint("<= 1.8.1")
-	versions := make([]*version.Version, len(tfVersions))
-
-	for i, tfvals := range tfVersions {
-		newVersion, err := version.NewVersion(tfvals)
-		if err == nil {
-			versions[i] = newVersion
+		version, err := version.NewVersion(matched[0])
+		if err != nil {
+			log.Err("error parsing version string: %s", err)
+			return nil
 		}
+		return version
 	}
 
-	if len(versions) == 0 {
-		log.Debug("Did not specify exact valid version in terraform configuration, found %q", requiredVersionSetting)
+	constraintStr := requiredVersionSetting
+	vc, err := version.NewConstraint(constraintStr)
+	if err != nil {
+		log.Err("Error parsing constraint string: %s", err)
 		return nil
 	}
 
-	sort.Sort(sort.Reverse(version.Collection(versions)))
-
-	for _, element := range versions {
-		if constraint.Check(element) && highestSupportedConstraint.Check(element) { // Validate a version against a constraint
-			tfversionStr := element.String()
-			if lib.ValidVersionFormat(tfversionStr) { //check if version format is correct
-				tfversion, _ := version.NewVersion(tfversionStr)
-				log.Info("Detected module requires version: %s", tfversionStr)
-				return tfversion
-			}
-		}
+	constrainedVersions := &releases.Versions{
+		Product:     product.Terraform,
+		Constraints: vc,
 	}
-	log.Debug("Could not match any valid terraform version with %q", requiredVersionSetting)
-	return nil
+	installCandidates, err := constrainedVersions.List(context.Background())
+	if err != nil {
+		log.Err("error listing available versions: %s", err)
+		return nil
+	}
+	if len(installCandidates) == 0 {
+		log.Err("no Terraform versions found for constraints %s", constraintStr)
+		return nil
+	}
+
+	// We want to select the highest version that satisfies the constraint.
+	versionDownloader := installCandidates[len(installCandidates)-1]
+
+	// Get the Version object from the versionDownloader.
+	downloadVersion := versionDownloader.(*releases.ExactVersion).Version
+
+	return downloadVersion
 }
 
 // See Client.EnsureVersion.
@@ -532,7 +513,15 @@ func MustConstraint(v string) version.Constraints {
 
 // ensureVersion returns the path to a terraform binary of version v.
 // It will download this version if we don't have it.
-func ensureVersion(log logging.SimpleLogging, dl Downloader, versions map[string]string, v *version.Version, binDir string, downloadURL string, downloadsAllowed bool) (string, error) {
+func ensureVersion(
+	log logging.SimpleLogging,
+	dl Downloader,
+	versions map[string]string,
+	v *version.Version,
+	binDir string,
+	downloadURL string,
+	downloadsAllowed bool,
+) (string, error) {
 	if binPath, ok := versions[v.String()]; ok {
 		return binPath, nil
 	}
@@ -554,21 +543,25 @@ func ensureVersion(log logging.SimpleLogging, dl Downloader, versions map[string
 		return dest, nil
 	}
 	if !downloadsAllowed {
-		return "", fmt.Errorf("Could not find terraform version %s in PATH or %s, and downloads are disabled", v.String(), binDir)
+		return "", fmt.Errorf(
+			"could not find terraform version %s in PATH or %s, and downloads are disabled",
+			v.String(),
+			binDir,
+		)
 	}
 
-	log.Info("Could not find terraform version %s in PATH or %s, downloading from %s", v.String(), binDir, downloadURL)
-	urlPrefix := fmt.Sprintf("%s/terraform/%s/terraform_%s", downloadURL, v.String(), v.String())
-	binURL := fmt.Sprintf("%s_%s_%s.zip", urlPrefix, runtime.GOOS, runtime.GOARCH)
-	checksumURL := fmt.Sprintf("%s_SHA256SUMS", urlPrefix)
-	fullSrcURL := fmt.Sprintf("%s?checksum=file:%s", binURL, checksumURL)
-	if err := dl.GetFile(dest, fullSrcURL); err != nil {
-		return "", errors.Wrapf(err, "downloading terraform version %s at %q", v.String(), fullSrcURL)
+	log.Info("could not find terraform version %s in PATH or %s", v.String(), binDir)
+
+	log.Info("using Hashicorp's 'hc-install' to download Terraform version %s from download URL %s", v.String(), downloadURL)
+	execPath, err := dl.Install(binDir, downloadURL, v)
+
+	if err != nil {
+		return "", errors.Wrapf(err, "error downloading terraform version %s", v.String())
 	}
 
-	log.Info("Downloaded terraform %s to %s", v.String(), dest)
-	versions[v.String()] = dest
-	return dest, nil
+	log.Info("Downloaded terraform %s to %s", v.String(), execPath)
+	versions[v.String()] = execPath
+	return execPath, nil
 }
 
 // generateRCFile generates a .terraformrc file containing config for tfeToken
@@ -632,13 +625,31 @@ var rcFileContents = `credentials "%s" {
 
 type DefaultDownloader struct{}
 
-// See go-getter.GetFile.
-func (d *DefaultDownloader) GetFile(dst, src string) error {
-	_, err := getter.GetFile(context.Background(), dst, src)
-	return err
+func (d *DefaultDownloader) Install(dir string, downloadURL string, v *version.Version) (string, error) {
+	installer := install.NewInstaller()
+	execPath, err := installer.Install(context.Background(), []src.Installable{
+		&releases.ExactVersion{
+			Product:    product.Terraform,
+			Version:    v,
+			InstallDir: dir,
+			ApiBaseURL: downloadURL,
+		},
+	})
+	if err != nil {
+		return "", err
+	}
+
+	// hc-install installs terraform binary as just "terraform".
+	// We need to rename it to terraform{version} to be consistent with current naming convention.
+	newPath := filepath.Join(dir, "terraform"+v.String())
+	if err := os.Rename(execPath, newPath); err != nil {
+		return "", err
+	}
+
+	return newPath, nil
 }
 
-// See go-getter.GetFile.
+// See go-getter.GetAny.
 func (d *DefaultDownloader) GetAny(dst, src string) error {
 	_, err := getter.GetAny(context.Background(), dst, src)
 	return err

server/core/terraform/terraform_client_test.go

@@ -17,7 +17,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
-	"runtime"
+	"reflect"
 	"strings"
 	"testing"
 	"time"
@@ -211,22 +211,18 @@ func TestNewClient_DefaultTFFlagDownload(t *testing.T) {
 	defer tempSetEnv(t, "PATH", "")()
 
 	mockDownloader := mocks.NewMockDownloader()
-	When(mockDownloader.GetFile(Any[string](), Any[string]())).Then(func(params []pegomock.Param) pegomock.ReturnValues {
-		err := os.WriteFile(params[0].(string), []byte("#!/bin/sh\necho '\nTerraform v0.11.10\n'"), 0700) // #nosec G306
-		return []pegomock.ReturnValue{err}
+	When(mockDownloader.Install(Any[string](), Any[string](), Any[*version.Version]())).Then(func(params []pegomock.Param) pegomock.ReturnValues {
+		binPath := filepath.Join(params[0].(string), "terraform0.11.10")
+		err := os.WriteFile(binPath, []byte("#!/bin/sh\necho '\nTerraform v0.11.10\n'"), 0700) // #nosec G306
+		return []pegomock.ReturnValue{binPath, err}
 	})
-	c, err := terraform.NewClient(logger, binDir, cacheDir, "", "", "0.11.10", cmd.DefaultTFVersionFlag, "https://my-mirror.releases.mycompany.com", mockDownloader, true, true, projectCmdOutputHandler)
+	c, err := terraform.NewClient(logger, binDir, cacheDir, "", "", "0.11.10", cmd.DefaultTFVersionFlag, cmd.DefaultTFDownloadURL, mockDownloader, true, true, projectCmdOutputHandler)
 	Ok(t, err)
 
 	Ok(t, err)
 	Equals(t, "0.11.10", c.DefaultVersion().String())
-	baseURL := "https://my-mirror.releases.mycompany.com/terraform/0.11.10"
-	expURL := fmt.Sprintf("%s/terraform_0.11.10_%s_%s.zip?checksum=file:%s/terraform_0.11.10_SHA256SUMS",
-		baseURL,
-		runtime.GOOS,
-		runtime.GOARCH,
-		baseURL)
-	mockDownloader.VerifyWasCalledEventually(Once(), 2*time.Second).GetFile(filepath.Join(tmp, "bin", "terraform0.11.10"), expURL)
+
+	mockDownloader.VerifyWasCalledEventually(Once(), 2*time.Second).Install(binDir, cmd.DefaultTFDownloadURL, version.Must(version.NewVersion("0.11.10")))
 
 	// Reset PATH so that it has sh.
 	Ok(t, os.Setenv("PATH", orig))
@@ -257,26 +253,21 @@ func TestRunCommandWithVersion_DLsTF(t *testing.T) {
 		RepoRelDir: ".",
 	}
 
+	v, err := version.NewVersion("99.99.99")
+	Ok(t, err)
+
 	mockDownloader := mocks.NewMockDownloader()
 	// Set up our mock downloader to write a fake tf binary when it's called.
-	baseURL := fmt.Sprintf("%s/terraform/99.99.99", cmd.DefaultTFDownloadURL)
-	expURL := fmt.Sprintf("%s/terraform_99.99.99_%s_%s.zip?checksum=file:%s/terraform_99.99.99_SHA256SUMS",
-		baseURL,
-		runtime.GOOS,
-		runtime.GOARCH,
-		baseURL)
-	When(mockDownloader.GetFile(filepath.Join(tmp, "bin", "terraform99.99.99"), expURL)).Then(func(params []pegomock.Param) pegomock.ReturnValues {
-		err := os.WriteFile(params[0].(string), []byte("#!/bin/sh\necho '\nTerraform v99.99.99\n'"), 0700) // #nosec G306
-		return []pegomock.ReturnValue{err}
+	When(mockDownloader.Install(binDir, cmd.DefaultTFDownloadURL, v)).Then(func(params []pegomock.Param) pegomock.ReturnValues {
+		binPath := filepath.Join(params[0].(string), "terraform99.99.99")
+		err := os.WriteFile(binPath, []byte("#!/bin/sh\necho '\nTerraform v99.99.99\n'"), 0700) // #nosec G306
+		return []pegomock.ReturnValue{binPath, err}
 	})
 
 	c, err := terraform.NewClient(logger, binDir, cacheDir, "", "", "0.11.10", cmd.DefaultTFVersionFlag, cmd.DefaultTFDownloadURL, mockDownloader, true, true, projectCmdOutputHandler)
 	Ok(t, err)
 	Equals(t, "0.11.10", c.DefaultVersion().String())
 
-	v, err := version.NewVersion("99.99.99")
-	Ok(t, err)
-
 	output, err := c.RunCommandWithVersion(ctx, tmp, []string{"terraform", "init"}, map[string]string{}, v, "")
 
 	Assert(t, err == nil, "err: %s: %s", err, output)
@@ -287,7 +278,7 @@ func TestRunCommandWithVersion_DLsTF(t *testing.T) {
 func TestEnsureVersion_downloaded(t *testing.T) {
 	logger := logging.NewNoopLogger(t)
 	RegisterMockTestingT(t)
-	tmp, binDir, cacheDir := mkSubDirs(t)
+	_, binDir, cacheDir := mkSubDirs(t)
 	projectCmdOutputHandler := jobmocks.NewMockProjectCommandOutputHandler()
 
 	mockDownloader := mocks.NewMockDownloader()
@@ -300,17 +291,49 @@ func TestEnsureVersion_downloaded(t *testing.T) {
 	v, err := version.NewVersion("99.99.99")
 	Ok(t, err)
 
+	When(mockDownloader.Install(binDir, cmd.DefaultTFDownloadURL, v)).Then(func(params []pegomock.Param) pegomock.ReturnValues {
+		binPath := filepath.Join(params[0].(string), "terraform99.99.99")
+		err := os.WriteFile(binPath, []byte("#!/bin/sh\necho '\nTerraform v99.99.99\n'"), 0700) // #nosec G306
+		return []pegomock.ReturnValue{binPath, err}
+	})
+
 	err = c.EnsureVersion(logger, v)
 
 	Ok(t, err)
 
-	baseURL := fmt.Sprintf("%s/terraform/99.99.99", cmd.DefaultTFDownloadURL)
-	expURL := fmt.Sprintf("%s/terraform_99.99.99_%s_%s.zip?checksum=file:%s/terraform_99.99.99_SHA256SUMS",
-		baseURL,
-		runtime.GOOS,
-		runtime.GOARCH,
-		baseURL)
-	mockDownloader.VerifyWasCalledEventually(Once(), 2*time.Second).GetFile(filepath.Join(tmp, "bin", "terraform99.99.99"), expURL)
+	mockDownloader.VerifyWasCalledEventually(Once(), 2*time.Second).Install(binDir, cmd.DefaultTFDownloadURL, v)
+}
+
+// Test that EnsureVersion downloads terraform from a custom URL.
+func TestEnsureVersion_downloaded_customURL(t *testing.T) {
+	logger := logging.NewNoopLogger(t)
+	RegisterMockTestingT(t)
+	_, binDir, cacheDir := mkSubDirs(t)
+	projectCmdOutputHandler := jobmocks.NewMockProjectCommandOutputHandler()
+
+	mockDownloader := mocks.NewMockDownloader()
+	downloadsAllowed := true
+	customURL := "http://releases.example.com"
+
+	c, err := terraform.NewTestClient(logger, binDir, cacheDir, "", "", "0.11.10", cmd.DefaultTFVersionFlag, customURL, mockDownloader, downloadsAllowed, true, projectCmdOutputHandler)
+	Ok(t, err)
+
+	Equals(t, "0.11.10", c.DefaultVersion().String())
+
+	v, err := version.NewVersion("99.99.99")
+	Ok(t, err)
+
+	When(mockDownloader.Install(binDir, customURL, v)).Then(func(params []pegomock.Param) pegomock.ReturnValues {
+		binPath := filepath.Join(params[0].(string), "terraform99.99.99")
+		err := os.WriteFile(binPath, []byte("#!/bin/sh\necho '\nTerraform v99.99.99\n'"), 0700) // #nosec G306
+		return []pegomock.ReturnValue{binPath, err}
+	})
+
+	err = c.EnsureVersion(logger, v)
+
+	Ok(t, err)
+
+	mockDownloader.VerifyWasCalledEventually(Once(), 2*time.Second).Install(binDir, customURL, v)
 }
 
 // Test that EnsureVersion throws an error when downloads are disabled
@@ -332,7 +355,7 @@ func TestEnsureVersion_downloaded_downloadingDisabled(t *testing.T) {
 	Ok(t, err)
 
 	err = c.EnsureVersion(logger, v)
-	ErrContains(t, "Could not find terraform version", err)
+	ErrContains(t, "could not find terraform version", err)
 	ErrContains(t, "downloads are disabled", err)
 	mockDownloader.VerifyWasCalled(Never())
 }
@@ -393,12 +416,6 @@ terraform {
 		// cannot use ~> 1.3 or ~> 1.0 since that is a moving target since it will always
 		// resolve to the latest terraform 1.x
 		"~> 1.3.0": "1.3.10",
-		// Since terraform version 1.8.2, terraform is not a single file download anymore and
-		// Atlantis fails to download version 1.8.2 and higher. So, as a short-term fix,
-		// we need to block any version higher than 1.8.1 until proper solution is implemented.
-		// More details on the issue here - https://github.com/runatlantis/atlantis/issues/4471
-		">= 1.3.0": "1.8.1",
-		">= 1.8.2": "",
 	}
 
 	type testCase struct {
@@ -497,3 +514,55 @@ terraform {
 		runDetectVersionTestCase(t, name+": Downloads Disabled", testCase, false)
 	}
 }
+
+func TestInstall(t *testing.T) {
+	d := &terraform.DefaultDownloader{}
+	RegisterMockTestingT(t)
+	_, binDir, _ := mkSubDirs(t)
+
+	v, _ := version.NewVersion("1.8.1")
+
+	newPath, err := d.Install(binDir, cmd.DefaultTFDownloadURL, v)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+
+	if _, err := os.Stat(newPath); os.IsNotExist(err) {
+		t.Errorf("Binary not found at %s", newPath)
+	}
+}
+
+func TestExtractExactRegex(t *testing.T) {
+	logger := logging.NewNoopLogger(t)
+	RegisterMockTestingT(t)
+	_, binDir, cacheDir := mkSubDirs(t)
+	projectCmdOutputHandler := jobmocks.NewMockProjectCommandOutputHandler()
+
+	mockDownloader := mocks.NewMockDownloader()
+
+	c, err := terraform.NewTestClient(logger, binDir, cacheDir, "", "", "0.11.10", cmd.DefaultTFVersionFlag, cmd.DefaultTFDownloadURL, mockDownloader, true, true, projectCmdOutputHandler)
+	Ok(t, err)
+
+	tests := []struct {
+		version string
+		want    []string
+	}{
+		{"= 1.2.3", []string{"1.2.3"}},
+		{"=1.2.3", []string{"1.2.3"}},
+		{"1.2.3", []string{"1.2.3"}},
+		{"v1.2.3", nil},
+		{">= 1.2.3", nil},
+		{">=1.2.3", nil},
+		{"<= 1.2.3", nil},
+		{"<=1.2.3", nil},
+		{"~> 1.2.3", nil},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.version, func(t *testing.T) {
+			if got := c.ExtractExactRegex(logger, tt.version); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("ExtractExactRegex() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

server/events/project_command_builder_test.go

@@ -234,7 +234,6 @@ terraform {
 	userConfig := defaultUserConfig
 
 	terraformClient := terraform_mocks.NewMockClient()
-	When(terraformClient.ListAvailableVersions(Any[logging.SimpleLogging]())).ThenReturn([]string{}, nil)
 
 	for _, c := range cases {
 		t.Run(c.Description, func(t *testing.T) {
@@ -618,7 +617,6 @@ projects:
 				}
 
 				terraformClient := terraform_mocks.NewMockClient()
-				When(terraformClient.ListAvailableVersions(Any[logging.SimpleLogging]())).ThenReturn([]string{}, nil)
 
 				builder := events.NewProjectCommandBuilder(
 					false,
@@ -807,7 +805,6 @@ projects:
 			}
 
 			terraformClient := terraform_mocks.NewMockClient()
-			When(terraformClient.ListAvailableVersions(Any[logging.SimpleLogging]())).ThenReturn([]string{}, nil)
 
 			builder := events.NewProjectCommandBuilder(
 				false,
@@ -1137,7 +1134,6 @@ projects:
 			}
 
 			terraformClient := terraform_mocks.NewMockClient()
-			When(terraformClient.ListAvailableVersions(Any[logging.SimpleLogging]())).ThenReturn([]string{}, nil)
 
 			builder := events.NewProjectCommandBuilder(
 				false,
@@ -1236,7 +1232,6 @@ func TestDefaultProjectCommandBuilder_BuildMultiApply(t *testing.T) {
 	scope, _, _ := metrics.NewLoggingScope(logger, "atlantis")
 
 	terraformClient := terraform_mocks.NewMockClient()
-	When(terraformClient.ListAvailableVersions(Any[logging.SimpleLogging]())).ThenReturn([]string{}, nil)
 
 	builder := events.NewProjectCommandBuilder(
 		false,
@@ -1323,7 +1318,6 @@ projects:
 	userConfig := defaultUserConfig
 
 	terraformClient := terraform_mocks.NewMockClient()
-	When(terraformClient.ListAvailableVersions(Any[logging.SimpleLogging]())).ThenReturn([]string{}, nil)
 
 	builder := events.NewProjectCommandBuilder(
 		false,
@@ -1412,7 +1406,6 @@ func TestDefaultProjectCommandBuilder_EscapeArgs(t *testing.T) {
 			}
 
 			terraformClient := terraform_mocks.NewMockClient()
-			When(terraformClient.ListAvailableVersions(Any[logging.SimpleLogging]())).ThenReturn([]string{}, nil)
 
 			builder := events.NewProjectCommandBuilder(
 				false,
@@ -1685,7 +1678,6 @@ projects:
 		}
 		scope, _, _ := metrics.NewLoggingScope(logger, "atlantis")
 		terraformClient := terraform_mocks.NewMockClient()
-		When(terraformClient.ListAvailableVersions(Any[logging.SimpleLogging]())).ThenReturn([]string{}, nil)
 
 		builder := events.NewProjectCommandBuilder(
 			false,
@@ -1755,7 +1747,6 @@ func TestDefaultProjectCommandBuilder_WithPolicyCheckEnabled_BuildAutoplanComman
 
 	globalCfg := valid.NewGlobalCfgFromArgs(globalCfgArgs)
 	terraformClient := terraform_mocks.NewMockClient()
-	When(terraformClient.ListAvailableVersions(Any[logging.SimpleLogging]())).ThenReturn([]string{}, nil)
 
 	builder := events.NewProjectCommandBuilder(
 		true,
@@ -1844,7 +1835,6 @@ func TestDefaultProjectCommandBuilder_BuildVersionCommand(t *testing.T) {
 		AllowAllRepoSettings: false,
 	}
 	terraformClient := terraform_mocks.NewMockClient()
-	When(terraformClient.ListAvailableVersions(Any[logging.SimpleLogging]())).ThenReturn([]string{}, nil)
 
 	builder := events.NewProjectCommandBuilder(
 		false,
@@ -1975,7 +1965,6 @@ func TestDefaultProjectCommandBuilder_BuildPlanCommands_Single_With_RestrictFile
 			}
 
 			terraformClient := terraform_mocks.NewMockClient()
-			When(terraformClient.ListAvailableVersions(Any[logging.SimpleLogging]())).ThenReturn([]string{}, nil)
 
 			builder := events.NewProjectCommandBuilder(
 				false, // policyChecksSupported
@@ -2087,7 +2076,6 @@ func TestDefaultProjectCommandBuilder_BuildPlanCommands_with_IncludeGitUntracked
 			}
 
 			terraformClient := terraform_mocks.NewMockClient()
-			When(terraformClient.ListAvailableVersions(Any[logging.SimpleLogging]())).ThenReturn([]string{}, nil)
 
 			builder := events.NewProjectCommandBuilder(
 				false, // policyChecksSupported

server/events/project_command_context_builder_test.go

@@ -48,7 +48,6 @@ func TestProjectCommandContextBuilder_PullStatus(t *testing.T) {
 	expectedPlanCmt := "Plan Comment"
 
 	terraformClient := terraform_mocks.NewMockClient()
-	When(terraformClient.ListAvailableVersions(commandCtx.Log))
 
 	t.Run("with project name defined", func(t *testing.T) {
 		When(mockCommentBuilder.BuildPlanComment(projRepoRelDir, projWorkspace, projName, []string{})).ThenReturn(expectedPlanCmt)