feat: Add --autoplan-file-list server flag. (#1475)

* Add `--autoplan-file-list` server flag to allow modifying the list of files that trigger planning.

* Set default value for `--autoplan-file-list` and add `terragrunt.hcl` to the list.

* Default is handled in server now. No need for it here.

* Fix syntax error test.

* Fix e2e tests

cmd/server.go

@@ -20,6 +20,7 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/docker/docker/pkg/fileutils"
 	homedir "github.com/mitchellh/go-homedir"
 	"github.com/pkg/errors"
 	"github.com/runatlantis/atlantis/server"
@@ -44,6 +45,7 @@ const (
 	AllowRepoConfigFlag        = "allow-repo-config"
 	AtlantisURLFlag            = "atlantis-url"
 	AutomergeFlag              = "automerge"
+	AutoplanFileListFlag       = "autoplan-file-list"
 	BitbucketBaseURLFlag       = "bitbucket-base-url"
 	BitbucketTokenFlag         = "bitbucket-token"
 	BitbucketUserFlag          = "bitbucket-user"
@@ -102,6 +104,7 @@ const (
 	// NOTE: Must manually set these as defaults in the setDefaults function.
 	DefaultADBasicUser      = ""
 	DefaultADBasicPassword  = ""
+	DefaultAutoplanFileList = "**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl"
 	DefaultCheckoutStrategy = "branch"
 	DefaultBitbucketBaseURL = bitbucketcloud.BaseURL
 	DefaultDataDir          = "~/.atlantis"
@@ -137,6 +140,13 @@ var stringFlags = map[string]stringFlag{
 	AtlantisURLFlag: {
 		description: "URL that Atlantis can be reached at. Defaults to http://$(hostname):$port where $port is from --" + PortFlag + ". Supports a base path ex. https://example.com/basepath.",
 	},
+	AutoplanFileListFlag: {
+		description: "Comma separated list of file patterns that Atlantis will use to check if a directory contains modified files that should trigger project planning." +
+			" Patterns use the dockerignore (https://docs.docker.com/engine/reference/builder/#dockerignore-file) syntax." +
+			" Use single quotes to avoid shell expansion of '*'. Defaults to '" + DefaultAutoplanFileList + "'." +
+			" A custom Workflow that uses autoplan 'when_modified' will ignore this value.",
+		defaultValue: DefaultAutoplanFileList,
+	},
 	BitbucketUserFlag: {
 		description: "Bitbucket username of API user.",
 	},
@@ -569,6 +579,9 @@ func (s *ServerCmd) run() error {
 }
 
 func (s *ServerCmd) setDefaults(c *server.UserConfig) {
+	if c.AutoplanFileList == "" {
+		c.AutoplanFileList = DefaultAutoplanFileList
+	}
 	if c.CheckoutStrategy == "" {
 		c.CheckoutStrategy = DefaultCheckoutStrategy
 	}
@@ -686,6 +699,11 @@ func (s *ServerCmd) validate(userConfig server.UserConfig) error {
 		return fmt.Errorf("if setting --%s, must set --%s", TFEHostnameFlag, TFETokenFlag)
 	}
 
+	_, patternErr := fileutils.NewPatternMatcher(strings.Split(userConfig.AutoplanFileList, ","))
+	if patternErr != nil {
+		return errors.Wrapf(patternErr, "invalid pattern in --%s, %s", AutoplanFileListFlag, userConfig.AutoplanFileList)
+	}
+
 	return nil
 }
 

cmd/server_test.go

@@ -28,6 +28,7 @@ import (
 	. "github.com/runatlantis/atlantis/testing"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
+	"gopkg.in/yaml.v2"
 )
 
 // passedConfig is set to whatever config ended up being passed to NewServer.
@@ -58,6 +59,7 @@ var testFlags = map[string]interface{}{
 	AllowForkPRsFlag:           true,
 	AllowRepoConfigFlag:        true,
 	AutomergeFlag:              true,
+	AutoplanFileListFlag:       "**/*.tf,**/*.yml",
 	BitbucketBaseURLFlag:       "https://bitbucket-base-url.com",
 	BitbucketTokenFlag:         "bitbucket-token",
 	BitbucketUserFlag:          "bitbucket-user",
@@ -167,11 +169,10 @@ func TestExecute_Flags(t *testing.T) {
 
 func TestExecute_ConfigFile(t *testing.T) {
 	t.Log("Should use all the values from the config file.")
-	var cfgContents string
-	for flag, value := range testFlags {
-		cfgContents += fmt.Sprintf("%s: %v\n", flag, value)
-	}
-	tmpFile := tempFile(t, cfgContents)
+	// Use yaml package to quote values that need quoting
+	cfgContents, yamlErr := yaml.Marshal(&testFlags)
+	Ok(t, yamlErr)
+	tmpFile := tempFile(t, string(cfgContents))
 	defer os.Remove(tmpFile) // nolint: errcheck
 	c := setup(map[string]interface{}{
 		ConfigFlag: tmpFile,
@@ -738,6 +739,53 @@ func TestExecute_RepoWhitelistDeprecation(t *testing.T) {
 	Equals(t, "*", passedConfig.RepoAllowlist)
 }
 
+func TestExecute_AutoplanFileList(t *testing.T) {
+	cases := []struct {
+		description string
+		flags       map[string]interface{}
+		expectErr   string
+	}{
+		{
+			"default value",
+			map[string]interface{}{
+				AutoplanFileListFlag: DefaultAutoplanFileList,
+			},
+			"",
+		},
+		{
+			"valid value",
+			map[string]interface{}{
+				AutoplanFileListFlag: "**/*.tf",
+			},
+			"",
+		},
+		{
+			"invalid exclusion pattern",
+			map[string]interface{}{
+				AutoplanFileListFlag: "**/*.yml,!",
+			},
+			"invalid pattern in --autoplan-file-list, **/*.yml,!: illegal exclusion pattern: \"!\"",
+		},
+		{
+			"invalid pattern",
+			map[string]interface{}{
+				AutoplanFileListFlag: "[^]",
+			},
+			"invalid pattern in --autoplan-file-list, [^]: syntax error in pattern",
+		},
+	}
+	for _, testCase := range cases {
+		t.Log("Should validate autoplan file list when " + testCase.description)
+		c := setupWithDefaults(testCase.flags, t)
+		err := c.Execute()
+		if testCase.expectErr != "" {
+			ErrEquals(t, testCase.expectErr, err)
+		} else {
+			Ok(t, err)
+		}
+	}
+}
+
 func setup(flags map[string]interface{}, t *testing.T) *cobra.Command {
 	vipr := viper.New()
 	for k, v := range flags {

runatlantis.io/docs/server-configuration.md

@@ -110,6 +110,30 @@ Values are chosen in this order:
   Automatically merge pull requests after all plans have been successfully applied.
   Defaults to `false`. See [Automerging](automerging.html) for more details.
 
+* ### `--autoplan-file-list`
+  ```bash
+  # NOTE: Use single quotes to avoid shell expansion of *.
+  atlantis server --autoplan-file-list='**/*.tf,project1/*.pkr.hcl'
+  ```
+  List of file patterns that Atlantis will use to check if a directory contains modified files that should trigger project planning.
+
+  Notes:
+  * Accepts a comma separated list, ex. `pattern1,pattern2`.
+  * Patterns use the [`.dockerignore` syntax](https://docs.docker.com/engine/reference/builder/#dockerignore-file)
+  * List of file patterns will be used by both automatic and manually run plans.
+  * When not set, defaults to all `.tf`, `.tfvars`, `.tfvars.json` and `terragrunt.hcl` files
+    (`--autoplan-file-list='**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl'`).
+  * Setting `--autoplan-file-list` will override the defaults. You **must** add `**/*.tf` and other defaults if you want to include them.
+  * A custom [Workflow](repo-level-atlantis-yaml.html#configuring-planning) that uses autoplan `when_modified` will ignore this value.
+
+  Examples:
+  * Autoplan when any `*.tf` or `*.tfvars` file is modified.
+    * `--autoplan-file-list='**/*.tf,**/*.tfvars'`
+  * Autoplan when any `*.tf` file is modified except in `project2/` directory
+    * `--autoplan-file-list='**/*.tf,!project2'`
+  * Autoplan when any `*.tf` files or `.yml` files in subfolder of `project1` is modified.
+    * `--autoplan-file-list='**/*.tf,project2/**/*.yml'`
+
 * ### `--azuredevops-webhook-password`
   ```bash
   atlantis server --azuredevops-webhook-password="password123"

server/events/project_command_builder.go

@@ -39,6 +39,7 @@ func NewProjectCommandBuilder(
 	commentBuilder CommentBuilder,
 	skipCloneNoChanges bool,
 	EnableRegExpCmd bool,
+	AutoplanFileList string,
 ) *DefaultProjectCommandBuilder {
 	projectCommandBuilder := &DefaultProjectCommandBuilder{
 		ParserValidator:    parserValidator,
@@ -50,6 +51,7 @@ func NewProjectCommandBuilder(
 		PendingPlanFinder:  pendingPlanFinder,
 		SkipCloneNoChanges: skipCloneNoChanges,
 		EnableRegExpCmd:    EnableRegExpCmd,
+		AutoplanFileList:   AutoplanFileList,
 		ProjectCommandContextBuilder: NewProjectCommandContextBulder(
 			policyChecksSupported,
 			commentBuilder,
@@ -104,6 +106,7 @@ type DefaultProjectCommandBuilder struct {
 	ProjectCommandContextBuilder ProjectCommandContextBuilder
 	SkipCloneNoChanges           bool
 	EnableRegExpCmd              bool
+	AutoplanFileList             string
 }
 
 // See ProjectCommandBuilder.BuildAutoplanCommands.
@@ -241,7 +244,10 @@ func (p *DefaultProjectCommandBuilder) buildPlanAllCommands(ctx *CommandContext,
 		// If there is no config file, then we'll plan each project that
 		// our algorithm determines was modified.
 		ctx.Log.Info("found no %s file", yaml.AtlantisYAMLFilename)
-		modifiedProjects := p.ProjectFinder.DetermineProjects(ctx.Log, modifiedFiles, ctx.Pull.BaseRepo.FullName, repoDir)
+		modifiedProjects := p.ProjectFinder.DetermineProjects(ctx.Log, modifiedFiles, ctx.Pull.BaseRepo.FullName, repoDir, p.AutoplanFileList)
+		if err != nil {
+			return nil, errors.Wrapf(err, "finding modified projects: %s", modifiedFiles)
+		}
 		ctx.Log.Info("automatically determined that there were %d projects modified in this pull request: %s", len(modifiedProjects), modifiedProjects)
 		for _, mp := range modifiedProjects {
 			ctx.Log.Debug("determining config for project at dir: %q", mp.Path)

server/events/project_command_builder_internal_test.go

@@ -592,6 +592,7 @@ projects:
 				&CommentParser{},
 				false,
 				false,
+				"**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl",
 			)
 
 			// We run a test for each type of command.
@@ -778,6 +779,7 @@ projects:
 				&CommentParser{},
 				false,
 				true,
+				"**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl",
 			)
 
 			// We run a test for each type of command, again specific projects
@@ -983,6 +985,7 @@ workflows:
 				&CommentParser{},
 				false,
 				false,
+				"**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl",
 			)
 
 			cmd := models.PolicyCheckCommand

server/events/project_command_builder_test.go

@@ -148,6 +148,7 @@ projects:
 				&events.CommentParser{},
 				false,
 				false,
+				"**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl",
 			)
 
 			ctxs, err := builder.BuildAutoplanCommands(&events.CommandContext{
@@ -377,6 +378,7 @@ projects:
 					&events.CommentParser{},
 					false,
 					true,
+					"**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl",
 				)
 
 				var actCtxs []models.ProjectCommandContext
@@ -518,6 +520,7 @@ projects:
 				&events.CommentParser{},
 				false,
 				false,
+				"**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl",
 			)
 
 			ctxs, err := builder.BuildPlanCommands(
@@ -597,6 +600,7 @@ func TestDefaultProjectCommandBuilder_BuildMultiApply(t *testing.T) {
 		&events.CommentParser{},
 		false,
 		false,
+		"**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl",
 	)
 
 	ctxs, err := builder.BuildApplyCommands(
@@ -669,6 +673,7 @@ projects:
 		&events.CommentParser{},
 		false,
 		false,
+		"**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl",
 	)
 
 	ctx := &events.CommandContext{
@@ -736,6 +741,7 @@ func TestDefaultProjectCommandBuilder_EscapeArgs(t *testing.T) {
 				&events.CommentParser{},
 				false,
 				false,
+				"**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl",
 			)
 
 			var actCtxs []models.ProjectCommandContext
@@ -907,6 +913,7 @@ projects:
 				&events.CommentParser{},
 				false,
 				false,
+				"**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl",
 			)
 
 			actCtxs, err := builder.BuildPlanCommands(
@@ -962,6 +969,7 @@ projects:
 		&events.CommentParser{},
 		true,
 		false,
+		"**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl",
 	)
 
 	var actCtxs []models.ProjectCommandContext
@@ -1005,6 +1013,7 @@ func TestDefaultProjectCommandBuilder_WithPolicyCheckEnabled_BuildAutoplanComman
 		&events.CommentParser{},
 		false,
 		false,
+		"**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl",
 	)
 
 	ctxs, err := builder.BuildAutoplanCommands(&events.CommandContext{

server/events/project_finder.go

@@ -17,7 +17,6 @@ import (
 	"os"
 	"path"
 	"path/filepath"
-	"regexp"
 	"strings"
 
 	"github.com/runatlantis/atlantis/server/events/yaml/valid"
@@ -34,7 +33,7 @@ type ProjectFinder interface {
 	// DetermineProjects returns the list of projects that were modified based on
 	// the modifiedFiles. The list will be de-duplicated.
 	// absRepoDir is the path to the cloned repo on disk.
-	DetermineProjects(log logging.SimpleLogging, modifiedFiles []string, repoFullName string, absRepoDir string) []models.Project
+	DetermineProjects(log logging.SimpleLogging, modifiedFiles []string, repoFullName string, absRepoDir string, autoplanFileList string) []models.Project
 	// DetermineProjectsViaConfig returns the list of projects that were modified
 	// based on modifiedFiles and the repo's config.
 	// absRepoDir is the path to the cloned repo on disk.
@@ -48,10 +47,10 @@ var ignoredFilenameFragments = []string{"terraform.tfstate", "terraform.tfstate.
 type DefaultProjectFinder struct{}
 
 // See ProjectFinder.DetermineProjects.
-func (p *DefaultProjectFinder) DetermineProjects(log logging.SimpleLogging, modifiedFiles []string, repoFullName string, absRepoDir string) []models.Project {
+func (p *DefaultProjectFinder) DetermineProjects(log logging.SimpleLogging, modifiedFiles []string, repoFullName string, absRepoDir string, autoplanFileList string) []models.Project {
 	var projects []models.Project
 
-	modifiedTerraformFiles := p.filterToTerraform(modifiedFiles)
+	modifiedTerraformFiles := p.filterToFileList(log, modifiedFiles, autoplanFileList)
 	if len(modifiedTerraformFiles) == 0 {
 		return projects
 	}
@@ -145,13 +144,23 @@ func (p *DefaultProjectFinder) DetermineProjectsViaConfig(log logging.SimpleLogg
 	return projects, nil
 }
 
-// filterToTerraform filters non-terraform files from files.
-func (p *DefaultProjectFinder) filterToTerraform(files []string) []string {
+// filterToFileList filters out files not included in the file list
+func (p *DefaultProjectFinder) filterToFileList(log logging.SimpleLogging, files []string, fileList string) []string {
 	var filtered []string
-	fileNameRe, _ := regexp.Compile(`^.*(\.tf|\.tfvars|\.tfvars.json)$`)
+	patterns := strings.Split(fileList, ",")
+	// Ignore pattern matcher error here as it was checked for errors in server validation
+	patternMatcher, _ := fileutils.NewPatternMatcher(patterns)
 
 	for _, fileName := range files {
-		if !p.shouldIgnore(fileName) && (fileNameRe.MatchString(fileName) || filepath.Base(fileName) == "terragrunt.hcl") {
+		if p.shouldIgnore(fileName) {
+			continue
+		}
+		match, err := patternMatcher.Matches(fileName)
+		if err != nil {
+			log.Debug("filter err for file %q: %s", fileName, err)
+			continue
+		}
+		if match {
 			filtered = append(filtered, fileName)
 		}
 	}

server/events/project_finder_test.go

@@ -107,112 +107,152 @@ func TestDetermineProjects(t *testing.T) {
 	noopLogger := logging.NewNoopLogger(t)
 	setupTmpRepos(t)
 
+	defaultAutoplanFileList := "**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl"
+
 	cases := []struct {
-		description     string
-		files           []string
-		expProjectPaths []string
-		repoDir         string
+		description      string
+		files            []string
+		expProjectPaths  []string
+		repoDir          string
+		autoplanFileList string
 	}{
 		{
 			"If no files were modified then should return an empty list",
 			nil,
 			nil,
 			nestedModules1,
+			defaultAutoplanFileList,
 		},
 		{
 			"Should ignore non .tf files and return an empty list",
 			[]string{"non-tf", "non.tf.suffix"},
 			nil,
 			nestedModules1,
+			defaultAutoplanFileList,
 		},
 		{
 			"Should ignore .tflint.hcl files and return an empty list",
 			[]string{".tflint.hcl", "project1/.tflint.hcl"},
 			nil,
 			nestedModules1,
+			defaultAutoplanFileList,
 		},
 		{
 			"Should plan in the parent directory from modules if that dir has a main.tf",
 			[]string{"project1/modules/main.tf"},
 			[]string{"project1"},
 			nestedModules1,
+			defaultAutoplanFileList,
 		},
 		{
 			"Should plan in the parent directory from modules if that dir has a main.tf",
 			[]string{"modules/main.tf"},
 			[]string{"."},
 			nestedModules2,
+			defaultAutoplanFileList,
 		},
 		{
 			"Should plan in the parent directory from modules when module is in a subdir if that dir has a main.tf",
 			[]string{"modules/subdir/main.tf"},
 			[]string{"."},
 			nestedModules2,
+			defaultAutoplanFileList,
 		},
 		{
 			"Should not plan in the parent directory from modules if that dir does not have a main.tf",
 			[]string{"modules/main.tf"},
 			[]string{},
 			topLevelModules,
+			defaultAutoplanFileList,
 		},
 		{
 			"Should not plan in the parent directory from modules if that dir does not have a main.tf",
 			[]string{"modules/main.tf", "project1/main.tf"},
 			[]string{"project1"},
 			topLevelModules,
+			defaultAutoplanFileList,
 		},
 		{
 			"Should ignore tfstate files and return an empty list",
 			[]string{"terraform.tfstate", "terraform.tfstate.backup", "parent/terraform.tfstate", "parent/terraform.tfstate.backup"},
 			nil,
 			nestedModules1,
+			defaultAutoplanFileList,
 		},
 		{
 			"Should return '.' when changed file is at root",
 			[]string{"a.tf"},
 			[]string{"."},
 			nestedModules2,
+			defaultAutoplanFileList,
 		},
 		{
 			"Should return directory when changed file is in a dir",
 			[]string{"project1/a.tf"},
 			[]string{"project1"},
 			nestedModules1,
+			defaultAutoplanFileList,
 		},
 		{
 			"Should return parent dir when changed file is in an env/ dir",
 			[]string{"env/staging.tfvars"},
 			[]string{"."},
 			envDir,
+			defaultAutoplanFileList,
 		},
 		{
 			"Should de-duplicate when multiple files changed in the same dir",
 			[]string{"env/staging.tfvars", "main.tf", "other.tf"},
 			[]string{"."},
 			"",
+			defaultAutoplanFileList,
 		},
 		{
 			"Should ignore changes in a dir that was deleted",
 			[]string{"wasdeleted/main.tf"},
 			[]string{},
 			"",
+			defaultAutoplanFileList,
 		},
 		{
 			"Should not ignore terragrunt.hcl files",
 			[]string{"terragrunt.hcl"},
 			[]string{"."},
 			nestedModules2,
+			defaultAutoplanFileList,
 		},
 		{
 			"Should find terragrunt.hcl file inside a nested directory",
 			[]string{"project1/terragrunt.hcl"},
 			[]string{"project1"},
 			nestedModules1,
+			defaultAutoplanFileList,
+		},
+		{
+			"Should find packer files and ignore default tf files",
+			[]string{"project1/image.pkr.hcl", "project2/main.tf"},
+			[]string{"project1"},
+			topLevelModules,
+			"**/*.pkr.hcl",
+		},
+		{
+			"Should find yaml files in addition to defaults",
+			[]string{"project1/ansible.yml", "project2/main.tf"},
+			[]string{"project1", "project2"},
+			topLevelModules,
+			"**/*.tf,**/*.yml",
+		},
+		{
+			"Should find yaml files unless excluded",
+			[]string{"project1/ansible.yml", "project2/config.yml"},
+			[]string{"project1"},
+			topLevelModules,
+			"**/*.yml,!project2/*.yml",
 		},
 	}
 	for _, c := range cases {
 		t.Run(c.description, func(t *testing.T) {
-			projects := m.DetermineProjects(noopLogger, c.files, modifiedRepo, c.repoDir)
+			projects := m.DetermineProjects(noopLogger, c.files, modifiedRepo, c.repoDir, c.autoplanFileList)
 
 			// Extract the paths from the projects. We use a slice here instead of a
 			// map so we can test whether there are duplicates returned.

server/events_controller_e2e_test.go

@@ -718,6 +718,7 @@ func setupE2E(t *testing.T, repoDir string) (server.EventsController, *vcsmocks.
 		commentParser,
 		false,
 		false,
+		"**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl",
 	)
 
 	showStepRunner, err := runtime.NewShowStepRunner(terraformClient, defaultTFVersion)

server/server.go

@@ -427,6 +427,7 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) {
 		commentParser,
 		userConfig.SkipCloneNoChanges,
 		userConfig.EnableRegExpCmd,
+		userConfig.AutoplanFileList,
 	)
 
 	showStepRunner, err := runtime.NewShowStepRunner(terraformClient, defaultTfVersion)

server/user_config.go

@@ -12,6 +12,7 @@ type UserConfig struct {
 	AllowRepoConfig            bool   `mapstructure:"allow-repo-config"`
 	AtlantisURL                string `mapstructure:"atlantis-url"`
 	Automerge                  bool   `mapstructure:"automerge"`
+	AutoplanFileList           string `mapstructure:"autoplan-file-list"`
 	AzureDevopsToken           string `mapstructure:"azuredevops-token"`
 	AzureDevopsUser            string `mapstructure:"azuredevops-user"`
 	AzureDevopsWebhookPassword string `mapstructure:"azuredevops-webhook-password"`