From f290780b61dd2e18c8abcd58404f4503ae19eb09 Mon Sep 17 00:00:00 2001 From: Brian Zoetewey Date: Thu, 22 Apr 2021 12:48:40 -0400 Subject: [PATCH] feat: Add `--autoplan-file-list` server flag. (#1475) * Add `--autoplan-file-list` server flag to allow modifying the list of files that trigger planning. * Set default value for `--autoplan-file-list` and add `terragrunt.hcl` to the list. * Default is handled in server now. No need for it here. * Fix syntax error test. * Fix e2e tests --- cmd/server.go | 18 ++++++ cmd/server_test.go | 58 +++++++++++++++++-- runatlantis.io/docs/server-configuration.md | 24 ++++++++ server/events/project_command_builder.go | 8 ++- .../project_command_builder_internal_test.go | 3 + server/events/project_command_builder_test.go | 9 +++ server/events/project_finder.go | 25 +++++--- server/events/project_finder_test.go | 50 ++++++++++++++-- server/events_controller_e2e_test.go | 1 + server/server.go | 1 + server/user_config.go | 1 + 11 files changed, 179 insertions(+), 19 deletions(-) diff --git a/cmd/server.go b/cmd/server.go index df705e7ba..059142d22 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -20,6 +20,7 @@ import ( "path/filepath" "strings" + "github.com/docker/docker/pkg/fileutils" homedir "github.com/mitchellh/go-homedir" "github.com/pkg/errors" "github.com/runatlantis/atlantis/server" @@ -44,6 +45,7 @@ const ( AllowRepoConfigFlag = "allow-repo-config" AtlantisURLFlag = "atlantis-url" AutomergeFlag = "automerge" + AutoplanFileListFlag = "autoplan-file-list" BitbucketBaseURLFlag = "bitbucket-base-url" BitbucketTokenFlag = "bitbucket-token" BitbucketUserFlag = "bitbucket-user" @@ -102,6 +104,7 @@ const ( // NOTE: Must manually set these as defaults in the setDefaults function. DefaultADBasicUser = "" DefaultADBasicPassword = "" + DefaultAutoplanFileList = "**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl" DefaultCheckoutStrategy = "branch" DefaultBitbucketBaseURL = bitbucketcloud.BaseURL DefaultDataDir = "~/.atlantis" @@ -137,6 +140,13 @@ var stringFlags = map[string]stringFlag{ AtlantisURLFlag: { description: "URL that Atlantis can be reached at. Defaults to http://$(hostname):$port where $port is from --" + PortFlag + ". Supports a base path ex. https://example.com/basepath.", }, + AutoplanFileListFlag: { + description: "Comma separated list of file patterns that Atlantis will use to check if a directory contains modified files that should trigger project planning." + + " Patterns use the dockerignore (https://docs.docker.com/engine/reference/builder/#dockerignore-file) syntax." + + " Use single quotes to avoid shell expansion of '*'. Defaults to '" + DefaultAutoplanFileList + "'." + + " A custom Workflow that uses autoplan 'when_modified' will ignore this value.", + defaultValue: DefaultAutoplanFileList, + }, BitbucketUserFlag: { description: "Bitbucket username of API user.", }, @@ -569,6 +579,9 @@ func (s *ServerCmd) run() error { } func (s *ServerCmd) setDefaults(c *server.UserConfig) { + if c.AutoplanFileList == "" { + c.AutoplanFileList = DefaultAutoplanFileList + } if c.CheckoutStrategy == "" { c.CheckoutStrategy = DefaultCheckoutStrategy } @@ -686,6 +699,11 @@ func (s *ServerCmd) validate(userConfig server.UserConfig) error { return fmt.Errorf("if setting --%s, must set --%s", TFEHostnameFlag, TFETokenFlag) } + _, patternErr := fileutils.NewPatternMatcher(strings.Split(userConfig.AutoplanFileList, ",")) + if patternErr != nil { + return errors.Wrapf(patternErr, "invalid pattern in --%s, %s", AutoplanFileListFlag, userConfig.AutoplanFileList) + } + return nil } diff --git a/cmd/server_test.go b/cmd/server_test.go index 675f3a4fa..4523d8cc8 100644 --- a/cmd/server_test.go +++ b/cmd/server_test.go @@ -28,6 +28,7 @@ import ( . "github.com/runatlantis/atlantis/testing" "github.com/spf13/cobra" "github.com/spf13/viper" + "gopkg.in/yaml.v2" ) // passedConfig is set to whatever config ended up being passed to NewServer. @@ -58,6 +59,7 @@ var testFlags = map[string]interface{}{ AllowForkPRsFlag: true, AllowRepoConfigFlag: true, AutomergeFlag: true, + AutoplanFileListFlag: "**/*.tf,**/*.yml", BitbucketBaseURLFlag: "https://bitbucket-base-url.com", BitbucketTokenFlag: "bitbucket-token", BitbucketUserFlag: "bitbucket-user", @@ -167,11 +169,10 @@ func TestExecute_Flags(t *testing.T) { func TestExecute_ConfigFile(t *testing.T) { t.Log("Should use all the values from the config file.") - var cfgContents string - for flag, value := range testFlags { - cfgContents += fmt.Sprintf("%s: %v\n", flag, value) - } - tmpFile := tempFile(t, cfgContents) + // Use yaml package to quote values that need quoting + cfgContents, yamlErr := yaml.Marshal(&testFlags) + Ok(t, yamlErr) + tmpFile := tempFile(t, string(cfgContents)) defer os.Remove(tmpFile) // nolint: errcheck c := setup(map[string]interface{}{ ConfigFlag: tmpFile, @@ -738,6 +739,53 @@ func TestExecute_RepoWhitelistDeprecation(t *testing.T) { Equals(t, "*", passedConfig.RepoAllowlist) } +func TestExecute_AutoplanFileList(t *testing.T) { + cases := []struct { + description string + flags map[string]interface{} + expectErr string + }{ + { + "default value", + map[string]interface{}{ + AutoplanFileListFlag: DefaultAutoplanFileList, + }, + "", + }, + { + "valid value", + map[string]interface{}{ + AutoplanFileListFlag: "**/*.tf", + }, + "", + }, + { + "invalid exclusion pattern", + map[string]interface{}{ + AutoplanFileListFlag: "**/*.yml,!", + }, + "invalid pattern in --autoplan-file-list, **/*.yml,!: illegal exclusion pattern: \"!\"", + }, + { + "invalid pattern", + map[string]interface{}{ + AutoplanFileListFlag: "[^]", + }, + "invalid pattern in --autoplan-file-list, [^]: syntax error in pattern", + }, + } + for _, testCase := range cases { + t.Log("Should validate autoplan file list when " + testCase.description) + c := setupWithDefaults(testCase.flags, t) + err := c.Execute() + if testCase.expectErr != "" { + ErrEquals(t, testCase.expectErr, err) + } else { + Ok(t, err) + } + } +} + func setup(flags map[string]interface{}, t *testing.T) *cobra.Command { vipr := viper.New() for k, v := range flags { diff --git a/runatlantis.io/docs/server-configuration.md b/runatlantis.io/docs/server-configuration.md index bc69b35d8..ad6165a85 100644 --- a/runatlantis.io/docs/server-configuration.md +++ b/runatlantis.io/docs/server-configuration.md @@ -110,6 +110,30 @@ Values are chosen in this order: Automatically merge pull requests after all plans have been successfully applied. Defaults to `false`. See [Automerging](automerging.html) for more details. +* ### `--autoplan-file-list` + ```bash + # NOTE: Use single quotes to avoid shell expansion of *. + atlantis server --autoplan-file-list='**/*.tf,project1/*.pkr.hcl' + ``` + List of file patterns that Atlantis will use to check if a directory contains modified files that should trigger project planning. + + Notes: + * Accepts a comma separated list, ex. `pattern1,pattern2`. + * Patterns use the [`.dockerignore` syntax](https://docs.docker.com/engine/reference/builder/#dockerignore-file) + * List of file patterns will be used by both automatic and manually run plans. + * When not set, defaults to all `.tf`, `.tfvars`, `.tfvars.json` and `terragrunt.hcl` files + (`--autoplan-file-list='**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl'`). + * Setting `--autoplan-file-list` will override the defaults. You **must** add `**/*.tf` and other defaults if you want to include them. + * A custom [Workflow](repo-level-atlantis-yaml.html#configuring-planning) that uses autoplan `when_modified` will ignore this value. + + Examples: + * Autoplan when any `*.tf` or `*.tfvars` file is modified. + * `--autoplan-file-list='**/*.tf,**/*.tfvars'` + * Autoplan when any `*.tf` file is modified except in `project2/` directory + * `--autoplan-file-list='**/*.tf,!project2'` + * Autoplan when any `*.tf` files or `.yml` files in subfolder of `project1` is modified. + * `--autoplan-file-list='**/*.tf,project2/**/*.yml'` + * ### `--azuredevops-webhook-password` ```bash atlantis server --azuredevops-webhook-password="password123" diff --git a/server/events/project_command_builder.go b/server/events/project_command_builder.go index a26f2e3b6..4da9ccc17 100644 --- a/server/events/project_command_builder.go +++ b/server/events/project_command_builder.go @@ -39,6 +39,7 @@ func NewProjectCommandBuilder( commentBuilder CommentBuilder, skipCloneNoChanges bool, EnableRegExpCmd bool, + AutoplanFileList string, ) *DefaultProjectCommandBuilder { projectCommandBuilder := &DefaultProjectCommandBuilder{ ParserValidator: parserValidator, @@ -50,6 +51,7 @@ func NewProjectCommandBuilder( PendingPlanFinder: pendingPlanFinder, SkipCloneNoChanges: skipCloneNoChanges, EnableRegExpCmd: EnableRegExpCmd, + AutoplanFileList: AutoplanFileList, ProjectCommandContextBuilder: NewProjectCommandContextBulder( policyChecksSupported, commentBuilder, @@ -104,6 +106,7 @@ type DefaultProjectCommandBuilder struct { ProjectCommandContextBuilder ProjectCommandContextBuilder SkipCloneNoChanges bool EnableRegExpCmd bool + AutoplanFileList string } // See ProjectCommandBuilder.BuildAutoplanCommands. @@ -241,7 +244,10 @@ func (p *DefaultProjectCommandBuilder) buildPlanAllCommands(ctx *CommandContext, // If there is no config file, then we'll plan each project that // our algorithm determines was modified. ctx.Log.Info("found no %s file", yaml.AtlantisYAMLFilename) - modifiedProjects := p.ProjectFinder.DetermineProjects(ctx.Log, modifiedFiles, ctx.Pull.BaseRepo.FullName, repoDir) + modifiedProjects := p.ProjectFinder.DetermineProjects(ctx.Log, modifiedFiles, ctx.Pull.BaseRepo.FullName, repoDir, p.AutoplanFileList) + if err != nil { + return nil, errors.Wrapf(err, "finding modified projects: %s", modifiedFiles) + } ctx.Log.Info("automatically determined that there were %d projects modified in this pull request: %s", len(modifiedProjects), modifiedProjects) for _, mp := range modifiedProjects { ctx.Log.Debug("determining config for project at dir: %q", mp.Path) diff --git a/server/events/project_command_builder_internal_test.go b/server/events/project_command_builder_internal_test.go index f721dff6b..24482df14 100644 --- a/server/events/project_command_builder_internal_test.go +++ b/server/events/project_command_builder_internal_test.go @@ -592,6 +592,7 @@ projects: &CommentParser{}, false, false, + "**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl", ) // We run a test for each type of command. @@ -778,6 +779,7 @@ projects: &CommentParser{}, false, true, + "**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl", ) // We run a test for each type of command, again specific projects @@ -983,6 +985,7 @@ workflows: &CommentParser{}, false, false, + "**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl", ) cmd := models.PolicyCheckCommand diff --git a/server/events/project_command_builder_test.go b/server/events/project_command_builder_test.go index 5c59f8cd4..87907eb8f 100644 --- a/server/events/project_command_builder_test.go +++ b/server/events/project_command_builder_test.go @@ -148,6 +148,7 @@ projects: &events.CommentParser{}, false, false, + "**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl", ) ctxs, err := builder.BuildAutoplanCommands(&events.CommandContext{ @@ -377,6 +378,7 @@ projects: &events.CommentParser{}, false, true, + "**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl", ) var actCtxs []models.ProjectCommandContext @@ -518,6 +520,7 @@ projects: &events.CommentParser{}, false, false, + "**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl", ) ctxs, err := builder.BuildPlanCommands( @@ -597,6 +600,7 @@ func TestDefaultProjectCommandBuilder_BuildMultiApply(t *testing.T) { &events.CommentParser{}, false, false, + "**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl", ) ctxs, err := builder.BuildApplyCommands( @@ -669,6 +673,7 @@ projects: &events.CommentParser{}, false, false, + "**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl", ) ctx := &events.CommandContext{ @@ -736,6 +741,7 @@ func TestDefaultProjectCommandBuilder_EscapeArgs(t *testing.T) { &events.CommentParser{}, false, false, + "**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl", ) var actCtxs []models.ProjectCommandContext @@ -907,6 +913,7 @@ projects: &events.CommentParser{}, false, false, + "**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl", ) actCtxs, err := builder.BuildPlanCommands( @@ -962,6 +969,7 @@ projects: &events.CommentParser{}, true, false, + "**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl", ) var actCtxs []models.ProjectCommandContext @@ -1005,6 +1013,7 @@ func TestDefaultProjectCommandBuilder_WithPolicyCheckEnabled_BuildAutoplanComman &events.CommentParser{}, false, false, + "**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl", ) ctxs, err := builder.BuildAutoplanCommands(&events.CommandContext{ diff --git a/server/events/project_finder.go b/server/events/project_finder.go index 679df16f8..d6feedc7f 100644 --- a/server/events/project_finder.go +++ b/server/events/project_finder.go @@ -17,7 +17,6 @@ import ( "os" "path" "path/filepath" - "regexp" "strings" "github.com/runatlantis/atlantis/server/events/yaml/valid" @@ -34,7 +33,7 @@ type ProjectFinder interface { // DetermineProjects returns the list of projects that were modified based on // the modifiedFiles. The list will be de-duplicated. // absRepoDir is the path to the cloned repo on disk. - DetermineProjects(log logging.SimpleLogging, modifiedFiles []string, repoFullName string, absRepoDir string) []models.Project + DetermineProjects(log logging.SimpleLogging, modifiedFiles []string, repoFullName string, absRepoDir string, autoplanFileList string) []models.Project // DetermineProjectsViaConfig returns the list of projects that were modified // based on modifiedFiles and the repo's config. // absRepoDir is the path to the cloned repo on disk. @@ -48,10 +47,10 @@ var ignoredFilenameFragments = []string{"terraform.tfstate", "terraform.tfstate. type DefaultProjectFinder struct{} // See ProjectFinder.DetermineProjects. -func (p *DefaultProjectFinder) DetermineProjects(log logging.SimpleLogging, modifiedFiles []string, repoFullName string, absRepoDir string) []models.Project { +func (p *DefaultProjectFinder) DetermineProjects(log logging.SimpleLogging, modifiedFiles []string, repoFullName string, absRepoDir string, autoplanFileList string) []models.Project { var projects []models.Project - modifiedTerraformFiles := p.filterToTerraform(modifiedFiles) + modifiedTerraformFiles := p.filterToFileList(log, modifiedFiles, autoplanFileList) if len(modifiedTerraformFiles) == 0 { return projects } @@ -145,13 +144,23 @@ func (p *DefaultProjectFinder) DetermineProjectsViaConfig(log logging.SimpleLogg return projects, nil } -// filterToTerraform filters non-terraform files from files. -func (p *DefaultProjectFinder) filterToTerraform(files []string) []string { +// filterToFileList filters out files not included in the file list +func (p *DefaultProjectFinder) filterToFileList(log logging.SimpleLogging, files []string, fileList string) []string { var filtered []string - fileNameRe, _ := regexp.Compile(`^.*(\.tf|\.tfvars|\.tfvars.json)$`) + patterns := strings.Split(fileList, ",") + // Ignore pattern matcher error here as it was checked for errors in server validation + patternMatcher, _ := fileutils.NewPatternMatcher(patterns) for _, fileName := range files { - if !p.shouldIgnore(fileName) && (fileNameRe.MatchString(fileName) || filepath.Base(fileName) == "terragrunt.hcl") { + if p.shouldIgnore(fileName) { + continue + } + match, err := patternMatcher.Matches(fileName) + if err != nil { + log.Debug("filter err for file %q: %s", fileName, err) + continue + } + if match { filtered = append(filtered, fileName) } } diff --git a/server/events/project_finder_test.go b/server/events/project_finder_test.go index 4f218371c..b4e523ae3 100644 --- a/server/events/project_finder_test.go +++ b/server/events/project_finder_test.go @@ -107,112 +107,152 @@ func TestDetermineProjects(t *testing.T) { noopLogger := logging.NewNoopLogger(t) setupTmpRepos(t) + defaultAutoplanFileList := "**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl" + cases := []struct { - description string - files []string - expProjectPaths []string - repoDir string + description string + files []string + expProjectPaths []string + repoDir string + autoplanFileList string }{ { "If no files were modified then should return an empty list", nil, nil, nestedModules1, + defaultAutoplanFileList, }, { "Should ignore non .tf files and return an empty list", []string{"non-tf", "non.tf.suffix"}, nil, nestedModules1, + defaultAutoplanFileList, }, { "Should ignore .tflint.hcl files and return an empty list", []string{".tflint.hcl", "project1/.tflint.hcl"}, nil, nestedModules1, + defaultAutoplanFileList, }, { "Should plan in the parent directory from modules if that dir has a main.tf", []string{"project1/modules/main.tf"}, []string{"project1"}, nestedModules1, + defaultAutoplanFileList, }, { "Should plan in the parent directory from modules if that dir has a main.tf", []string{"modules/main.tf"}, []string{"."}, nestedModules2, + defaultAutoplanFileList, }, { "Should plan in the parent directory from modules when module is in a subdir if that dir has a main.tf", []string{"modules/subdir/main.tf"}, []string{"."}, nestedModules2, + defaultAutoplanFileList, }, { "Should not plan in the parent directory from modules if that dir does not have a main.tf", []string{"modules/main.tf"}, []string{}, topLevelModules, + defaultAutoplanFileList, }, { "Should not plan in the parent directory from modules if that dir does not have a main.tf", []string{"modules/main.tf", "project1/main.tf"}, []string{"project1"}, topLevelModules, + defaultAutoplanFileList, }, { "Should ignore tfstate files and return an empty list", []string{"terraform.tfstate", "terraform.tfstate.backup", "parent/terraform.tfstate", "parent/terraform.tfstate.backup"}, nil, nestedModules1, + defaultAutoplanFileList, }, { "Should return '.' when changed file is at root", []string{"a.tf"}, []string{"."}, nestedModules2, + defaultAutoplanFileList, }, { "Should return directory when changed file is in a dir", []string{"project1/a.tf"}, []string{"project1"}, nestedModules1, + defaultAutoplanFileList, }, { "Should return parent dir when changed file is in an env/ dir", []string{"env/staging.tfvars"}, []string{"."}, envDir, + defaultAutoplanFileList, }, { "Should de-duplicate when multiple files changed in the same dir", []string{"env/staging.tfvars", "main.tf", "other.tf"}, []string{"."}, "", + defaultAutoplanFileList, }, { "Should ignore changes in a dir that was deleted", []string{"wasdeleted/main.tf"}, []string{}, "", + defaultAutoplanFileList, }, { "Should not ignore terragrunt.hcl files", []string{"terragrunt.hcl"}, []string{"."}, nestedModules2, + defaultAutoplanFileList, }, { "Should find terragrunt.hcl file inside a nested directory", []string{"project1/terragrunt.hcl"}, []string{"project1"}, nestedModules1, + defaultAutoplanFileList, + }, + { + "Should find packer files and ignore default tf files", + []string{"project1/image.pkr.hcl", "project2/main.tf"}, + []string{"project1"}, + topLevelModules, + "**/*.pkr.hcl", + }, + { + "Should find yaml files in addition to defaults", + []string{"project1/ansible.yml", "project2/main.tf"}, + []string{"project1", "project2"}, + topLevelModules, + "**/*.tf,**/*.yml", + }, + { + "Should find yaml files unless excluded", + []string{"project1/ansible.yml", "project2/config.yml"}, + []string{"project1"}, + topLevelModules, + "**/*.yml,!project2/*.yml", }, } for _, c := range cases { t.Run(c.description, func(t *testing.T) { - projects := m.DetermineProjects(noopLogger, c.files, modifiedRepo, c.repoDir) + projects := m.DetermineProjects(noopLogger, c.files, modifiedRepo, c.repoDir, c.autoplanFileList) // Extract the paths from the projects. We use a slice here instead of a // map so we can test whether there are duplicates returned. diff --git a/server/events_controller_e2e_test.go b/server/events_controller_e2e_test.go index b6fdb17bf..e72829abd 100644 --- a/server/events_controller_e2e_test.go +++ b/server/events_controller_e2e_test.go @@ -718,6 +718,7 @@ func setupE2E(t *testing.T, repoDir string) (server.EventsController, *vcsmocks. commentParser, false, false, + "**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl", ) showStepRunner, err := runtime.NewShowStepRunner(terraformClient, defaultTFVersion) diff --git a/server/server.go b/server/server.go index 902b6ff67..d872e68d5 100644 --- a/server/server.go +++ b/server/server.go @@ -427,6 +427,7 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) { commentParser, userConfig.SkipCloneNoChanges, userConfig.EnableRegExpCmd, + userConfig.AutoplanFileList, ) showStepRunner, err := runtime.NewShowStepRunner(terraformClient, defaultTfVersion) diff --git a/server/user_config.go b/server/user_config.go index d91b24ac8..32be1d5c4 100644 --- a/server/user_config.go +++ b/server/user_config.go @@ -12,6 +12,7 @@ type UserConfig struct { AllowRepoConfig bool `mapstructure:"allow-repo-config"` AtlantisURL string `mapstructure:"atlantis-url"` Automerge bool `mapstructure:"automerge"` + AutoplanFileList string `mapstructure:"autoplan-file-list"` AzureDevopsToken string `mapstructure:"azuredevops-token"` AzureDevopsUser string `mapstructure:"azuredevops-user"` AzureDevopsWebhookPassword string `mapstructure:"azuredevops-webhook-password"`