From f2609fd70bc26ea1cb065d6ced65cbaf6ce96ed2 Mon Sep 17 00:00:00 2001 From: Marc Barlo Date: Fri, 7 Feb 2020 14:21:53 -0800 Subject: [PATCH] Add parallel plans via repo config --- server/events/command_runner.go | 60 ++++++++++++++++++++- server/events/models/models.go | 4 +- server/events/project_command_builder.go | 50 +++++++++-------- server/events/yaml/raw/repo_cfg.go | 26 ++++++--- server/events/yaml/raw/repo_cfg_test.go | 68 +++++++++++++++--------- server/events/yaml/valid/repo_cfg.go | 9 ++-- 6 files changed, 156 insertions(+), 61 deletions(-) diff --git a/server/events/command_runner.go b/server/events/command_runner.go index 9f63690ab..708fd864e 100644 --- a/server/events/command_runner.go +++ b/server/events/command_runner.go @@ -15,6 +15,7 @@ package events import ( "fmt" + "sync" "github.com/google/go-github/v28/github" "github.com/mcdafydd/go-azuredevops/azuredevops" @@ -134,7 +135,15 @@ func (c *DefaultCommandRunner) RunAutoplanCommand(baseRepo models.Repo, headRepo return } - result := c.runProjectCmds(projectCmds, models.PlanCommand) + // Run our plan commands in parallel if enabled + var result CommandResult + if c.parallelPlansEnabled(ctx, projectCmds) { + ctx.Log.Info("Running plans in parallel") + result = c.runProjectCmdsParallel(projectCmds, models.PlanCommand) + } else { + result = c.runProjectCmds(projectCmds, models.PlanCommand) + } + if c.automergeEnabled(ctx, projectCmds) && result.HasErrors() { ctx.Log.Info("deleting plans because there were errors and automerge requires all plans succeed") c.deletePlans(ctx) @@ -244,7 +253,14 @@ func (c *DefaultCommandRunner) RunCommentCommand(baseRepo models.Repo, maybeHead return } - result := c.runProjectCmds(projectCmds, cmd.Name) + // Run our plan commands in parallel if enabled + var result CommandResult + if cmd.Name == models.PlanCommand && c.parallelPlansEnabled(ctx, projectCmds) { + ctx.Log.Info("Running plans in parallel") + result = c.runProjectCmdsParallel(projectCmds, cmd.Name) + } else { + result = c.runProjectCmds(projectCmds, cmd.Name) + } if cmd.Name == models.PlanCommand && c.automergeEnabled(ctx, projectCmds) && result.HasErrors() { ctx.Log.Info("deleting plans because there were errors and automerge requires all plans succeed") c.deletePlans(ctx) @@ -329,6 +345,41 @@ func (c *DefaultCommandRunner) automerge(ctx *CommandContext, pullStatus models. } } +func (c *DefaultCommandRunner) runProjectCmdsParallel(cmds []models.ProjectCommandContext, cmdName models.CommandName) CommandResult { + var results []models.ProjectResult + var wg sync.WaitGroup + mux := &sync.Mutex{} + + wg.Add(len(cmds)) + for _, pCmd := range cmds { + pCmd := pCmd + var execute func() + + switch cmdName { + case models.PlanCommand: + execute = func() { + defer wg.Done() + res := c.ProjectCommandRunner.Plan(pCmd) + mux.Lock() + results = append(results, res) + mux.Unlock() + } + case models.ApplyCommand: + execute = func() { + defer wg.Done() + res := c.ProjectCommandRunner.Apply(pCmd) + mux.Lock() + results = append(results, res) + mux.Unlock() + } + } + go execute() + } + + wg.Wait() + return CommandResult{ProjectResults: results} +} + func (c *DefaultCommandRunner) runProjectCmds(cmds []models.ProjectCommandContext, cmdName models.CommandName) CommandResult { var results []models.ProjectResult for _, pCmd := range cmds { @@ -477,6 +528,11 @@ func (c *DefaultCommandRunner) automergeEnabled(ctx *CommandContext, projectCmds (len(projectCmds) > 0 && projectCmds[0].AutomergeEnabled) } +// parallelPlansEnabled returns true if parallel plans is enabled in this context. +func (c *DefaultCommandRunner) parallelPlansEnabled(ctx *CommandContext, projectCmds []models.ProjectCommandContext) bool { + return len(projectCmds) > 0 && projectCmds[0].ParallelPlansEnabled +} + // automergeComment is the comment that gets posted when Atlantis automatically // merges the PR. var automergeComment = `Automatically merging because all plans have been successfully applied.` diff --git a/server/events/models/models.go b/server/events/models/models.go index 8e1ec5379..07367ccd8 100644 --- a/server/events/models/models.go +++ b/server/events/models/models.go @@ -305,9 +305,11 @@ type ProjectCommandContext struct { // ApplyRequirements is the list of requirements that must be satisfied // before we will run the apply stage. ApplyRequirements []string - // AutoplanEnabled is true if automerge is enabled for the repo that this + // AutomergeEnabled is true if automerge is enabled for the repo that this // project is in. AutomergeEnabled bool + // ParallelPlansEnabled is true if parallel plans is enabled for this project. + ParallelPlansEnabled bool // AutoplanEnabled is true if autoplanning is enabled for this project. AutoplanEnabled bool // BaseRepo is the repository that the pull request will be merged into. diff --git a/server/events/project_command_builder.go b/server/events/project_command_builder.go index f88085da4..622b71f24 100644 --- a/server/events/project_command_builder.go +++ b/server/events/project_command_builder.go @@ -25,6 +25,8 @@ const ( DefaultWorkspace = "default" // DefaultAutomergeEnabled is the default for the automerge setting. DefaultAutomergeEnabled = false + // DefaultParallelPlansEnabled is the default for the parallel plans setting. + DefaultParallelPlansEnabled = false ) //go:generate pegomock generate -m --use-experimental-model-gen --package mocks -o mocks/mock_project_command_builder.go ProjectCommandBuilder @@ -141,7 +143,7 @@ func (p *DefaultProjectCommandBuilder) buildPlanAllCommands(ctx *CommandContext, for _, mp := range matchingProjects { ctx.Log.Debug("determining config for project at dir: %q workspace: %q", mp.Dir, mp.Workspace) mergedCfg := p.GlobalCfg.MergeProjectCfg(ctx.Log, ctx.BaseRepo.ID(), mp, repoCfg) - projCtxs = append(projCtxs, p.buildCtx(ctx, models.PlanCommand, mergedCfg, commentFlags, repoCfg.Automerge, verbose, repoDir)) + projCtxs = append(projCtxs, p.buildCtx(ctx, models.PlanCommand, mergedCfg, commentFlags, repoCfg.Automerge, repoCfg.ParallelPlans, verbose, repoDir)) } } else { // If there is no config file, then we'll plan each project that @@ -152,7 +154,7 @@ func (p *DefaultProjectCommandBuilder) buildPlanAllCommands(ctx *CommandContext, for _, mp := range modifiedProjects { ctx.Log.Debug("determining config for project at dir: %q", mp.Path) pCfg := p.GlobalCfg.DefaultProjCfg(ctx.Log, ctx.BaseRepo.ID(), mp.Path, DefaultWorkspace) - projCtxs = append(projCtxs, p.buildCtx(ctx, models.PlanCommand, pCfg, commentFlags, DefaultAutomergeEnabled, verbose, repoDir)) + projCtxs = append(projCtxs, p.buildCtx(ctx, models.PlanCommand, pCfg, commentFlags, DefaultAutomergeEnabled, repoCfg.ParallelPlans, verbose, repoDir)) } } @@ -283,10 +285,12 @@ func (p *DefaultProjectCommandBuilder) buildProjectCommandCtx( } automerge := DefaultAutomergeEnabled + parallelPlans := DefaultParallelPlansEnabled if repoCfgPtr != nil { automerge = repoCfgPtr.Automerge + parallelPlans = repoCfgPtr.ParallelPlans } - return p.buildCtx(ctx, cmd, projCfg, commentFlags, automerge, verbose, repoDir), nil + return p.buildCtx(ctx, cmd, projCfg, commentFlags, automerge, parallelPlans, verbose, repoDir), nil } // getCfg returns the atlantis.yaml config (if it exists) for this project. If @@ -375,6 +379,7 @@ func (p *DefaultProjectCommandBuilder) buildCtx(ctx *CommandContext, projCfg valid.MergedProjectCfg, commentArgs []string, automergeEnabled bool, + parallelPlansEnabled bool, verbose bool, absRepoDir string) models.ProjectCommandContext { @@ -393,25 +398,26 @@ func (p *DefaultProjectCommandBuilder) buildCtx(ctx *CommandContext, } return models.ProjectCommandContext{ - ApplyCmd: p.CommentBuilder.BuildApplyComment(projCfg.RepoRelDir, projCfg.Workspace, projCfg.Name), - BaseRepo: ctx.BaseRepo, - EscapedCommentArgs: p.escapeArgs(commentArgs), - AutomergeEnabled: automergeEnabled, - AutoplanEnabled: projCfg.AutoplanEnabled, - Steps: steps, - HeadRepo: ctx.HeadRepo, - Log: ctx.Log, - PullMergeable: ctx.PullMergeable, - Pull: ctx.Pull, - ProjectName: projCfg.Name, - ApplyRequirements: projCfg.ApplyRequirements, - RePlanCmd: p.CommentBuilder.BuildPlanComment(projCfg.RepoRelDir, projCfg.Workspace, projCfg.Name, commentArgs), - RepoRelDir: projCfg.RepoRelDir, - RepoConfigVersion: projCfg.RepoCfgVersion, - TerraformVersion: projCfg.TerraformVersion, - User: ctx.User, - Verbose: verbose, - Workspace: projCfg.Workspace, + ApplyCmd: p.CommentBuilder.BuildApplyComment(projCfg.RepoRelDir, projCfg.Workspace, projCfg.Name), + BaseRepo: ctx.BaseRepo, + EscapedCommentArgs: p.escapeArgs(commentArgs), + AutomergeEnabled: automergeEnabled, + ParallelPlansEnabled: parallelPlansEnabled, + AutoplanEnabled: projCfg.AutoplanEnabled, + Steps: steps, + HeadRepo: ctx.HeadRepo, + Log: ctx.Log, + PullMergeable: ctx.PullMergeable, + Pull: ctx.Pull, + ProjectName: projCfg.Name, + ApplyRequirements: projCfg.ApplyRequirements, + RePlanCmd: p.CommentBuilder.BuildPlanComment(projCfg.RepoRelDir, projCfg.Workspace, projCfg.Name, commentArgs), + RepoRelDir: projCfg.RepoRelDir, + RepoConfigVersion: projCfg.RepoCfgVersion, + TerraformVersion: projCfg.TerraformVersion, + User: ctx.User, + Verbose: verbose, + Workspace: projCfg.Workspace, } } diff --git a/server/events/yaml/raw/repo_cfg.go b/server/events/yaml/raw/repo_cfg.go index c33c03cba..6f53667cd 100644 --- a/server/events/yaml/raw/repo_cfg.go +++ b/server/events/yaml/raw/repo_cfg.go @@ -10,12 +10,16 @@ import ( // DefaultAutomerge is the default setting for automerge. const DefaultAutomerge = false +// DefaultParallelPlans is the default setting for parallel plans +const DefaultParallelPlans = false + // RepoCfg is the raw schema for repo-level atlantis.yaml config. type RepoCfg struct { - Version *int `yaml:"version,omitempty"` - Projects []Project `yaml:"projects,omitempty"` - Workflows map[string]Workflow `yaml:"workflows,omitempty"` - Automerge *bool `yaml:"automerge,omitempty"` + Version *int `yaml:"version,omitempty"` + Projects []Project `yaml:"projects,omitempty"` + Workflows map[string]Workflow `yaml:"workflows,omitempty"` + Automerge *bool `yaml:"automerge,omitempty"` + ParallelPlans *bool `yaml:"parallel_plans,omitempty"` } func (r RepoCfg) Validate() error { @@ -52,10 +56,16 @@ func (r RepoCfg) ToValid() valid.RepoCfg { automerge = *r.Automerge } + parallelPlans := DefaultParallelPlans + if r.ParallelPlans != nil { + parallelPlans = *r.ParallelPlans + } + return valid.RepoCfg{ - Version: *r.Version, - Projects: validProjects, - Workflows: validWorkflows, - Automerge: automerge, + Version: *r.Version, + Projects: validProjects, + Workflows: validWorkflows, + Automerge: automerge, + ParallelPlans: parallelPlans, } } diff --git a/server/events/yaml/raw/repo_cfg_test.go b/server/events/yaml/raw/repo_cfg_test.go index 8984f71d2..6ddf47d17 100644 --- a/server/events/yaml/raw/repo_cfg_test.go +++ b/server/events/yaml/raw/repo_cfg_test.go @@ -111,11 +111,22 @@ func TestConfig_UnmarshalYAML(t *testing.T) { }, expErr: "yaml: unmarshal errors:\n line 2: cannot unmarshal !!str `notabool` into bool", }, + { + description: "parallel plans not a boolean", + input: "version: 3\nparallel_plans: notabool", + exp: raw.RepoCfg{ + Version: nil, + Projects: nil, + Workflows: nil, + }, + expErr: "yaml: unmarshal errors:\n line 2: cannot unmarshal !!str `notabool` into bool", + }, { description: "should use values if set", input: ` version: 3 automerge: true +parallel_plans: true projects: - dir: mydir workspace: myworkspace @@ -132,8 +143,9 @@ workflows: apply: steps: []`, exp: raw.RepoCfg{ - Version: Int(3), - Automerge: Bool(true), + Version: Int(3), + Automerge: Bool(true), + ParallelPlans: Bool(true), Projects: []raw.Project{ { Dir: String("mydir"), @@ -236,38 +248,43 @@ func TestConfig_ToValid(t *testing.T) { }, }, { - description: "automerge omitted", + description: "automerge and parallel_plans omitted", input: raw.RepoCfg{ Version: Int(2), }, exp: valid.RepoCfg{ - Version: 2, - Automerge: false, - Workflows: map[string]valid.Workflow{}, + Version: 2, + Automerge: false, + ParallelPlans: false, + Workflows: map[string]valid.Workflow{}, }, }, { - description: "automerge true", + description: "automerge and parallel_plans true", input: raw.RepoCfg{ - Version: Int(2), - Automerge: Bool(true), + Version: Int(2), + Automerge: Bool(true), + ParallelPlans: Bool(true), }, exp: valid.RepoCfg{ - Version: 2, - Automerge: true, - Workflows: map[string]valid.Workflow{}, + Version: 2, + Automerge: true, + ParallelPlans: true, + Workflows: map[string]valid.Workflow{}, }, }, { - description: "automerge false", + description: "automerge and parallel_plans false", input: raw.RepoCfg{ - Version: Int(2), - Automerge: Bool(false), + Version: Int(2), + Automerge: Bool(false), + ParallelPlans: Bool(false), }, exp: valid.RepoCfg{ - Version: 2, - Automerge: false, - Workflows: map[string]valid.Workflow{}, + Version: 2, + Automerge: false, + ParallelPlans: false, + Workflows: map[string]valid.Workflow{}, }, }, { @@ -282,8 +299,9 @@ func TestConfig_ToValid(t *testing.T) { }, }, exp: valid.RepoCfg{ - Version: 2, - Automerge: false, + Version: 2, + Automerge: false, + ParallelPlans: false, Workflows: map[string]valid.Workflow{ "myworkflow": { Name: "myworkflow", @@ -302,8 +320,9 @@ func TestConfig_ToValid(t *testing.T) { { description: "everything set", input: raw.RepoCfg{ - Version: Int(2), - Automerge: Bool(true), + Version: Int(2), + Automerge: Bool(true), + ParallelPlans: Bool(true), Workflows: map[string]raw.Workflow{ "myworkflow": { Apply: &raw.Stage{ @@ -329,8 +348,9 @@ func TestConfig_ToValid(t *testing.T) { }, }, exp: valid.RepoCfg{ - Version: 2, - Automerge: true, + Version: 2, + Automerge: true, + ParallelPlans: true, Workflows: map[string]valid.Workflow{ "myworkflow": { Name: "myworkflow", diff --git a/server/events/yaml/valid/repo_cfg.go b/server/events/yaml/valid/repo_cfg.go index 8fa20aeba..09cd89c5c 100644 --- a/server/events/yaml/valid/repo_cfg.go +++ b/server/events/yaml/valid/repo_cfg.go @@ -7,10 +7,11 @@ import version "github.com/hashicorp/go-version" // RepoCfg is the atlantis.yaml config after it's been parsed and validated. type RepoCfg struct { // Version is the version of the atlantis YAML file. - Version int - Projects []Project - Workflows map[string]Workflow - Automerge bool + Version int + Projects []Project + Workflows map[string]Workflow + Automerge bool + ParallelPlans bool } func (r RepoCfg) FindProjectsByDirWorkspace(repoRelDir string, workspace string) []Project {