Add back the parallel_plan repo cfg option

+ Default parallel plans breaks in a lot of autoplan scenarios i.e. with shared workspaces
+ Added a set of e2e tests to check parallel plans and applies
+ Also added an option to the Terraform client to turn off the plugin cache. This was breaking the e2e test suite (never seen this error in practice)

server/events/command_runner.go

@@ -145,7 +145,14 @@ func (c *DefaultCommandRunner) RunAutoplanCommand(baseRepo models.Repo, headRepo
 		ctx.Log.Warn("unable to update commit status: %s", err)
 	}
 
-	result := c.runProjectCmdsParallel(projectCmds, models.PlanCommand)
+	// Only run commands in parallel if enabled
+	var result CommandResult
+	if c.parallelPlanEnabled(ctx, projectCmds) {
+		ctx.Log.Info("Running plans in parallel")
+		result = c.runProjectCmdsParallel(projectCmds, models.PlanCommand)
+	} else {
+		result = c.runProjectCmds(projectCmds, models.PlanCommand)
+	}
 
 	if c.automergeEnabled(ctx, projectCmds) && result.HasErrors() {
 		ctx.Log.Info("deleting plans because there were errors and automerge requires all plans succeed")
@@ -257,14 +264,18 @@ func (c *DefaultCommandRunner) RunCommentCommand(baseRepo models.Repo, maybeHead
 		return
 	}
 
-	// Only run applies in parallel if enabled
+	// Only run commands in parallel if enabled
 	var result CommandResult
-	if cmd.Name == models.ApplyCommand && !c.parallelApplyEnabled(ctx, projectCmds) {
-		result = c.runProjectCmds(projectCmds, cmd.Name)
-	} else {
-		ctx.Log.Info("Running commands in parallel")
+	if cmd.Name == models.ApplyCommand && c.parallelApplyEnabled(ctx, projectCmds) {
+		ctx.Log.Info("Running applies in parallel")
 		result = c.runProjectCmdsParallel(projectCmds, cmd.Name)
+	} else if cmd.Name == models.PlanCommand && c.parallelPlanEnabled(ctx, projectCmds) {
+		ctx.Log.Info("Running plans in parallel")
+		result = c.runProjectCmdsParallel(projectCmds, cmd.Name)
+	} else {
+		result = c.runProjectCmds(projectCmds, cmd.Name)
 	}
+
 	if cmd.Name == models.PlanCommand && c.automergeEnabled(ctx, projectCmds) && result.HasErrors() {
 		ctx.Log.Info("deleting plans because there were errors and automerge requires all plans succeed")
 		c.deletePlans(ctx)
@@ -546,6 +557,11 @@ func (c *DefaultCommandRunner) parallelApplyEnabled(ctx *CommandContext, project
 	return len(projectCmds) > 0 && projectCmds[0].ParallelApplyEnabled
 }
 
+// parallelPlanEnabled returns true if parallel plan is enabled in this context.
+func (c *DefaultCommandRunner) parallelPlanEnabled(ctx *CommandContext, projectCmds []models.ProjectCommandContext) bool {
+	return len(projectCmds) > 0 && projectCmds[0].ParallelPlanEnabled
+}
+
 // automergeComment is the comment that gets posted when Atlantis automatically
 // merges the PR.
 var automergeComment = `Automatically merging because all plans have been successfully applied.`

server/events/models/models.go

@@ -310,6 +310,8 @@ type ProjectCommandContext struct {
 	AutomergeEnabled bool
 	// ParallelApplyEnabled is true if parallel apply is enabled for this project.
 	ParallelApplyEnabled bool
+	// ParallelPlanEnabled is true if parallel plan is enabled for this project.
+	ParallelPlanEnabled bool
 	// AutoplanEnabled is true if autoplanning is enabled for this project.
 	AutoplanEnabled bool
 	// BaseRepo is the repository that the pull request will be merged into.

server/events/project_command_builder.go

@@ -27,6 +27,8 @@ const (
 	DefaultAutomergeEnabled = false
 	// DefaultParallelApplyEnabled is the default for the parallel apply setting.
 	DefaultParallelApplyEnabled = false
+	// DefaultParallelPlanEnabled is the default for the parallel plan setting.
+	DefaultParallelPlanEnabled = false
 )
 
 //go:generate pegomock generate -m --use-experimental-model-gen --package mocks -o mocks/mock_project_command_builder.go ProjectCommandBuilder
@@ -143,7 +145,7 @@ func (p *DefaultProjectCommandBuilder) buildPlanAllCommands(ctx *CommandContext,
 		for _, mp := range matchingProjects {
 			ctx.Log.Debug("determining config for project at dir: %q workspace: %q", mp.Dir, mp.Workspace)
 			mergedCfg := p.GlobalCfg.MergeProjectCfg(ctx.Log, ctx.BaseRepo.ID(), mp, repoCfg)
-			projCtxs = append(projCtxs, p.buildCtx(ctx, models.PlanCommand, mergedCfg, commentFlags, repoCfg.Automerge, repoCfg.ParallelApply, verbose, repoDir))
+			projCtxs = append(projCtxs, p.buildCtx(ctx, models.PlanCommand, mergedCfg, commentFlags, repoCfg.Automerge, repoCfg.ParallelApply, repoCfg.ParallelPlan, verbose, repoDir))
 		}
 	} else {
 		// If there is no config file, then we'll plan each project that
@@ -154,7 +156,7 @@ func (p *DefaultProjectCommandBuilder) buildPlanAllCommands(ctx *CommandContext,
 		for _, mp := range modifiedProjects {
 			ctx.Log.Debug("determining config for project at dir: %q", mp.Path)
 			pCfg := p.GlobalCfg.DefaultProjCfg(ctx.Log, ctx.BaseRepo.ID(), mp.Path, DefaultWorkspace)
-			projCtxs = append(projCtxs, p.buildCtx(ctx, models.PlanCommand, pCfg, commentFlags, DefaultAutomergeEnabled, DefaultParallelApplyEnabled, verbose, repoDir))
+			projCtxs = append(projCtxs, p.buildCtx(ctx, models.PlanCommand, pCfg, commentFlags, DefaultAutomergeEnabled, DefaultParallelApplyEnabled, DefaultParallelPlanEnabled, verbose, repoDir))
 		}
 	}
 
@@ -286,11 +288,13 @@ func (p *DefaultProjectCommandBuilder) buildProjectCommandCtx(
 
 	automerge := DefaultAutomergeEnabled
 	parallelApply := DefaultParallelApplyEnabled
+	parallelPlan := DefaultParallelPlanEnabled
 	if repoCfgPtr != nil {
 		automerge = repoCfgPtr.Automerge
 		parallelApply = repoCfgPtr.ParallelApply
+		parallelPlan = repoCfgPtr.ParallelPlan
 	}
-	return p.buildCtx(ctx, cmd, projCfg, commentFlags, automerge, parallelApply, verbose, repoDir), nil
+	return p.buildCtx(ctx, cmd, projCfg, commentFlags, automerge, parallelApply, parallelPlan, verbose, repoDir), nil
 }
 
 // getCfg returns the atlantis.yaml config (if it exists) for this project. If
@@ -380,6 +384,7 @@ func (p *DefaultProjectCommandBuilder) buildCtx(ctx *CommandContext,
 	commentArgs []string,
 	automergeEnabled bool,
 	parallelApplyEnabled bool,
+	parallelPlanEnabled bool,
 	verbose bool,
 	absRepoDir string) models.ProjectCommandContext {
 
@@ -403,6 +408,7 @@ func (p *DefaultProjectCommandBuilder) buildCtx(ctx *CommandContext,
 		EscapedCommentArgs:   p.escapeArgs(commentArgs),
 		AutomergeEnabled:     automergeEnabled,
 		ParallelApplyEnabled: parallelApplyEnabled,
+		ParallelPlanEnabled:  parallelPlanEnabled,
 		AutoplanEnabled:      projCfg.AutoplanEnabled,
 		Steps:                steps,
 		HeadRepo:             ctx.HeadRepo,

server/events/terraform/terraform_client.go

@@ -65,6 +65,9 @@ type DefaultClient struct {
 
 	// versionsLock is used to ensure versions isn't being concurrently written to.
 	versionsLock *sync.Mutex
+
+	// usePluginCache determines whether or not to set the TF_PLUGIN_CACHE_DIR env var
+	usePluginCache bool
 }
 
 //go:generate pegomock generate -m --use-experimental-model-gen --package mocks -o mocks/mock_downloader.go Downloader
@@ -107,7 +110,8 @@ func NewClient(
 	defaultVersionStr string,
 	defaultVersionFlagName string,
 	tfDownloadURL string,
-	tfDownloader Downloader) (*DefaultClient, error) {
+	tfDownloader Downloader,
+	usePluginCache bool) (*DefaultClient, error) {
 	var finalDefaultVersion *version.Version
 	var localVersion *version.Version
 	versions := make(map[string]string)
@@ -179,6 +183,7 @@ func NewClient(
 		downloadBaseURL:         tfDownloadURL,
 		versionsLock:            &versionsLock,
 		versions:                versions,
+		usePluginCache:          usePluginCache,
 	}, nil
 }
 
@@ -259,11 +264,13 @@ func (c *DefaultClient) prepCmd(log *logging.SimpleLogger, v *version.Version, w
 		// Will de-emphasize specific commands to run in output.
 		"TF_IN_AUTOMATION=true",
 		// Cache plugins so terraform init runs faster.
-		fmt.Sprintf("TF_PLUGIN_CACHE_DIR=%s", c.terraformPluginCacheDir),
 		fmt.Sprintf("WORKSPACE=%s", workspace),
 		fmt.Sprintf("ATLANTIS_TERRAFORM_VERSION=%s", v.String()),
 		fmt.Sprintf("DIR=%s", path),
 	}
+	if c.usePluginCache {
+		envVars = append(envVars, fmt.Sprintf("TF_PLUGIN_CACHE_DIR=%s", c.terraformPluginCacheDir))
+	}
 	// Append current Atlantis process's environment variables, ex.
 	// AWS_ACCESS_KEY.
 	envVars = append(envVars, os.Environ()...)

server/events/terraform/terraform_client_internal_test.go

@@ -94,6 +94,7 @@ func TestDefaultClient_RunCommandWithVersion_EnvVars(t *testing.T) {
 		defaultVersion:          v,
 		terraformPluginCacheDir: tmp,
 		overrideTF:              "echo",
+		usePluginCache:          true,
 	}
 
 	args := []string{
@@ -143,6 +144,7 @@ func TestDefaultClient_RunCommandAsync_Success(t *testing.T) {
 		defaultVersion:          v,
 		terraformPluginCacheDir: tmp,
 		overrideTF:              "echo",
+		usePluginCache:          true,
 	}
 
 	args := []string{

server/events/terraform/terraform_client_test.go

@@ -68,7 +68,7 @@ is 0.11.13. You can update by downloading from www.terraform.io/downloads.html
 	Ok(t, err)
 	defer tempSetEnv(t, "PATH", fmt.Sprintf("%s:%s", tmp, os.Getenv("PATH")))()
 
-	c, err := terraform.NewClient(nil, tmp, "", "", "", cmd.DefaultTFVersionFlag, cmd.DefaultTFDownloadURL, nil)
+	c, err := terraform.NewClient(nil, tmp, "", "", "", cmd.DefaultTFVersionFlag, cmd.DefaultTFDownloadURL, nil, true)
 	Ok(t, err)
 
 	Ok(t, err)
@@ -96,7 +96,7 @@ is 0.11.13. You can update by downloading from www.terraform.io/downloads.html
 	Ok(t, err)
 	defer tempSetEnv(t, "PATH", fmt.Sprintf("%s:%s", tmp, os.Getenv("PATH")))()
 
-	c, err := terraform.NewClient(nil, tmp, "", "", "0.11.10", cmd.DefaultTFVersionFlag, cmd.DefaultTFDownloadURL, nil)
+	c, err := terraform.NewClient(nil, tmp, "", "", "0.11.10", cmd.DefaultTFVersionFlag, cmd.DefaultTFDownloadURL, nil, true)
 	Ok(t, err)
 
 	Ok(t, err)
@@ -116,7 +116,7 @@ func TestNewClient_NoTF(t *testing.T) {
 	// Set PATH to only include our empty directory.
 	defer tempSetEnv(t, "PATH", tmp)()
 
-	_, err := terraform.NewClient(nil, tmp, "", "", "", cmd.DefaultTFVersionFlag, cmd.DefaultTFDownloadURL, nil)
+	_, err := terraform.NewClient(nil, tmp, "", "", "", cmd.DefaultTFVersionFlag, cmd.DefaultTFDownloadURL, nil, true)
 	ErrEquals(t, "terraform not found in $PATH. Set --default-tf-version or download terraform from https://www.terraform.io/downloads.html", err)
 }
 
@@ -133,7 +133,7 @@ func TestNewClient_DefaultTFFlagInPath(t *testing.T) {
 	Ok(t, err)
 	defer tempSetEnv(t, "PATH", fmt.Sprintf("%s:%s", tmp, os.Getenv("PATH")))()
 
-	c, err := terraform.NewClient(nil, tmp, "", "", "0.11.10", cmd.DefaultTFVersionFlag, cmd.DefaultTFDownloadURL, nil)
+	c, err := terraform.NewClient(nil, tmp, "", "", "0.11.10", cmd.DefaultTFVersionFlag, cmd.DefaultTFDownloadURL, nil, true)
 	Ok(t, err)
 
 	Ok(t, err)
@@ -157,7 +157,7 @@ func TestNewClient_DefaultTFFlagInBinDir(t *testing.T) {
 	Ok(t, err)
 	defer tempSetEnv(t, "PATH", fmt.Sprintf("%s:%s", tmp, os.Getenv("PATH")))()
 
-	c, err := terraform.NewClient(logging.NewNoopLogger(), tmp, "", "", "0.11.10", cmd.DefaultTFVersionFlag, cmd.DefaultTFDownloadURL, nil)
+	c, err := terraform.NewClient(logging.NewNoopLogger(), tmp, "", "", "0.11.10", cmd.DefaultTFVersionFlag, cmd.DefaultTFDownloadURL, nil, true)
 	Ok(t, err)
 
 	Ok(t, err)
@@ -183,7 +183,7 @@ func TestNewClient_DefaultTFFlagDownload(t *testing.T) {
 		err := ioutil.WriteFile(params[0].(string), []byte("#!/bin/sh\necho '\nTerraform v0.11.10\n'"), 0755)
 		return []pegomock.ReturnValue{err}
 	})
-	c, err := terraform.NewClient(nil, tmp, "", "", "0.11.10", cmd.DefaultTFVersionFlag, "https://my-mirror.releases.mycompany.com", mockDownloader)
+	c, err := terraform.NewClient(nil, tmp, "", "", "0.11.10", cmd.DefaultTFVersionFlag, "https://my-mirror.releases.mycompany.com", mockDownloader, true)
 	Ok(t, err)
 
 	Ok(t, err)
@@ -207,7 +207,7 @@ func TestNewClient_DefaultTFFlagDownload(t *testing.T) {
 func TestNewClient_BadVersion(t *testing.T) {
 	tmp, cleanup := TempDir(t)
 	defer cleanup()
-	_, err := terraform.NewClient(nil, tmp, "", "", "malformed", cmd.DefaultTFVersionFlag, cmd.DefaultTFDownloadURL, nil)
+	_, err := terraform.NewClient(nil, tmp, "", "", "malformed", cmd.DefaultTFVersionFlag, cmd.DefaultTFDownloadURL, nil, true)
 	ErrEquals(t, "Malformed version: malformed", err)
 }
 
@@ -230,7 +230,7 @@ func TestRunCommandWithVersion_DLsTF(t *testing.T) {
 		return []pegomock.ReturnValue{err}
 	})
 
-	c, err := terraform.NewClient(nil, tmp, "", "", "0.11.10", cmd.DefaultTFVersionFlag, cmd.DefaultTFDownloadURL, mockDownloader)
+	c, err := terraform.NewClient(nil, tmp, "", "", "0.11.10", cmd.DefaultTFVersionFlag, cmd.DefaultTFDownloadURL, mockDownloader, true)
 	Ok(t, err)
 	Equals(t, "0.11.10", c.DefaultVersion().String())
 
@@ -249,7 +249,7 @@ func TestEnsureVersion_downloaded(t *testing.T) {
 
 	mockDownloader := mocks.NewMockDownloader()
 
-	c, err := terraform.NewClient(nil, tmp, "", "", "0.11.10", cmd.DefaultTFVersionFlag, cmd.DefaultTFDownloadURL, mockDownloader)
+	c, err := terraform.NewClient(nil, tmp, "", "", "0.11.10", cmd.DefaultTFVersionFlag, cmd.DefaultTFDownloadURL, mockDownloader, true)
 	Ok(t, err)
 
 	Equals(t, "0.11.10", c.DefaultVersion().String())

server/events/yaml/raw/repo_cfg.go

@@ -13,6 +13,9 @@ const DefaultAutomerge = false
 // DefaultParallelApply is the default setting for parallel apply
 const DefaultParallelApply = false
 
+// DefaultParallelPlan is the default setting for parallel plan
+const DefaultParallelPlan = false
+
 // RepoCfg is the raw schema for repo-level atlantis.yaml config.
 type RepoCfg struct {
 	Version       *int                `yaml:"version,omitempty"`
@@ -20,6 +23,7 @@ type RepoCfg struct {
 	Workflows     map[string]Workflow `yaml:"workflows,omitempty"`
 	Automerge     *bool               `yaml:"automerge,omitempty"`
 	ParallelApply *bool               `yaml:"parallel_apply,omitempty"`
+	ParallelPlan  *bool               `yaml:"parallel_plan,omitempty"`
 }
 
 func (r RepoCfg) Validate() error {
@@ -61,11 +65,17 @@ func (r RepoCfg) ToValid() valid.RepoCfg {
 		parallelApply = *r.ParallelApply
 	}
 
+	parallelPlan := DefaultParallelPlan
+	if r.ParallelPlan != nil {
+		parallelPlan = *r.ParallelPlan
+	}
+
 	return valid.RepoCfg{
 		Version:       *r.Version,
 		Projects:      validProjects,
 		Workflows:     validWorkflows,
 		Automerge:     automerge,
 		ParallelApply: parallelApply,
+		ParallelPlan:  parallelPlan,
 	}
 }

server/events/yaml/raw/repo_cfg_test.go

@@ -127,6 +127,7 @@ func TestConfig_UnmarshalYAML(t *testing.T) {
 version: 3
 automerge: true
 parallel_apply: true
+parallel_plan: false
 projects:
 - dir: mydir
   workspace: myworkspace
@@ -146,6 +147,7 @@ workflows:
 				Version:       Int(3),
 				Automerge:     Bool(true),
 				ParallelApply: Bool(true),
+				ParallelPlan:  Bool(false),
 				Projects: []raw.Project{
 					{
 						Dir:              String("mydir"),

server/events/yaml/valid/repo_cfg.go

@@ -12,6 +12,7 @@ type RepoCfg struct {
 	Workflows     map[string]Workflow
 	Automerge     bool
 	ParallelApply bool
+	ParallelPlan  bool
 }
 
 func (r RepoCfg) FindProjectsByDirWorkspace(repoRelDir string, workspace string) []Project {

server/events_controller_e2e_test.go

@@ -60,9 +60,12 @@ func TestGitHubWorkflow(t *testing.T) {
 		ExpAutomerge bool
 		// ExpAutoplan is true if we expect Atlantis to autoplan.
 		ExpAutoplan bool
+		// ExpParallel is true if we expect Atlantis to run parallel plans or applies.
+		ExpParallel bool
 		// ExpReplies is a list of files containing the expected replies that
-		// Atlantis writes to the pull request in order.
-		ExpReplies []string
+		// Atlantis writes to the pull request in order. A reply from a parallel operation
+		// will be matched using a substring check.
+		ExpReplies [][]string
 	}{
 		{
 			Description:   "simple",
@@ -71,10 +74,10 @@ func TestGitHubWorkflow(t *testing.T) {
 			Comments: []string{
 				"atlantis apply",
 			},
-			ExpReplies: []string{
-				"exp-output-autoplan.txt",
-				"exp-output-apply.txt",
-				"exp-output-merge.txt",
+			ExpReplies: [][]string{
+				[]string{"exp-output-autoplan.txt"},
+				[]string{"exp-output-apply.txt"},
+				[]string{"exp-output-merge.txt"},
 			},
 			ExpAutoplan: true,
 		},
@@ -87,11 +90,11 @@ func TestGitHubWorkflow(t *testing.T) {
 				"atlantis plan",
 				"atlantis apply",
 			},
-			ExpReplies: []string{
-				"exp-output-autoplan.txt",
-				"exp-output-autoplan.txt",
-				"exp-output-apply.txt",
-				"exp-output-merge.txt",
+			ExpReplies: [][]string{
+				[]string{"exp-output-autoplan.txt"},
+				[]string{"exp-output-autoplan.txt"},
+				[]string{"exp-output-apply.txt"},
+				[]string{"exp-output-merge.txt"},
 			},
 		},
 		{
@@ -103,11 +106,11 @@ func TestGitHubWorkflow(t *testing.T) {
 				"atlantis plan -- -var var=overridden",
 				"atlantis apply",
 			},
-			ExpReplies: []string{
-				"exp-output-autoplan.txt",
-				"exp-output-atlantis-plan-var-overridden.txt",
-				"exp-output-apply-var.txt",
-				"exp-output-merge.txt",
+			ExpReplies: [][]string{
+				[]string{"exp-output-autoplan.txt"},
+				[]string{"exp-output-atlantis-plan-var-overridden.txt"},
+				[]string{"exp-output-apply-var.txt"},
+				[]string{"exp-output-merge.txt"},
 			},
 		},
 		{
@@ -121,13 +124,13 @@ func TestGitHubWorkflow(t *testing.T) {
 				"atlantis apply -w default",
 				"atlantis apply -w new_workspace",
 			},
-			ExpReplies: []string{
-				"exp-output-autoplan.txt",
-				"exp-output-atlantis-plan.txt",
-				"exp-output-atlantis-plan-new-workspace.txt",
-				"exp-output-apply-var-default-workspace.txt",
-				"exp-output-apply-var-new-workspace.txt",
-				"exp-output-merge-workspaces.txt",
+			ExpReplies: [][]string{
+				[]string{"exp-output-autoplan.txt"},
+				[]string{"exp-output-atlantis-plan.txt"},
+				[]string{"exp-output-atlantis-plan-new-workspace.txt"},
+				[]string{"exp-output-apply-var-default-workspace.txt"},
+				[]string{"exp-output-apply-var-new-workspace.txt"},
+				[]string{"exp-output-merge-workspaces.txt"},
 			},
 		},
 		{
@@ -140,12 +143,12 @@ func TestGitHubWorkflow(t *testing.T) {
 				"atlantis plan -w new_workspace -- -var var=new_workspace",
 				"atlantis apply",
 			},
-			ExpReplies: []string{
-				"exp-output-autoplan.txt",
-				"exp-output-atlantis-plan.txt",
-				"exp-output-atlantis-plan-new-workspace.txt",
-				"exp-output-apply-var-all.txt",
-				"exp-output-merge-workspaces.txt",
+			ExpReplies: [][]string{
+				[]string{"exp-output-autoplan.txt"},
+				[]string{"exp-output-atlantis-plan.txt"},
+				[]string{"exp-output-atlantis-plan-new-workspace.txt"},
+				[]string{"exp-output-apply-var-all.txt"},
+				[]string{"exp-output-merge-workspaces.txt"},
 			},
 		},
 		{
@@ -157,11 +160,11 @@ func TestGitHubWorkflow(t *testing.T) {
 				"atlantis apply -w staging",
 				"atlantis apply -w default",
 			},
-			ExpReplies: []string{
-				"exp-output-autoplan.txt",
-				"exp-output-apply-staging.txt",
-				"exp-output-apply-default.txt",
-				"exp-output-merge.txt",
+			ExpReplies: [][]string{
+				[]string{"exp-output-autoplan.txt"},
+				[]string{"exp-output-apply-staging.txt"},
+				[]string{"exp-output-apply-default.txt"},
+				[]string{"exp-output-merge.txt"},
 			},
 		},
 		{
@@ -172,10 +175,10 @@ func TestGitHubWorkflow(t *testing.T) {
 			Comments: []string{
 				"atlantis apply",
 			},
-			ExpReplies: []string{
-				"exp-output-autoplan.txt",
-				"exp-output-apply-all.txt",
-				"exp-output-merge.txt",
+			ExpReplies: [][]string{
+				[]string{"exp-output-autoplan.txt"},
+				[]string{"exp-output-apply-all.txt"},
+				[]string{"exp-output-merge.txt"},
 			},
 		},
 		{
@@ -187,11 +190,11 @@ func TestGitHubWorkflow(t *testing.T) {
 				"atlantis plan",
 				"atlantis apply",
 			},
-			ExpReplies: []string{
-				"exp-output-autoplan.txt",
-				"exp-output-autoplan.txt",
-				"exp-output-apply-all.txt",
-				"exp-output-merge.txt",
+			ExpReplies: [][]string{
+				[]string{"exp-output-autoplan.txt"},
+				[]string{"exp-output-autoplan.txt"},
+				[]string{"exp-output-apply-all.txt"},
+				[]string{"exp-output-merge.txt"},
 			},
 		},
 		{
@@ -202,10 +205,10 @@ func TestGitHubWorkflow(t *testing.T) {
 			Comments: []string{
 				"atlantis apply -d staging",
 			},
-			ExpReplies: []string{
-				"exp-output-autoplan-only-staging.txt",
-				"exp-output-apply-staging.txt",
-				"exp-output-merge-only-staging.txt",
+			ExpReplies: [][]string{
+				[]string{"exp-output-autoplan-only-staging.txt"},
+				[]string{"exp-output-apply-staging.txt"},
+				[]string{"exp-output-merge-only-staging.txt"},
 			},
 		},
 		{
@@ -219,12 +222,12 @@ func TestGitHubWorkflow(t *testing.T) {
 				"atlantis apply -d staging",
 				"atlantis apply -d production",
 			},
-			ExpReplies: []string{
-				"exp-output-plan-staging.txt",
-				"exp-output-plan-production.txt",
-				"exp-output-apply-staging.txt",
-				"exp-output-apply-production.txt",
-				"exp-output-merge-all-dirs.txt",
+			ExpReplies: [][]string{
+				[]string{"exp-output-plan-staging.txt"},
+				[]string{"exp-output-plan-production.txt"},
+				[]string{"exp-output-apply-staging.txt"},
+				[]string{"exp-output-apply-production.txt"},
+				[]string{"exp-output-merge-all-dirs.txt"},
 			},
 		},
 		{
@@ -236,11 +239,11 @@ func TestGitHubWorkflow(t *testing.T) {
 				"atlantis apply -d staging",
 				"atlantis apply -d production",
 			},
-			ExpReplies: []string{
-				"exp-output-autoplan.txt",
-				"exp-output-apply-staging.txt",
-				"exp-output-apply-production.txt",
-				"exp-output-merge.txt",
+			ExpReplies: [][]string{
+				[]string{"exp-output-autoplan.txt"},
+				[]string{"exp-output-apply-staging.txt"},
+				[]string{"exp-output-apply-production.txt"},
+				[]string{"exp-output-merge.txt"},
 			},
 		},
 		{
@@ -252,11 +255,11 @@ func TestGitHubWorkflow(t *testing.T) {
 				"atlantis apply -p staging",
 				"atlantis apply -p default",
 			},
-			ExpReplies: []string{
-				"exp-output-autoplan.txt",
-				"exp-output-apply-staging.txt",
-				"exp-output-apply-default.txt",
-				"exp-output-merge.txt",
+			ExpReplies: [][]string{
+				[]string{"exp-output-autoplan.txt"},
+				[]string{"exp-output-apply-staging.txt"},
+				[]string{"exp-output-apply-default.txt"},
+				[]string{"exp-output-merge.txt"},
 			},
 		},
 		{
@@ -270,12 +273,12 @@ func TestGitHubWorkflow(t *testing.T) {
 				"atlantis apply -p staging",
 				"atlantis apply -p default",
 			},
-			ExpReplies: []string{
-				"exp-output-plan-staging.txt",
-				"exp-output-plan-default.txt",
-				"exp-output-apply-staging.txt",
-				"exp-output-apply-default.txt",
-				"exp-output-merge.txt",
+			ExpReplies: [][]string{
+				[]string{"exp-output-plan-staging.txt"},
+				[]string{"exp-output-plan-default.txt"},
+				[]string{"exp-output-apply-staging.txt"},
+				[]string{"exp-output-apply-default.txt"},
+				[]string{"exp-output-merge.txt"},
 			},
 		},
 		{
@@ -288,12 +291,12 @@ func TestGitHubWorkflow(t *testing.T) {
 				"atlantis apply -d dir1",
 				"atlantis apply -d dir2",
 			},
-			ExpReplies: []string{
-				"exp-output-autoplan.txt",
-				"exp-output-apply-dir1.txt",
-				"exp-output-apply-dir2.txt",
-				"exp-output-automerge.txt",
-				"exp-output-merge.txt",
+			ExpReplies: [][]string{
+				[]string{"exp-output-autoplan.txt"},
+				[]string{"exp-output-apply-dir1.txt"},
+				[]string{"exp-output-apply-dir2.txt"},
+				[]string{"exp-output-automerge.txt"},
+				[]string{"exp-output-merge.txt"},
 			},
 		},
 		{
@@ -306,11 +309,26 @@ func TestGitHubWorkflow(t *testing.T) {
 				"atlantis apply -w staging",
 				"atlantis apply -w default",
 			},
-			ExpReplies: []string{
-				"exp-output-autoplan.txt",
-				"exp-output-apply-staging-workspace.txt",
-				"exp-output-apply-default-workspace.txt",
-				"exp-output-merge.txt",
+			ExpReplies: [][]string{
+				[]string{"exp-output-autoplan.txt"},
+				[]string{"exp-output-apply-staging-workspace.txt"},
+				[]string{"exp-output-apply-default-workspace.txt"},
+				[]string{"exp-output-merge.txt"},
+			},
+		},
+		{
+			Description:   "workspaces parallel with atlantis.yaml",
+			RepoDir:       "workspace-parallel-yaml",
+			ModifiedFiles: []string{"production/main.tf", "staging/main.tf"},
+			ExpAutoplan:   true,
+			ExpParallel:   true,
+			Comments: []string{
+				"atlantis apply",
+			},
+			ExpReplies: [][]string{
+				[]string{"exp-output-autoplan-staging.txt", "exp-output-autoplan-production.txt"},
+				[]string{"exp-output-apply-all-staging.txt", "exp-output-apply-all-production.txt"},
+				[]string{"exp-output-merge.txt"},
 			},
 		},
 	}
@@ -364,7 +382,7 @@ func TestGitHubWorkflow(t *testing.T) {
 			_, _, actReplies := vcsClient.VerifyWasCalled(Times(expNumReplies)).CreateComment(AnyRepo(), AnyInt(), AnyString()).GetAllCapturedArguments()
 			Assert(t, len(c.ExpReplies) == len(actReplies), "missing expected replies, got %d but expected %d", len(actReplies), len(c.ExpReplies))
 			for i, expReply := range c.ExpReplies {
-				assertCommentEquals(t, expReply, actReplies[i], c.RepoDir)
+				assertCommentEquals(t, expReply, actReplies[i], c.RepoDir, c.ExpParallel)
 			}
 
 			if c.ExpAutomerge {
@@ -400,7 +418,7 @@ func setupE2E(t *testing.T, repoDir string) (server.EventsController, *vcsmocks.
 		GithubUser: "github-user",
 		GitlabUser: "gitlab-user",
 	}
-	terraformClient, err := terraform.NewClient(logger, dataDir, "", "", "", "default-tf-version", "https://releases.hashicorp.com", &NoopTFDownloader{})
+	terraformClient, err := terraform.NewClient(logger, dataDir, "", "", "", "default-tf-version", "https://releases.hashicorp.com", &NoopTFDownloader{}, false)
 	Ok(t, err)
 	boltdb, err := db.New(dataDir)
 	Ok(t, err)
@@ -617,10 +635,8 @@ func runCmd(t *testing.T, dir string, name string, args ...string) string {
 	return string(cpOut)
 }
 
-func assertCommentEquals(t *testing.T, expFile string, act string, repoDir string) {
+func assertCommentEquals(t *testing.T, expReplies []string, act string, repoDir string, parallel bool) {
 	t.Helper()
-	exp, err := ioutil.ReadFile(filepath.Join(absRepoPath(t, repoDir), expFile))
-	Ok(t, err)
 
 	// Replace all 'Creation complete after 0s [id=2135833172528078362]' strings with
 	// 'Creation complete after *s [id=*******************]' so we can do a comparison.
@@ -633,29 +649,45 @@ func assertCommentEquals(t *testing.T, expFile string, act string, repoDir strin
 	resourceRegex := regexp.MustCompile(`null_resource\.simple(\[\d])?\d?:.*`)
 	act = resourceRegex.ReplaceAllString(act, "null_resource.simple:")
 
-	expStr := string(exp)
-	// My editor adds a newline to all the files, so if the actual comment
-	// doesn't end with a newline then strip the last newline from the file's
-	// contents.
-	if !strings.HasSuffix(act, "\n") {
-		expStr = strings.TrimSuffix(expStr, "\n")
+	// For parallel plans and applies, do a substring match since output may be out of order
+	var replyMatchesExpected func(string, string) bool
+	if parallel {
+		replyMatchesExpected = func(act string, expStr string) bool {
+			return strings.Contains(act, expStr)
+		}
+	} else {
+		replyMatchesExpected = func(act string, expStr string) bool {
+			return expStr == act
+		}
 	}
 
-	if expStr != act {
-		// If in CI, we write the diff to the console. Otherwise we write the diff
-		// to file so we can use our local diff viewer.
-		if os.Getenv("CI") == "true" {
-			t.Logf("exp: %s, got: %s", expStr, act)
-			t.FailNow()
-		} else {
-			actFile := filepath.Join(absRepoPath(t, repoDir), expFile+".act")
-			err := ioutil.WriteFile(actFile, []byte(act), 0600)
-			Ok(t, err)
-			cwd, err := os.Getwd()
-			Ok(t, err)
-			rel, err := filepath.Rel(cwd, actFile)
-			Ok(t, err)
-			t.Errorf("%q was different, wrote actual comment to %q", expFile, rel)
+	for _, expFile := range expReplies {
+		exp, err := ioutil.ReadFile(filepath.Join(absRepoPath(t, repoDir), expFile))
+		Ok(t, err)
+		expStr := string(exp)
+		// My editor adds a newline to all the files, so if the actual comment
+		// doesn't end with a newline then strip the last newline from the file's
+		// contents.
+		if !strings.HasSuffix(act, "\n") {
+			expStr = strings.TrimSuffix(expStr, "\n")
+		}
+
+		if !replyMatchesExpected(act, expStr) {
+			// If in CI, we write the diff to the console. Otherwise we write the diff
+			// to file so we can use our local diff viewer.
+			if os.Getenv("CI") == "true" {
+				t.Logf("exp: %s, got: %s", expStr, act)
+				t.FailNow()
+			} else {
+				actFile := filepath.Join(absRepoPath(t, repoDir), expFile+".act")
+				err := ioutil.WriteFile(actFile, []byte(act), 0600)
+				Ok(t, err)
+				cwd, err := os.Getwd()
+				Ok(t, err)
+				rel, err := filepath.Rel(cwd, actFile)
+				Ok(t, err)
+				t.Errorf("%q was different, wrote actual comment to %q", expFile, rel)
+			}
 		}
 	}
 }

server/server.go

@@ -221,7 +221,8 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) {
 		userConfig.DefaultTFVersion,
 		config.DefaultTFVersionFlag,
 		userConfig.TFDownloadURL,
-		&terraform.DefaultDownloader{})
+		&terraform.DefaultDownloader{},
+		true)
 	// The flag.Lookup call is to detect if we're running in a unit test. If we
 	// are, then we don't error out because we don't have/want terraform
 	// installed on our CI system where the unit tests run.

server/testfixtures/test-repos/workspace-parallel-yaml/atlantis.yaml

@@ -0,0 +1,12 @@
+version: 3
+parallel_plan: false
+parallel_apply: true
+projects:
+  - dir: production
+    workspace: production
+    autoplan:
+      when_modified: ["**/*.tf*"]
+  - dir: staging
+    workspace: staging
+    autoplan:
+      when_modified: ["**/*.tf*"]

server/testfixtures/test-repos/workspace-parallel-yaml/exp-output-apply-all-production.txt

@@ -0,0 +1,21 @@
+<details><summary>Show Output</summary>
+
+```diff
+null_resource.this: Creating...
+null_resource.this: Creation complete after *s [id=*******************]
+
+Apply complete! Resources: 1 added, 0 changed, 0 destroyed.
+
+The state of your infrastructure has been saved to the path
+below. This state is required to modify and destroy your
+infrastructure, so keep it safe. To inspect the complete state
+use the `terraform show` command.
+
+State path: terraform.tfstate
+
+Outputs:
+
+workspace = production
+
+```
+</details>

server/testfixtures/test-repos/workspace-parallel-yaml/exp-output-apply-all-staging.txt

@@ -0,0 +1,21 @@
+<details><summary>Show Output</summary>
+
+```diff
+null_resource.this: Creating...
+null_resource.this: Creation complete after *s [id=*******************]
+
+Apply complete! Resources: 1 added, 0 changed, 0 destroyed.
+
+The state of your infrastructure has been saved to the path
+below. This state is required to modify and destroy your
+infrastructure, so keep it safe. To inspect the complete state
+use the `terraform show` command.
+
+State path: terraform.tfstate
+
+Outputs:
+
+workspace = staging
+
+```
+</details>

server/testfixtures/test-repos/workspace-parallel-yaml/exp-output-autoplan-production.txt

@@ -0,0 +1,25 @@
+<details><summary>Show Output</summary>
+
+```diff
+
+An execution plan has been generated and is shown below.
+Resource actions are indicated with the following symbols:
++ create
+
+Terraform will perform the following actions:
+
+  # null_resource.this will be created
++ resource "null_resource" "this" {
+      + id = (known after apply)
+    }
+
+Plan: 1 to add, 0 to change, 0 to destroy.
+
+```
+
+* :arrow_forward: To **apply** this plan, comment:
+    * `atlantis apply -d production -w production`
+* :put_litter_in_its_place: To **delete** this plan click [here](lock-url)
+* :repeat: To **plan** this project again, comment:
+    * `atlantis plan -d production -w production`
+</details>

server/testfixtures/test-repos/workspace-parallel-yaml/exp-output-autoplan-staging.txt

@@ -0,0 +1,25 @@
+<details><summary>Show Output</summary>
+
+```diff
+
+An execution plan has been generated and is shown below.
+Resource actions are indicated with the following symbols:
++ create
+
+Terraform will perform the following actions:
+
+  # null_resource.this will be created
++ resource "null_resource" "this" {
+      + id = (known after apply)
+    }
+
+Plan: 1 to add, 0 to change, 0 to destroy.
+
+```
+
+* :arrow_forward: To **apply** this plan, comment:
+    * `atlantis apply -d staging -w staging`
+* :put_litter_in_its_place: To **delete** this plan click [here](lock-url)
+* :repeat: To **plan** this project again, comment:
+    * `atlantis plan -d staging -w staging`
+</details>

server/testfixtures/test-repos/workspace-parallel-yaml/exp-output-merge.txt

@@ -0,0 +1,4 @@
+Locks and plans deleted for the projects and workspaces modified in this pull request:
+
+- dir: `production` workspace: `production`
+- dir: `staging` workspace: `staging`

server/testfixtures/test-repos/workspace-parallel-yaml/production/main.tf

@@ -0,0 +1,5 @@
+resource "null_resource" "this" {
+}
+output "workspace" {
+  value = "${terraform.workspace}"
+}

server/testfixtures/test-repos/workspace-parallel-yaml/staging/main.tf

@@ -0,0 +1,5 @@
+resource "null_resource" "this" {
+}
+output "workspace" {
+  value = "${terraform.workspace}"
+}