Return error if workspace doesn't exist (#61)

* Return error if workspace doesn't exist
* Default data dir to ~/.atlantis. Don't clean workspaces
* Run goimports
This commit is contained in:
Luke Kysow
2017-07-01 16:17:22 -07:00
committed by GitHub
parent 0daacbd29a
commit 8fbe662185
21 changed files with 187 additions and 347 deletions

View File

@@ -36,3 +36,5 @@ dist: ## Package up everything in static/ using go-bindata-assetfs so it can be
vendor-status:
@govendor status
fmt: ## Run goimports (which also formats)
goimports -w $$(find . -type f -name '*.go' -not -path "./vendor/*")

View File

@@ -1,8 +1,9 @@
package cmd
import (
"github.com/spf13/cobra"
"os"
"github.com/spf13/cobra"
)
var RootCmd = &cobra.Command{

View File

@@ -31,7 +31,6 @@ const (
planS3PrefixFlag = "plan-s3-prefix"
portFlag = "port"
requireApprovalFlag = "require-approval"
scratchDirFlag = "scratch-dir"
sshKeyFlag = "ssh-key"
)
@@ -56,7 +55,7 @@ var stringFlags = []stringFlag{
{
name: dataDirFlag,
description: "Path to directory to store Atlantis data.",
value: "/var/lib/atlantis",
value: "~/.atlantis",
},
{
name: ghHostnameFlag,
@@ -103,11 +102,6 @@ var stringFlags = []stringFlag{
description: "How to store plan files: file or s3. If set to file, will store plan files on disk in the directory specified by data-dir.",
value: "file",
},
{
name: scratchDirFlag,
description: "Path to directory to use as a temporary workspace for checking out repos.",
value: "/tmp/atlantis",
},
{
name: sshKeyFlag,
description: "Path to SSH key used for GitHub.",

View File

@@ -2,6 +2,7 @@ package cmd
import (
"fmt"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)

View File

@@ -102,7 +102,7 @@ func (b Backend) Unlock(project models.Project, env string) (*models.ProjectLock
Key: map[string]*dynamodb.AttributeValue{
"LockKey": {S: aws.String(key)},
},
TableName: aws.String(b.LockTable),
TableName: aws.String(b.LockTable),
ReturnValues: aws.String("ALL_OLD"),
}
output, err := b.DB.DeleteItem(params)

View File

@@ -1,9 +1,10 @@
package middleware
import (
"net/http"
"github.com/hootsuite/atlantis/logging"
"github.com/urfave/negroni"
"net/http"
)
func NewNon200Logger(logger *logging.SimpleLogger) *FailedRequestLogger {

View File

@@ -27,8 +27,8 @@ type User struct {
type ProjectLock struct {
Project Project
Pull PullRequest
User User
Pull PullRequest
User User
Env string
Time time.Time
}

View File

@@ -1,13 +1,14 @@
package file
import (
"github.com/hootsuite/atlantis/models"
"github.com/hootsuite/atlantis/plan"
"github.com/pkg/errors"
"io/ioutil"
"os"
"path/filepath"
"strconv"
"github.com/hootsuite/atlantis/models"
"github.com/hootsuite/atlantis/plan"
"github.com/pkg/errors"
)
type Backend struct {

View File

@@ -108,7 +108,7 @@ func (b *Backend) SavePlan(path string, project models.Project, env string, pull
func (b *Backend) DeletePlan(project models.Project, env string, pullNum int) error {
_, err := b.s3.DeleteObject(&s3.DeleteObjectInput{
Bucket: aws.String(b.bucket),
Key: aws.String(b.path(project, env, pullNum)),
Key: aws.String(b.path(project, env, pullNum)),
})
return err
}

View File

@@ -9,8 +9,6 @@ import (
"path/filepath"
"strconv"
version "github.com/hashicorp/go-version"
"github.com/hootsuite/atlantis/locking"
"github.com/hootsuite/atlantis/plan"
@@ -21,7 +19,6 @@ type ApplyExecutor struct {
github *GithubClient
githubStatus *GithubStatus
awsConfig *AWSConfig
scratchDir string
sshKey string
terraform *TerraformClient
githubCommentRenderer *GithubCommentRenderer
@@ -30,7 +27,8 @@ type ApplyExecutor struct {
planBackend plan.Backend
preRun *prerun.PreRun
configReader *ConfigReader
concurrentRunLocker *ConcurrentRunLocker
concurrentRunLocker *ConcurrentRunLocker
workspace *Workspace
}
/** Result Types **/
@@ -81,13 +79,19 @@ func (a *ApplyExecutor) execute(ctx *CommandContext, github *GithubClient) {
func (a *ApplyExecutor) setupAndApply(ctx *CommandContext) ExecutionResult {
if a.requireApproval {
if approved, res := a.isApproved(ctx); !approved {
approved, res := a.isApproved(ctx)
if !approved {
return res
}
}
// todo: reclone repo and switch branch, don't assume it's already there
repoDir := filepath.Join(a.scratchDir, ctx.Repo.FullName, strconv.Itoa(ctx.Pull.Num))
repoDir, err := a.workspace.GetWorkspace(ctx)
if err != nil {
ctx.Log.Err(err.Error())
a.githubStatus.Update(ctx.Repo, ctx.Pull, Error, ApplyStep)
return ExecutionResult{SetupError: GeneralError{errors.New("Workspace missing, please plan again")}}
}
plans, err := a.planBackend.CopyPlans(repoDir, ctx.Repo.FullName, ctx.Command.environment, ctx.Pull.Num)
if err != nil {
errMsg := fmt.Sprintf("failed to get plans: %s", err)

File diff suppressed because one or more lines are too long

View File

@@ -7,7 +7,7 @@ import (
// ConcurrentRunLocker is used to prevent multiple runs and commands from occurring at the same time for a single
// repo, pull, and environment
type ConcurrentRunLocker struct{
type ConcurrentRunLocker struct {
mutex sync.Mutex
locks map[string]interface{}
}

View File

@@ -1,9 +1,10 @@
package server_test
import (
. "github.com/hootsuite/atlantis/testing_util"
"testing"
"github.com/hootsuite/atlantis/server"
. "github.com/hootsuite/atlantis/testing_util"
)
var repo = "repo/owner"

View File

@@ -3,9 +3,10 @@ package server
import (
"fmt"
"strings"
"github.com/google/go-github/github"
"github.com/hootsuite/atlantis/models"
"strings"
)
type Status int
@@ -40,9 +41,9 @@ func (s Status) String() string {
func (g *GithubStatus) Update(repo models.Repo, pull models.PullRequest, status Status, step string) error {
repoStatus := github.RepoStatus{
State: github.String(status.String()),
State: github.String(status.String()),
Description: github.String(fmt.Sprintf("%s %s", strings.Title(step), strings.Title(status.String()))),
Context: github.String(statusContext)}
Context: github.String(statusContext)}
return g.client.UpdateStatus(repo, pull, &repoStatus)
}

View File

@@ -2,16 +2,13 @@ package server
import (
"fmt"
"io/ioutil"
"os"
"os/exec"
"path"
"path/filepath"
"strings"
version "github.com/hashicorp/go-version"
"github.com/hootsuite/atlantis/locking"
"github.com/hootsuite/atlantis/logging"
"github.com/hootsuite/atlantis/models"
"github.com/hootsuite/atlantis/plan"
"github.com/hootsuite/atlantis/prerun"
@@ -23,18 +20,18 @@ type PlanExecutor struct {
github *GithubClient
githubStatus *GithubStatus
awsConfig *AWSConfig
scratchDir string
s3Bucket string
sshKey string
terraform *TerraformClient
githubCommentRenderer *GithubCommentRenderer
lockingClient *locking.Client
// LockURL is a function that given a lock id will return a url for lock view
LockURL func(id string) (url string)
planBackend plan.Backend
preRun *prerun.PreRun
configReader *ConfigReader
LockURL func(id string) (url string)
planBackend plan.Backend
preRun *prerun.PreRun
configReader *ConfigReader
concurrentRunLocker *ConcurrentRunLocker
workspace *Workspace
}
/** Result Types **/
@@ -115,51 +112,9 @@ func (p *PlanExecutor) setupAndPlan(ctx *CommandContext) ExecutionResult {
return ExecutionResult{SetupError: GeneralError{errors.New("Plan Failed: we determined that no terraform projects were modified")}}
}
// set up our workspace by cloning the repo
cloneDir := fmt.Sprintf("%s/%s/%d", p.scratchDir, ctx.Repo.FullName, ctx.Pull.Num)
ctx.Log.Info("cleaning clone directory %q", cloneDir)
if err := os.RemoveAll(cloneDir); err != nil {
ctx.Log.Warn("failed to clean dir %q before cloning, attempting to continue: %v", cloneDir, err)
}
// create the directory and parents if necessary
ctx.Log.Info("creating dir %q", cloneDir)
if err := os.MkdirAll(cloneDir, 0755); err != nil {
ctx.Log.Warn("failed to create dir %q prior to cloning, attempting to continue: %v", cloneDir, err)
}
// Check if ssh key is set and create git ssh wrapper
cloneCmd := exec.Command("git", "clone", ctx.Repo.SSHURL, cloneDir)
if p.sshKey != "" {
err := GenerateSSHWrapper()
if err != nil {
return p.setupError(ctx, errors.Wrap(err, "creating git ssh wrapper"))
}
cloneCmd.Env = []string{
fmt.Sprintf("GIT_SSH=%s", defaultSSHWrapper),
fmt.Sprintf("PKEY=%s", p.sshKey),
}
}
// git clone the repo
ctx.Log.Info("git cloning %q into %q", ctx.Repo.SSHURL, cloneDir)
if output, err := cloneCmd.CombinedOutput(); err != nil {
return p.setupError(ctx, fmt.Errorf("cloning %s: %s: %s", ctx.Repo.SSHURL, err, string(output)))
}
// check out the branch for this PR
ctx.Log.Info("checking out branch %q", ctx.Pull.Branch)
checkoutCmd := exec.Command("git", "checkout", ctx.Pull.Branch)
checkoutCmd.Dir = cloneDir
if err := checkoutCmd.Run(); err != nil {
return p.setupError(ctx, errors.Wrapf(err, "checking out branch %s", ctx.Pull.Branch))
}
//workspace.Initialize(ctx.Repo, ctx.Pull.Num)
// todo: update how we clean the workspace based on the new way of storing plans
planFilesPrefix := fmt.Sprintf("%s_%d", strings.Replace(ctx.Repo.FullName, "/", "_", -1), ctx.Pull.Num)
if err := p.CleanWorkspace(ctx.Log, planFilesPrefix, p.scratchDir, cloneDir, projects); err != nil {
return p.setupError(ctx, errors.Wrap(err, "cleaning workspace"))
cloneDir, err := p.workspace.Clone(ctx)
if err != nil {
return ExecutionResult{SetupError: GeneralError{fmt.Errorf("Plan Failed: setting up workspace: %s", err)}}
}
tfEnv := ctx.Command.environment
@@ -210,7 +165,7 @@ func (p *PlanExecutor) setupAndPlan(ctx *CommandContext) ExecutionResult {
ctx.Log.Info("Pre run output: \n%s", preRunOutput)
}
generatePlanResponse := p.plan(ctx, cloneDir, p.scratchDir, project, p.sshKey, terraformPlanExtraArgs)
generatePlanResponse := p.plan(ctx, cloneDir, project, p.sshKey, terraformPlanExtraArgs)
generatePlanResponse.Path = project.Path
planOutputs = append(planOutputs, generatePlanResponse)
}
@@ -223,7 +178,6 @@ func (p *PlanExecutor) setupAndPlan(ctx *CommandContext) ExecutionResult {
func (p *PlanExecutor) plan(
ctx *CommandContext,
repoDir string,
planOutDir string,
project models.Project,
sshKey string,
terraformArgs []string) PathResult {
@@ -372,31 +326,6 @@ func (p *PlanExecutor) getProjectPath(modifiedFilePath string) string {
return dir
}
// CleanWorkspace deletes all .terraform/ folders from the project folders and cleans up any plans in the output directory
func (p *PlanExecutor) CleanWorkspace(log *logging.SimpleLogger, deleteFilesPrefix string, planOutDir string, repoDir string, projects []models.Project) error {
log.Info("cleaning workspace directory %q", planOutDir)
// delete .terraform directories
for _, project := range projects {
os.RemoveAll(filepath.Join(repoDir, project.Path, ".terraform"))
}
// delete old plan files
files, err := ioutil.ReadDir(planOutDir)
if err != nil {
return err
}
for _, file := range files {
if strings.HasPrefix(file.Name(), deleteFilesPrefix) {
log.Info("deleting file %q", file.Name())
fullPath := filepath.Join(planOutDir, file.Name())
if err := os.Remove(fullPath); err != nil {
log.Warn("failed to remove plan file %q", fullPath)
}
}
}
return nil
}
func (p *PlanExecutor) setupError(ctx *CommandContext, err error) ExecutionResult {
ctx.Log.Err(err.Error())
p.githubStatus.Update(ctx.Repo, ctx.Pull, Error, PlanStep)

View File

@@ -1,8 +1,9 @@
package server
import (
. "github.com/hootsuite/atlantis/testing_util"
"testing"
. "github.com/hootsuite/atlantis/testing_util"
)
var p PlanExecutor

View File

@@ -8,7 +8,6 @@ import (
version "github.com/hashicorp/go-version"
"github.com/pkg/errors"
yaml "gopkg.in/yaml.v2"
)

View File

@@ -1,31 +1,48 @@
package server
import (
"github.com/hootsuite/atlantis/locking"
"github.com/hootsuite/atlantis/models"
"github.com/pkg/errors"
"github.com/hootsuite/atlantis/plan"
"bytes"
"fmt"
"strings"
"text/template"
"bytes"
"github.com/hootsuite/atlantis/locking"
"github.com/hootsuite/atlantis/models"
"github.com/hootsuite/atlantis/plan"
"github.com/pkg/errors"
)
type PullClosedExecutor struct {
locking *locking.Client
github *GithubClient
locking *locking.Client
github *GithubClient
planBackend plan.Backend
workspace *Workspace
}
type templatedProject struct {
Path string
Envs string
}
var pullClosedTemplate = template.Must(template.New("").Parse("Locks and plans deleted for the projects and environments modified in this pull request:\n" +
"{{ range . }}\n" +
"- path: `{{ .Path }}` {{ .Envs }}{{ end }}"))
"{{ range . }}\n" +
"- path: `{{ .Path }}` {{ .Envs }}{{ end }}"))
func (p *PullClosedExecutor) CleanUpPull(repo models.Repo, pull models.PullRequest) error {
// delete the workspace
if err := p.workspace.Delete(repo, pull); err != nil {
return errors.Wrap(err, "cleaning workspace")
}
// delete plans
err := p.planBackend.DeletePlansByPull(repo.FullName, pull.Num)
if err != nil {
return errors.Wrap(err, "cleaning up plans")
}
// finally, delete locks. We do this last because when someone
// unlocks a project, right now we don't actually delete the plan
// so we might have plans laying around but no locks
locks, err := p.locking.UnlockByPull(repo.FullName, pull.Num)
if err != nil {
return errors.Wrap(err, "cleaning up locks")
@@ -36,11 +53,6 @@ func (p *PullClosedExecutor) CleanUpPull(repo models.Repo, pull models.PullReque
return nil
}
err = p.planBackend.DeletePlansByPull(repo.FullName, pull.Num)
if err != nil {
return errors.Wrap(err, "cleaning up plans")
}
templateData := p.buildTemplateData(locks)
var buf bytes.Buffer
if err = pullClosedTemplate.Execute(&buf, templateData); err != nil {

View File

@@ -3,16 +3,15 @@ package server
import (
"context"
"fmt"
"io/ioutil"
"log"
"net/http"
"net/url"
"os"
"os/user"
"strings"
"time"
"io/ioutil"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/elazarl/go-bindata-assetfs"
"github.com/google/go-github/github"
@@ -26,10 +25,10 @@ import (
"github.com/hootsuite/atlantis/plan"
"github.com/hootsuite/atlantis/plan/file"
"github.com/hootsuite/atlantis/plan/s3"
"github.com/hootsuite/atlantis/prerun"
"github.com/pkg/errors"
"github.com/urfave/cli"
"github.com/urfave/negroni"
"github.com/hootsuite/atlantis/prerun"
)
const (
@@ -70,7 +69,6 @@ type ServerConfig struct {
PlanBackend string `mapstructure:"plan-backend"`
RequireApproval bool `mapstructure:"require-approval"`
SSHKey string `mapstructure:"ssh-key"`
ScratchDir string `mapstructure:"scratch-dir"`
}
type CommandContext struct {
@@ -109,8 +107,17 @@ func (g GeneralError) Template() *CompiledTemplate {
// todo: /end
func NewServer(config ServerConfig) (*Server, error) {
// if ~ was used in data-dir convert that to actual home directory otherwise we'll
// create a directory call "~" instead of actually using home
if strings.HasPrefix(config.DataDir, "~/") {
user, err := user.Current()
if err != nil {
return nil, errors.Wrap(err, "determining current user")
}
config.DataDir = user.HomeDir + strings.TrimPrefix(config.DataDir, "~")
}
tp := github.BasicAuthTransport{
Username: strings.TrimSpace(config.GithubUser),
Password: strings.TrimSpace(config.GithubPassword),
@@ -167,11 +174,14 @@ func NewServer(config ServerConfig) (*Server, error) {
preRun := &prerun.PreRun{}
configReader := &ConfigReader{}
concurrentRunLocker := NewConcurrentRunLocker()
workspace := &Workspace{
dataDir: config.DataDir,
sshKey: config.SSHKey,
}
applyExecutor := &ApplyExecutor{
github: githubClient,
githubStatus: githubStatus,
awsConfig: awsConfig,
scratchDir: config.ScratchDir,
sshKey: config.SSHKey,
terraform: terraformClient,
githubCommentRenderer: githubComments,
@@ -179,28 +189,30 @@ func NewServer(config ServerConfig) (*Server, error) {
requireApproval: config.RequireApproval,
planBackend: planBackend,
preRun: preRun,
configReader: configReader,
concurrentRunLocker: concurrentRunLocker,
configReader: configReader,
concurrentRunLocker: concurrentRunLocker,
workspace: workspace,
}
planExecutor := &PlanExecutor{
github: githubClient,
githubStatus: githubStatus,
awsConfig: awsConfig,
scratchDir: config.ScratchDir,
sshKey: config.SSHKey,
terraform: terraformClient,
githubCommentRenderer: githubComments,
lockingClient: lockingClient,
planBackend: planBackend,
preRun: preRun,
configReader: configReader,
concurrentRunLocker: concurrentRunLocker,
configReader: configReader,
concurrentRunLocker: concurrentRunLocker,
workspace: workspace,
}
helpExecutor := &HelpExecutor{}
pullClosedExecutor := &PullClosedExecutor{
planBackend: planBackend,
github: githubClient,
locking: lockingClient,
workspace: workspace,
}
logger := logging.NewSimpleLogger("server", log.New(os.Stderr, "", log.LstdFlags), false, logging.ToLogLevel(config.LogLevel))
eventParser := &EventParser{}

View File

@@ -1,13 +0,0 @@
package server
import "io/ioutil"
// todo: make this object oriented
const defaultSSHWrapper = "/tmp/git-ssh.sh"
// Create git ssh wrapper
func GenerateSSHWrapper() error {
d1 := []byte("#!/bin/sh\nif [ -z \"$PKEY\" ]; then\n# if PKEY is not specified, run ssh using default keyfile\nssh -oStrictHostKeyChecking=no \"$@\"\nelse\nssh -oStrictHostKeyChecking=no -i \"$PKEY\" \"$@\"\nfi")
return ioutil.WriteFile(defaultSSHWrapper, d1, 0755)
}

92
server/workspace.go Normal file
View File

@@ -0,0 +1,92 @@
package server
import (
"fmt"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"strconv"
"github.com/hootsuite/atlantis/models"
"github.com/pkg/errors"
)
const defaultSSHWrapper = "/tmp/git-ssh.sh"
const workspacePrefix = "repos"
type Workspace struct {
dataDir string
sshKey string
}
func (w *Workspace) Clone(ctx *CommandContext) (string, error) {
cloneDir := w.cloneDir(ctx)
// this is safe to do because we lock runs on repo/pull/env so no one else is using this workspace
ctx.Log.Info("cleaning clone directory %q", cloneDir)
if err := os.RemoveAll(cloneDir); err != nil {
return "", errors.Wrap(err, "deleting old workspace")
}
// create the directory and parents if necessary
ctx.Log.Info("creating dir %q", cloneDir)
if err := os.MkdirAll(cloneDir, 0755); err != nil {
return "", errors.Wrap(err, "creating new workspace")
}
// Check if ssh key is set and create git ssh wrapper
cloneCmd := exec.Command("git", "clone", ctx.Repo.SSHURL, cloneDir)
if w.sshKey != "" {
// todo: is this still needed?
err := w.generateSSHWrapper()
if err != nil {
return "", errors.Wrap(err, "creating git ssh wrapper")
}
cloneCmd.Env = []string{
fmt.Sprintf("GIT_SSH=%s", defaultSSHWrapper),
fmt.Sprintf("PKEY=%s", w.sshKey),
}
}
// clone the repo
ctx.Log.Info("git cloning %q into %q", ctx.Repo.SSHURL, cloneDir)
if output, err := cloneCmd.CombinedOutput(); err != nil {
return "", errors.Wrapf(err, "cloning %s: %s", ctx.Repo.SSHURL, string(output))
}
// check out the branch for this PR
ctx.Log.Info("checking out branch %q", ctx.Pull.Branch)
checkoutCmd := exec.Command("git", "checkout", ctx.Pull.Branch)
checkoutCmd.Dir = cloneDir
if err := checkoutCmd.Run(); err != nil {
return "", errors.Wrapf(err, "checking out branch %s", ctx.Pull.Branch)
}
return cloneDir, nil
}
func (w *Workspace) GetWorkspace(ctx *CommandContext) (string, error) {
repoDir := w.cloneDir(ctx)
if _, err := os.Stat(repoDir); err != nil {
return "", errors.Wrap(err, "checking if workspace exists")
}
return repoDir, nil
}
// Delete deletes the workspace for this repo and pull
func (w *Workspace) Delete(repo models.Repo, pull models.PullRequest) error {
return os.RemoveAll(w.repoPullDir(repo, pull))
}
func (w *Workspace) repoPullDir(repo models.Repo, pull models.PullRequest) string {
return filepath.Join(w.dataDir, workspacePrefix, repo.FullName, strconv.Itoa(pull.Num))
}
func (w *Workspace) cloneDir(ctx *CommandContext) string {
return filepath.Join(w.repoPullDir(ctx.Repo, ctx.Pull), ctx.Command.environment)
}
func (w *Workspace) generateSSHWrapper() error {
d1 := []byte("#!/bin/sh\nif [ -z \"$PKEY\" ]; then\n# if PKEY is not specified, run ssh using default keyfile\nssh -oStrictHostKeyChecking=no \"$@\"\nelse\nssh -oStrictHostKeyChecking=no -i \"$PKEY\" \"$@\"\nfi")
return ioutil.WriteFile(defaultSSHWrapper, d1, 0755)
}