diff --git a/cmd/server.go b/cmd/server.go index 1ba59b322..034283f47 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -36,7 +36,7 @@ var stringFlags = []stringFlag{ { name: ghHostnameFlag, description: "Hostname of Github installation.", - value: "api.github.com", + value: "api.github.com", }, { name: logLevelFlag, diff --git a/cmd/version.go b/cmd/version.go index 505484c2e..d23669cf7 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -3,15 +3,14 @@ package cmd import ( "fmt" "github.com/spf13/cobra" + "github.com/spf13/viper" ) -const version = "0.1.0" - var versionCmd = &cobra.Command{ Use: "version", Short: "Print the current atlantis version", Run: func(cmd *cobra.Command, args []string) { - fmt.Printf("atlantis %s\n", version) + fmt.Printf("atlantis %s\n", viper.Get("version")) }, } diff --git a/locking/boltdb/boltdb.go b/locking/boltdb/boltdb.go index abf36af2b..766b60aeb 100644 --- a/locking/boltdb/boltdb.go +++ b/locking/boltdb/boltdb.go @@ -1,19 +1,19 @@ package boltdb import ( - "github.com/boltdb/bolt" - "fmt" - "encoding/json" - "github.com/pkg/errors" "bytes" + "encoding/json" + "fmt" + "github.com/boltdb/bolt" + "github.com/hootsuite/atlantis/models" + "github.com/pkg/errors" "os" - "time" "path" - "github.com/hootsuite/atlantis/locking" + "time" ) type Backend struct { - db *bolt.DB + db *bolt.DB bucket []byte } @@ -21,14 +21,14 @@ const bucketName = "runLocks" func New(dataDir string) (*Backend, error) { if err := os.MkdirAll(dataDir, 0755); err != nil { - return nil, errors.Wrap(err,"creating data dir") + return nil, errors.Wrap(err, "creating data dir") } db, err := bolt.Open(path.Join(dataDir, "atlantis.db"), 0600, &bolt.Options{Timeout: 1 * time.Second}) if err != nil { if err.Error() == "timeout" { return nil, errors.New("starting BoltDB: timeout (a possible cause is another Atlantis instance already running)") } - return nil, errors.Wrap(err,"starting BoltDB") + return nil, errors.Wrap(err, "starting BoltDB") } err = db.Update(func(tx *bolt.Tx) error { if _, err := tx.CreateBucketIfNotExists([]byte(bucketName)); err != nil { @@ -37,7 +37,7 @@ func New(dataDir string) (*Backend, error) { return nil }) if err != nil { - return nil, errors.Wrap(err,"starting BoltDB") + return nil, errors.Wrap(err, "starting BoltDB") } // todo: close BoltDB when server is sigtermed return &Backend{db, []byte(bucketName)}, nil @@ -48,104 +48,111 @@ func NewWithDB(db *bolt.DB, bucket string) (*Backend, error) { return &Backend{db, []byte(bucket)}, nil } -func (b Backend) TryLock(run locking.Run) (locking.TryLockResponse, error) { - var response locking.TryLockResponse - newRunSerialized, _ := json.Marshal(run) - lockID := run.StateKey() +func (b *Backend) TryLock(project models.Project, env string, pullNum int) (bool, int, error) { + // return variables + var lockAcquired bool + var lockingPullNum int + key := b.key(project, env) + newLock := models.ProjectLock{ + PullNum: pullNum, + Project: project, + Time: time.Now(), + Env: env, + } + newLockSerialized, _ := json.Marshal(newLock) transactionErr := b.db.Update(func(tx *bolt.Tx) error { bucket := tx.Bucket(b.bucket) // if there is no run at that key then we're free to create the lock - lockingRunSerialized := bucket.Get([]byte(lockID)) - if lockingRunSerialized == nil { - bucket.Put([]byte(lockID), newRunSerialized) // not a readonly bucketName so okay to ignore error - response = locking.TryLockResponse{ - LockAcquired: true, - LockingRun: run, - LockID: lockID, - } + currLockSerialized := bucket.Get([]byte(key)) + if currLockSerialized == nil { + bucket.Put([]byte(key), newLockSerialized) // not a readonly bucketName so okay to ignore error + lockAcquired = true + lockingPullNum = pullNum return nil } // otherwise the lock fails, return to caller the run that's holding the lock - var lockingRun locking.Run - if err := b.deserialize(lockingRunSerialized, &lockingRun); err != nil { - return errors.Wrap(err, "failed to deserialize run") - } - response = locking.TryLockResponse{ - LockAcquired: false, - LockingRun: lockingRun, - LockID: lockID, + var currLock models.ProjectLock + if err := json.Unmarshal(currLockSerialized, &currLock); err != nil { + return errors.Wrap(err, "failed to deserialize current lock") } + lockAcquired = false + lockingPullNum = currLock.PullNum return nil }) if transactionErr != nil { - return response, errors.Wrap(transactionErr, "DB transaction failed") + return false, lockingPullNum, errors.Wrap(transactionErr, "DB transaction failed") } - return response, nil + return lockAcquired, lockingPullNum, nil } -func (b Backend) Unlock(lockID string) error { +func (b Backend) Unlock(p models.Project, env string) error { + key := b.key(p, env) err := b.db.Update(func(tx *bolt.Tx) error { locks := tx.Bucket(b.bucket) - return locks.Delete([]byte(lockID)) + return locks.Delete([]byte(key)) }) return errors.Wrap(err, "DB transaction failed") } -func (b Backend) ListLocks() (map[string]locking.Run, error) { - m := make(map[string]locking.Run) - bytes := make(map[string][]byte) - +func (b Backend) List() ([]models.ProjectLock, error) { + var locks []models.ProjectLock + var locksBytes [][]byte err := b.db.View(func(tx *bolt.Tx) error { bucket := tx.Bucket(b.bucket) c := bucket.Cursor() for k, v := c.First(); k != nil; k, v = c.Next() { - bytes[string(k)] = v + locksBytes = append(locksBytes, v) } return nil }) if err != nil { - return m, errors.Wrap(err, "DB transaction failed") + return locks, errors.Wrap(err, "DB transaction failed") } // deserialize bytes into the proper objects - for k, v := range bytes { - var run locking.Run - if err := b.deserialize(v, &run); err != nil { - return m, errors.Wrap(err, fmt.Sprintf("failed to deserialize run at key %q", string(k))) + for k, v := range locksBytes { + var lock models.ProjectLock + if err := json.Unmarshal(v, &lock); err != nil { + return locks, errors.Wrap(err, fmt.Sprintf("failed to deserialize lock at key %q", string(k))) } - m[k] = run + locks = append(locks, lock) } - return m, nil + return locks, nil } -func (b Backend) FindLocksForPull(repoFullName string, pullNum int) ([]string, error) { - var ids []string +func (b Backend) UnlockByPull(repoFullName string, pullNum int) error { + // get the locks that match that pull request + var locks []models.ProjectLock err := b.db.View(func(tx *bolt.Tx) error { c := tx.Bucket(b.bucket).Cursor() - // the key for each lock is repoFullName/path/env so we can scan through all entries - // and get the locks for that repo. Then we can check if the lock is for the right pull - prefix := []byte(repoFullName + "/") - for k, v := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, v = c.Next() { - var run locking.Run - if err := b.deserialize(v, &run); err != nil { - return errors.Wrapf(err, "failed to deserialize run at key %q", string(k)) + // we can use the repoFullName as a prefix search since that's the first part of the key + for k, v := c.Seek([]byte(repoFullName)); k != nil && bytes.HasPrefix(k, []byte(repoFullName)); k, v = c.Next() { + var lock models.ProjectLock + if err := json.Unmarshal(v, &lock); err != nil { + return errors.Wrapf(err, "failed to deserialize lock at key %q", string(k)) } - if run.PullNum == pullNum { - ids = append(ids, string(k)) + if lock.PullNum == pullNum { + locks = append(locks, lock) } } - return nil }) - return ids, err + + // delete the locks + for _, lock := range locks { + if err = b.Unlock(lock.Project, lock.Env); err != nil { + return errors.Wrapf(err, "unlocking repo %s, path %s, env %s", lock.Project.RepoFullName, lock.Project.Path, lock.Env) + } + } + return nil } -func (b Backend) deserialize(bs []byte, run *locking.Run) error { - return json.Unmarshal(bs, run) +func (b Backend) key(p models.Project, env string) string { + return fmt.Sprintf("%s/%s/%s", p.RepoFullName, p.Path, env) } diff --git a/locking/boltdb/boltdb_test.go b/locking/boltdb/boltdb_test.go index c40f0faef..1efce5955 100644 --- a/locking/boltdb/boltdb_test.go +++ b/locking/boltdb/boltdb_test.go @@ -1,34 +1,28 @@ package boltdb_test import ( - "github.com/hootsuite/atlantis/locking" . "github.com/hootsuite/atlantis/testing_util" "github.com/boltdb/bolt" + "github.com/pkg/errors" + "io/ioutil" "os" "testing" - "io/ioutil" - "github.com/pkg/errors" - "time" "github.com/hootsuite/atlantis/locking/boltdb" + "github.com/hootsuite/atlantis/models" ) var lockBucket = "bucket" -var run = locking.Run{ - RepoFullName: "owner/repo", - Path: "parent/child", - Env: "default", - PullNum: 1, - User: "user", - Timestamp: time.Now(), -} +var project = models.NewProject("owner/repo", "parent/child") +var env = "default" +var pullNum = 1 func TestListNoLocks(t *testing.T) { t.Log("listing locks when there are none should return an empty list") db, b := newTestDB() defer cleanupDB(db) - ls, err := b.ListLocks() + ls, err := b.List() Ok(t, err) Equals(t, 0, len(ls)) } @@ -37,12 +31,11 @@ func TestListOneLock(t *testing.T) { t.Log("listing locks when there is one should return it") db, b := newTestDB() defer cleanupDB(db) - _, err := b.TryLock(run) + _, _, err := b.TryLock(project, env, pullNum) Ok(t, err) - ls, err := b.ListLocks() + ls, err := b.List() Ok(t, err) Equals(t, 1, len(ls)) - Equals(t, run, ls[run.StateKey()]) } func TestListMultipleLocks(t *testing.T) { @@ -51,42 +44,40 @@ func TestListMultipleLocks(t *testing.T) { defer cleanupDB(db) // add multiple locks - _, err := b.TryLock(run) - Ok(t, err) + repos := []string{ + "owner/repo1", + "owner/repo2", + "owner/repo3", + "owner/repo4", + } - run2 := run - run2.RepoFullName = "different" - _, err = b.TryLock(run2) - Ok(t, err) - - run3 := run - run3.RepoFullName = "different again" - _, err = b.TryLock(run3) - Ok(t, err) - - run4 := run - run4.RepoFullName = "different again again" - _, err = b.TryLock(run4) - Ok(t, err) - - ls, err := b.ListLocks() + for _, r := range repos { + _, _, err := b.TryLock(models.NewProject(r, "path"), env, pullNum) + Ok(t, err) + } + ls, err := b.List() Ok(t, err) Equals(t, 4, len(ls)) - Equals(t, run, ls[run.StateKey()]) - Equals(t, run2, ls[run2.StateKey()]) - Equals(t, run3, ls[run3.StateKey()]) - Equals(t, run4, ls[run4.StateKey()]) + for _, r := range repos { + found := false + for _, l := range ls { + if l.Project.RepoFullName == r { + found = true + } + } + Assert(t, found == true, "expected %s in %v", r, ls) + } } func TestListAddRemove(t *testing.T) { t.Log("listing after adding and removing should return none") db, b := newTestDB() defer cleanupDB(db) - _, err := b.TryLock(run) + _, _, err := b.TryLock(project, env, pullNum) Ok(t, err) - b.Unlock(run.StateKey()) + b.Unlock(project, env) - ls, err := b.ListLocks() + ls, err := b.List() Ok(t, err) Equals(t, 0, len(ls)) } @@ -95,62 +86,50 @@ func TestLockingNoLocks(t *testing.T) { t.Log("with no locks yet, lock should succeed") db, b := newTestDB() defer cleanupDB(db) - r, err := b.TryLock(run) + acquired, lockingPullNum, err := b.TryLock(project, env, pullNum) Ok(t, err) - Equals(t, true, r.LockAcquired) - Equals(t, run, r.LockingRun) - Equals(t, run.StateKey(), r.LockID) + Equals(t, true, acquired) + Equals(t, pullNum, lockingPullNum) } func TestLockingExistingLock(t *testing.T) { t.Log("if there is an existing lock, lock should...") db, b := newTestDB() defer cleanupDB(db) - _, err := b.TryLock(run) + _, _, err := b.TryLock(project, env, pullNum) Ok(t, err) - t.Log("...succeed if the new run has a different path") + t.Log("...succeed if the new project has a different path") { - new := run - new.Path = "different/path" - r, err := b.TryLock(new) + acquired, lockingPull, err := b.TryLock(models.NewProject(project.RepoFullName, "different/path"), env, pullNum) Ok(t, err) - Equals(t, true, r.LockAcquired) - Equals(t, new, r.LockingRun) - Equals(t, new.StateKey(), r.LockID) + Equals(t, true, acquired) + Equals(t, pullNum, lockingPull) } - t.Log("...succeed if the new run has a different environment") + t.Log("...succeed if the new project has a different environment") { - new := run - new.Env = "different-env" - r, err := b.TryLock(new) + acquired, lockingPull, err := b.TryLock(project, "different-env", pullNum) Ok(t, err) - Equals(t, true, r.LockAcquired) - Equals(t, new, r.LockingRun) - Equals(t, new.StateKey(), r.LockID) + Equals(t, true, acquired) + Equals(t, pullNum, lockingPull) } - t.Log("...succeed if the new run has a different repoName") + t.Log("...succeed if the new project has a different repoName") { - new := run - new.RepoFullName = "new/repo" - r, err := b.TryLock(new) + acquired, lockingPull, err := b.TryLock(models.NewProject("different/repo", project.Path), env, pullNum) Ok(t, err) - Equals(t, true, r.LockAcquired) - Equals(t, new, r.LockingRun) - Equals(t, new.StateKey(), r.LockID) + Equals(t, true, acquired) + Equals(t, pullNum, lockingPull) } - t.Log("...not succeed if the new run only has a different pullNum") + t.Log("...not succeed if the new project only has a different pullNum") { - new := run - new.PullNum = run.PullNum + 1 - r, err := b.TryLock(new) + newPull := pullNum + 1 + acquired, lockingPull, err := b.TryLock(project, env, newPull) Ok(t, err) - Equals(t, false, r.LockAcquired) - Equals(t, run, r.LockingRun) - Equals(t, run.StateKey(), r.LockID) + Equals(t, false, acquired) + Equals(t, lockingPull, pullNum) } } @@ -159,7 +138,7 @@ func TestUnlockingNoLocks(t *testing.T) { db, b := newTestDB() defer cleanupDB(db) - Ok(t, b.Unlock("any-lock-id")) + Ok(t, b.Unlock(project, env)) } func TestUnlocking(t *testing.T) { @@ -167,20 +146,20 @@ func TestUnlocking(t *testing.T) { db, b := newTestDB() defer cleanupDB(db) - b.TryLock(run) - Ok(t, b.Unlock(run.StateKey())) + b.TryLock(project, env, pullNum) + Ok(t, b.Unlock(project, env)) // should be no locks listed - ls, err := b.ListLocks() + ls, err := b.List() Ok(t, err) Equals(t, 0, len(ls)) // should be able to re-lock that repo with a new pull num - new := run - new.PullNum = run.PullNum + 1 - r, err := b.TryLock(new) + newPull := pullNum + 1 + acquired, lockingPull, err := b.TryLock(project, env, newPull) Ok(t, err) - Equals(t, true, r.LockAcquired) + Equals(t, true, acquired) + Equals(t, newPull, lockingPull) } func TestUnlockingMultiple(t *testing.T) { @@ -188,107 +167,114 @@ func TestUnlockingMultiple(t *testing.T) { db, b := newTestDB() defer cleanupDB(db) - b.TryLock(run) + b.TryLock(project, env, pullNum) - new := run + new := project new.RepoFullName = "new/repo" - b.TryLock(new) + b.TryLock(project, env, pullNum) - new2 := run + new2 := project new2.Path = "new/path" - b.TryLock(new2) + b.TryLock(project, env, pullNum) - new3 := run - new3.Env = "new/env" - b.TryLock(new3) + new3Env := "new-env" + b.TryLock(project, new3Env, pullNum) // now try and unlock them - Ok(t, b.Unlock(new3.StateKey())) - Ok(t, b.Unlock(new2.StateKey())) - Ok(t, b.Unlock(new.StateKey())) - Ok(t, b.Unlock(run.StateKey())) + Ok(t, b.Unlock(project, new3Env)) + Ok(t, b.Unlock(new2, env)) + Ok(t, b.Unlock(new, env)) + Ok(t, b.Unlock(project, env)) // should be none left - ls, err := b.ListLocks() + ls, err := b.List() Ok(t, err) Equals(t, 0, len(ls)) } -func TestFindLocksNone(t *testing.T) { - t.Log("find should return no locks when there are none") +func TestUnlockByPullNone(t *testing.T) { + t.Log("UnlockByPull should be successful when there are no locks") db, b := newTestDB() defer cleanupDB(db) - ls, err := b.FindLocksForPull("any/repo", 1) + err := b.UnlockByPull("any/repo", 1) Ok(t, err) - Equals(t, 0, len(ls)) } -func TestFindLocksOne(t *testing.T) { - t.Log("with one lock find should...") +func TestUnlockByPullOne(t *testing.T) { + t.Log("with one lock, UnlockByPull should...") db, b := newTestDB() defer cleanupDB(db) - _, err := b.TryLock(run) + _, _, err := b.TryLock(project, env, pullNum) Ok(t, err) - t.Log("...return no locks from the same repo but different pull num") + t.Log("...delete nothing when its the same repo but a different pull") { - ls, err := b.FindLocksForPull(run.RepoFullName, run.PullNum + 1) + err := b.UnlockByPull(project.RepoFullName, pullNum+1) Ok(t, err) - Equals(t, 0, len(ls)) - } - t.Log("...return no locks from a different repo but the same pull num") - { - ls, err := b.FindLocksForPull(run.RepoFullName + "dif", run.PullNum) - Ok(t, err) - Equals(t, 0, len(ls)) - } - t.Log("...return the one lock when called with that repo and pull num") - { - ls, err := b.FindLocksForPull(run.RepoFullName, run.PullNum) + ls, err := b.List() Ok(t, err) Equals(t, 1, len(ls)) - Equals(t, run.StateKey(), ls[0]) + } + t.Log("...delete nothing when its the same pull but a different repo") + { + err := b.UnlockByPull("different/repo", pullNum) + Ok(t, err) + ls, err := b.List() + Ok(t, err) + Equals(t, 1, len(ls)) + } + t.Log("...delete the lock when its the same repo and pull") + { + err := b.UnlockByPull(project.RepoFullName, pullNum) + Ok(t, err) + ls, err := b.List() + Ok(t, err) + Equals(t, 0, len(ls)) } } -func TestFindLocksAfterUnlock(t *testing.T) { - t.Log("after locking and unlocking find should return no locks") +func TestUnlockByPullAfterUnlock(t *testing.T) { + t.Log("after locking and unlocking, UnlockByPull should be successful") db, b := newTestDB() defer cleanupDB(db) - _, err := b.TryLock(run) + _, _, err := b.TryLock(project, env, pullNum) Ok(t, err) - Ok(t, b.Unlock(run.StateKey())) + Ok(t, b.Unlock(project, env)) - ls, err := b.FindLocksForPull(run.RepoFullName, run.PullNum) + err = b.UnlockByPull(project.RepoFullName, pullNum) + Ok(t, err) + ls, err := b.List() Ok(t, err) Equals(t, 0, len(ls)) } -func TestFindMultipleMatching(t *testing.T) { - t.Log("find should return all matching lock ids") +func TestUnlockByPullMatching(t *testing.T) { + t.Log("UnlockByPull should delete all locks in that repo and pull num") db, b := newTestDB() defer cleanupDB(db) - _, err := b.TryLock(run) + _, _, err := b.TryLock(project, env, pullNum) Ok(t, err) // add additional locks with the same repo and pull num but different paths/envs - new := run + new := project new.Path = "dif/path" - _, err = b.TryLock(new) + _, _, err = b.TryLock(new, env, pullNum) Ok(t, err) - new2 := run - new2.Env = "new-env" - _, err = b.TryLock(new2) + _, _, err = b.TryLock(project, "new-env", pullNum) Ok(t, err) - // should get all of them back - ls, err := b.FindLocksForPull(run.RepoFullName, run.PullNum) + // there should now be 3 + ls, err := b.List() Ok(t, err) Equals(t, 3, len(ls)) - ContainsStr(t, run.StateKey(), ls) - ContainsStr(t, new.StateKey(), ls) - ContainsStr(t, new2.StateKey(), ls) + + // should all be unlocked + err = b.UnlockByPull(project.RepoFullName, pullNum) + Ok(t, err) + ls, err = b.List() + Ok(t, err) + Equals(t, 0, len(ls)) } // newTestDB returns a TestDB using a temporary path. @@ -322,4 +308,3 @@ func cleanupDB(db *bolt.DB) { os.Remove(db.Path()) db.Close() } - diff --git a/locking/dynamodb/dynamodb.go b/locking/dynamodb/dynamodb.go index 9206ec212..a35ef3033 100644 --- a/locking/dynamodb/dynamodb.go +++ b/locking/dynamodb/dynamodb.go @@ -1,18 +1,16 @@ package dynamodb import ( - "github.com/aws/aws-sdk-go/service/dynamodb" - "github.com/aws/aws-sdk-go/aws/client" - "github.com/aws/aws-sdk-go/aws" - "github.com/pkg/errors" - "encoding/json" "fmt" - "encoding/hex" - "github.com/aws/aws-sdk-go/service/dynamodb/dynamodbiface" - "time" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/client" + "github.com/aws/aws-sdk-go/service/dynamodb" "github.com/aws/aws-sdk-go/service/dynamodb/dynamodbattribute" + "github.com/aws/aws-sdk-go/service/dynamodb/dynamodbiface" + "github.com/hootsuite/atlantis/models" + "github.com/pkg/errors" "strconv" - "github.com/hootsuite/atlantis/locking" + "time" ) type Backend struct { @@ -20,104 +18,114 @@ type Backend struct { LockTable string } -type dynamoRun struct { - LockID string +// dynamoLock duplicates the fields of models.ProjectLock and adds LocksKey +// so everything is a top-level field for serialization and then querying +// in DynamodB and also so any changes to models.ProjectLock won't affect +// how we're storing our data or will at least cause a compile error +type dynamoLock struct { + LockKey string RepoFullName string Path string - Env string PullNum int - User string - Timestamp time.Time + Env string + Time time.Time } -func New(lockTable string, p client.ConfigProvider) *Backend { - return &Backend{ +func New(lockTable string, p client.ConfigProvider) Backend { + return Backend{ DB: dynamodb.New(p), LockTable: lockTable, } } -func (d *Backend) TryLock(run locking.Run) (locking.TryLockResponse, error) { - var r locking.TryLockResponse - newRunSerialized, err := d.toDynamoItem(run) +func (b Backend) key(project models.Project, env string) string { + return fmt.Sprintf("%s/%s/%s", project.RepoFullName, project.Path, env) +} + +func (b Backend) TryLock(project models.Project, env string, pullNum int) (bool, int, error) { + key := b.key(project, env) + newDynamoLock := dynamoLock{ + LockKey: key, + RepoFullName: project.RepoFullName, + Path: project.Path, + PullNum: pullNum, + Env: env, + Time: time.Now(), + } + newLockSerialized, err := dynamodbattribute.MarshalMap(newDynamoLock) if err != nil { - return r, errors.Wrap(err, "serializing") + return false, 0, errors.Wrap(err, "serializing") } // check if there is an existing lock getItemParams := &dynamodb.GetItemInput{ Key: map[string]*dynamodb.AttributeValue{ - "LockID": { - S: aws.String(run.StateKey()), + "LockKey": { + S: aws.String(key), }, }, - TableName: aws.String(d.LockTable), + TableName: aws.String(b.LockTable), ConsistentRead: aws.Bool(true), } - item, err := d.DB.GetItem(getItemParams) + item, err := b.DB.GetItem(getItemParams) if err != nil { - return r, errors.Wrap(err, "checking if lock exists") + return false, 0, errors.Wrap(err, "checking if lock exists") } - // if there is already a lock then we can't acquire a lock. Return the existing lock + var currLock dynamoLock if len(item.Item) != 0 { - var dynamoRun dynamoRun - if err := dynamodbattribute.UnmarshalMap(item.Item, &dynamoRun); err != nil { - return r, errors.Wrap(err,"found an existing lock at that id but it could not be deserialized. We suggest manually deleting this key from DynamoDB") + if err := dynamodbattribute.UnmarshalMap(item.Item, &currLock); err != nil { + return false, 0, errors.Wrap(err, "found an existing lock at that key but it could not be deserialized. We suggest manually deleting this key from DynamoDB") } - lockingRun := d.fromDynamoItem(dynamoRun) - return locking.TryLockResponse{ - LockAcquired: false, - LockingRun: lockingRun, - LockID: run.StateKey(), - }, nil + return false, currLock.PullNum, nil } // else we should be able to lock putItem := &dynamodb.PutItemInput{ - Item: newRunSerialized, - TableName: aws.String(d.LockTable), + Item: newLockSerialized, + TableName: aws.String(b.LockTable), // this will ensure that we don't insert the new item in a race situation // where someone has written this key just after our read - ConditionExpression: aws.String("attribute_not_exists(LockID)"), + ConditionExpression: aws.String("attribute_not_exists(LockKey)"), } - if _, err := d.DB.PutItem(putItem); err != nil { - return r, errors.Wrap(err, "writing lock") + if _, err := b.DB.PutItem(putItem); err != nil { + return false, 0, errors.Wrap(err, "writing lock") } - return locking.TryLockResponse{ - LockAcquired: true, - LockingRun: run, - LockID: run.StateKey(), - }, nil + return true, pullNum, nil } -func (d *Backend) Unlock(lockID string) error { +func (b Backend) Unlock(project models.Project, env string) error { + key := b.key(project, env) params := &dynamodb.DeleteItemInput{ Key: map[string]*dynamodb.AttributeValue{ - "LockID": {S: aws.String(lockID)}, + "LockKey": {S: aws.String(key)}, }, - TableName: aws.String(d.LockTable), + TableName: aws.String(b.LockTable), } - _, err := d.DB.DeleteItem(params) + _, err := b.DB.DeleteItem(params) return errors.Wrap(err, "deleting lock") } -func (d *Backend) ListLocks() (map[string]locking.Run, error) { - params := &dynamodb.ScanInput{ - TableName: aws.String(d.LockTable), - } - - m := make(map[string]locking.Run) +func (b Backend) List() ([]models.ProjectLock, error) { + var locks []models.ProjectLock var err, internalErr error - err = d.DB.ScanPages(params, func(out *dynamodb.ScanOutput, lastPage bool) bool { - var runs []dynamoRun - if err := dynamodbattribute.UnmarshalListOfMaps(out.Items, &runs); err != nil { - internalErr = errors.Wrap(err,"deserializing locks") + params := &dynamodb.ScanInput{ + TableName: aws.String(b.LockTable), + } + err = b.DB.ScanPages(params, func(out *dynamodb.ScanOutput, lastPage bool) bool { + var dynamoLocks []dynamoLock + if err := dynamodbattribute.UnmarshalListOfMaps(out.Items, &dynamoLocks); err != nil { + internalErr = errors.Wrap(err, "deserializing locks") return false } - for _, run := range runs { - m[run.LockID] = d.fromDynamoItem(run) + for _, lock := range dynamoLocks { + locks = append(locks, models.ProjectLock{ + PullNum: lock.PullNum, + Project: models.NewProject(lock.RepoFullName, lock.Path), + Env: lock.Env, + Time: lock.Time, + }) } return lastPage }) @@ -125,10 +133,10 @@ func (d *Backend) ListLocks() (map[string]locking.Run, error) { if err == nil && internalErr != nil { err = internalErr } - return m, errors.Wrap(err, "scanning dynamodb") + return locks, errors.Wrap(err, "scanning dynamodb") } -func (d *Backend) FindLocksForPull(repoFullName string, pullNum int) ([]string, error) { +func (b Backend) UnlockByPull(repoFullName string, pullNum int) error { params := &dynamodb.ScanInput{ ExpressionAttributeValues: map[string]*dynamodb.AttributeValue{ ":pullNum": { @@ -139,77 +147,31 @@ func (d *Backend) FindLocksForPull(repoFullName string, pullNum int) ([]string, }, }, FilterExpression: aws.String("RepoFullName = :repoFullName and PullNum = :pullNum"), - TableName: aws.String(d.LockTable), + TableName: aws.String(b.LockTable), } - var ids []string + // scan DynamoDB for locks that match the pull request + var locks []dynamoLock var err, internalErr error - err = d.DB.ScanPages(params, func(out *dynamodb.ScanOutput, lastPage bool) bool { - var runs []dynamoRun - if err := dynamodbattribute.UnmarshalListOfMaps(out.Items, &runs); err != nil { - internalErr = errors.Wrap(err,"deserializing locks") + err = b.DB.ScanPages(params, func(out *dynamodb.ScanOutput, lastPage bool) bool { + if err := dynamodbattribute.UnmarshalListOfMaps(out.Items, &locks); err != nil { + internalErr = errors.Wrap(err, "deserializing locks") return false } - for _, run := range runs { - ids = append(ids, run.LockID) - } return lastPage }) - - if err == nil && internalErr != nil { + if err == nil { err = internalErr } - return ids, errors.Wrap(err,"scanning dynamodb") -} - -func (d *Backend) deserializeItem(item map[string]*dynamodb.AttributeValue) (string, locking.Run, error) { - var lockID string - var run locking.Run - - lockIDItem, ok := item["LockID"] - if !ok || lockIDItem == nil { - return lockID, run, fmt.Errorf("lock did not have expected key 'LockID'") - } - lockID = string(hex.EncodeToString(lockIDItem.B)) - runItem, ok := item["Run"] - if !ok || runItem == nil { - return lockID, run, fmt.Errorf("lock did not have expected key 'Run'") + if err != nil { + return errors.Wrap(err, "scanning dynamodb") } - if err := d.deserialize(runItem.B, &run); err != nil { - return lockID, run, fmt.Errorf("deserializing run at key %q: %s", lockID, err) - } - return lockID, run, nil -} - -func (d *Backend) deserialize(bs []byte, run *locking.Run) error { - return json.Unmarshal(bs, run) -} - -func (d *Backend) serialize(run locking.Run) ([]byte, error) { - return json.Marshal(run) -} - -func (d *Backend) toDynamoItem(run locking.Run) (map[string]*dynamodb.AttributeValue, error) { - item := dynamoRun{ - LockID: run.StateKey(), - PullNum: run.PullNum, - RepoFullName: run.RepoFullName, - Env: run.Env, - Path: run.Path, - Timestamp: run.Timestamp, - User: run.User, - } - return dynamodbattribute.MarshalMap(item) -} - -func (d *Backend) fromDynamoItem(dynamoRun dynamoRun) locking.Run { - return locking.Run{ - User: dynamoRun.User, - Timestamp: dynamoRun.Timestamp, - Path: dynamoRun.Path, - Env: dynamoRun.Env, - RepoFullName: dynamoRun.RepoFullName, - PullNum: dynamoRun.PullNum, + // now we can unlock all of them + for _, lock := range locks { + if err := b.Unlock(models.NewProject(lock.RepoFullName, lock.Path), lock.Env); err != nil { + return errors.Wrapf(err, "unlocking repo %s, path %s, env %s", lock.RepoFullName, lock.Path, lock.Env) + } } + return nil } diff --git a/locking/locking.go b/locking/locking.go index 1b297e9c2..43152d663 100644 --- a/locking/locking.go +++ b/locking/locking.go @@ -1,35 +1,70 @@ package locking import ( - "time" + "errors" "fmt" + "github.com/hootsuite/atlantis/models" + "regexp" ) -type Run struct { - RepoFullName string - Path string - Env string - PullNum int - User string - Timestamp time.Time -} - -// StateKey returns the unique key to identify the set of infrastructure being modified by this run. -// Returns `{fullRepoName}/{tfProjectPath}/{environment}`. -// Used in locking to determine what part of the infrastructure is locked. -func (r Run) StateKey() string { - return fmt.Sprintf("%s/%s/%s", r.RepoFullName, r.Path, r.Env) +type Backend interface { + TryLock(project models.Project, env string, pullNum int) (bool, int, error) + Unlock(project models.Project, env string) error + List() ([]models.ProjectLock, error) + UnlockByPull(repoFullName string, pullNum int) error } type TryLockResponse struct { - LockAcquired bool - LockingRun Run // what is currently holding the lock - LockID string + LockAcquired bool + LockingPullNum int + LockKey string } -type Backend interface { - TryLock(run Run) (TryLockResponse, error) - Unlock(lockID string) error - ListLocks() (map[string]Run, error) - FindLocksForPull(repoFullName string, pullNum int) ([]string, error) +type Client struct { + backend Backend +} + +func NewClient(backend Backend) *Client { + return &Client{ + backend: backend, + } +} + +// keyRegex matches and captures {repoFullName}/{path}/{env} where path can have multiple /'s in it +var keyRegex = regexp.MustCompile(`^(.*?\/.*?)\/(.*)\/(.*)$`) + +func (c *Client) TryLock(p models.Project, env string, pullNum int) (TryLockResponse, error) { + lockAcquired, lockingPullNum, err := c.backend.TryLock(p, env, pullNum) + if err != nil { + return TryLockResponse{}, err + } + return TryLockResponse{lockAcquired, lockingPullNum, c.key(p, env)}, nil +} + +func (c *Client) Unlock(key string) error { + matches := keyRegex.FindStringSubmatch(key) + if len(matches) != 4 { + return errors.New("invalid key format") + } + return c.backend.Unlock(models.Project{matches[1], matches[2]}, matches[3]) +} + +func (c *Client) List() (map[string]models.ProjectLock, error) { + m := make(map[string]models.ProjectLock) + locks, err := c.backend.List() + if err != nil { + return m, err + } + for _, lock := range locks { + m[c.key(lock.Project, lock.Env)] = lock + } + return m, nil +} + +func (c *Client) UnlockByPull(repoFullName string, pullNum int) error { + return c.backend.UnlockByPull(repoFullName, pullNum) +} + +func (c *Client) key(p models.Project, env string) string { + return fmt.Sprintf("%s/%s/%s", p.RepoFullName, p.Path, env) } diff --git a/logging/simple_logger.go b/logging/simple_logger.go index ef22e19b4..1a9fd14a1 100644 --- a/logging/simple_logger.go +++ b/logging/simple_logger.go @@ -26,9 +26,9 @@ const ( func NewSimpleLogger(source string, log *log.Logger, keepHistory bool, level LogLevel) *SimpleLogger { return &SimpleLogger{ - Source: source, - Log: log, - Level: level, + Source: source, + Log: log, + Level: level, KeepHistory: keepHistory, } } diff --git a/middleware/middleware.go b/middleware/middleware.go index fb474f043..b3873b3d9 100644 --- a/middleware/middleware.go +++ b/middleware/middleware.go @@ -1,9 +1,9 @@ package middleware import ( - "net/http" - "github.com/urfave/negroni" "github.com/hootsuite/atlantis/logging" + "github.com/urfave/negroni" + "net/http" ) func NewNon200Logger(logger *logging.SimpleLogger) *FailedRequestLogger { @@ -11,7 +11,7 @@ func NewNon200Logger(logger *logging.SimpleLogger) *FailedRequestLogger { } // FailedRequestLogger logs the request when a response code >= 400 is sent -type FailedRequestLogger struct{ +type FailedRequestLogger struct { logger *logging.SimpleLogger } diff --git a/models/models.go b/models/models.go new file mode 100644 index 000000000..5f0cefb0b --- /dev/null +++ b/models/models.go @@ -0,0 +1,54 @@ +package models + +import ( + paths "path" + "time" +) + +type Repo struct { + FullName string + Owner string + Name string + SSHURL string +} + +type PullRequest struct { + Num int + HeadCommit string + BaseCommit string + Link string + Branch string + Author string +} + +type User struct { + Username string +} + +type ProjectLock struct { + Project Project + PullNum int + Env string + Time time.Time +} + +// Project represents a Terraform project. +// Since there may be multiple Terraform projects in a single repo we also include Path +type Project struct { + RepoFullName string + // Path to project root in the repo. + // If "." then project is at root. + // Never ends in "/". + Path string +} + +func NewProject(repoFullName string, path string) Project { + path = paths.Clean(path) + if path == "/" { + path = "." + } + return Project{ + RepoFullName: repoFullName, + Path: path, + } +} diff --git a/server/apply_executor.go b/server/apply_executor.go index 992c803f2..24a8a5fb4 100644 --- a/server/apply_executor.go +++ b/server/apply_executor.go @@ -14,13 +14,20 @@ import ( "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" "github.com/hootsuite/atlantis/locking" - "time" + "github.com/hootsuite/atlantis/models" + "strconv" ) type ApplyExecutor struct { - BaseExecutor - requireApproval bool - atlantisGithubUser string + github *GithubClient + awsConfig *AWSConfig + scratchDir string + s3Bucket string + sshKey string + terraform *TerraformClient + githubCommentRenderer *GithubCommentRenderer + lockingClient *locking.Client + requireApproval bool } /** Result Types **/ @@ -54,82 +61,85 @@ func (n NoPlansFailure) Template() *CompiledTemplate { return NoPlansFailureTmpl } -func (a *ApplyExecutor) execute(ctx *ExecutionContext, pullCtx *PullRequestContext) ExecutionResult { - res := a.setupAndApply(ctx, pullCtx) +func (a *ApplyExecutor) execute(ctx *CommandContext, github *GithubClient) { + res := a.setupAndApply(ctx) res.Command = Apply - return res + comment := a.githubCommentRenderer.render(res, ctx.Log.History.String(), ctx.Command.verbose) + github.CreateComment(ctx, comment) } -func (a *ApplyExecutor) setupAndApply(ctx *ExecutionContext, pullCtx *PullRequestContext) ExecutionResult { - a.github.UpdateStatus(pullCtx, PendingStatus, "Applying...") +func (a *ApplyExecutor) setupAndApply(ctx *CommandContext) ExecutionResult { + a.github.UpdateStatus(ctx.Repo, ctx.Pull, PendingStatus, "Applying...") if a.requireApproval { - ok, err := a.github.PullIsApproved(pullCtx) + ok, err := a.github.PullIsApproved(ctx.Repo, ctx.Pull) if err != nil { msg := fmt.Sprintf("failed to determine if pull request was approved: %v", err) - ctx.log.Err(msg) - a.github.UpdateStatus(pullCtx, ErrorStatus, "Apply Error") + ctx.Log.Err(msg) + a.github.UpdateStatus(ctx.Repo, ctx.Pull, ErrorStatus, "Apply Error") return ExecutionResult{SetupError: GeneralError{errors.New(msg)}} } if !ok { - ctx.log.Info("pull request was not approved") - a.github.UpdateStatus(pullCtx, FailureStatus, "Apply Failed") + ctx.Log.Info("pull request was not approved") + a.github.UpdateStatus(ctx.Repo, ctx.Pull, FailureStatus, "Apply Failed") return ExecutionResult{SetupFailure: PullNotApprovedFailure{}} } } - planPaths, err := a.downloadPlans(ctx.repoFullName, ctx.pullNum, ctx.command.environment, a.scratchDir, a.awsConfig, a.s3Bucket) + planPaths, err := a.downloadPlans(ctx.Repo.FullName, ctx.Pull.Num, ctx.Command.environment, a.scratchDir, a.awsConfig, a.s3Bucket) if err != nil { errMsg := fmt.Sprintf("failed to download plans: %v", err) - ctx.log.Err(errMsg) - a.github.UpdateStatus(pullCtx, ErrorStatus, "Apply Error") + ctx.Log.Err(errMsg) + a.github.UpdateStatus(ctx.Repo, ctx.Pull, ErrorStatus, "Apply Error") return ExecutionResult{SetupError: GeneralError{errors.New(errMsg)}} } // If there are no plans found for the pull request if len(planPaths) == 0 { failure := "found 0 plans for this pull request" - ctx.log.Warn(failure) - a.github.UpdateStatus(pullCtx, FailureStatus, "Apply Failure") + ctx.Log.Warn(failure) + a.github.UpdateStatus(ctx.Repo, ctx.Pull, FailureStatus, "Apply Failure") return ExecutionResult{SetupFailure: NoPlansFailure{}} } //runLog = append(runLog, fmt.Sprintf("-> Downloaded plans: %v", planPaths)) applyOutputs := []PathResult{} for _, planPath := range planPaths { - output := a.apply(ctx, pullCtx, planPath) + output := a.apply(ctx, planPath) output.Path = planPath applyOutputs = append(applyOutputs, output) } - a.updateGithubStatus(pullCtx, applyOutputs) + a.updateGithubStatus(ctx, applyOutputs) return ExecutionResult{PathResults: applyOutputs} } -func (a *ApplyExecutor) apply(ctx *ExecutionContext, pullCtx *PullRequestContext, planPath string) PathResult { - //runLog = append(runLog, fmt.Sprintf("-> Running apply %s", planPath)) +func (a *ApplyExecutor) apply(ctx *CommandContext, planPath string) PathResult { planName := path.Base(planPath) - planSubDir := a.determinePlanSubDir(planName, ctx.pullNum) - planDir := filepath.Join(a.scratchDir, ctx.repoFullName, fmt.Sprintf("%v", ctx.pullNum), planSubDir) + planSubDir := a.determinePlanSubDir(planName, ctx.Pull.Num) + // todo: don't assume repo is cloned here + repoDir := filepath.Join(a.scratchDir, ctx.Repo.FullName, strconv.Itoa(ctx.Pull.Num)) + planDir := filepath.Join(repoDir, planSubDir) + project := models.NewProject(ctx.Repo.FullName, planSubDir) execPath := NewExecutionPath(planDir, planSubDir) var config Config var remoteStatePath string // check if config file is found, if not we continue the run if config.Exists(execPath.Absolute) { - ctx.log.Info("Config file found in %s", execPath.Absolute) + ctx.Log.Info("Config file found in %s", execPath.Absolute) err := config.Read(execPath.Absolute) if err != nil { msg := fmt.Sprintf("Error reading config file: %v", err) - ctx.log.Err(msg) + ctx.Log.Err(msg) return PathResult{ Status: "error", Result: GeneralError{errors.New(msg)}, } } // need to use the remote state path and backend to do remote configure - err = PreRun(&config, ctx.log, execPath.Absolute, ctx.command) + err = PreRun(&config, ctx.Log, execPath.Absolute, ctx.Command) if err != nil { msg := fmt.Sprintf("pre run failed: %v", err) - ctx.log.Err(msg) + ctx.Log.Err(msg) return PathResult{ Status: "error", Result: GeneralError{errors.New(msg)}, @@ -146,10 +156,10 @@ func (a *ApplyExecutor) apply(ctx *ExecutionContext, pullCtx *PullRequestContext // NOTE: THIS CODE IS TO SUPPORT TERRAFORM PROJECTS THAT AREN'T USING ATLANTIS CONFIG FILE. if config.StashPath == "" { // configure remote state - statePath, err := a.terraform.ConfigureRemoteState(ctx.log, execPath, ctx.command.environment, a.sshKey) + statePath, err := a.terraform.ConfigureRemoteState(ctx.Log, repoDir, project, ctx.Command.environment, a.sshKey) if err != nil { msg := fmt.Sprintf("failed to set up remote state: %v", err) - ctx.log.Err(msg) + ctx.Log.Err(msg) return PathResult{ Status: "error", Result: GeneralError{errors.New(msg)}, @@ -158,72 +168,62 @@ func (a *ApplyExecutor) apply(ctx *ExecutionContext, pullCtx *PullRequestContext remoteStatePath = statePath } else { // use state path from config file - remoteStatePath = generateStatePath(config.StashPath, ctx.command.environment) + remoteStatePath = generateStatePath(config.StashPath, ctx.Command.environment) } if remoteStatePath != "" { - tfEnv := ctx.command.environment + tfEnv := ctx.Command.environment if tfEnv == "" { tfEnv = "default" } - run := locking.Run{ - RepoFullName: pullCtx.repoFullName, - Path: execPath.Relative, - Env: tfEnv, - PullNum: pullCtx.number, - User: pullCtx.terraformApplier, - Timestamp: time.Now(), - } - lockAttempt, err := a.lockingBackend.TryLock(run) + lockAttempt, err := a.lockingClient.TryLock(project, tfEnv, ctx.Pull.Num) if err != nil { return PathResult{ Status: "error", Result: GeneralError{fmt.Errorf("failed to acquire lock: %s", err)}, } } - if lockAttempt.LockAcquired != true && lockAttempt.LockingRun.PullNum != pullCtx.number { + if lockAttempt.LockAcquired != true && lockAttempt.LockingPullNum != ctx.Pull.Num { return PathResult{ Status: "error", - Result: GeneralError{fmt.Errorf("failed to acquire lock: lock held by pull request #%d", lockAttempt.LockingRun.PullNum)}, + Result: GeneralError{fmt.Errorf("failed to acquire lock: lock held by pull request #%d", lockAttempt.LockingPullNum)}, } } } // need to get auth data from assumed role // todo: de-duplicate calls to assumeRole - //runLog = append(runLog, "-> Assuming role prior to running apply") - a.awsConfig.AWSSessionName = ctx.pullCreator + a.awsConfig.AWSSessionName = ctx.User.Username awsSession, err := a.awsConfig.CreateAWSSession() if err != nil { - ctx.log.Err(err.Error()) + ctx.Log.Err(err.Error()) return PathResult{ Status: "error", Result: GeneralError{err}, } } - //runLog = append(runLog, "-> Assumed AWS role successfully") credVals, err := awsSession.Config.Credentials.Get() if err != nil { msg := fmt.Sprintf("failed to get assumed role credentials: %v", err) - ctx.log.Err(msg) + ctx.Log.Err(msg) return PathResult{ Status: "error", Result: GeneralError{errors.New(msg)}, } } - ctx.log.Info("running apply from %q", execPath.Relative) + ctx.Log.Info("running apply from %q", execPath.Relative) - terraformApplyCmdArgs, output, err := a.terraform.RunTerraformCommand(execPath, []string{"apply", "-no-color", planPath}, []string{ + terraformApplyCmdArgs, output, err := a.terraform.RunTerraformCommand(execPath.Absolute, []string{"apply", "-no-color", planPath}, []string{ fmt.Sprintf("AWS_ACCESS_KEY_ID=%s", credVals.AccessKeyID), fmt.Sprintf("AWS_SECRET_ACCESS_KEY=%s", credVals.SecretAccessKey), fmt.Sprintf("AWS_SESSION_TOKEN=%s", credVals.SessionToken), }) //runLog = append(runLog, "```\n Apply output:\n", fmt.Sprintf("```bash\n%s\n", string(out[:]))) if err != nil { - ctx.log.Err("failed to apply: %v %s", err, output) + ctx.Log.Err("failed to apply: %v %s", err, output) return PathResult{ Status: "failure", Result: ApplyFailure{Command: strings.Join(terraformApplyCmdArgs, " "), Output: output, ErrorMessage: err.Error()}, @@ -292,3 +292,27 @@ func (a *ApplyExecutor) determinePlanSubDir(planName string, pullNum int) string dirsStr := match[1] // in form dir_subdir_subsubdir return filepath.Clean(strings.Replace(dirsStr, "_", "/", -1)) } + +func (a *ApplyExecutor) updateGithubStatus(ctx *CommandContext, pathResults []PathResult) { + // the status will be the worst result + worstResult := a.worstResult(pathResults) + if worstResult == "success" { + a.github.UpdateStatus(ctx.Repo, ctx.Pull, SuccessStatus, "Apply Succeeded") + } else if worstResult == "failure" { + a.github.UpdateStatus(ctx.Repo, ctx.Pull, FailureStatus, "Apply Failed") + } else { + a.github.UpdateStatus(ctx.Repo, ctx.Pull, ErrorStatus, "Apply Error") + } +} + +func (a *ApplyExecutor) worstResult(results []PathResult) string { + var worst string = "success" + for _, result := range results { + if result.Status == "error" { + return result.Status + } else if result.Status == "failure" { + worst = result.Status + } + } + return worst +} diff --git a/server/atlantis_config_file_test.go b/server/atlantis_config_file_test.go index 741a021f9..efc98e58b 100644 --- a/server/atlantis_config_file_test.go +++ b/server/atlantis_config_file_test.go @@ -1,10 +1,10 @@ package server import ( + . "github.com/hootsuite/atlantis/testing_util" "io/ioutil" "os" "testing" - . "github.com/hootsuite/atlantis/testing_util" ) var tempConfigFile = "/tmp/" + AtlantisConfigFile diff --git a/server/base_executor.go b/server/base_executor.go deleted file mode 100644 index 3ce3232bf..000000000 --- a/server/base_executor.go +++ /dev/null @@ -1,109 +0,0 @@ -package server - -import ( - "path/filepath" - "github.com/hootsuite/atlantis/locking" -) - -type BaseExecutor struct { - github *GithubClient - awsConfig *AWSConfig - scratchDir string - s3Bucket string - sshKey string - ghComments *GithubCommentRenderer - terraform *TerraformClient - githubCommentRenderer *GithubCommentRenderer - lockingBackend locking.Backend -} - -type PullRequestContext struct { - repoFullName string - head string - base string - number int - pullRequestLink string - terraformApplier string - terraformApplierEmail string -} - -type ExecutionResult struct { - SetupError Templater - SetupFailure Templater - PathResults []PathResult - Command CommandType -} - -type PathResult struct { - Path string - Status string // todo: this should be an enum for success/error/failure - Result Templater -} - -type ExecutionPath struct { - // Absolute is the full path on the OS where we will execute. - // Will never end with a '/'. - Absolute string - // Relative is the path relative to the repo root. - // Will never end with a '/'. - Relative string -} - -func NewExecutionPath(absolutePath string, relativePath string) ExecutionPath { - return ExecutionPath{filepath.Clean(absolutePath), filepath.Clean(relativePath)} -} - -func (b *BaseExecutor) updateGithubStatus(pullCtx *PullRequestContext, pathResults []PathResult) { - // the status will be the worst result - worstResult := b.worstResult(pathResults) - if worstResult == "success" { - b.github.UpdateStatus(pullCtx, SuccessStatus, "Plan Succeeded") - } else if worstResult == "failure" { - b.github.UpdateStatus(pullCtx, FailureStatus, "Plan Failed") - } else { - b.github.UpdateStatus(pullCtx, ErrorStatus, "Plan Error") - } -} - -func (b *BaseExecutor) worstResult(results []PathResult) string { - var worst string = "success" - for _, result := range results { - if result.Status == "error" { - return result.Status - } else if result.Status == "failure" { - worst = result.Status - } - } - return worst -} - -func (b *BaseExecutor) Exec(f func(*ExecutionContext, *PullRequestContext) ExecutionResult, ctx *ExecutionContext, github *GithubClient) { - pullCtx := b.githubContext(ctx) - result := f(ctx, pullCtx) - comment := b.githubCommentRenderer.render(result, ctx.log.History.String(), ctx.command.verbose) - github.CreateComment(pullCtx, comment) -} - -func (b *BaseExecutor) githubContext(ctx *ExecutionContext) *PullRequestContext { - return &PullRequestContext{ - repoFullName: ctx.repoFullName, - head: ctx.head, - base: ctx.base, - number: ctx.pullNum, - pullRequestLink: ctx.pullLink, - terraformApplier: ctx.requesterUsername, - terraformApplierEmail: ctx.requesterEmail, - } -} - -type Templater interface { - Template() *CompiledTemplate -} - -type GeneralError struct { - Error error -} - -func (g GeneralError) Template() *CompiledTemplate { - return GeneralErrorTmpl -} diff --git a/server/bindata_assetfs.go b/server/bindata_assetfs.go index 8120d6452..64ff1f59a 100644 --- a/server/bindata_assetfs.go +++ b/server/bindata_assetfs.go @@ -12,10 +12,10 @@ package server import ( - "github.com/elazarl/go-bindata-assetfs" "bytes" "compress/gzip" "fmt" + "github.com/elazarl/go-bindata-assetfs" "io" "io/ioutil" "os" @@ -267,12 +267,12 @@ func AssetNames() []string { // _bindata is a table, holding each asset generator, mapped to its name. var _bindata = map[string]func() (*asset, error){ - "static/atlantis-icon.png": staticAtlantisIconPng, - "static/atlantis-icon_512.png": staticAtlantisIcon_512Png, - "static/css/custom.css": staticCssCustomCss, - "static/css/normalize.css": staticCssNormalizeCss, - "static/css/skeleton.css": staticCssSkeletonCss, - "static/images/atlantis-icon.png": staticImagesAtlantisIconPng, + "static/atlantis-icon.png": staticAtlantisIconPng, + "static/atlantis-icon_512.png": staticAtlantisIcon_512Png, + "static/css/custom.css": staticCssCustomCss, + "static/css/normalize.css": staticCssNormalizeCss, + "static/css/skeleton.css": staticCssSkeletonCss, + "static/images/atlantis-icon.png": staticImagesAtlantisIconPng, "static/images/atlantis-icon_512.png": staticImagesAtlantisIcon_512Png, } @@ -315,18 +315,19 @@ type bintree struct { Func func() (*asset, error) Children map[string]*bintree } + var _bintree = &bintree{nil, map[string]*bintree{ - "static": &bintree{nil, map[string]*bintree{ - "atlantis-icon.png": &bintree{staticAtlantisIconPng, map[string]*bintree{}}, - "atlantis-icon_512.png": &bintree{staticAtlantisIcon_512Png, map[string]*bintree{}}, - "css": &bintree{nil, map[string]*bintree{ - "custom.css": &bintree{staticCssCustomCss, map[string]*bintree{}}, - "normalize.css": &bintree{staticCssNormalizeCss, map[string]*bintree{}}, - "skeleton.css": &bintree{staticCssSkeletonCss, map[string]*bintree{}}, + "static": {nil, map[string]*bintree{ + "atlantis-icon.png": {staticAtlantisIconPng, map[string]*bintree{}}, + "atlantis-icon_512.png": {staticAtlantisIcon_512Png, map[string]*bintree{}}, + "css": {nil, map[string]*bintree{ + "custom.css": {staticCssCustomCss, map[string]*bintree{}}, + "normalize.css": {staticCssNormalizeCss, map[string]*bintree{}}, + "skeleton.css": {staticCssSkeletonCss, map[string]*bintree{}}, }}, - "images": &bintree{nil, map[string]*bintree{ - "atlantis-icon.png": &bintree{staticImagesAtlantisIconPng, map[string]*bintree{}}, - "atlantis-icon_512.png": &bintree{staticImagesAtlantisIcon_512Png, map[string]*bintree{}}, + "images": {nil, map[string]*bintree{ + "atlantis-icon.png": {staticImagesAtlantisIconPng, map[string]*bintree{}}, + "atlantis-icon_512.png": {staticImagesAtlantisIcon_512Png, map[string]*bintree{}}, }}, }}, }} @@ -378,7 +379,6 @@ func _filePath(dir, name string) string { return filepath.Join(append([]string{dir}, strings.Split(cannonicalName, "/")...)...) } - func assetFS() *assetfs.AssetFS { for k := range _bintree.Children { return &assetfs.AssetFS{Asset: Asset, AssetDir: AssetDir, AssetInfo: AssetInfo, Prefix: k} diff --git a/server/executor.go b/server/executor.go deleted file mode 100644 index 18f77f7c5..000000000 --- a/server/executor.go +++ /dev/null @@ -1,5 +0,0 @@ -package server - -type Executor interface { - execute(ctx *ExecutionContext, pullCtx *PullRequestContext) ExecutionResult -} diff --git a/server/github_client.go b/server/github_client.go index 3be567247..49c583f37 100644 --- a/server/github_client.go +++ b/server/github_client.go @@ -1,10 +1,10 @@ package server import ( + "context" "fmt" "github.com/google/go-github/github" - "context" - "strings" + "github.com/hootsuite/atlantis/models" ) type GithubClient struct { @@ -16,21 +16,21 @@ const ( statusContext = "Atlantis" PendingStatus = "pending" SuccessStatus = "success" - ErrorStatus = "error" + ErrorStatus = "error" FailureStatus = "failure" ) -func (g *GithubClient) UpdateStatus(ctx *PullRequestContext, status string, description string) { +func (g *GithubClient) UpdateStatus(repo models.Repo, pull models.PullRequest, status string, description string) { repoStatus := github.RepoStatus{State: github.String(status), Description: github.String(description), Context: github.String(statusContext)} - owner, repo := g.repoFullNameToOwnerAndRepo(ctx.repoFullName) - g.client.Repositories.CreateStatus(g.ctx, owner, repo, ctx.head, &repoStatus) + g.client.Repositories.CreateStatus(g.ctx, repo.Owner, repo.Name, pull.HeadCommit, &repoStatus) // todo: deal with error updating status } -func (g *GithubClient) GetModifiedFiles(ctx *PullRequestContext) ([]string, error) { - var files = []string{} - owner, repo := g.repoFullNameToOwnerAndRepo(ctx.repoFullName) - comparison, _, err := g.client.Repositories.CompareCommits(g.ctx, owner, repo, ctx.base, ctx.head) +// GetModifiedFiles returns the names of files that were modified in the pull request. +// The names include the path to the file from the repo root, ex. parent/child/file.txt +func (g *GithubClient) GetModifiedFiles(repo models.Repo, pull models.PullRequest) ([]string, error) { + var files []string + comparison, _, err := g.client.Repositories.CompareCommits(g.ctx, repo.Owner, repo.Name, pull.BaseCommit, pull.HeadCommit) if err != nil { return files, err } @@ -40,40 +40,15 @@ func (g *GithubClient) GetModifiedFiles(ctx *PullRequestContext) ([]string, erro return files, nil } -func (g *GithubClient) CreateComment(ctx *PullRequestContext, comment string) error { - owner, repo := g.repoFullNameToOwnerAndRepo(ctx.repoFullName) - _, _, err := g.client.Issues.CreateComment(g.ctx, owner, repo, ctx.number, &github.IssueComment{Body: &comment}) +func (g *GithubClient) CreateComment(ctx *CommandContext, comment string) error { + _, _, err := g.client.Issues.CreateComment(g.ctx, ctx.Repo.Owner, ctx.Repo.Name, ctx.Pull.Num, &github.IssueComment{Body: &comment}) return err } -// CommentExists searches through comments on a pull request and returns true if one matches matcher -func (g *GithubClient) CommentExists(ctx *PullRequestContext, matcher func(*github.IssueComment) bool) (bool, error) { - opt := &github.IssueListCommentsOptions{} - // need to loop since there may be multiple pages of comments - for { - owner, repo := g.repoFullNameToOwnerAndRepo(ctx.repoFullName) - comments, resp, err := g.client.Issues.ListComments(g.ctx, owner, repo, ctx.number, opt) - if err != nil { - return false, fmt.Errorf("failed to retrieve comments: %v", err) - } - for _, comment := range comments { - if matcher(comment) { - return true, nil - } - } - if resp.NextPage == 0 { - break - } - opt.ListOptions.Page = resp.NextPage - } - return false, nil -} - -func (g *GithubClient) PullIsApproved(ctx *PullRequestContext) (bool, error) { +func (g *GithubClient) PullIsApproved(repo models.Repo, pull models.PullRequest) (bool, error) { // todo: move back to using g.client.PullRequests.ListReviews when we update our GitHub enterprise version // to where we don't need to include the custom accept header - owner, repo := g.repoFullNameToOwnerAndRepo(ctx.repoFullName) - u := fmt.Sprintf("repos/%v/%v/pulls/%d/reviews", owner, repo, ctx.number) + u := fmt.Sprintf("repos/%v/%v/pulls/%d/reviews", repo.Owner, repo.Name, pull.Num) req, err := g.client.NewRequest("GET", u, nil) if err != nil { return false, err @@ -93,17 +68,6 @@ func (g *GithubClient) PullIsApproved(ctx *PullRequestContext) (bool, error) { return false, nil } -func (g *GithubClient) GetPullRequest(repoFullName string, number int) (*github.PullRequest, *github.Response, error) { - owner, repo := g.repoFullNameToOwnerAndRepo(repoFullName) - return g.client.PullRequests.Get(g.ctx, owner, repo, number) -} - -// repoFullNameToOwnerAndRepo splits up a repository full name which contains the organization and repo name separated by / -// into its two parts: organization and repo name. ex baxterthehacker/public-repo => (baxterthehacker, public-repo) -func (g *GithubClient) repoFullNameToOwnerAndRepo(fullName string) (string, string) { - split := strings.SplitN(fullName, "/", 2) - if len(split) != 2 { - return fmt.Sprintf("repo name %s could not be split into organization and name", fullName), "" - } - return split[0], split[1] +func (g *GithubClient) GetPullRequest(repo models.Repo, num int) (*github.PullRequest, *github.Response, error) { + return g.client.PullRequests.Get(g.ctx, repo.Owner, repo.Name, num) } diff --git a/server/help_executor.go b/server/help_executor.go index a94d17163..b801795de 100644 --- a/server/help_executor.go +++ b/server/help_executor.go @@ -2,12 +2,10 @@ package server import "github.com/spf13/viper" -type HelpExecutor struct { - BaseExecutor -} +type HelpExecutor struct{} var helpComment = "```cmake\n" + -`atlantis - Terraform collaboration tool that enables you to collaborate on infrastructure + `atlantis - Terraform collaboration tool that enables you to collaborate on infrastructure safely and securely. (v` + viper.GetString("version") + `) Usage: atlantis [environment] [--verbose] @@ -32,18 +30,8 @@ atlantis apply staging atlantis apply ` -func (h *HelpExecutor) execute(ctx *ExecutionContext, github *GithubClient) { - pullCtx := &PullRequestContext{ - repoFullName: ctx.repoFullName, - head: ctx.head, - base: ctx.base, - number: ctx.pullNum, - pullRequestLink: ctx.pullLink, - terraformApplier: ctx.requesterUsername, - terraformApplierEmail: ctx.requesterEmail, - } - github.CreateComment(pullCtx, helpComment) - - ctx.log.Info("generating help comment....") +func (h *HelpExecutor) execute(ctx *CommandContext, github *GithubClient) { + ctx.Log.Info("generating help comment....") + github.CreateComment(ctx, helpComment) return } diff --git a/server/plan_executor.go b/server/plan_executor.go index 10ed1263e..ba85c32dd 100644 --- a/server/plan_executor.go +++ b/server/plan_executor.go @@ -3,19 +3,27 @@ package server import ( "errors" "fmt" + "github.com/hootsuite/atlantis/locking" + "github.com/hootsuite/atlantis/logging" + "github.com/hootsuite/atlantis/models" "io/ioutil" "os" "os/exec" + "path" "path/filepath" "strings" - "github.com/hootsuite/atlantis/locking" - "time" - "github.com/hootsuite/atlantis/logging" ) // PlanExecutor handles everything related to running the Terraform plan including integration with S3, Terraform, and Github type PlanExecutor struct { - BaseExecutor + github *GithubClient + awsConfig *AWSConfig + scratchDir string + s3Bucket string + sshKey string + terraform *TerraformClient + githubCommentRenderer *GithubCommentRenderer + lockingClient *locking.Client // DeleteLockURL is a function that given a lock id will return a url for deleting the lock DeleteLockURL func(id string) (url string) } @@ -23,7 +31,7 @@ type PlanExecutor struct { /** Result Types **/ type PlanSuccess struct { TerraformOutput string - LockURL string + LockURL string } func (p PlanSuccess) Template() *CompiledTemplate { @@ -61,37 +69,38 @@ func (e EnvironmentFailure) Template() *CompiledTemplate { return EnvironmentErrorTmpl } -func (p *PlanExecutor) execute(ctx *ExecutionContext, pullCtx *PullRequestContext) ExecutionResult { - res := p.setupAndPlan(ctx, pullCtx) +func (p *PlanExecutor) execute(ctx *CommandContext, github *GithubClient) { + res := p.setupAndPlan(ctx) res.Command = Plan - return res + comment := p.githubCommentRenderer.render(res, ctx.Log.History.String(), ctx.Command.verbose) + github.CreateComment(ctx, comment) } -func (p *PlanExecutor) setupAndPlan(ctx *ExecutionContext, pullCtx *PullRequestContext) ExecutionResult { - p.github.UpdateStatus(pullCtx, "pending", "Planning...") +func (p *PlanExecutor) setupAndPlan(ctx *CommandContext) ExecutionResult { + p.github.UpdateStatus(ctx.Repo, ctx.Pull, "pending", "Planning...") // todo: lock when cloning or somehow separate workspaces // clean the directory where we're going to clone - cloneDir := fmt.Sprintf("%s/%s/%d", p.scratchDir, ctx.repoFullName, ctx.pullNum) - ctx.log.Info("cleaning clone directory %q", cloneDir) + cloneDir := fmt.Sprintf("%s/%s/%d", p.scratchDir, ctx.Repo.FullName, ctx.Pull.Num) + ctx.Log.Info("cleaning clone directory %q", cloneDir) if err := os.RemoveAll(cloneDir); err != nil { - ctx.log.Warn("failed to clean dir %q before cloning, attempting to continue: %v", cloneDir, err) + ctx.Log.Warn("failed to clean dir %q before cloning, attempting to continue: %v", cloneDir, err) } // create the directory and parents if necessary - ctx.log.Info("creating dir %q", cloneDir) + ctx.Log.Info("creating dir %q", cloneDir) if err := os.MkdirAll(cloneDir, 0755); err != nil { - ctx.log.Warn("failed to create dir %q prior to cloning, attempting to continue: %v", cloneDir, err) + ctx.Log.Warn("failed to create dir %q prior to cloning, attempting to continue: %v", cloneDir, err) } // Check if ssh key is set and create git ssh wrapper - cloneCmd := exec.Command("git", "clone", ctx.repoSSHUrl, cloneDir) + cloneCmd := exec.Command("git", "clone", ctx.Repo.SSHURL, cloneDir) if p.sshKey != "" { err := GenerateSSHWrapper() if err != nil { errMsg := fmt.Sprintf("failed to create git ssh wrapper: %v", err) - ctx.log.Err(errMsg) - p.github.UpdateStatus(pullCtx, ErrorStatus, "Plan Error") + ctx.Log.Err(errMsg) + p.github.UpdateStatus(ctx.Repo, ctx.Pull, ErrorStatus, "Plan Error") return ExecutionResult{SetupError: GeneralError{errors.New(errMsg)}} } @@ -102,53 +111,53 @@ func (p *PlanExecutor) setupAndPlan(ctx *ExecutionContext, pullCtx *PullRequestC } // git clone the repo - ctx.log.Info("git cloning %q into %q", ctx.repoSSHUrl, cloneDir) + ctx.Log.Info("git cloning %q into %q", ctx.Repo.SSHURL, cloneDir) if output, err := cloneCmd.CombinedOutput(); err != nil { - errMsg := fmt.Sprintf("failed to clone repository %q: %v: %s", ctx.repoSSHUrl, err, string(output)) - ctx.log.Err(errMsg) - p.github.UpdateStatus(pullCtx, ErrorStatus, "Plan Error") + errMsg := fmt.Sprintf("failed to clone repository %q: %v: %s", ctx.Repo.SSHURL, err, string(output)) + ctx.Log.Err(errMsg) + p.github.UpdateStatus(ctx.Repo, ctx.Pull, ErrorStatus, "Plan Error") return ExecutionResult{SetupError: GeneralError{errors.New(errMsg)}} } // check out the branch for this PR - ctx.log.Info("checking out branch %q", ctx.branch) - checkoutCmd := exec.Command("git", "checkout", ctx.branch) + ctx.Log.Info("checking out branch %q", ctx.Pull.Branch) + checkoutCmd := exec.Command("git", "checkout", ctx.Pull.Branch) checkoutCmd.Dir = cloneDir if err := checkoutCmd.Run(); err != nil { - errMsg := fmt.Sprintf("failed to git checkout branch %q: %v", ctx.branch, err) - ctx.log.Err(errMsg) - p.github.UpdateStatus(pullCtx, ErrorStatus, "Plan Error") + errMsg := fmt.Sprintf("failed to git checkout branch %q: %v", ctx.Pull.Branch, err) + ctx.Log.Err(errMsg) + p.github.UpdateStatus(ctx.Repo, ctx.Pull, ErrorStatus, "Plan Error") return ExecutionResult{SetupError: GeneralError{errors.New(errMsg)}} } - ctx.log.Info("listing modified files from pull request") - modifiedFiles, err := p.github.GetModifiedFiles(pullCtx) + ctx.Log.Info("listing modified files from pull request") + modifiedFiles, err := p.github.GetModifiedFiles(ctx.Repo, ctx.Pull) if err != nil { errMsg := fmt.Sprintf("failed to retrieve list of modified files from GitHub: %v", err) - ctx.log.Err(errMsg) - p.github.UpdateStatus(pullCtx, ErrorStatus, "Plan Error") + ctx.Log.Err(errMsg) + p.github.UpdateStatus(ctx.Repo, ctx.Pull, ErrorStatus, "Plan Error") return ExecutionResult{SetupError: GeneralError{errors.New(errMsg)}} } modifiedTerraformFiles := p.filterToTerraform(modifiedFiles) if len(modifiedTerraformFiles) == 0 { - ctx.log.Info("no modified terraform files found, exiting") - p.github.UpdateStatus(pullCtx, FailureStatus, "Plan Failed") + ctx.Log.Info("no modified terraform files found, exiting") + p.github.UpdateStatus(ctx.Repo, ctx.Pull, FailureStatus, "Plan Failed") return ExecutionResult{SetupError: GeneralError{errors.New("Plan Failed: no modified terraform files found")}} } - ctx.log.Debug("Found %d modified terraform files: %v", len(modifiedTerraformFiles), modifiedTerraformFiles) + ctx.Log.Debug("Found %d modified terraform files: %v", len(modifiedTerraformFiles), modifiedTerraformFiles) - execPaths := p.DetermineExecPaths(p.trimSuffix(cloneDir, "/"), modifiedTerraformFiles) - if len(execPaths) == 0 { - ctx.log.Info("no exec paths found, exiting") - p.github.UpdateStatus(pullCtx, FailureStatus, "Plan Failed") - return ExecutionResult{SetupError: GeneralError{errors.New("Plan Failed: there were no paths to run plan in")}} + projects := p.ModifiedProjects(ctx.Repo.FullName, modifiedTerraformFiles) + if len(projects) == 0 { + ctx.Log.Info("no Terraform projects were modified") + p.github.UpdateStatus(ctx.Repo, ctx.Pull, FailureStatus, "Plan Failed") + return ExecutionResult{SetupError: GeneralError{errors.New("Plan Failed: we determined that no terraform projects were modified")}} } - planFilesPrefix := fmt.Sprintf("%s_%d", strings.Replace(ctx.repoFullName, "/", "_", -1), ctx.pullNum) - if err := p.CleanWorkspace(ctx.log, planFilesPrefix, p.scratchDir, execPaths); err != nil { + planFilesPrefix := fmt.Sprintf("%s_%d", strings.Replace(ctx.Repo.FullName, "/", "_", -1), ctx.Pull.Num) + if err := p.CleanWorkspace(ctx.Log, planFilesPrefix, p.scratchDir, cloneDir, projects); err != nil { errMsg := fmt.Sprintf("failed to clean workspace, aborting: %v", err) - ctx.log.Err(errMsg) - p.github.UpdateStatus(pullCtx, ErrorStatus, "Plan Error") + ctx.Log.Err(errMsg) + p.github.UpdateStatus(ctx.Repo, ctx.Pull, ErrorStatus, "Plan Error") return ExecutionResult{SetupError: GeneralError{errors.New(errMsg)}} } s3Client := NewS3Client(p.awsConfig, p.s3Bucket, "plans") @@ -156,25 +165,26 @@ func (p *PlanExecutor) setupAndPlan(ctx *ExecutionContext, pullCtx *PullRequestC var config Config // run `terraform plan` in each plan path and collect the results planOutputs := []PathResult{} - for _, path := range execPaths { + for _, project := range projects { // todo: not sure it makes sense to be generating the output filename and plan name here - tfPlanFilename := p.GenerateOutputFilename(cloneDir, path, ctx.command.environment) - tfPlanName := fmt.Sprintf("%s_%d%s", strings.Replace(ctx.repoFullName, "/", "_", -1), ctx.pullNum, tfPlanFilename) - s3Key := fmt.Sprintf("%s/%s", ctx.repoFullName, tfPlanName) + tfPlanFilename := p.GenerateOutputFilename(project, ctx.Command.environment) + tfPlanName := fmt.Sprintf("%s_%d%s", strings.Replace(ctx.Repo.FullName, "/", "_", -1), ctx.Pull.Num, tfPlanFilename) + s3Key := fmt.Sprintf("%s/%s", ctx.Repo.FullName, tfPlanName) // check if config file is found, if not we continue the run - if config.Exists(path.Absolute) { - ctx.log.Info("Config file found in %s", path.Absolute) - err := config.Read(path.Absolute) + absolutePath := filepath.Join(cloneDir, project.Path) + if config.Exists(absolutePath) { + ctx.Log.Info("Config file found in %s", absolutePath) + err := config.Read(absolutePath) if err != nil { errMsg := fmt.Sprintf("Error reading config file: %v", err) - ctx.log.Err(errMsg) + ctx.Log.Err(errMsg) return ExecutionResult{SetupError: GeneralError{errors.New(errMsg)}} } // need to use the remote state path and backend to do remote configure - err = PreRun(&config, ctx.log, path.Absolute, ctx.command) + err = PreRun(&config, ctx.Log, absolutePath, ctx.Command) if err != nil { errMsg := fmt.Sprintf("pre run failed: %v", err) - ctx.log.Err(errMsg) + ctx.Log.Err(errMsg) return ExecutionResult{SetupError: GeneralError{errors.New(errMsg)}} } @@ -185,41 +195,31 @@ func (p *PlanExecutor) setupAndPlan(ctx *ExecutionContext, pullCtx *PullRequestC p.terraform.tfExecutableName = "terraform" } } - generatePlanResponse := p.plan(ctx.log, pullCtx, cloneDir, p.scratchDir, tfPlanName, s3Client, path, ctx.command.environment, s3Key, p.sshKey, ctx.pullCreator, config.StashPath) - generatePlanResponse.Path = path.Relative + generatePlanResponse := p.plan(ctx, cloneDir, p.scratchDir, tfPlanName, s3Client, project, s3Key, p.sshKey, config.StashPath) + generatePlanResponse.Path = project.Path planOutputs = append(planOutputs, generatePlanResponse) } - p.updateGithubStatus(pullCtx, planOutputs) + p.updateGithubStatus(ctx, planOutputs) return ExecutionResult{PathResults: planOutputs} } // plan runs the steps necessary to run `terraform plan`. If there is an error, the error message will be encapsulated in error // and the GeneratePlanResponse struct will also contain the full log including the error -func (p *PlanExecutor) plan(log *logging.SimpleLogger, - pullCtx *PullRequestContext, +func (p *PlanExecutor) plan( + ctx *CommandContext, repoDir string, planOutDir string, tfPlanName string, s3Client S3Client, - path ExecutionPath, - tfEnvName string, + project models.Project, s3Key string, sshKey string, - pullRequestCreator string, stashPath string) PathResult { - log.Info("generating plan for path %q", path) - run := locking.Run{ - RepoFullName: pullCtx.repoFullName, - Path: path.Relative, - Env: tfEnvName, - PullNum: pullCtx.number, - User: pullCtx.terraformApplier, - Timestamp: time.Now(), - } + ctx.Log.Info("generating plan for path %q", project.Path) // NOTE: THIS CODE IS TO SUPPORT TERRAFORM PROJECTS THAT AREN'T USING ATLANTIS CONFIG FILE. if stashPath == "" { - _, err := p.terraform.ConfigureRemoteState(log, path, tfEnvName, sshKey) + _, err := p.terraform.ConfigureRemoteState(ctx.Log, repoDir, project, ctx.Command.environment, sshKey) if err != nil { return PathResult{ Status: "error", @@ -227,34 +227,39 @@ func (p *PlanExecutor) plan(log *logging.SimpleLogger, } } } - lockAttempt, err := p.lockingBackend.TryLock(run) + // todo: setting environment to default should be done elsewhere + tfEnv := ctx.Command.environment + if tfEnv == "" { + tfEnv = "default" + } + lockAttempt, err := p.lockingClient.TryLock(project, ctx.Command.environment, ctx.Pull.Num) if err != nil { return PathResult{ - Status:" failure", + Status: " failure", Result: GeneralError{fmt.Errorf("failed to lock state: %v", err)}, } } // the run is locked unless the locking run is the same pull id as this run - if lockAttempt.LockAcquired == false && lockAttempt.LockingRun.PullNum != pullCtx.number { + if lockAttempt.LockAcquired == false && lockAttempt.LockingPullNum != ctx.Pull.Num { return PathResult{ Status: "failure", - Result: RunLockedFailure{lockAttempt.LockingRun.PullNum}, + Result: RunLockedFailure{lockAttempt.LockingPullNum}, } } // Run terraform plan - log.Info("running terraform plan in directory %q", path.Relative) + ctx.Log.Info("running terraform plan in directory %q", project.Path) tfPlanCmd := []string{"plan", "-refresh", "-no-color"} // Generate terraform plan filename tfPlanOutputPath := filepath.Join(planOutDir, tfPlanName) // Generate terraform plan arguments - if tfEnvName != "" { - tfEnvFileName := filepath.Join("env", tfEnvName+".tfvars") - if _, err := os.Stat(filepath.Join(path.Absolute, tfEnvFileName)); err == nil { + if ctx.Command.environment != "" { + tfEnvFileName := filepath.Join("env", ctx.Command.environment+".tfvars") + if _, err := os.Stat(filepath.Join(repoDir, project.Path, tfEnvFileName)); err == nil { tfPlanCmd = append(tfPlanCmd, "-var-file", tfEnvFileName, "-out", tfPlanOutputPath) } else { - log.Err("environment file %q not found", tfEnvFileName) + ctx.Log.Err("environment file %q not found", tfEnvFileName) return PathResult{ Status: "failure", Result: EnvironmentFileNotFoundFailure{tfEnvFileName}, @@ -265,10 +270,10 @@ func (p *PlanExecutor) plan(log *logging.SimpleLogger, } // set pull request creator as the session name - p.awsConfig.AWSSessionName = pullRequestCreator + p.awsConfig.AWSSessionName = ctx.Pull.Author awsSession, err := p.awsConfig.CreateAWSSession() if err != nil { - log.Err(err.Error()) + ctx.Log.Err(err.Error()) return PathResult{ Status: "error", Result: GeneralError{err}, @@ -278,14 +283,14 @@ func (p *PlanExecutor) plan(log *logging.SimpleLogger, credVals, err := awsSession.Config.Credentials.Get() if err != nil { err = fmt.Errorf("failed to get assumed role credentials: %v", err) - log.Err(err.Error()) + ctx.Log.Err(err.Error()) return PathResult{ Status: "error", Result: GeneralError{err}, } } - terraformPlanCmdArgs, output, err := p.terraform.RunTerraformCommand(path, tfPlanCmd, []string{ + terraformPlanCmdArgs, output, err := p.terraform.RunTerraformCommand(filepath.Join(repoDir, project.Path), tfPlanCmd, []string{ fmt.Sprintf("AWS_ACCESS_KEY_ID=%s", credVals.AccessKeyID), fmt.Sprintf("AWS_SECRET_ACCESS_KEY=%s", credVals.SecretAccessKey), fmt.Sprintf("AWS_SESSION_TOKEN=%s", credVals.SessionToken), @@ -299,10 +304,10 @@ func (p *PlanExecutor) plan(log *logging.SimpleLogger, Command: strings.Join(terraformPlanCmdArgs, " "), Output: output, } - log.Err("error running terraform plan: %v", output) - log.Info("unlocking state since plan failed") - if err := p.lockingBackend.Unlock(lockAttempt.LockID); err != nil { - log.Err("error unlocking state: %v", err) + ctx.Log.Err("error running terraform plan: %v", output) + ctx.Log.Info("unlocking state since plan failed") + if err := p.lockingClient.Unlock(lockAttempt.LockKey); err != nil { + ctx.Log.Err("error unlocking state: %v", err) } return PathResult{ Status: "failure", @@ -310,12 +315,12 @@ func (p *PlanExecutor) plan(log *logging.SimpleLogger, } } // Upload plan to S3 - log.Info("uploading plan to S3 with key %q", s3Key) + ctx.Log.Info("uploading plan to S3 with key %q", s3Key) if err := UploadPlanFile(s3Client, s3Key, tfPlanOutputPath); err != nil { err = fmt.Errorf("failed to upload to S3: %v", err) - log.Err(err.Error()) - if err := p.lockingBackend.Unlock(lockAttempt.LockID); err != nil { - log.Err("error unlocking state: %v", err) + ctx.Log.Err(err.Error()) + if err := p.lockingClient.Unlock(lockAttempt.LockKey); err != nil { + ctx.Log.Err("error unlocking state: %v", err) } return PathResult{ Status: "error", @@ -324,22 +329,22 @@ func (p *PlanExecutor) plan(log *logging.SimpleLogger, } // Delete local plan file planFilePath := fmt.Sprintf("%s/%s", planOutDir, tfPlanName) - log.Info("deleting local plan file %q", planFilePath) + ctx.Log.Info("deleting local plan file %q", planFilePath) if err := os.Remove(planFilePath); err != nil { - log.Err("failed to delete local plan file %q", planFilePath, err) + ctx.Log.Err("failed to delete local plan file %q", planFilePath, err) // todo: return an error } return PathResult{ Status: "success", Result: PlanSuccess{ TerraformOutput: output, - LockURL: p.DeleteLockURL(lockAttempt.LockID), + LockURL: p.DeleteLockURL(lockAttempt.LockKey), }, } } func (p *PlanExecutor) filterToTerraform(files []string) []string { - var out = []string{} + var out []string for _, fileName := range files { if !p.isInExcludeList(fileName) && strings.Contains(fileName, ".tf") { out = append(out, fileName) @@ -359,46 +364,40 @@ func (p *PlanExecutor) trimSuffix(s, suffix string) string { return s } -func (p *PlanExecutor) removeDuplicates(paths []ExecutionPath) []ExecutionPath { - deDuped := []ExecutionPath{} - seen := map[ExecutionPath]bool{} - for _, path := range paths { - if _, ok := seen[path]; !ok { - deDuped = append(deDuped, path) - seen[path] = true +// ModifiedProjects returns the list of Terraform projects that have been changed due to the +// modified files +func (p *PlanExecutor) ModifiedProjects(repoFullName string, modifiedFiles []string) []models.Project { + var projects []models.Project + seenPaths := make(map[string]bool) + for _, modifiedFile := range modifiedFiles { + path := p.getProjectPath(modifiedFile) + if _, ok := seenPaths[path]; !ok { + projects = append(projects, models.NewProject(repoFullName, path)) + seenPaths[path] = true } } - return deDuped + return projects } -// DetermineExecPaths returns the list of directories in which we'll need to run `terraform plan` -func (p *PlanExecutor) DetermineExecPaths(repoPath string, modifiedFiles []string) []ExecutionPath { - var paths []ExecutionPath - for _, modifiedFile := range modifiedFiles { - relative := p.getRelativePlanPath(modifiedFile) - absolute := filepath.Join(repoPath, relative) + "/" - paths = append(paths, NewExecutionPath(absolute, relative)) - } - return p.removeDuplicates(paths) -} - -func (p *PlanExecutor) getRelativePlanPath(modifiedFilePath string) string { - parentDir := filepath.Dir(modifiedFilePath) - if filepath.Base(parentDir) == "env" { +// getProjectPath returns the path to the project relative to the repo root +// if the project is at the root returns "." +func (p *PlanExecutor) getProjectPath(modifiedFilePath string) string { + dir := path.Dir(modifiedFilePath) + if path.Base(dir) == "env" { // if the modified file was inside an env/ directory, we treat this specially and // run plan one level up - return filepath.Dir(parentDir) + return path.Dir(dir) } - return parentDir + return dir } -// CleanWorkspace deletes all .terraform/ folders from the plan paths and cleans up any plans in the output directory -func (p *PlanExecutor) CleanWorkspace(log *logging.SimpleLogger, deleteFilesPrefix string, planOutDir string, execPaths []ExecutionPath) error { - log.Info("cleaning workspace directory %q and plan paths %v", planOutDir, execPaths) +// CleanWorkspace deletes all .terraform/ folders from the project folders and cleans up any plans in the output directory +func (p *PlanExecutor) CleanWorkspace(log *logging.SimpleLogger, deleteFilesPrefix string, planOutDir string, repoDir string, projects []models.Project) error { + log.Info("cleaning workspace directory %q", planOutDir) // delete .terraform directories - for _, path := range execPaths { - os.RemoveAll(filepath.Join(path.Absolute, ".terraform")) + for _, project := range projects { + os.RemoveAll(filepath.Join(repoDir, project.Path, ".terraform")) } // delete old plan files files, err := ioutil.ReadDir(planOutDir) @@ -423,14 +422,14 @@ func (p *PlanExecutor) DeleteLocalPlanFile(path string) error { // GenerateOutputFilename determines the name of the plan that will be stored in s3 // if we're executing inside a sub directory, there will be a leading underscore -func (p *PlanExecutor) GenerateOutputFilename(repoDir string, execPath ExecutionPath, tfEnvName string) string { +func (p *PlanExecutor) GenerateOutputFilename(project models.Project, tfEnvName string) string { prefix := "" - if execPath.Relative != "." { + if project.Path != "." { // If not executing at repo root, need to encode the sub dir in the name of the output file. // We do this by substituting / for _ // We also add an _ because this gets appended to a larger path // todo: refactor the path handling so it's all in one place - prefix = "_" + strings.Replace(execPath.Relative, "/", "_", -1) + prefix = "_" + strings.Replace(project.Path, "/", "_", -1) } suffix := "" if tfEnvName != "" { @@ -443,3 +442,27 @@ func (p *PlanExecutor) GenerateOutputFilename(repoDir string, execPath Execution func generateStatePath(path string, tfEnvName string) string { return strings.Replace(path, "$ENVIRONMENT", tfEnvName, -1) } + +func (p *PlanExecutor) updateGithubStatus(ctx *CommandContext, pathResults []PathResult) { + // the status will be the worst result + worstResult := p.worstResult(pathResults) + if worstResult == "success" { + p.github.UpdateStatus(ctx.Repo, ctx.Pull, SuccessStatus, "Plan Succeeded") + } else if worstResult == "failure" { + p.github.UpdateStatus(ctx.Repo, ctx.Pull, FailureStatus, "Plan Failed") + } else { + p.github.UpdateStatus(ctx.Repo, ctx.Pull, ErrorStatus, "Plan Error") + } +} + +func (p *PlanExecutor) worstResult(results []PathResult) string { + var worst string = "success" + for _, result := range results { + if result.Status == "error" { + return result.Status + } else if result.Status == "failure" { + worst = result.Status + } + } + return worst +} diff --git a/server/plan_executor_test.go b/server/plan_executor_test.go index e0836bcb9..47548c17b 100644 --- a/server/plan_executor_test.go +++ b/server/plan_executor_test.go @@ -1,40 +1,41 @@ package server import ( - "reflect" + "github.com/hootsuite/atlantis/models" + . "github.com/hootsuite/atlantis/testing_util" "testing" ) -func TestDetermineExecPaths(t *testing.T) { - runTest(t, "should handle empty plan paths", "/repoPath", []string{}, []ExecutionPath{}) - runTest(t, "should extract path and append to repoPath", "/repoPath", []string{"a/b/c.ext"}, []ExecutionPath{{"/repoPath/a/b", "a/b"}}) - runTest(t, "should return repoPath if at root", "/repoPath", []string{"a.ext"}, []ExecutionPath{{"/repoPath", "."}}) - runTest(t, "should handle repoPath with trailing slash", "/repoPath/", []string{"a.ext"}, []ExecutionPath{{"/repoPath", "."}}) - runTest(t, "should set plan dir one level up from env/ directories", "/repoPath/", []string{"env/a.ext"}, []ExecutionPath{{"/repoPath", "."}}) - runTest(t, "should set plan dir one level up from env/ directories and deduplicate plans.", "/repoPath/", []string{"env/a.ext", "b.ext"}, []ExecutionPath{{"/repoPath", "."}}) - runTest(t, "should de-depluciate", "/repoPath/", []string{"a/b/c.ext", "a/b/d.ext"}, []ExecutionPath{{"/repoPath/a/b", "a/b"}}) +var p PlanExecutor + +func TestModifiedProjects(t *testing.T) { + runTest(t, "should handle no files modified", []string{}, []string{}) + runTest(t, "should handle files at root", []string{"root.tf"}, []string{"."}) + runTest(t, "should de-duplicate files at root", []string{"root1.tf", "root2.tf"}, []string{"."}) + runTest(t, "should handle sub directories", []string{"sub/dir/file.tf"}, []string{"sub/dir"}) + runTest(t, "should de-duplicate files in sub directories", []string{"sub/dir/file.tf", "sub/dir/file2.tf"}, []string{"sub/dir"}) + runTest(t, "should handle nested sub directories", []string{"root.tf", "sub/child.tf", "sub/sub/child.tf"}, []string{".", "sub", "sub/sub"}) + runTest(t, "should use parent of env/ dirs", []string{"env/env.tf"}, []string{"."}) + runTest(t, "should use parent of env/ dirs in sub dirs", []string{"sub/env/env.tf"}, []string{"sub"}) } -func runTest(t *testing.T, testDescrip string, repoPath string, filesChanged []string, expected []ExecutionPath) { - p := PlanExecutor{} - plans := p.DetermineExecPaths(repoPath, filesChanged) - if !reflect.DeepEqual(expected, plans) { - t.Errorf("%s: expected %v, got %v", testDescrip, expected, plans) +func runTest(t *testing.T, testDescrip string, filesChanged []string, expectedPaths []string) { + projects := p.ModifiedProjects("owner/repo", filesChanged) + for i, p := range projects { + t.Log(testDescrip) + Equals(t, expectedPaths[i], p.Path) } } func TestGenerateOutputFilename(t *testing.T) { - runTestGetOutputFilename(t, "should handle empty plan path", "/repoPath", NewExecutionPath("", ""), "env", ".tfplan.env") - runTestGetOutputFilename(t, "should handle empty environment", "/repoPath", NewExecutionPath("", ""), "", ".tfplan") - runTestGetOutputFilename(t, "should prepend underscore on relative paths", "/repoPath", NewExecutionPath("", "a/b"), "", "_a_b.tfplan") - runTestGetOutputFilename(t, "should work with relative path and environment", "/repoPath", NewExecutionPath("", "a/b"), "env", "_a_b.tfplan.env") - runTestGetOutputFilename(t, "should exec path at root", "/a/b", NewExecutionPath("", "."), "env", ".tfplan.env") + runTestGetOutputFilename(t, "should handle root", ".", "env", ".tfplan.env") + runTestGetOutputFilename(t, "should handle empty environment", ".", "", ".tfplan") + runTestGetOutputFilename(t, "should prepend underscore on relative paths", "a/b", "", "_a_b.tfplan") + runTestGetOutputFilename(t, "should prepend underscore on relative paths and env", "a/b", "env", "_a_b.tfplan.env") } -func runTestGetOutputFilename(t *testing.T, testDescrip string, repoPath string, tfPlanPath ExecutionPath, tfEnvName string, expected string) { - p := PlanExecutor{} - outputFileName := p.GenerateOutputFilename(repoPath, tfPlanPath, tfEnvName) - if !reflect.DeepEqual(expected, outputFileName) { - t.Errorf("%s: expected %v, got %v", testDescrip, expected, outputFileName) - } +func runTestGetOutputFilename(t *testing.T, testDescrip string, path string, env string, expected string) { + t.Log(testDescrip) + outputFileName := p.GenerateOutputFilename(models.NewProject("owner/repo", path), env) + Equals(t, expected, outputFileName) } diff --git a/server/pre_run.go b/server/pre_run.go index 8543e395d..e8a52efdd 100644 --- a/server/pre_run.go +++ b/server/pre_run.go @@ -3,17 +3,18 @@ package server import ( "bufio" "fmt" + "github.com/hootsuite/atlantis/logging" "io/ioutil" "os" "os/exec" - "github.com/hootsuite/atlantis/logging" ) const InlineShebang = "/bin/sh -e" +// todo: make OO // PreRun is a function that will determine whether -func PreRun(c *Config, log *logging.SimpleLogger, execPath string, command *Command) error { - log.Info("Staring pre run in %s", execPath) +func PreRun(c *Config, log *logging.SimpleLogger, path string, command *Command) error { + log.Info("Staring pre run in %s", path) var execScript string if command.commandType == Plan { @@ -44,7 +45,7 @@ func PreRun(c *Config, log *logging.SimpleLogger, execPath string, command *Comm if c.TerraformVersion != "" { os.Setenv("ATLANTIS_TERRAFORM_VERSION", c.TerraformVersion) } - os.Setenv("WORKSPACE", execPath) + os.Setenv("WORKSPACE", path) output, err := execute(execScript) if err != nil { return err diff --git a/server/pre_run_test.go b/server/pre_run_test.go index c545e3cab..3eb6313fb 100644 --- a/server/pre_run_test.go +++ b/server/pre_run_test.go @@ -1,11 +1,11 @@ package server import ( + "github.com/hootsuite/atlantis/logging" + . "github.com/hootsuite/atlantis/testing_util" "log" "os" "testing" - . "github.com/hootsuite/atlantis/testing_util" - "github.com/hootsuite/atlantis/logging" ) var level logging.LogLevel = logging.Info diff --git a/server/request_parser.go b/server/request_parser.go index c32f1df89..905cc9159 100644 --- a/server/request_parser.go +++ b/server/request_parser.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "github.com/google/go-github/github" + "github.com/hootsuite/atlantis/models" "regexp" ) @@ -24,6 +25,7 @@ type Command struct { } func (r *RequestParser) determineCommand(comment *github.IssueCommentEvent) (*Command, error) { + // for legacy, also support "run" instead of atlantis atlantisCommentRegex := `^(?:run|atlantis) (plan|apply|help)([[:blank:]])?([a-zA-Z0-9_-]+)?\s*(--verbose)?$` runPlanMatcher := regexp.MustCompile(atlantisCommentRegex) @@ -37,7 +39,7 @@ func (r *RequestParser) determineCommand(comment *github.IssueCommentEvent) (*Co return nil, errors.New("key 'comment.comment.body' is null") } - // extract the command and environment. ex. for "run plan staging", the command is "plan", and the environment is "staging" + // extract the command and environment. ex. for "atlantis plan staging", the command is "plan", and the environment is "staging" match := runPlanMatcher.FindStringSubmatch(*commentBody) if len(match) < 5 { var truncated = *commentBody @@ -65,13 +67,19 @@ func (r *RequestParser) determineCommand(comment *github.IssueCommentEvent) (*Co return command, nil } -func (r *RequestParser) extractCommentData(comment *github.IssueCommentEvent, params *ExecutionContext) error { - missingField := "" - +func (r *RequestParser) extractCommentData(comment *github.IssueCommentEvent, ctx *CommandContext) error { repoFullName := comment.Repo.FullName if repoFullName == nil { return errors.New("key 'comment.repo.full_name' is null") } + repoOwner := comment.Repo.Owner.Login + if repoOwner == nil { + return errors.New("key 'comment.repo.owner.login' is null") + } + repoName := comment.Repo.Name + if repoName == nil { + return errors.New("key 'comment.repo.name' is null") + } pullNum := comment.Issue.Number if pullNum == nil { return errors.New("key 'comment.issue.number' is null") @@ -84,10 +92,6 @@ func (r *RequestParser) extractCommentData(comment *github.IssueCommentEvent, pa if commentorUsername == nil { return errors.New("key 'comment.comment.user.login' is null") } - commentorEmail := comment.Comment.User.Email - if commentorEmail == nil { - commentorEmail = &missingField - } repoSSHURL := comment.Repo.SSHURL if repoSSHURL == nil { return errors.New("key 'comment.repo.sshurl' is null") @@ -96,17 +100,23 @@ func (r *RequestParser) extractCommentData(comment *github.IssueCommentEvent, pa if htmlURL == nil { return errors.New("key 'comment.issue.htmlUrl' is null") } - params.requesterUsername = *commentorUsername - params.requesterEmail = *commentorEmail - params.repoFullName = *repoFullName - params.pullNum = *pullNum - params.pullCreator = *pullCreator - params.repoSSHUrl = *repoSSHURL - params.htmlUrl = *htmlURL + ctx.Repo = models.Repo{ + FullName: *repoFullName, + Owner: *repoOwner, + Name: *repoName, + SSHURL: *repoSSHURL, + } + ctx.User = models.User{ + Username: *commentorUsername, + } + ctx.Pull = models.PullRequest{ + Num: *pullNum, + } + return nil } -func (r *RequestParser) extractPullData(pull *github.PullRequest, params *ExecutionContext) error { +func (r *RequestParser) extractPullData(pull *github.PullRequest, params *CommandContext) error { commit := pull.Head.SHA if commit == nil { return errors.New("key 'pull.head.sha' is null") @@ -123,9 +133,21 @@ func (r *RequestParser) extractPullData(pull *github.PullRequest, params *Execut if branch == nil { return errors.New("key 'pull.head.ref' is null") } - params.branch = *branch - params.head = *commit - params.base = *base - params.pullLink = *pullLink + authorUsername := pull.User.Login + if authorUsername == nil { + return errors.New("key 'pull.user.login' is null") + } + num := pull.Number + if num == nil { + return errors.New("key 'pull.num' is null") + } + params.Pull = models.PullRequest{ + BaseCommit: *base, + Author: *authorUsername, + Branch: *branch, + HeadCommit: *commit, + Link: *pullLink, + Num: *num, + } return nil } diff --git a/server/s3_client.go b/server/s3_client.go index 4edb04627..a1222cc5b 100644 --- a/server/s3_client.go +++ b/server/s3_client.go @@ -48,6 +48,7 @@ func NewS3Client(awsConfig *AWSConfig, bucketName string, prefix string) S3Clien return _s3.GetS3Info() } +// todo: make OO func UploadPlanFile(s S3Client, key string, outputFilePath string) error { file, err := os.Open(outputFilePath) if err != nil { @@ -75,7 +76,7 @@ func UploadPlanFile(s S3Client, key string, outputFilePath string) error { ContentLength: aws.Int64(size), ContentType: aws.String(fileType), Metadata: map[string]*string{ - "Key": aws.String("MetadataValue"), //required + "Key": aws.String("MetadataValue"), //required //todo: I don't think this is required }, } diff --git a/server/server.go b/server/server.go index 0636dfccd..dddad0869 100644 --- a/server/server.go +++ b/server/server.go @@ -2,27 +2,30 @@ package server import ( "context" + "encoding/json" "fmt" "log" "net/http" "net/url" "os" "strings" - "encoding/json" - "github.com/google/go-github/github" - "github.com/urfave/cli" - "github.com/hootsuite/atlantis/recovery" - "github.com/hootsuite/atlantis/locking" - "github.com/urfave/negroni" - "github.com/hootsuite/atlantis/middleware" - "github.com/hootsuite/atlantis/logging" "github.com/elazarl/go-bindata-assetfs" + "github.com/google/go-github/github" "github.com/gorilla/mux" - "github.com/pkg/errors" - "io/ioutil" - "github.com/hootsuite/atlantis/locking/dynamodb" + "github.com/hootsuite/atlantis/locking" "github.com/hootsuite/atlantis/locking/boltdb" + "github.com/hootsuite/atlantis/locking/dynamodb" + "github.com/hootsuite/atlantis/logging" + "github.com/hootsuite/atlantis/middleware" + "github.com/hootsuite/atlantis/models" + "github.com/hootsuite/atlantis/recovery" + "github.com/pkg/errors" + "github.com/urfave/cli" + "github.com/urfave/negroni" + "io/ioutil" + "path/filepath" + "time" ) const ( @@ -31,7 +34,7 @@ const ( LockingDynamoDBBackend = "dynamodb" ) -// WebhookServer listens for Github webhooks and runs the necessary Atlantis command +// Server listens for Github webhooks and runs the necessary Atlantis command type Server struct { router *mux.Router port int @@ -46,16 +49,16 @@ type Server struct { logger *logging.SimpleLogger githubComments *GithubCommentRenderer requestParser *RequestParser - lockingBackend locking.Backend + lockingClient *locking.Client atlantisURL string } // the mapstructure tags correspond to flags in cmd/server.go type ServerConfig struct { - GitHubHostname string `mapstructure:"gh-hostname"` - GitHubUser string `mapstructure:"gh-user"` - GitHubPassword string `mapstructure:"gh-password"` - SSHKey string `mapstructure:"ssh-key"` + GitHubHostname string `mapstructure:"gh-hostname"` + GitHubUser string `mapstructure:"gh-user"` + GitHubPassword string `mapstructure:"gh-password"` + SSHKey string `mapstructure:"ssh-key"` AssumeRole string `mapstructure:"aws-assume-role-arn"` Port int `mapstructure:"port"` ScratchDir string `mapstructure:"scratch-dir"` @@ -69,22 +72,51 @@ type ServerConfig struct { LockingDynamoDBTable string `mapstructure:"locking-dynamodb-table"` } -type ExecutionContext struct { - repoFullName string - pullNum int - requesterUsername string - requesterEmail string - comment string - repoSSHUrl string - head string - // commit base sha - base string - pullLink string - branch string - htmlUrl string - pullCreator string - command *Command - log *logging.SimpleLogger +// todo: rename to Command +type CommandContext struct { + Repo models.Repo + Pull models.PullRequest + User models.User + Command *Command + Log *logging.SimpleLogger +} + +type ExecutionResult struct { + SetupError Templater + SetupFailure Templater + PathResults []PathResult + Command CommandType +} + +type PathResult struct { + Path string + Status string // todo: this should be an enum for success/error/failure + Result Templater +} + +type ExecutionPath struct { + // Absolute is the full path on the OS where we will execute. + // Will never end with a '/'. + Absolute string + // Relative is the path relative to the repo root. + // Will never end with a '/'. + Relative string +} + +func NewExecutionPath(absolutePath string, relativePath string) ExecutionPath { + return ExecutionPath{filepath.Clean(absolutePath), filepath.Clean(relativePath)} +} + +type Templater interface { + Template() *CompiledTemplate +} + +type GeneralError struct { + Error error +} + +func (g GeneralError) Template() *CompiledTemplate { + return GeneralErrorTmpl } func NewServer(config ServerConfig) (*Server, error) { @@ -108,34 +140,42 @@ func NewServer(config ServerConfig) (*Server, error) { AWSRegion: config.AWSRegion, AWSRoleArn: config.AssumeRole, } - var lockingBackend locking.Backend + var lockingClient *locking.Client if config.LockingBackend == LockingDynamoDBBackend { session, err := awsConfig.CreateAWSSession() if err != nil { return nil, errors.Wrap(err, "creating aws session for DynamoDB") } - lockingBackend = dynamodb.New(config.LockingDynamoDBTable, session) + lockingClient = locking.NewClient(dynamodb.New(config.LockingDynamoDBTable, session)) } else { - var err error - lockingBackend, err = boltdb.New(config.DataDir) + backend, err := boltdb.New(config.DataDir) if err != nil { return nil, err } + lockingClient = locking.NewClient(backend) } - baseExecutor := BaseExecutor{ + applyExecutor := &ApplyExecutor{ github: githubClient, awsConfig: awsConfig, scratchDir: config.ScratchDir, s3Bucket: config.S3Bucket, sshKey: config.SSHKey, - ghComments: githubComments, terraform: terraformClient, githubCommentRenderer: githubComments, - lockingBackend: lockingBackend, + lockingClient: lockingClient, + requireApproval: config.RequireApproval, } - applyExecutor := &ApplyExecutor{BaseExecutor: baseExecutor, requireApproval: config.RequireApproval, atlantisGithubUser: config.GitHubUser} - planExecutor := &PlanExecutor{BaseExecutor: baseExecutor} - helpExecutor := &HelpExecutor{BaseExecutor: baseExecutor} + planExecutor := &PlanExecutor{ + github: githubClient, + awsConfig: awsConfig, + scratchDir: config.ScratchDir, + s3Bucket: config.S3Bucket, + sshKey: config.SSHKey, + terraform: terraformClient, + githubCommentRenderer: githubComments, + lockingClient: lockingClient, + } + helpExecutor := &HelpExecutor{} logger := logging.NewSimpleLogger("server", log.New(os.Stderr, "", log.LstdFlags), false, logging.ToLogLevel(config.LogLevel)) router := mux.NewRouter() return &Server{ @@ -152,7 +192,7 @@ func NewServer(config ServerConfig) (*Server, error) { logger: logger, githubComments: githubComments, requestParser: &RequestParser{}, - lockingBackend: lockingBackend, + lockingClient: lockingClient, atlantisURL: config.AtlantisURL, }, nil } @@ -172,7 +212,7 @@ func (s *Server) Start() error { // function that planExecutor can use to construct delete lock urls // injecting this here because this is the earliest routes are created - s.planExecutor.DeleteLockURL = func (lockID string) string { + s.planExecutor.DeleteLockURL = func(lockID string) string { // ignoring error since guaranteed to succeed if "id" is specified u, _ := deleteLockRoute.URL("id", url.QueryEscape(lockID)) return s.atlantisURL + u.RequestURI() @@ -189,23 +229,27 @@ func (s *Server) Start() error { } func (s *Server) index(w http.ResponseWriter, r *http.Request) { - locks, err := s.lockingBackend.ListLocks() + locks, err := s.lockingClient.List() if err != nil { w.WriteHeader(http.StatusServiceUnavailable) fmt.Fprintf(w, "Could not retrieve locks: %s", err) return } - type runLock struct { - locking.Run - UnlockURL string + type lock struct { + UnlockURL string + RepoFullName string + PullNum int + Time time.Time } - var results []runLock + var results []lock for id, v := range locks { u, _ := s.router.Get(deleteLockRoute).URL("id", url.QueryEscape(id)) - results = append(results, runLock{ - v, - u.String(), + results = append(results, lock{ + UnlockURL: u.String(), + RepoFullName: v.Project.RepoFullName, + PullNum: v.PullNum, + Time: v.Time, }) } indexTemplate.Execute(w, results) @@ -222,7 +266,7 @@ func (s *Server) deleteLock(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusBadRequest) fmt.Fprint(w, "invalid lock id") } - if err := s.lockingBackend.Unlock(idUnencoded); err != nil { + if err := s.lockingClient.Unlock(idUnencoded); err != nil { w.WriteHeader(http.StatusBadRequest) fmt.Fprintf(w, "Failed to unlock: %s", err) return @@ -238,7 +282,7 @@ func (s *Server) postHooks(w http.ResponseWriter, r *http.Request) { bytes, err := ioutil.ReadAll(r.Body) if err != nil { w.WriteHeader(http.StatusBadRequest) - fmt.Fprintf(w,"could not read body: %s\n", err) + fmt.Fprintf(w, "could not read body: %s\n", err) return } @@ -261,42 +305,27 @@ func (s *Server) postHooks(w http.ResponseWriter, r *http.Request) { func (s *Server) handlePullClosedEvent(w http.ResponseWriter, pullEvent github.PullRequestEvent, githubReqID string) { repo := *pullEvent.Repo.FullName pullNum := *pullEvent.PullRequest.Number - locks, err := s.lockingBackend.FindLocksForPull(repo, pullNum) + s.logger.Debug("Unlocking locks for repo %s and pull %d %s", repo, pullNum, githubReqID) + err := s.lockingClient.UnlockByPull(repo, pullNum) if err != nil { - s.logger.Err("finding locks for repo %s pull %d: %s", repo, pullNum, err) + s.logger.Err("unlocking locks for repo %s pull %d: %v", repo, pullNum, err) w.WriteHeader(http.StatusServiceUnavailable) - fmt.Fprintf(w, "Error finding locks: %s\n", err) + fmt.Fprintf(w, "Error unlocking locks: %v\n", err) return } - - s.logger.Debug("Unlocking locks %v %s", locks, githubReqID) - var errors []error - for _, l := range locks { - if err := s.lockingBackend.Unlock(l); err != nil { - errors = append(errors, err) - } - } - - if len(errors) != 0 { - s.logger.Err("unlocking locks for repo %s pull %d: %v", repo, pullNum, errors) - w.WriteHeader(http.StatusServiceUnavailable) - fmt.Fprintf(w, "Error unlocking locks: %v\n", errors) - return - } - - fmt.Fprintln(w,"Locks unlocked") + fmt.Fprintln(w, "Locks unlocked") } func (s *Server) handleCommentCreatedEvent(w http.ResponseWriter, comment github.IssueCommentEvent, githubReqID string) { // determine if the comment matches a plan or apply command - ctx := &ExecutionContext{} + ctx := &CommandContext{} command, err := s.requestParser.determineCommand(&comment) if err != nil { s.logger.Debug("Ignoring request: %v %s", err, githubReqID) fmt.Fprintln(w, "Ignoring") return } - ctx.command = command + ctx.Command = command if err = s.requestParser.extractCommentData(&comment, ctx); err != nil { s.logger.Err("Failed parsing event: %v %s", err, githubReqID) @@ -308,32 +337,32 @@ func (s *Server) handleCommentCreatedEvent(w http.ResponseWriter, comment github go s.executeCommand(ctx) } -func (s *Server) executeCommand(ctx *ExecutionContext) { - src := fmt.Sprintf("%s/pull/%d", ctx.repoFullName, ctx.pullNum) +func (s *Server) executeCommand(ctx *CommandContext) { + src := fmt.Sprintf("%s/pull/%d", ctx.Repo.FullName, ctx.Pull.Num) // it's safe to reuse the underlying logger s.logger.Log - ctx.log = logging.NewSimpleLogger(src, s.logger.Log, true, s.logger.Level) + ctx.Log = logging.NewSimpleLogger(src, s.logger.Log, true, s.logger.Level) defer s.recover(ctx) // we've got data from the comment, now we need to get data from the actual PR - pull, _, err := s.githubClient.GetPullRequest(ctx.repoFullName, ctx.pullNum) + pull, _, err := s.githubClient.GetPullRequest(ctx.Repo, ctx.Pull.Num) if err != nil { - ctx.log.Err("pull request data api call failed: %v", err) + ctx.Log.Err("pull request data api call failed: %v", err) return } if err := s.requestParser.extractPullData(pull, ctx); err != nil { - ctx.log.Err("failed to extract required fields from comment data: %v", err) + ctx.Log.Err("failed to extract required fields from comment data: %v", err) return } - switch ctx.command.commandType { + switch ctx.Command.commandType { case Plan: - s.planExecutor.Exec(s.planExecutor.execute, ctx, s.githubClient) + s.planExecutor.execute(ctx, s.githubClient) case Apply: - s.applyExecutor.Exec(s.applyExecutor.execute, ctx, s.githubClient) + s.applyExecutor.execute(ctx, s.githubClient) case Help: s.helpExecutor.execute(ctx, s.githubClient) default: - ctx.log.Err("failed to determine desired command, neither plan nor apply") + ctx.Log.Err("failed to determine desired command, neither plan nor apply") } } @@ -346,11 +375,10 @@ func (s *Server) isPullClosedEvent(event github.PullRequestEvent) bool { } // recover logs and creates a comment on the pull request for panics -func (s *Server) recover(ctx *ExecutionContext) { +func (s *Server) recover(ctx *CommandContext) { if err := recover(); err != nil { - ghCtx := s.planExecutor.githubContext(ctx) // this won't have every field but it has the ones needed by CreateComment stack := recovery.Stack(3) - s.githubClient.CreateComment(ghCtx, fmt.Sprintf("**Error: goroutine panic. This is a bug.**\n```\n%s\n%s```", err, stack)) - ctx.log.Err("PANIC: %s\n%s", err, stack) + s.githubClient.CreateComment(ctx, fmt.Sprintf("**Error: goroutine panic. This is a bug.**\n```\n%s\n%s```", err, stack)) + ctx.Log.Err("PANIC: %s\n%s", err, stack) } } diff --git a/server/terraform_client.go b/server/terraform_client.go index 6fb54c994..ee01d1b35 100644 --- a/server/terraform_client.go +++ b/server/terraform_client.go @@ -2,52 +2,55 @@ package server import ( "fmt" + "github.com/hootsuite/atlantis/logging" + "github.com/hootsuite/atlantis/models" "os" "os/exec" "path/filepath" "regexp" - "github.com/hootsuite/atlantis/logging" ) type TerraformClient struct { tfExecutableName string } -func (t *TerraformClient) ConfigureRemoteState(log *logging.SimpleLogger, execPath ExecutionPath, tfEnvName string, sshKey string) (string, error) { - log.Info("setting up remote state in directory %q", execPath.Absolute) +func (t *TerraformClient) ConfigureRemoteState(log *logging.SimpleLogger, repoDir string, project models.Project, env string, sshKey string) (string, error) { + absolutePath := filepath.Join(repoDir, project.Path) + log.Info("setting up remote state in directory %q", absolutePath) var remoteSetupCmdArgs []string // Check if setup file exists - setupFileInfo, err := os.Stat(filepath.Join(execPath.Absolute, "setup.sh")) + setupFileInfo, err := os.Stat(filepath.Join(absolutePath, "setup.sh")) if err != nil { - return "", fmt.Errorf("setup.sh file doesn't exist in terraform plan path %q: %v", execPath.Relative, err) + return "", fmt.Errorf("setup.sh file doesn't exist in terraform plan path %q: %v", project.Path, err) } // Check if setup file is executed if setupFileInfo.Mode() != os.FileMode(0755) { return "", fmt.Errorf("setup file isn't executable, required permissions are 0755") } // Check if environment is specified - if tfEnvName == "" { + if env == "" { // Check if env/ folder exist and environment isn't specified - if _, err := os.Stat(filepath.Join(execPath.Absolute, "env")); err == nil { + // todo: make env a constant (environmentDirName) + if _, err := os.Stat(filepath.Join(absolutePath, "env")); err == nil { log.Info("environment directory exists but no environment was supplied") return "", nil } } else { // Check if the environment file exists - envFile := tfEnvName + ".tfvars" - envPath := filepath.Join(execPath.Absolute, "env") + envFile := env + ".tfvars" + envPath := filepath.Join(absolutePath, "env") if _, err := os.Stat(filepath.Join(envPath, envFile)); err != nil { return "", fmt.Errorf("environment file %q not found in %q", envFile, envPath) } } // Set environment file parameter for ./setup.sh - if tfEnvName != "" { - remoteSetupCmdArgs = append(remoteSetupCmdArgs, "-e", tfEnvName) + if env != "" { + remoteSetupCmdArgs = append(remoteSetupCmdArgs, "-e", env) } remoteSetupCmd := exec.Command("./setup.sh", remoteSetupCmdArgs...) - remoteSetupCmd.Dir = execPath.Absolute + remoteSetupCmd.Dir = absolutePath // Check if ssh key is set if sshKey != "" { // Fixing a bug when git isn't found in path when environment variables are set for command @@ -71,9 +74,9 @@ func (t *TerraformClient) ConfigureRemoteState(log *logging.SimpleLogger, execPa return remoteStatePath, nil } -func (t *TerraformClient) RunTerraformCommand(path ExecutionPath, tfPlanCmd []string, tfEnvVars []string) ([]string, string, error) { +func (t *TerraformClient) RunTerraformCommand(path string, tfPlanCmd []string, tfEnvVars []string) ([]string, string, error) { terraformCmd := exec.Command(t.tfExecutableName, tfPlanCmd...) - terraformCmd.Dir = path.Absolute + terraformCmd.Dir = path terraformCmd.Env = tfEnvVars out, err := terraformCmd.CombinedOutput() output := string(out) diff --git a/server/testing_util.go b/server/testing_util.go deleted file mode 100644 index 40ee98355..000000000 --- a/server/testing_util.go +++ /dev/null @@ -1,38 +0,0 @@ -package server - -// taken from https://github.com/benbjohnson/testing - -import ( - "fmt" - "path/filepath" - "reflect" - "runtime" - "testing" -) - -// assert fails the test if the condition is false. -func assert(tb testing.TB, condition bool, msg string, v ...interface{}) { - if !condition { - _, file, line, _ := runtime.Caller(1) - fmt.Printf("\033[31m%s:%d: "+msg+"\033[39m\n\n", append([]interface{}{filepath.Base(file), line}, v...)...) - tb.FailNow() - } -} - -// ok fails the test if an err is not nil. -func ok(tb testing.TB, err error) { - if err != nil { - _, file, line, _ := runtime.Caller(1) - fmt.Printf("\033[31m%s:%d: unexpected error: %s\033[39m\n\n", filepath.Base(file), line, err.Error()) - tb.FailNow() - } -} - -// equals fails the test if exp is not equal to act. -func equals(tb testing.TB, exp, act interface{}) { - if !reflect.DeepEqual(exp, act) { - _, file, line, _ := runtime.Caller(1) - fmt.Printf("\033[31m%s:%d:\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n", filepath.Base(file), line, exp, act) - tb.FailNow() - } -} diff --git a/server/util.go b/server/util.go deleted file mode 100644 index 2e6388af1..000000000 --- a/server/util.go +++ /dev/null @@ -1,11 +0,0 @@ -package server - -// ContainsStr returns true if slice has a value equal to search and false otherwise -func ContainsStr(slice []string, search string) bool { - for _, v := range slice { - if v == search { - return true - } - } - return false -} diff --git a/server/web_templates.go b/server/web_templates.go index c60369dd4..a41e32ed0 100644 --- a/server/web_templates.go +++ b/server/web_templates.go @@ -16,12 +16,12 @@ var indexTemplate = template.Must(template.New("index.html.tmpl").Parse(` - +
- +

atlantis

{{ end }} {{ else }} diff --git a/static/atlantis-icon.png b/static/atlantis-icon.png deleted file mode 100644 index f98f15a09..000000000 Binary files a/static/atlantis-icon.png and /dev/null differ diff --git a/static/atlantis-icon_512.png b/static/atlantis-icon_512.png deleted file mode 100644 index 0b57dfd90..000000000 Binary files a/static/atlantis-icon_512.png and /dev/null differ diff --git a/testing_util/testing_util.go b/testing_util/testing_util.go index 38e9d619f..660c6d7b1 100644 --- a/testing_util/testing_util.go +++ b/testing_util/testing_util.go @@ -38,13 +38,13 @@ func Equals(tb testing.TB, exp, act interface{}) { } // Contains fails the test if the slice doesn't contain the expected element -func ContainsStr(tb testing.TB, exp string, act []string) { - for _, v := range act { - if v == exp { +func Contains(tb testing.TB, exp interface{}, slice []interface{}) { + for _, v := range slice { + if reflect.DeepEqual(v, exp) { return } } _, file, line, _ := runtime.Caller(1) - fmt.Printf("\033[31m%s:%d:\n\n\texp: %#v\n\n\tin: %#v\033[39m\n\n", filepath.Base(file), line, exp, act) + fmt.Printf("\033[31m%s:%d:\n\n\texp: %#v\n\n\twas not in: %#v\033[39m\n\n", filepath.Base(file), line, exp, slice) tb.FailNow() }