From d1d1539ced062ccb8dc2a0368b4e4c802b6799b6 Mon Sep 17 00:00:00 2001 From: Nish Krishnan Date: Fri, 18 Mar 2022 15:18:13 -0700 Subject: [PATCH] feat: Add stats support for basic operations (#2147) --- cmd/server.go | 9 + cmd/server_test.go | 1 + go.mod | 15 +- go.sum | 19 +- runatlantis.io/docs/server-configuration.md | 6 + .../docs/server-side-repo-config.md | 24 +- runatlantis.io/docs/stats.md | 11 + .../controllers/events/events_controller.go | 264 ++++++++++++++---- .../events/events_controller_e2e_test.go | 6 + .../events/events_controller_test.go | 57 ++-- server/controllers/jobs_controller.go | 37 ++- server/core/config/raw/global_cfg.go | 6 +- server/core/config/raw/metrics.go | 44 +++ server/core/config/raw/metrics_test.go | 116 ++++++++ server/core/config/valid/global_cfg.go | 10 + server/events/apply_command_runner_test.go | 4 + server/events/command_context.go | 2 + server/events/command_runner.go | 17 ++ server/events/command_runner_test.go | 3 + .../instrumented_project_command_builder.go | 76 +++++ .../instrumented_project_command_runner.go | 58 ++++ .../instrumented_pull_closed_executor.go | 53 ++++ server/events/models/models.go | 9 + server/events/project_command_builder.go | 46 ++- .../project_command_builder_internal_test.go | 34 ++- server/events/project_command_builder_test.go | 62 +++- .../events/project_command_context_builder.go | 49 +++- server/events/vcs/instrumented_client.go | 237 ++++++++++++++++ server/metrics/common.go | 8 + server/metrics/debug.go | 93 ++++++ server/metrics/scope.go | 63 +++++ server/scheduled/executor_service.go | 100 +++++++ server/scheduled/runtime_stats.go | 128 +++++++++ server/server.go | 121 +++++--- server/user_config.go | 1 + 35 files changed, 1642 insertions(+), 147 deletions(-) create mode 100644 runatlantis.io/docs/stats.md create mode 100644 server/core/config/raw/metrics.go create mode 100644 server/core/config/raw/metrics_test.go create mode 100644 server/events/instrumented_project_command_builder.go create mode 100644 server/events/instrumented_project_command_runner.go create mode 100644 server/events/instrumented_pull_closed_executor.go create mode 100644 server/events/vcs/instrumented_client.go create mode 100644 server/metrics/common.go create mode 100644 server/metrics/debug.go create mode 100644 server/metrics/scope.go create mode 100644 server/scheduled/executor_service.go create mode 100644 server/scheduled/runtime_stats.go diff --git a/cmd/server.go b/cmd/server.go index bf8537265..934322032 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -80,6 +80,7 @@ const ( HidePrevPlanComments = "hide-prev-plan-comments" LogLevelFlag = "log-level" ParallelPoolSize = "parallel-pool-size" + StatsNamespace = "stats-namespace" AllowDraftPRs = "allow-draft-prs" PortFlag = "port" RepoConfigFlag = "repo-config" @@ -120,6 +121,7 @@ const ( DefaultGitlabHostname = "gitlab.com" DefaultLogLevel = "info" DefaultParallelPoolSize = 15 + DefaultStatsNamespace = "atlantis" DefaultPort = 4141 DefaultTFDownloadURL = "https://releases.hashicorp.com" DefaultTFEHostname = "app.terraform.io" @@ -259,6 +261,10 @@ var stringFlags = map[string]stringFlag{ description: "Log level. Either debug, info, warn, or error.", defaultValue: DefaultLogLevel, }, + StatsNamespace: { + description: "Namespace for aggregating stats.", + defaultValue: DefaultStatsNamespace, + }, RepoConfigFlag: { description: "Path to a repo config file, used to customize how Atlantis runs on each repo. See runatlantis.io/docs for more details.", }, @@ -652,6 +658,9 @@ func (s *ServerCmd) setDefaults(c *server.UserConfig) { if c.ParallelPoolSize == 0 { c.ParallelPoolSize = DefaultParallelPoolSize } + if c.StatsNamespace == "" { + c.StatsNamespace = DefaultStatsNamespace + } if c.Port == 0 { c.Port = DefaultPort } diff --git a/cmd/server_test.go b/cmd/server_test.go index 95d82aa31..19b1f8e1e 100644 --- a/cmd/server_test.go +++ b/cmd/server_test.go @@ -85,6 +85,7 @@ var testFlags = map[string]interface{}{ GitlabUserFlag: "gitlab-user", GitlabWebhookSecretFlag: "gitlab-secret", LogLevelFlag: "debug", + StatsNamespace: "atlantis", AllowDraftPRs: true, PortFlag: 8181, ParallelPoolSize: 100, diff --git a/go.mod b/go.mod index 3b613f738..0d64a01d5 100644 --- a/go.mod +++ b/go.mod @@ -80,7 +80,7 @@ require ( github.com/huandu/xstrings v1.3.1 // indirect github.com/imdario/mergo v0.3.11 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect - github.com/jmespath/go-jmespath v0.3.1-0.20200310193758-2437e8417af5 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/klauspost/compress v1.11.2 // indirect github.com/kr/pretty v0.2.0 // indirect github.com/leodido/go-urn v1.2.0 // indirect @@ -93,7 +93,6 @@ require ( github.com/mitchellh/go-wordwrap v1.0.0 // indirect github.com/mitchellh/mapstructure v1.4.3 // indirect github.com/mitchellh/reflectwalk v1.0.0 // indirect - github.com/onsi/ginkgo v1.14.0 // indirect github.com/onsi/gomega v1.10.1 // indirect github.com/pelletier/go-toml v1.9.4 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect @@ -101,7 +100,7 @@ require ( github.com/shopspring/decimal v1.2.0 // indirect github.com/shurcooL/graphql v0.0.0-20181231061246-d48a9a75455f // indirect github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect - github.com/sirupsen/logrus v1.6.1-0.20200528085638-6699a89a232f // indirect + github.com/sirupsen/logrus v1.8.1 // indirect github.com/spf13/afero v1.6.0 // indirect github.com/spf13/cast v1.4.1 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect @@ -109,7 +108,7 @@ require ( github.com/ulikunitz/xz v0.5.8 // indirect github.com/zclconf/go-cty v1.5.1 // indirect go.opencensus.io v0.23.0 // indirect - go.uber.org/atomic v1.7.0 // indirect + go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.6.0 // indirect golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d // indirect @@ -128,3 +127,11 @@ require ( gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect gotest.tools v2.2.0+incompatible // indirect ) + +require ( + github.com/cactus/go-statsd-client/statsd v0.0.0-20200623234511-94959e3146b2 + github.com/twmb/murmur3 v1.1.6 // indirect + github.com/uber-go/tally v3.4.3+incompatible +) + +require github.com/onsi/ginkgo v1.14.0 // indirect diff --git a/go.sum b/go.sum index 117395ca7..afea2beff 100644 --- a/go.sum +++ b/go.sum @@ -95,6 +95,8 @@ github.com/bradleyfalzon/ghinstallation/v2 v2.0.4 h1:tXKVfhE7FcSkhkv0UwkLvPDeZ4k github.com/bradleyfalzon/ghinstallation/v2 v2.0.4/go.mod h1:B40qPqJxWE0jDZgOR1JmaMy+4AY1eBP+IByOvqyAKp0= github.com/briandowns/spinner v0.0.0-20170614154858-48dbb65d7bd5 h1:osZyZB7J4kE1tKLeaUjV6+uZVBfS835T0I/RxmwWw1w= github.com/briandowns/spinner v0.0.0-20170614154858-48dbb65d7bd5/go.mod h1:hw/JEQBIE+c/BLI4aKM8UU8v+ZqrD3h7HC27kKt8JQU= +github.com/cactus/go-statsd-client/statsd v0.0.0-20200623234511-94959e3146b2 h1:GgJnJEJYymy/lx+1zXOO2TvGPRQJJ9vz4onxnA9gF3k= +github.com/cactus/go-statsd-client/statsd v0.0.0-20200623234511-94959e3146b2/go.mod h1:l/bIBLeOl9eX+wxJAzxS4TveKRtAqlyDpHjhkfO0MEI= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -314,8 +316,10 @@ github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NH github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= -github.com/jmespath/go-jmespath v0.3.1-0.20200310193758-2437e8417af5 h1:1G6l+WClVmbflmgW0Wsr6a50KeKCQcYKv/vUjtQUHuw= -github.com/jmespath/go-jmespath v0.3.1-0.20200310193758-2437e8417af5/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -455,8 +459,8 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5I github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.1-0.20200528085638-6699a89a232f h1:qqqIhBDFUBrbMezIyJkKWIpf+E5CdObleGMjW1s19Hg= -github.com/sirupsen/logrus v1.6.1-0.20200528085638-6699a89a232f/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= @@ -485,6 +489,10 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= +github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= +github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= +github.com/uber-go/tally v3.4.3+incompatible h1:Oq25FXV8cWHPRo+EPeNdbN3LfuozC9mDK2/4vZ1k38U= +github.com/uber-go/tally v3.4.3+incompatible/go.mod h1:YDTIBxdXyOU/sCWilKB4bgyufu1cEi0jdVnRdxvjnmU= github.com/ulikunitz/xz v0.5.8 h1:ERv8V6GKqVi23rgu5cj9pVfVzJbOqAY2Ntl88O6c2nQ= github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/urfave/cli v1.22.5 h1:lNq9sAHXK2qfdI8W+GRItjCEkI+2oR4d+MEHy1CKXoU= @@ -517,8 +525,9 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= diff --git a/runatlantis.io/docs/server-configuration.md b/runatlantis.io/docs/server-configuration.md index 54155f388..f9e3db3dd 100644 --- a/runatlantis.io/docs/server-configuration.md +++ b/runatlantis.io/docs/server-configuration.md @@ -600,6 +600,12 @@ Values are chosen in this order: ``` File containing x509 private key matching `--ssl-cert-file`. +* ### `--stats-namespace` + ```bash + atlantis server --stats-namespace="myatlantis" + ``` + Namespace for emitting stats/metrics. See (stats.html#Metrics/Stats) + * ### `--tf-download-url` ```bash atlantis server --tf-download-url="https://releases.company.com" diff --git a/runatlantis.io/docs/server-side-repo-config.md b/runatlantis.io/docs/server-side-repo-config.md index c617edf13..6e99deffd 100644 --- a/runatlantis.io/docs/server-side-repo-config.md +++ b/runatlantis.io/docs/server-side-repo-config.md @@ -1,16 +1,18 @@ -# Server Side Repo Config -A Server-Side Repo Config file is used to control per-repo behaviour +# Server Side Config +A Server-Side Config file is used for more groups of server config that can't reasonably be expressed through flags. + +One such usecase is to control per-repo behaviour and what users can do in repo-level `atlantis.yaml` files. [[toc]] -## Do I Need A Server-Side Repo Config File? +## Do I Need A Server-Side Config File? You do not need a server-side repo config file unless you want to customize some aspect of Atlantis on a per-repo basis. Read through the [use-cases](#use-cases) to determine if you need it. -## Enabling Server Side Repo Config +## Enabling Server Side Config To use server side repo config create a config file, ex. `repos.yaml`, and pass it to the `atlantis server` command via the `--repo-config` flag, ex. `--repo-config=path/to/repos.yaml`. @@ -453,3 +455,17 @@ If you set a workflow with the key `default`, it will override this. | name | string | none | yes | unique name for the policy set | | path | string | none | yes | path to the rego policies directory | | source | string | none | yes | only `local` is supported at this time | + + +### Metrics + +| Key | Type | Default | Required | Description | +|------------------------|-----------------|---------|-----------|------------------------------------------| +| statsd | Statsd(#Statsd) | none | no | Statsd metrics provider | + +### Statsd + +| Key | Type | Default | Required | Description | +| ------ | ------ | ------- | -------- | -------------------------------------- | +| host | string | none | yes | statsd host ip address | +| port | string | none | yes | statsd port | diff --git a/runatlantis.io/docs/stats.md b/runatlantis.io/docs/stats.md new file mode 100644 index 000000000..d43d8978d --- /dev/null +++ b/runatlantis.io/docs/stats.md @@ -0,0 +1,11 @@ +# Metrics/Stats + +Atlantis exposes a set of metrics for each of its operations including errors, successes, and latencies. + +::: warning NOTE +Only statsd is supported currently, but it should be relatively straightforward to add other providers such as prometheus. +::: + +## Configuration + +Metrics are configured through the (server-side-repo-config.html#Metrics). \ No newline at end of file diff --git a/server/controllers/events/events_controller.go b/server/controllers/events/events_controller.go index cd4d097ae..f2697edff 100644 --- a/server/controllers/events/events_controller.go +++ b/server/controllers/events/events_controller.go @@ -29,6 +29,7 @@ import ( "github.com/runatlantis/atlantis/server/events/vcs/bitbucketcloud" "github.com/runatlantis/atlantis/server/events/vcs/bitbucketserver" "github.com/runatlantis/atlantis/server/logging" + "github.com/uber-go/tally" gitlab "github.com/xanzy/go-gitlab" ) @@ -48,6 +49,7 @@ type VCSEventsController struct { CommandRunner events.CommandRunner PullCleaner events.PullCleaner Logger logging.SimpleLogging + Scope tally.Scope Parser events.EventParsing CommentParser events.CommentParsing ApplyDisabled bool @@ -136,6 +138,16 @@ func (e *VCSEventsController) Post(w http.ResponseWriter, r *http.Request) { e.respond(w, logging.Debug, http.StatusBadRequest, "Ignoring request") } +type HTTPError struct { + err error + code int +} + +type HTTPResponse struct { + body string + err HTTPError +} + func (e *VCSEventsController) handleGithubPost(w http.ResponseWriter, r *http.Request) { // Validate the request against the optional webhook secret. payload, err := e.GithubRequestValidator.Validate(r, e.GithubWebhookSecret) @@ -143,20 +155,41 @@ func (e *VCSEventsController) handleGithubPost(w http.ResponseWriter, r *http.Re e.respond(w, logging.Warn, http.StatusBadRequest, err.Error()) return } - e.Logger.Debug("request valid") githubReqID := "X-Github-Delivery=" + r.Header.Get("X-Github-Delivery") + logger := e.Logger.With("gh-request-id", githubReqID) + scope := e.Scope.SubScope("github.event") + + logger.Debug("request valid") + event, _ := github.ParseWebHook(github.WebHookType(r), payload) + + var resp HTTPResponse + switch event := event.(type) { case *github.IssueCommentEvent: - e.Logger.Debug("handling as comment event") - e.HandleGithubCommentEvent(w, event, githubReqID) + resp = e.HandleGithubCommentEvent(event, githubReqID, logger) + scope = scope.SubScope(fmt.Sprintf("comment.%s", *event.Action)) case *github.PullRequestEvent: - e.Logger.Debug("handling as pull request event") - e.HandleGithubPullRequestEvent(w, event, githubReqID) + resp = e.HandleGithubPullRequestEvent(logger, event, githubReqID) + scope = scope.SubScope(fmt.Sprintf("pr.%s", *event.Action)) default: - e.respond(w, logging.Debug, http.StatusOK, "Ignoring unsupported event %s", githubReqID) + resp = HTTPResponse{ + body: fmt.Sprintf("Ignoring unsupported event %s", githubReqID), + } } + + if resp.err.code != 0 { + logger.Err("error handling gh post code: %d err: %s", resp.err.code, resp.err.err.Error()) + scope.Counter(fmt.Sprintf("error_%d", resp.err.code)).Inc(1) + w.WriteHeader(resp.err.code) + fmt.Fprintln(w, resp.err.err.Error()) + return + } + + scope.Counter(fmt.Sprintf("success_%d", http.StatusOK)).Inc(1) + w.WriteHeader(http.StatusOK) + fmt.Fprintln(w, resp.body) } func (e *VCSEventsController) handleBitbucketCloudPost(w http.ResponseWriter, r *http.Request) { @@ -247,21 +280,29 @@ func (e *VCSEventsController) handleAzureDevopsPost(w http.ResponseWriter, r *ht // HandleGithubCommentEvent handles comment events from GitHub where Atlantis // commands can come from. It's exported to make testing easier. -func (e *VCSEventsController) HandleGithubCommentEvent(w http.ResponseWriter, event *github.IssueCommentEvent, githubReqID string) { +func (e *VCSEventsController) HandleGithubCommentEvent(event *github.IssueCommentEvent, githubReqID string, logger logging.SimpleLogging) HTTPResponse { if event.GetAction() != "created" { - e.respond(w, logging.Debug, http.StatusOK, "Ignoring comment event since action was not created %s", githubReqID) - return + return HTTPResponse{ + body: fmt.Sprintf("Ignoring comment event since action was not created %s", githubReqID), + } } baseRepo, user, pullNum, err := e.Parser.ParseGithubIssueCommentEvent(event) + + wrapped := errors.Wrapf(err, "Failed parsing event: %s", githubReqID) if err != nil { - e.respond(w, logging.Error, http.StatusBadRequest, "Failed parsing event: %v %s", err, githubReqID) - return + return HTTPResponse{ + body: wrapped.Error(), + err: HTTPError{ + code: http.StatusBadRequest, + err: wrapped, + }, + } } // We pass in nil for maybeHeadRepo because the head repo data isn't // available in the GithubIssueComment event. - e.handleCommentEvent(w, baseRepo, nil, nil, user, pullNum, event.Comment.GetBody(), models.Github) + return e.handleCommentEvent(logger, baseRepo, nil, nil, user, pullNum, event.Comment.GetBody(), models.Github) } // HandleBitbucketCloudCommentEvent handles comment events from Bitbucket. @@ -271,7 +312,18 @@ func (e *VCSEventsController) HandleBitbucketCloudCommentEvent(w http.ResponseWr e.respond(w, logging.Error, http.StatusBadRequest, "Error parsing pull data: %s %s=%s", err, bitbucketCloudRequestIDHeader, reqID) return } - e.handleCommentEvent(w, baseRepo, &headRepo, &pull, user, pull.Num, comment, models.BitbucketCloud) + resp := e.handleCommentEvent(e.Logger, baseRepo, &headRepo, &pull, user, pull.Num, comment, models.BitbucketCloud) + + //TODO: move this to the outer most function similar to github + lvl := logging.Debug + code := http.StatusOK + msg := resp.body + if resp.err.code != 0 { + lvl = logging.Error + code = resp.err.code + msg = resp.err.err.Error() + } + e.respond(w, lvl, code, msg) } // HandleBitbucketServerCommentEvent handles comment events from Bitbucket. @@ -281,7 +333,18 @@ func (e *VCSEventsController) HandleBitbucketServerCommentEvent(w http.ResponseW e.respond(w, logging.Error, http.StatusBadRequest, "Error parsing pull data: %s %s=%s", err, bitbucketCloudRequestIDHeader, reqID) return } - e.handleCommentEvent(w, baseRepo, &headRepo, &pull, user, pull.Num, comment, models.BitbucketCloud) + resp := e.handleCommentEvent(e.Logger, baseRepo, &headRepo, &pull, user, pull.Num, comment, models.BitbucketCloud) + + //TODO: move this to the outer most function similar to github + lvl := logging.Debug + code := http.StatusOK + msg := resp.body + if resp.err.code != 0 { + lvl = logging.Error + code = resp.err.code + msg = resp.err.err.Error() + } + e.respond(w, lvl, code, msg) } func (e *VCSEventsController) handleBitbucketCloudPullRequestEvent(w http.ResponseWriter, eventType string, body []byte, reqID string) { @@ -292,7 +355,18 @@ func (e *VCSEventsController) handleBitbucketCloudPullRequestEvent(w http.Respon } pullEventType := e.Parser.GetBitbucketCloudPullEventType(eventType) e.Logger.Info("identified event as type %q", pullEventType.String()) - e.handlePullRequestEvent(w, baseRepo, headRepo, pull, user, pullEventType) + resp := e.handlePullRequestEvent(e.Logger, baseRepo, headRepo, pull, user, pullEventType) + + //TODO: move this to the outer most function similar to github + lvl := logging.Debug + code := http.StatusOK + msg := resp.body + if resp.err.code != 0 { + lvl = logging.Error + code = resp.err.code + msg = resp.err.err.Error() + } + e.respond(w, lvl, code, msg) } func (e *VCSEventsController) handleBitbucketServerPullRequestEvent(w http.ResponseWriter, eventType string, body []byte, reqID string) { @@ -303,23 +377,40 @@ func (e *VCSEventsController) handleBitbucketServerPullRequestEvent(w http.Respo } pullEventType := e.Parser.GetBitbucketServerPullEventType(eventType) e.Logger.Info("identified event as type %q", pullEventType.String()) - e.handlePullRequestEvent(w, baseRepo, headRepo, pull, user, pullEventType) + resp := e.handlePullRequestEvent(e.Logger, baseRepo, headRepo, pull, user, pullEventType) + + //TODO: move this to the outer most function similar to github + lvl := logging.Debug + code := http.StatusOK + msg := resp.body + if resp.err.code != 0 { + lvl = logging.Error + code = resp.err.code + msg = resp.err.err.Error() + } + e.respond(w, lvl, code, msg) } // HandleGithubPullRequestEvent will delete any locks associated with the pull // request if the event is a pull request closed event. It's exported to make // testing easier. -func (e *VCSEventsController) HandleGithubPullRequestEvent(w http.ResponseWriter, pullEvent *github.PullRequestEvent, githubReqID string) { +func (e *VCSEventsController) HandleGithubPullRequestEvent(logger logging.SimpleLogging, pullEvent *github.PullRequestEvent, githubReqID string) HTTPResponse { pull, pullEventType, baseRepo, headRepo, user, err := e.Parser.ParseGithubPullEvent(pullEvent) if err != nil { - e.respond(w, logging.Error, http.StatusBadRequest, "Error parsing pull data: %s %s", err, githubReqID) - return + wrapped := errors.Wrapf(err, "Error parsing pull data: %s %s", err, githubReqID) + return HTTPResponse{ + body: wrapped.Error(), + err: HTTPError{ + code: http.StatusBadRequest, + err: wrapped, + }, + } } - e.Logger.Info("identified event as type %q", pullEventType.String()) - e.handlePullRequestEvent(w, baseRepo, headRepo, pull, user, pullEventType) + logger.Debug("identified event as type %q", pullEventType.String()) + return e.handlePullRequestEvent(logger, baseRepo, headRepo, pull, user, pullEventType) } -func (e *VCSEventsController) handlePullRequestEvent(w http.ResponseWriter, baseRepo models.Repo, headRepo models.Repo, pull models.PullRequest, user models.User, eventType models.PullRequestEventType) { +func (e *VCSEventsController) handlePullRequestEvent(logger logging.SimpleLogging, baseRepo models.Repo, headRepo models.Repo, pull models.PullRequest, user models.User, eventType models.PullRequestEventType) HTTPResponse { if !e.RepoAllowlistChecker.IsAllowlisted(baseRepo.FullName, baseRepo.VCSHost.Hostname) { // If the repo isn't allowlisted and we receive an opened pull request // event we comment back on the pull request that the repo isn't @@ -328,10 +419,16 @@ func (e *VCSEventsController) handlePullRequestEvent(w http.ResponseWriter, base if eventType == models.OpenedPullEvent { e.commentNotAllowlisted(baseRepo, pull.Num) } - e.respond(w, logging.Debug, http.StatusForbidden, - "Ignoring pull request event from non-allowlisted repo \"%s/%s\"", - baseRepo.VCSHost.Hostname, baseRepo.FullName) - return + + err := errors.Errorf("Pull request event from non-allowlisted repo \"%s/%s\"", baseRepo.VCSHost.Hostname, baseRepo.FullName) + + return HTTPResponse{ + body: err.Error(), + err: HTTPError{ + code: http.StatusForbidden, + err: err, + }, + } } switch eventType { @@ -341,30 +438,37 @@ func (e *VCSEventsController) handlePullRequestEvent(w http.ResponseWriter, base // Respond with success and then actually execute the command asynchronously. // We use a goroutine so that this function returns and the connection is // closed. - fmt.Fprintln(w, "Processing...") - - e.Logger.Info("executing autoplan") if !e.TestingMode { go e.CommandRunner.RunAutoplanCommand(baseRepo, headRepo, pull, user) } else { // When testing we want to wait for everything to complete. e.CommandRunner.RunAutoplanCommand(baseRepo, headRepo, pull, user) } - return + return HTTPResponse{ + body: "Processing...", + } case models.ClosedPullEvent: // If the pull request was closed, we delete locks. if err := e.PullCleaner.CleanUpPull(baseRepo, pull); err != nil { - e.respond(w, logging.Error, http.StatusInternalServerError, "Error cleaning pull request: %s", err) - return + return HTTPResponse{ + body: err.Error(), + err: HTTPError{ + code: http.StatusForbidden, + err: err, + }, + } + } + logger.Info("deleted locks and workspace for repo %s, pull %d", baseRepo.FullName, pull.Num) + return HTTPResponse{ + body: "Pull request cleaned successfully", } - e.Logger.Info("deleted locks and workspace for repo %s, pull %d", baseRepo.FullName, pull.Num) - fmt.Fprintln(w, "Pull request cleaned successfully") - return case models.OtherPullEvent: // Else we ignore the event. - e.respond(w, logging.Debug, http.StatusOK, "Ignoring non-actionable pull request event") - return + return HTTPResponse{ + body: "Ignoring non-actionable pull request event", + } } + return HTTPResponse{} } func (e *VCSEventsController) handleGitlabPost(w http.ResponseWriter, r *http.Request) { @@ -400,10 +504,21 @@ func (e *VCSEventsController) HandleGitlabCommentEvent(w http.ResponseWriter, ev e.respond(w, logging.Error, http.StatusBadRequest, "Error parsing webhook: %s", err) return } - e.handleCommentEvent(w, baseRepo, &headRepo, nil, user, event.MergeRequest.IID, event.ObjectAttributes.Note, models.Gitlab) + resp := e.handleCommentEvent(e.Logger, baseRepo, &headRepo, nil, user, event.MergeRequest.IID, event.ObjectAttributes.Note, models.Gitlab) + + //TODO: move this to the outer most function similar to github + lvl := logging.Debug + code := http.StatusOK + msg := resp.body + if resp.err.code != 0 { + lvl = logging.Error + code = resp.err.code + msg = resp.err.err.Error() + } + e.respond(w, lvl, code, msg) } -func (e *VCSEventsController) handleCommentEvent(w http.ResponseWriter, baseRepo models.Repo, maybeHeadRepo *models.Repo, maybePull *models.PullRequest, user models.User, pullNum int, comment string, vcsHost models.VCSHostType) { +func (e *VCSEventsController) handleCommentEvent(logger logging.SimpleLogging, baseRepo models.Repo, maybeHeadRepo *models.Repo, maybePull *models.PullRequest, user models.User, pullNum int, comment string, vcsHost models.VCSHostType) HTTPResponse { parseResult := e.CommentParser.Parse(comment, vcsHost) if parseResult.Ignore { truncated := comment @@ -411,17 +526,25 @@ func (e *VCSEventsController) handleCommentEvent(w http.ResponseWriter, baseRepo if len(truncated) > truncateLen { truncated = comment[:truncateLen] + "..." } - e.respond(w, logging.Debug, http.StatusOK, "Ignoring non-command comment: %q", truncated) - return + return HTTPResponse{ + body: fmt.Sprintf("Ignoring non-command comment: %q", truncated), + } } - e.Logger.Info("parsed comment as %s", parseResult.Command) + logger.Info("parsed comment as %s", parseResult.Command) // At this point we know it's a command we're not supposed to ignore, so now // we check if this repo is allowed to run commands in the first place. if !e.RepoAllowlistChecker.IsAllowlisted(baseRepo.FullName, baseRepo.VCSHost.Hostname) { e.commentNotAllowlisted(baseRepo, pullNum) - e.respond(w, logging.Warn, http.StatusForbidden, "Repo not allowlisted") - return + + err := errors.New("Repo not allowlisted") + return HTTPResponse{ + body: err.Error(), + err: HTTPError{ + err: err, + code: http.StatusForbidden, + }, + } } // If the command isn't valid or doesn't require processing, ex. @@ -430,14 +553,14 @@ func (e *VCSEventsController) handleCommentEvent(w http.ResponseWriter, baseRepo // variable to comment back on the pull request. if parseResult.CommentResponse != "" { if err := e.VCSClient.CreateComment(baseRepo, pullNum, parseResult.CommentResponse, ""); err != nil { - e.Logger.Err("unable to comment on pull request: %s", err) + logger.Err("unable to comment on pull request: %s", err) + } + return HTTPResponse{ + body: "Commenting back on pull request", } - e.respond(w, logging.Info, http.StatusOK, "Commenting back on pull request") - return } - e.Logger.Debug("executing command") - fmt.Fprintln(w, "Processing...") + logger.Debug("executing command") if !e.TestingMode { // Respond with success and then actually execute the command asynchronously. // We use a goroutine so that this function returns and the connection is @@ -447,6 +570,10 @@ func (e *VCSEventsController) handleCommentEvent(w http.ResponseWriter, baseRepo // When testing we want to wait for everything to complete. e.CommandRunner.RunCommentCommand(baseRepo, maybeHeadRepo, maybePull, user, pullNum, parseResult.Command) } + + return HTTPResponse{ + body: "Processing...", + } } // HandleGitlabMergeRequestEvent will delete any locks associated with the pull @@ -459,7 +586,18 @@ func (e *VCSEventsController) HandleGitlabMergeRequestEvent(w http.ResponseWrite return } e.Logger.Info("identified event as type %q", pullEventType.String()) - e.handlePullRequestEvent(w, baseRepo, headRepo, pull, user, pullEventType) + resp := e.handlePullRequestEvent(e.Logger, baseRepo, headRepo, pull, user, pullEventType) + + //TODO: move this to the outer most function similar to github + lvl := logging.Debug + code := http.StatusOK + msg := resp.body + if resp.err.code != 0 { + lvl = logging.Error + code = resp.err.code + msg = resp.err.err.Error() + } + e.respond(w, lvl, code, msg) } // HandleAzureDevopsPullRequestCommentedEvent handles comment events from Azure DevOps where Atlantis @@ -491,7 +629,18 @@ func (e *VCSEventsController) HandleAzureDevopsPullRequestCommentedEvent(w http. e.respond(w, logging.Error, http.StatusBadRequest, "Error parsing pull request repository field: %s; %s", err, azuredevopsReqID) return } - e.handleCommentEvent(w, baseRepo, nil, nil, user, resource.PullRequest.GetPullRequestID(), string(strippedComment), models.AzureDevops) + resp := e.handleCommentEvent(e.Logger, baseRepo, nil, nil, user, resource.PullRequest.GetPullRequestID(), string(strippedComment), models.AzureDevops) + + //TODO: move this to the outer most function similar to github + lvl := logging.Debug + code := http.StatusOK + msg := resp.body + if resp.err.code != 0 { + lvl = logging.Error + code = resp.err.code + msg = resp.err.err.Error() + } + e.respond(w, lvl, code, msg) } // HandleAzureDevopsPullRequestEvent will delete any locks associated with the pull @@ -521,7 +670,18 @@ func (e *VCSEventsController) HandleAzureDevopsPullRequestEvent(w http.ResponseW return } e.Logger.Info("identified event as type %q", pullEventType.String()) - e.handlePullRequestEvent(w, baseRepo, headRepo, pull, user, pullEventType) + resp := e.handlePullRequestEvent(e.Logger, baseRepo, headRepo, pull, user, pullEventType) + + //TODO: move this to the outer most function similar to github + lvl := logging.Debug + code := http.StatusOK + msg := resp.body + if resp.err.code != 0 { + lvl = logging.Error + code = resp.err.code + msg = resp.err.err.Error() + } + e.respond(w, lvl, code, msg) } // supportsHost returns true if h is in e.SupportedVCSHosts and false otherwise. diff --git a/server/controllers/events/events_controller_e2e_test.go b/server/controllers/events/events_controller_e2e_test.go index 34401dac6..319900f53 100644 --- a/server/controllers/events/events_controller_e2e_test.go +++ b/server/controllers/events/events_controller_e2e_test.go @@ -36,6 +36,7 @@ import ( "github.com/runatlantis/atlantis/server/events/webhooks" jobmocks "github.com/runatlantis/atlantis/server/jobs/mocks" "github.com/runatlantis/atlantis/server/logging" + "github.com/runatlantis/atlantis/server/metrics" . "github.com/runatlantis/atlantis/testing" ) @@ -916,6 +917,7 @@ func setupE2E(t *testing.T, repoDir string) (events_controllers.VCSEventsControl WorkingDir: workingDir, PostWorkflowHookRunner: mockPostWorkflowHookRunner, } + statsScope, _, _ := metrics.NewLoggingScope(logger, "atlantis") projectCommandBuilder := events.NewProjectCommandBuilder( userConfig.EnablePolicyChecksFlag, @@ -930,6 +932,8 @@ func setupE2E(t *testing.T, repoDir string) (events_controllers.VCSEventsControl false, false, "**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl,**/.terraform.lock.hcl", + statsScope, + logger, ) showStepRunner, err := runtime.NewShowStepRunner(terraformClient, defaultTFVersion) @@ -1079,6 +1083,7 @@ func setupE2E(t *testing.T, repoDir string) (events_controllers.VCSEventsControl GitlabMergeRequestGetter: e2eGitlabGetter, Logger: logger, GlobalCfg: globalCfg, + StatsScope: statsScope, AllowForkPRs: allowForkPRs, AllowForkPRsFlag: "allow-fork-prs", CommentCommandRunnerByCmd: commentCommandRunnerByCmd, @@ -1103,6 +1108,7 @@ func setupE2E(t *testing.T, repoDir string) (events_controllers.VCSEventsControl LogStreamResourceCleaner: projectCmdOutputHandler, }, Logger: logger, + Scope: statsScope, Parser: eventParser, CommentParser: commentParser, GithubWebhookSecret: nil, diff --git a/server/controllers/events/events_controller_test.go b/server/controllers/events/events_controller_test.go index dff2ffbb8..92da51fe1 100644 --- a/server/controllers/events/events_controller_test.go +++ b/server/controllers/events/events_controller_test.go @@ -17,15 +17,6 @@ import ( "bytes" "errors" "fmt" - "io" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "reflect" - "strings" - "testing" - . "github.com/petergtz/pegomock" events_controllers "github.com/runatlantis/atlantis/server/controllers/events" "github.com/runatlantis/atlantis/server/controllers/events/mocks" @@ -35,8 +26,17 @@ import ( "github.com/runatlantis/atlantis/server/events/models" vcsmocks "github.com/runatlantis/atlantis/server/events/vcs/mocks" "github.com/runatlantis/atlantis/server/logging" + "github.com/runatlantis/atlantis/server/metrics" . "github.com/runatlantis/atlantis/testing" gitlab "github.com/xanzy/go-gitlab" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "reflect" + "strings" + "testing" ) const githubHeader = "X-Github-Event" @@ -193,8 +193,11 @@ func TestPost_GitlabCommentNotAllowlisted(t *testing.T) { t.Log("when the event is a gitlab comment from a repo that isn't allowlisted we comment with an error") RegisterMockTestingT(t) vcsClient := vcsmocks.NewMockClient() + logger := logging.NewNoopLogger(t) + scope, _, _ := metrics.NewLoggingScope(logger, "null") e := events_controllers.VCSEventsController{ - Logger: logging.NewNoopLogger(t), + Logger: logger, + Scope: scope, CommentParser: &events.CommentParser{}, GitlabRequestParserValidator: &events_controllers.DefaultGitlabRequestParserValidator{}, Parser: &events.EventParser{}, @@ -221,8 +224,11 @@ func TestPost_GitlabCommentNotAllowlistedWithSilenceErrors(t *testing.T) { t.Log("when the event is a gitlab comment from a repo that isn't allowlisted and we are silencing errors, do not comment with an error") RegisterMockTestingT(t) vcsClient := vcsmocks.NewMockClient() + logger := logging.NewNoopLogger(t) + scope, _, _ := metrics.NewLoggingScope(logger, "null") e := events_controllers.VCSEventsController{ - Logger: logging.NewNoopLogger(t), + Logger: logger, + Scope: scope, CommentParser: &events.CommentParser{}, GitlabRequestParserValidator: &events_controllers.DefaultGitlabRequestParserValidator{}, Parser: &events.EventParser{}, @@ -250,8 +256,11 @@ func TestPost_GithubCommentNotAllowlisted(t *testing.T) { t.Log("when the event is a github comment from a repo that isn't allowlisted we comment with an error") RegisterMockTestingT(t) vcsClient := vcsmocks.NewMockClient() + logger := logging.NewNoopLogger(t) + scope, _, _ := metrics.NewLoggingScope(logger, "null") e := events_controllers.VCSEventsController{ - Logger: logging.NewNoopLogger(t), + Logger: logger, + Scope: scope, GithubRequestValidator: &events_controllers.DefaultGithubRequestValidator{}, CommentParser: &events.CommentParser{}, Parser: &events.EventParser{}, @@ -279,8 +288,11 @@ func TestPost_GithubCommentNotAllowlistedWithSilenceErrors(t *testing.T) { t.Log("when the event is a github comment from a repo that isn't allowlisted and we are silencing errors, do not comment with an error") RegisterMockTestingT(t) vcsClient := vcsmocks.NewMockClient() + logger := logging.NewNoopLogger(t) + scope, _, _ := metrics.NewLoggingScope(logger, "null") e := events_controllers.VCSEventsController{ - Logger: logging.NewNoopLogger(t), + Logger: logger, + Scope: scope, GithubRequestValidator: &events_controllers.DefaultGithubRequestValidator{}, CommentParser: &events.CommentParser{}, Parser: &events.EventParser{}, @@ -408,7 +420,7 @@ func TestPost_GithubPullRequestNotAllowlisted(t *testing.T) { When(v.Validate(req, secret)).ThenReturn([]byte(event), nil) w := httptest.NewRecorder() e.Post(w, req) - ResponseContains(t, w, http.StatusForbidden, "Ignoring pull request event from non-allowlisted repo") + ResponseContains(t, w, http.StatusForbidden, "Pull request event from non-allowlisted repo") } func TestPost_GitlabMergeRequestNotAllowlisted(t *testing.T) { @@ -427,7 +439,7 @@ func TestPost_GitlabMergeRequestNotAllowlisted(t *testing.T) { w := httptest.NewRecorder() e.Post(w, req) - ResponseContains(t, w, http.StatusForbidden, "Ignoring pull request event from non-allowlisted repo") + ResponseContains(t, w, http.StatusForbidden, "Pull request event from non-allowlisted repo") } func TestPost_GithubPullRequestUnsupportedAction(t *testing.T) { @@ -476,9 +488,12 @@ func TestPost_AzureDevopsPullRequestIgnoreEvent(t *testing.T) { vcsmock := vcsmocks.NewMockClient() repoAllowlistChecker, err := events.NewRepoAllowlistChecker("*") Ok(t, err) + logger := logging.NewNoopLogger(t) + scope, _, _ := metrics.NewLoggingScope(logger, "null") e := events_controllers.VCSEventsController{ TestingMode: true, - Logger: logging.NewNoopLogger(t), + Logger: logger, + Scope: scope, ApplyDisabled: false, AzureDevopsWebhookBasicUser: user, AzureDevopsWebhookBasicPassword: secret, @@ -632,6 +647,8 @@ func TestPost_BBServerPullClosed(t *testing.T) { pullCleaner := emocks.NewMockPullCleaner() allowlist, err := events.NewRepoAllowlistChecker("*") Ok(t, err) + logger := logging.NewNoopLogger(t) + scope, _, _ := metrics.NewLoggingScope(logger, "null") ec := &events_controllers.VCSEventsController{ PullCleaner: pullCleaner, Parser: &events.EventParser{ @@ -642,7 +659,8 @@ func TestPost_BBServerPullClosed(t *testing.T) { RepoAllowlistChecker: allowlist, SupportedVCSHosts: []models.VCSHostType{models.BitbucketServer}, VCSClient: nil, - Logger: logging.NewNoopLogger(t), + Logger: logger, + Scope: scope, } // Build HTTP request. @@ -760,9 +778,12 @@ func setup(t *testing.T) (events_controllers.VCSEventsController, *mocks.MockGit vcsmock := vcsmocks.NewMockClient() repoAllowlistChecker, err := events.NewRepoAllowlistChecker("*") Ok(t, err) + logger := logging.NewNoopLogger(t) + scope, _, _ := metrics.NewLoggingScope(logger, "null") e := events_controllers.VCSEventsController{ TestingMode: true, - Logger: logging.NewNoopLogger(t), + Logger: logger, + Scope: scope, GithubRequestValidator: v, Parser: p, CommentParser: cp, diff --git a/server/controllers/jobs_controller.go b/server/controllers/jobs_controller.go index 9c512b20e..a41e44446 100644 --- a/server/controllers/jobs_controller.go +++ b/server/controllers/jobs_controller.go @@ -10,6 +10,8 @@ import ( "github.com/runatlantis/atlantis/server/controllers/websocket" "github.com/runatlantis/atlantis/server/core/db" "github.com/runatlantis/atlantis/server/logging" + "github.com/runatlantis/atlantis/server/metrics" + "github.com/uber-go/tally" ) type JobIDKeyGenerator struct{} @@ -32,14 +34,15 @@ type JobsController struct { Db *db.BoltDB WsMux *websocket.Multiplexor KeyGenerator JobIDKeyGenerator + StatsScope tally.Scope } -func (j *JobsController) GetProjectJobs(w http.ResponseWriter, r *http.Request) { +func (j *JobsController) getProjectJobs(w http.ResponseWriter, r *http.Request) error { jobID, err := j.KeyGenerator.Generate(r) if err != nil { j.respond(w, logging.Error, http.StatusBadRequest, err.Error()) - return + return err } viewData := templates.ProjectJobData{ @@ -48,17 +51,39 @@ func (j *JobsController) GetProjectJobs(w http.ResponseWriter, r *http.Request) CleanedBasePath: j.AtlantisURL.Path, } - if err = j.ProjectJobsTemplate.Execute(w, viewData); err != nil { + return j.ProjectJobsTemplate.Execute(w, viewData) +} + +func (j *JobsController) GetProjectJobs(w http.ResponseWriter, r *http.Request) { + errorCounter := j.StatsScope.SubScope("getprojectjobs").Counter(metrics.ExecutionErrorMetric) + err := j.getProjectJobs(w, r) + if err != nil { j.Logger.Err(err.Error()) + errorCounter.Inc(1) } } -func (j *JobsController) GetProjectJobsWS(w http.ResponseWriter, r *http.Request) { +func (j *JobsController) getProjectJobsWS(w http.ResponseWriter, r *http.Request) error { err := j.WsMux.Handle(w, r) if err != nil { - j.respond(w, logging.Error, http.StatusBadRequest, err.Error()) - return + j.respond(w, logging.Error, http.StatusInternalServerError, err.Error()) + return err + } + + return nil +} + +func (j *JobsController) GetProjectJobsWS(w http.ResponseWriter, r *http.Request) { + jobsMetric := j.StatsScope.SubScope("getprojectjobs") + errorCounter := jobsMetric.Counter(metrics.ExecutionErrorMetric) + executionTime := jobsMetric.Timer(metrics.ExecutionTimeMetric).Start() + defer executionTime.Stop() + + err := j.getProjectJobsWS(w, r) + + if err != nil { + errorCounter.Inc(1) } } diff --git a/server/core/config/raw/global_cfg.go b/server/core/config/raw/global_cfg.go index a1d27fe74..260b10c02 100644 --- a/server/core/config/raw/global_cfg.go +++ b/server/core/config/raw/global_cfg.go @@ -15,6 +15,7 @@ type GlobalCfg struct { Repos []Repo `yaml:"repos" json:"repos"` Workflows map[string]Workflow `yaml:"workflows" json:"workflows"` PolicySets PolicySets `yaml:"policies" json:"policies"` + Metrics Metrics `yaml:"metrics" json:"metrics"` } // Repo is the raw schema for repos in the server-side repo config. @@ -34,7 +35,9 @@ type Repo struct { func (g GlobalCfg) Validate() error { err := validation.ValidateStruct(&g, validation.Field(&g.Repos), - validation.Field(&g.Workflows)) + validation.Field(&g.Workflows), + validation.Field(&g.Metrics), + ) if err != nil { return err } @@ -129,6 +132,7 @@ func (g GlobalCfg) ToValid(defaultCfg valid.GlobalCfg) valid.GlobalCfg { Repos: repos, Workflows: workflows, PolicySets: g.PolicySets.ToValid(), + Metrics: g.Metrics.ToValid(), } } diff --git a/server/core/config/raw/metrics.go b/server/core/config/raw/metrics.go new file mode 100644 index 000000000..368c6a4db --- /dev/null +++ b/server/core/config/raw/metrics.go @@ -0,0 +1,44 @@ +package raw + +import ( + validation "github.com/go-ozzo/ozzo-validation" + "github.com/go-ozzo/ozzo-validation/is" + "github.com/runatlantis/atlantis/server/core/config/valid" +) + +type Metrics struct { + Statsd *Statsd `yaml:"statsd" json:"statsd"` +} + +type Statsd struct { + Port string `yaml:"port" json:"port"` + Host string `yaml:"host" json:"host"` +} + +func (s *Statsd) Validate() error { + return validation.ValidateStruct(s, + validation.Field(&s.Host, validation.Required), + validation.Field(&s.Port, validation.Required), + validation.Field(&s.Host, is.IP), + validation.Field(&s.Port, is.Int)) +} + +func (m Metrics) Validate() error { + return validation.ValidateStruct(&m, + validation.Field(&m.Statsd), + ) +} + +func (m Metrics) ToValid() valid.Metrics { + // we've already validated at this point + if m.Statsd != nil { + return valid.Metrics{ + Statsd: &valid.Statsd{ + Host: m.Statsd.Host, + Port: m.Statsd.Port, + }, + } + } + + return valid.Metrics{} +} diff --git a/server/core/config/raw/metrics_test.go b/server/core/config/raw/metrics_test.go new file mode 100644 index 000000000..8bf5744ed --- /dev/null +++ b/server/core/config/raw/metrics_test.go @@ -0,0 +1,116 @@ +package raw_test + +import ( + "encoding/json" + "testing" + + "github.com/runatlantis/atlantis/server/core/config/raw" + "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v2" +) + +func TestMetrics_Unmarshal(t *testing.T) { + t.Run("yaml", func(t *testing.T) { + + rawYaml := ` +statsd: + host: 127.0.0.1 + port: 8125 +` + + var result raw.Metrics + + err := yaml.UnmarshalStrict([]byte(rawYaml), &result) + assert.NoError(t, err) + }) + + t.Run("json", func(t *testing.T) { + rawJSON := ` +{ + "statsd": { + "host": "127.0.0.1", + "port": "8125" + } +} +` + + var result raw.Metrics + + err := json.Unmarshal([]byte(rawJSON), &result) + assert.NoError(t, err) + }) +} +func TestMetrics_Validate_Success(t *testing.T) { + + cases := []struct { + description string + subject raw.Metrics + }{ + { + description: "success", + subject: raw.Metrics{ + Statsd: &raw.Statsd{ + Host: "127.0.0.1", + Port: "8125", + }, + }, + }, + { + description: "missing stats", + }, + } + + for _, c := range cases { + t.Run(c.description, func(t *testing.T) { + assert.NoError(t, c.subject.Validate()) + }) + } + +} +func TestMetrics_Validate_Error(t *testing.T) { + cases := []struct { + description string + subject raw.Metrics + }{ + { + description: "missing host", + subject: raw.Metrics{ + Statsd: &raw.Statsd{ + Port: "8125", + }, + }, + }, + { + description: "missing port", + subject: raw.Metrics{ + Statsd: &raw.Statsd{ + Host: "127.0.0.1", + }, + }, + }, + { + description: "invalid port", + subject: raw.Metrics{ + Statsd: &raw.Statsd{ + Host: "127.0.0.1", + Port: "string", + }, + }, + }, + { + description: "invalid host", + subject: raw.Metrics{ + Statsd: &raw.Statsd{ + Host: "127.0.1", + Port: "8125", + }, + }, + }, + } + + for _, c := range cases { + t.Run(c.description, func(t *testing.T) { + assert.Error(t, c.subject.Validate()) + }) + } +} diff --git a/server/core/config/valid/global_cfg.go b/server/core/config/valid/global_cfg.go index 0518fe7fa..fe17e4b62 100644 --- a/server/core/config/valid/global_cfg.go +++ b/server/core/config/valid/global_cfg.go @@ -35,6 +35,16 @@ type GlobalCfg struct { Repos []Repo Workflows map[string]Workflow PolicySets PolicySets + Metrics Metrics +} + +type Metrics struct { + Statsd *Statsd +} + +type Statsd struct { + Port string + Host string } // Repo is the final parsed version of server-side repo config. diff --git a/server/events/apply_command_runner_test.go b/server/events/apply_command_runner_test.go index 20f946fb9..09442966f 100644 --- a/server/events/apply_command_runner_test.go +++ b/server/events/apply_command_runner_test.go @@ -11,6 +11,7 @@ import ( "github.com/runatlantis/atlantis/server/events/models" "github.com/runatlantis/atlantis/server/events/models/fixtures" "github.com/runatlantis/atlantis/server/logging" + "github.com/runatlantis/atlantis/server/metrics" ) func TestApplyCommandRunner_IsLocked(t *testing.T) { @@ -46,6 +47,8 @@ func TestApplyCommandRunner_IsLocked(t *testing.T) { t.Run(c.Description, func(t *testing.T) { vcsClient := setup(t) + scopeNull, _, _ := metrics.NewLoggingScope(logger, "atlantis") + pull := &github.PullRequest{ State: github.String("open"), } @@ -56,6 +59,7 @@ func TestApplyCommandRunner_IsLocked(t *testing.T) { ctx := &events.CommandContext{ User: fixtures.User, Log: logging.NewNoopLogger(t), + Scope: scopeNull, Pull: modelPull, HeadRepo: fixtures.GithubRepo, Trigger: events.Comment, diff --git a/server/events/command_context.go b/server/events/command_context.go index fcea22dfe..723ade459 100644 --- a/server/events/command_context.go +++ b/server/events/command_context.go @@ -15,6 +15,7 @@ package events import ( "github.com/runatlantis/atlantis/server/events/models" "github.com/runatlantis/atlantis/server/logging" + "github.com/uber-go/tally" ) // CommandTrigger represents the how the command was triggered @@ -37,6 +38,7 @@ type CommandContext struct { // See https://help.github.com/articles/about-pull-request-merges/. HeadRepo models.Repo Pull models.PullRequest + Scope tally.Scope // User is the user that triggered this command. User models.User Log logging.SimpleLogging diff --git a/server/events/command_runner.go b/server/events/command_runner.go index 4e1738274..ebcca2a89 100644 --- a/server/events/command_runner.go +++ b/server/events/command_runner.go @@ -24,7 +24,9 @@ import ( "github.com/runatlantis/atlantis/server/events/models" "github.com/runatlantis/atlantis/server/events/vcs" "github.com/runatlantis/atlantis/server/logging" + "github.com/runatlantis/atlantis/server/metrics" "github.com/runatlantis/atlantis/server/recovery" + "github.com/uber-go/tally" gitlab "github.com/xanzy/go-gitlab" ) @@ -97,6 +99,7 @@ type DefaultCommandRunner struct { EventParser EventParsing Logger logging.SimpleLogging GlobalCfg valid.GlobalCfg + StatsScope tally.Scope // AllowForkPRs controls whether we operate on pull requests from forks. AllowForkPRs bool // ParallelPoolSize controls the size of the wait group used to run @@ -138,9 +141,14 @@ func (c *DefaultCommandRunner) RunAutoplanCommand(baseRepo models.Repo, headRepo log.Err("Unable to fetch pull status, this is likely a bug.", err) } + scope := c.StatsScope.SubScope("autoplan") + timer := scope.Timer(metrics.ExecutionTimeMetric).Start() + defer timer.Stop() + ctx := &CommandContext{ User: user, Log: log, + Scope: scope, Pull: pull, HeadRepo: headRepo, PullStatus: status, @@ -213,6 +221,14 @@ func (c *DefaultCommandRunner) RunCommentCommand(baseRepo models.Repo, maybeHead log := c.buildLogger(baseRepo.FullName, pullNum) defer c.logPanics(baseRepo, pullNum, log) + scope := c.StatsScope.SubScope("comment") + + if cmd != nil { + scope = scope.SubScope(cmd.Name.String()) + } + timer := scope.Timer(metrics.ExecutionTimeMetric).Start() + defer timer.Stop() + // Check if the user who commented has the permissions to execute the 'plan' or 'apply' commands ok, err := c.checkUserPermissions(baseRepo, user, cmd) if err != nil { @@ -242,6 +258,7 @@ func (c *DefaultCommandRunner) RunCommentCommand(baseRepo models.Repo, maybeHead PullStatus: status, HeadRepo: headRepo, Trigger: Comment, + Scope: scope, } if !c.validateCtxAndComment(ctx) { diff --git a/server/events/command_runner_test.go b/server/events/command_runner_test.go index bc8b6d9b7..ded60c635 100644 --- a/server/events/command_runner_test.go +++ b/server/events/command_runner_test.go @@ -24,6 +24,7 @@ import ( "github.com/runatlantis/atlantis/server/core/db" "github.com/runatlantis/atlantis/server/events/vcs" "github.com/runatlantis/atlantis/server/logging" + "github.com/runatlantis/atlantis/server/metrics" "github.com/google/go-github/v31/github" . "github.com/petergtz/pegomock" @@ -192,6 +193,7 @@ func setup(t *testing.T) *vcsmocks.MockClient { When(postWorkflowHooksCommandRunner.RunPostHooks(matchers.AnyPtrToEventsCommandContext())).ThenReturn(nil) globalCfg := valid.NewGlobalCfgFromArgs(valid.GlobalCfgArgs{}) + scope, _, _ := metrics.NewLoggingScope(logger, "atlantis") ch = events.DefaultCommandRunner{ VCSClient: vcsClient, @@ -201,6 +203,7 @@ func setup(t *testing.T) *vcsmocks.MockClient { GitlabMergeRequestGetter: gitlabGetter, AzureDevopsPullGetter: azuredevopsGetter, Logger: logger, + StatsScope: scope, GlobalCfg: globalCfg, AllowForkPRs: false, AllowForkPRsFlag: "allow-fork-prs-flag", diff --git a/server/events/instrumented_project_command_builder.go b/server/events/instrumented_project_command_builder.go new file mode 100644 index 000000000..e51cbe5c9 --- /dev/null +++ b/server/events/instrumented_project_command_builder.go @@ -0,0 +1,76 @@ +package events + +import ( + "github.com/runatlantis/atlantis/server/events/models" + "github.com/runatlantis/atlantis/server/logging" + "github.com/runatlantis/atlantis/server/metrics" +) + +type InstrumentedProjectCommandBuilder struct { + ProjectCommandBuilder + Logger logging.SimpleLogging +} + +func (b *InstrumentedProjectCommandBuilder) BuildApplyCommands(ctx *CommandContext, comment *CommentCommand) ([]models.ProjectCommandContext, error) { + scope := ctx.Scope.SubScope("builder") + + timer := scope.Timer(metrics.ExecutionTimeMetric).Start() + defer timer.Stop() + + executionSuccess := scope.Counter(metrics.ExecutionSuccessMetric) + executionError := scope.Counter(metrics.ExecutionErrorMetric) + + projectCmds, err := b.ProjectCommandBuilder.BuildApplyCommands(ctx, comment) + + if err != nil { + executionError.Inc(1) + b.Logger.Err("Error building apply commands: %s", err) + } else { + executionSuccess.Inc(1) + } + + return projectCmds, err + +} +func (b *InstrumentedProjectCommandBuilder) BuildAutoplanCommands(ctx *CommandContext) ([]models.ProjectCommandContext, error) { + scope := ctx.Scope.SubScope("builder") + + timer := scope.Timer(metrics.ExecutionTimeMetric).Start() + defer timer.Stop() + + executionSuccess := scope.Counter(metrics.ExecutionSuccessMetric) + executionError := scope.Counter(metrics.ExecutionErrorMetric) + + projectCmds, err := b.ProjectCommandBuilder.BuildAutoplanCommands(ctx) + + if err != nil { + executionError.Inc(1) + b.Logger.Err("Error building auto plan commands: %s", err) + } else { + executionSuccess.Inc(1) + } + + return projectCmds, err + +} +func (b *InstrumentedProjectCommandBuilder) BuildPlanCommands(ctx *CommandContext, comment *CommentCommand) ([]models.ProjectCommandContext, error) { + scope := ctx.Scope.SubScope("builder") + + timer := scope.Timer(metrics.ExecutionTimeMetric).Start() + defer timer.Stop() + + executionSuccess := scope.Counter(metrics.ExecutionSuccessMetric) + executionError := scope.Counter(metrics.ExecutionErrorMetric) + + projectCmds, err := b.ProjectCommandBuilder.BuildPlanCommands(ctx, comment) + + if err != nil { + executionError.Inc(1) + b.Logger.Err("Error building plan commands: %s", err) + } else { + executionSuccess.Inc(1) + } + + return projectCmds, err + +} diff --git a/server/events/instrumented_project_command_runner.go b/server/events/instrumented_project_command_runner.go new file mode 100644 index 000000000..98df05d2b --- /dev/null +++ b/server/events/instrumented_project_command_runner.go @@ -0,0 +1,58 @@ +package events + +import ( + "github.com/runatlantis/atlantis/server/events/models" + "github.com/runatlantis/atlantis/server/metrics" +) + +type InstrumentedProjectCommandRunner struct { + ProjectCommandRunner +} + +func (p *InstrumentedProjectCommandRunner) Plan(ctx models.ProjectCommandContext) models.ProjectResult { + return RunAndEmitStats("plan", ctx, p.ProjectCommandRunner.Plan) +} + +func (p *InstrumentedProjectCommandRunner) PolicyCheck(ctx models.ProjectCommandContext) models.ProjectResult { + return RunAndEmitStats("policy check", ctx, p.ProjectCommandRunner.PolicyCheck) +} + +func (p *InstrumentedProjectCommandRunner) Apply(ctx models.ProjectCommandContext) models.ProjectResult { + return RunAndEmitStats("apply", ctx, p.ProjectCommandRunner.Apply) +} + +func RunAndEmitStats(commandName string, ctx models.ProjectCommandContext, execute func(ctx models.ProjectCommandContext) models.ProjectResult) models.ProjectResult { + + // ensures we are differentiating between project level command and overall command + ctx.SetScope("project") + + scope := ctx.Scope + logger := ctx.Log + + executionTime := scope.Timer(metrics.ExecutionTimeMetric).Start() + defer executionTime.Stop() + + executionSuccess := scope.Counter(metrics.ExecutionSuccessMetric) + executionError := scope.Counter(metrics.ExecutionErrorMetric) + executionFailure := scope.Counter(metrics.ExecutionFailureMetric) + + result := execute(ctx) + + if result.Error != nil { + executionError.Inc(1) + logger.Err("Error running %s operation: %s", commandName, result.Error.Error()) + return result + } + + if result.Failure != "" { + executionFailure.Inc(1) + logger.Err("Failure running %s operation: %s", commandName, result.Failure) + return result + } + + logger.Info("%s success. output available at: %s", commandName, ctx.Pull.URL) + + executionSuccess.Inc(1) + return result + +} diff --git a/server/events/instrumented_pull_closed_executor.go b/server/events/instrumented_pull_closed_executor.go new file mode 100644 index 000000000..87fd73bbf --- /dev/null +++ b/server/events/instrumented_pull_closed_executor.go @@ -0,0 +1,53 @@ +package events + +import ( + "strconv" + + "github.com/runatlantis/atlantis/server/events/models" + "github.com/runatlantis/atlantis/server/logging" + "github.com/runatlantis/atlantis/server/metrics" + "github.com/uber-go/tally" +) + +type InstrumentedPullClosedExecutor struct { + scope tally.Scope + log logging.SimpleLogging + cleaner PullCleaner +} + +func NewInstrumentedPullClosedExecutor( + scope tally.Scope, log logging.SimpleLogging, cleaner PullCleaner, +) PullCleaner { + + return &InstrumentedPullClosedExecutor{ + scope: scope.SubScope("pullclosed.cleanup"), + log: log, + cleaner: cleaner, + } +} + +func (e *InstrumentedPullClosedExecutor) CleanUpPull(repo models.Repo, pull models.PullRequest) error { + log := e.log.With( + "repository", repo.FullName, + "pull-num", strconv.Itoa(pull.Num), + ) + + executionSuccess := e.scope.Counter(metrics.ExecutionSuccessMetric) + executionError := e.scope.Counter(metrics.ExecutionErrorMetric) + executionTime := e.scope.Timer(metrics.ExecutionTimeMetric).Start() + defer executionTime.Stop() + + log.Info("Initiating cleanup of pull data.") + + err := e.cleaner.CleanUpPull(repo, pull) + + if err != nil { + executionError.Inc(1) + log.Err("error during cleanup of pull data", err) + return err + } + + executionSuccess.Inc(1) + + return nil +} diff --git a/server/events/models/models.go b/server/events/models/models.go index 287c7a9ff..d43e203df 100644 --- a/server/events/models/models.go +++ b/server/events/models/models.go @@ -26,6 +26,7 @@ import ( "github.com/hashicorp/go-version" "github.com/runatlantis/atlantis/server/logging" + "github.com/uber-go/tally" "github.com/pkg/errors" "github.com/runatlantis/atlantis/server/core/config/valid" @@ -375,6 +376,8 @@ type ProjectCommandContext struct { HeadRepo Repo // Log is a logger that's been set up for this context. Log logging.SimpleLogging + // Scope is the scope for reporting stats setup for this context + Scope tally.Scope // PullReqStatus holds state about the PR that requires additional computation outside models.PullRequest PullReqStatus PullReqStatus // CurrentProjectPlanStatus is the status of the current project prior to this command. @@ -415,6 +418,12 @@ type ProjectCommandContext struct { JobID string } +// SetScope sets the scope of the stats object field. Note: we deliberately set this on the value +// instead of a pointer since we want scopes to mirror our function stack +func (p ProjectCommandContext) SetScope(scope string) { + p.Scope = p.Scope.SubScope(scope) //nolint +} + // GetShowResultFileName returns the filename (not the path) to store the tf show result func (p ProjectCommandContext) GetShowResultFileName() string { if p.ProjectName == "" { diff --git a/server/events/project_command_builder.go b/server/events/project_command_builder.go index eb121213c..d9f8657b6 100644 --- a/server/events/project_command_builder.go +++ b/server/events/project_command_builder.go @@ -5,6 +5,8 @@ import ( "os" "github.com/runatlantis/atlantis/server/core/config/valid" + "github.com/runatlantis/atlantis/server/logging" + "github.com/uber-go/tally" "github.com/pkg/errors" "github.com/runatlantis/atlantis/server/core/config" @@ -29,6 +31,43 @@ const ( DefaultDeleteSourceBranchOnMerge = false ) +func NewInstrumentedProjectCommandBuilder( + policyChecksSupported bool, + parserValidator *config.ParserValidator, + projectFinder ProjectFinder, + vcsClient vcs.Client, + workingDir WorkingDir, + workingDirLocker WorkingDirLocker, + globalCfg valid.GlobalCfg, + pendingPlanFinder *DefaultPendingPlanFinder, + commentBuilder CommentBuilder, + skipCloneNoChanges bool, + EnableRegExpCmd bool, + AutoplanFileList string, + scope tally.Scope, + logger logging.SimpleLogging, +) *InstrumentedProjectCommandBuilder { + return &InstrumentedProjectCommandBuilder{ + ProjectCommandBuilder: NewProjectCommandBuilder( + policyChecksSupported, + parserValidator, + projectFinder, + vcsClient, + workingDir, + workingDirLocker, + globalCfg, + pendingPlanFinder, + commentBuilder, + skipCloneNoChanges, + EnableRegExpCmd, + AutoplanFileList, + scope, + logger, + ), + Logger: logger, + } +} + func NewProjectCommandBuilder( policyChecksSupported bool, parserValidator *config.ParserValidator, @@ -42,8 +81,10 @@ func NewProjectCommandBuilder( skipCloneNoChanges bool, EnableRegExpCmd bool, AutoplanFileList string, + scope tally.Scope, + logger logging.SimpleLogging, ) *DefaultProjectCommandBuilder { - projectCommandBuilder := &DefaultProjectCommandBuilder{ + return &DefaultProjectCommandBuilder{ ParserValidator: parserValidator, ProjectFinder: projectFinder, VCSClient: vcsClient, @@ -57,10 +98,9 @@ func NewProjectCommandBuilder( ProjectCommandContextBuilder: NewProjectCommandContextBuilder( policyChecksSupported, commentBuilder, + scope, ), } - - return projectCommandBuilder } type ProjectPlanCommandBuilder interface { diff --git a/server/events/project_command_builder_internal_test.go b/server/events/project_command_builder_internal_test.go index 9f484b11c..9b1ca4289 100644 --- a/server/events/project_command_builder_internal_test.go +++ b/server/events/project_command_builder_internal_test.go @@ -14,12 +14,14 @@ import ( vcsmocks "github.com/runatlantis/atlantis/server/events/vcs/mocks" "github.com/runatlantis/atlantis/server/logging" logging_matchers "github.com/runatlantis/atlantis/server/logging/mocks/matchers" + "github.com/runatlantis/atlantis/server/metrics" . "github.com/runatlantis/atlantis/testing" ) // Test different permutations of global and repo config. func TestBuildProjectCmdCtx(t *testing.T) { logger := logging.NewNoopLogger(t) + statsScope, _, _ := metrics.NewLoggingScope(logging.NewNoopLogger(t), "atlantis") emptyPolicySets := valid.PolicySets{ Version: nil, PolicySets: []valid.PolicySet{}, @@ -66,6 +68,7 @@ workflows: AutoplanEnabled: true, HeadRepo: models.Repo{}, Log: logger, + Scope: statsScope, PullReqStatus: models.PullReqStatus{ Mergeable: true, }, @@ -118,6 +121,7 @@ projects: AutoplanEnabled: true, HeadRepo: models.Repo{}, Log: logger, + Scope: statsScope, PullReqStatus: models.PullReqStatus{ Mergeable: true, }, @@ -172,6 +176,7 @@ projects: AutoplanEnabled: true, HeadRepo: models.Repo{}, Log: logger, + Scope: statsScope, PullReqStatus: models.PullReqStatus{ Mergeable: true, }, @@ -234,6 +239,7 @@ projects: AutoplanEnabled: true, HeadRepo: models.Repo{}, Log: logger, + Scope: statsScope, PullReqStatus: models.PullReqStatus{ Mergeable: true, }, @@ -383,6 +389,7 @@ workflows: AutoplanEnabled: true, HeadRepo: models.Repo{}, Log: logger, + Scope: statsScope, PullReqStatus: models.PullReqStatus{ Mergeable: true, }, @@ -441,6 +448,7 @@ projects: AutoplanEnabled: true, HeadRepo: models.Repo{}, Log: logger, + Scope: statsScope, PullReqStatus: models.PullReqStatus{ Mergeable: true, }, @@ -502,6 +510,7 @@ workflows: AutoplanEnabled: true, HeadRepo: models.Repo{}, Log: logger, + Scope: statsScope, PullReqStatus: models.PullReqStatus{ Mergeable: true, }, @@ -547,6 +556,7 @@ projects: AutoplanEnabled: true, HeadRepo: models.Repo{}, Log: logger, + Scope: statsScope, PullReqStatus: models.PullReqStatus{ Mergeable: true, }, @@ -615,13 +625,16 @@ projects: false, false, "**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl,**/.terraform.lock.hcl", + statsScope, + logger, ) // We run a test for each type of command. for _, cmd := range []models.CommandName{models.PlanCommand, models.ApplyCommand} { t.Run(cmd.String(), func(t *testing.T) { ctxs, err := builder.buildProjectCommandCtx(&CommandContext{ - Log: logger, + Log: logger, + Scope: statsScope, Pull: models.PullRequest{ BaseRepo: baseRepo, }, @@ -674,6 +687,7 @@ projects: } func TestBuildProjectCmdCtx_WithRegExpCmdEnabled(t *testing.T) { + statsScope, _, _ := metrics.NewLoggingScope(logging.NewNoopLogger(t), "atlantis") emptyPolicySets := valid.PolicySets{ Version: nil, PolicySets: []valid.PolicySet{}, @@ -746,6 +760,7 @@ projects: AutoplanEnabled: true, HeadRepo: models.Repo{}, Log: logging.NewNoopLogger(t), + Scope: statsScope, PullReqStatus: models.PullReqStatus{ Mergeable: true, }, @@ -796,6 +811,9 @@ projects: Ok(t, os.WriteFile(filepath.Join(tmp, "atlantis.yaml"), []byte(c.repoCfg), 0600)) } + logger := logging.NewNoopLogger(t) + statsScope, _, _ := metrics.NewLoggingScope(logging.NewNoopLogger(t), "atlantis") + builder := NewProjectCommandBuilder( false, parser, @@ -809,6 +827,8 @@ projects: false, true, "**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl,**/.terraform.lock.hcl", + statsScope, + logger, ) // We run a test for each type of command, again specific projects @@ -818,7 +838,8 @@ projects: Pull: models.PullRequest{ BaseRepo: baseRepo, }, - Log: logging.NewNoopLogger(t), + Log: logging.NewNoopLogger(t), + Scope: statsScope, PullRequestStatus: models.PullReqStatus{ Mergeable: true, }, @@ -870,6 +891,7 @@ projects: func TestBuildProjectCmdCtx_WithPolicCheckEnabled(t *testing.T) { logger := logging.NewNoopLogger(t) + statsScope, _, _ := metrics.NewLoggingScope(logging.NewNoopLogger(t), "atlantis") emptyPolicySets := valid.PolicySets{ Version: nil, PolicySets: []valid.PolicySet{}, @@ -906,6 +928,7 @@ repos: AutoplanEnabled: true, HeadRepo: models.Repo{}, Log: logger, + Scope: statsScope, PullReqStatus: models.PullReqStatus{ Mergeable: true, }, @@ -963,6 +986,7 @@ workflows: AutoplanEnabled: true, HeadRepo: models.Repo{}, Log: logger, + Scope: statsScope, PullReqStatus: models.PullReqStatus{ Mergeable: true, }, @@ -1018,6 +1042,7 @@ workflows: if c.repoCfg != "" { Ok(t, os.WriteFile(filepath.Join(tmp, "atlantis.yaml"), []byte(c.repoCfg), 0600)) } + statsScope, _, _ := metrics.NewLoggingScope(logging.NewNoopLogger(t), "atlantis") builder := NewProjectCommandBuilder( true, @@ -1032,12 +1057,15 @@ workflows: false, false, "**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl,**/.terraform.lock.hcl", + statsScope, + logger, ) cmd := models.PolicyCheckCommand t.Run(cmd.String(), func(t *testing.T) { ctxs, err := builder.buildProjectCommandCtx(&CommandContext{ - Log: logger, + Log: logger, + Scope: statsScope, Pull: models.PullRequest{ BaseRepo: baseRepo, }, diff --git a/server/events/project_command_builder_test.go b/server/events/project_command_builder_test.go index 59aaf869d..fe7b4d6c6 100644 --- a/server/events/project_command_builder_test.go +++ b/server/events/project_command_builder_test.go @@ -16,6 +16,7 @@ import ( "github.com/runatlantis/atlantis/server/events/models" vcsmocks "github.com/runatlantis/atlantis/server/events/vcs/mocks" "github.com/runatlantis/atlantis/server/logging" + "github.com/runatlantis/atlantis/server/metrics" . "github.com/runatlantis/atlantis/testing" ) @@ -118,6 +119,7 @@ projects: } logger := logging.NewNoopLogger(t) + scope, _, _ := metrics.NewLoggingScope(logger, "atlantis") for _, c := range cases { t.Run(c.Description, func(t *testing.T) { @@ -156,13 +158,16 @@ projects: false, false, "**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl,**/.terraform.lock.hcl", + scope, + logger, ) ctxs, err := builder.BuildAutoplanCommands(&events.CommandContext{ PullRequestStatus: models.PullReqStatus{ Mergeable: true, }, - Log: logger, + Log: logger, + Scope: scope, }) Ok(t, err) Equals(t, len(c.exp), len(ctxs)) @@ -379,6 +384,7 @@ projects: } logger := logging.NewNoopLogger(t) + scope, _, _ := metrics.NewLoggingScope(logger, "atlantis") for _, c := range cases { // NOTE: we're testing both plan and apply here. @@ -420,16 +426,19 @@ projects: false, true, "**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl,**/.terraform.lock.hcl", + scope, + logger, ) var actCtxs []models.ProjectCommandContext var err error if cmdName == models.PlanCommand { actCtxs, err = builder.BuildPlanCommands(&events.CommandContext{ - Log: logger, + Log: logger, + Scope: scope, }, &c.Cmd) } else { - actCtxs, err = builder.BuildApplyCommands(&events.CommandContext{Log: logger}, &c.Cmd) + actCtxs, err = builder.BuildApplyCommands(&events.CommandContext{Log: logger, Scope: scope}, &c.Cmd) } if c.ExpErr != "" { @@ -535,6 +544,7 @@ projects: } logger := logging.NewNoopLogger(t) + scope, _, _ := metrics.NewLoggingScope(logger, "atlantis") for name, c := range cases { t.Run(name, func(t *testing.T) { RegisterMockTestingT(t) @@ -571,11 +581,14 @@ projects: false, false, "**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl,**/.terraform.lock.hcl", + scope, + logger, ) ctxs, err := builder.BuildPlanCommands( &events.CommandContext{ - Log: logger, + Log: logger, + Scope: scope, }, &events.CommentCommand{ RepoRelDir: "", @@ -644,6 +657,7 @@ func TestDefaultProjectCommandBuilder_BuildMultiApply(t *testing.T) { ApprovedReq: false, UnDivergedReq: false, } + scope, _, _ := metrics.NewLoggingScope(logger, "atlantis") builder := events.NewProjectCommandBuilder( false, @@ -658,11 +672,14 @@ func TestDefaultProjectCommandBuilder_BuildMultiApply(t *testing.T) { false, false, "**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl,**/.terraform.lock.hcl", + scope, + logger, ) ctxs, err := builder.BuildApplyCommands( &events.CommandContext{ - Log: logger, + Log: logger, + Scope: scope, }, &events.CommentCommand{ RepoRelDir: "", @@ -724,6 +741,8 @@ projects: ApprovedReq: false, UnDivergedReq: false, } + logger := logging.NewNoopLogger(t) + scope, _, _ := metrics.NewLoggingScope(logger, "atlantis") builder := events.NewProjectCommandBuilder( false, @@ -738,13 +757,16 @@ projects: false, false, "**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl,**/.terraform.lock.hcl", + scope, + logger, ) ctx := &events.CommandContext{ HeadRepo: models.Repo{}, Pull: models.PullRequest{}, User: models.User{}, - Log: logging.NewNoopLogger(t), + Log: logger, + Scope: scope, } _, err = builder.BuildPlanCommands(ctx, &events.CommentCommand{ RepoRelDir: ".", @@ -778,6 +800,7 @@ func TestDefaultProjectCommandBuilder_EscapeArgs(t *testing.T) { } logger := logging.NewNoopLogger(t) + scope, _, _ := metrics.NewLoggingScope(logger, "atlantis") for _, c := range cases { t.Run(strings.Join(c.ExtraArgs, " "), func(t *testing.T) { @@ -813,12 +836,15 @@ func TestDefaultProjectCommandBuilder_EscapeArgs(t *testing.T) { false, false, "**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl,**/.terraform.lock.hcl", + scope, + logger, ) var actCtxs []models.ProjectCommandContext var err error actCtxs, err = builder.BuildPlanCommands(&events.CommandContext{ - Log: logger, + Log: logger, + Scope: scope, }, &events.CommentCommand{ RepoRelDir: ".", Flags: c.ExtraArgs, @@ -949,6 +975,7 @@ projects: } logger := logging.NewNoopLogger(t) + scope, _, _ := metrics.NewLoggingScope(logger, "atlantis") for name, testCase := range testCases { t.Run(name, func(t *testing.T) { @@ -992,11 +1019,14 @@ projects: false, false, "**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl,**/.terraform.lock.hcl", + scope, + logger, ) actCtxs, err := builder.BuildPlanCommands( &events.CommandContext{ - Log: logger, + Log: logger, + Scope: scope, }, &events.CommentCommand{ RepoRelDir: "", @@ -1041,6 +1071,7 @@ projects: ApprovedReq: false, UnDivergedReq: false, } + scope, _, _ := metrics.NewLoggingScope(logger, "atlantis") builder := events.NewProjectCommandBuilder( false, @@ -1055,6 +1086,8 @@ projects: true, false, "**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl,**/.terraform.lock.hcl", + scope, + logger, ) var actCtxs []models.ProjectCommandContext @@ -1064,6 +1097,7 @@ projects: Pull: models.PullRequest{}, User: models.User{}, Log: logger, + Scope: scope, PullRequestStatus: models.PullReqStatus{ Mergeable: true, }, @@ -1081,6 +1115,7 @@ func TestDefaultProjectCommandBuilder_WithPolicyCheckEnabled_BuildAutoplanComman defer cleanup() logger := logging.NewNoopLogger(t) + scope, _, _ := metrics.NewLoggingScope(logger, "atlantis") workingDir := mocks.NewMockWorkingDir() When(workingDir.Clone(matchers.AnyPtrToLoggingSimpleLogger(), matchers.AnyModelsRepo(), matchers.AnyModelsPullRequest(), AnyString())).ThenReturn(tmpDir, false, nil) @@ -1109,13 +1144,16 @@ func TestDefaultProjectCommandBuilder_WithPolicyCheckEnabled_BuildAutoplanComman false, false, "**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl,**/.terraform.lock.hcl", + scope, + logger, ) ctxs, err := builder.BuildAutoplanCommands(&events.CommandContext{ PullRequestStatus: models.PullReqStatus{ Mergeable: true, }, - Log: logger, + Log: logger, + Scope: scope, }) Ok(t, err) @@ -1166,6 +1204,7 @@ func TestDefaultProjectCommandBuilder_BuildVersionCommand(t *testing.T) { ThenReturn(tmpDir, nil) logger := logging.NewNoopLogger(t) + scope, _, _ := metrics.NewLoggingScope(logger, "atlantis") globalCfgArgs := valid.GlobalCfgArgs{ AllowRepoCfg: false, @@ -1187,11 +1226,14 @@ func TestDefaultProjectCommandBuilder_BuildVersionCommand(t *testing.T) { false, false, "**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl,**/.terraform.lock.hcl", + scope, + logger, ) ctxs, err := builder.BuildVersionCommands( &events.CommandContext{ - Log: logger, + Log: logger, + Scope: scope, }, &events.CommentCommand{ RepoRelDir: "", diff --git a/server/events/project_command_context_builder.go b/server/events/project_command_context_builder.go index 719bbfb62..08299f353 100644 --- a/server/events/project_command_context_builder.go +++ b/server/events/project_command_context_builder.go @@ -9,9 +9,10 @@ import ( "github.com/hashicorp/terraform-config-inspect/tfconfig" "github.com/runatlantis/atlantis/server/core/config/valid" "github.com/runatlantis/atlantis/server/events/models" + "github.com/uber-go/tally" ) -func NewProjectCommandContextBuilder(policyCheckEnabled bool, commentBuilder CommentBuilder) ProjectCommandContextBuilder { +func NewProjectCommandContextBuilder(policyCheckEnabled bool, commentBuilder CommentBuilder, scope tally.Scope) ProjectCommandContextBuilder { projectCommandContextBuilder := &DefaultProjectCommandContextBuilder{ CommentBuilder: commentBuilder, } @@ -23,7 +24,10 @@ func NewProjectCommandContextBuilder(policyCheckEnabled bool, commentBuilder Com } } - return projectCommandContextBuilder + return &CommandScopedStatsProjectCommandContextBuilder{ + ProjectCommandContextBuilder: projectCommandContextBuilder, + ProjectCounter: scope.Counter("projects"), + } } type ProjectCommandContextBuilder interface { @@ -38,6 +42,43 @@ type ProjectCommandContextBuilder interface { ) []models.ProjectCommandContext } +// CommandScopedStatsProjectCommandContextBuilder ensures that project command context contains a scoped stats +// object relevant to the command it applies to. +type CommandScopedStatsProjectCommandContextBuilder struct { + ProjectCommandContextBuilder + // Conciously making this global since it gets flushed periodically anyways + ProjectCounter tally.Counter +} + +// BuildProjectContext builds the context and injects the appropriate command level scope after the fact. +func (cb *CommandScopedStatsProjectCommandContextBuilder) BuildProjectContext( + ctx *CommandContext, + cmdName models.CommandName, + prjCfg valid.MergedProjectCfg, + commentFlags []string, + repoDir string, + automerge, deleteSourceBranchOnMerge, parallelApply, parallelPlan, verbose bool, +) (projectCmds []models.ProjectCommandContext) { + cb.ProjectCounter.Inc(1) + + cmds := cb.ProjectCommandContextBuilder.BuildProjectContext( + ctx, cmdName, prjCfg, commentFlags, repoDir, automerge, deleteSourceBranchOnMerge, parallelApply, parallelPlan, verbose, + ) + + projectCmds = []models.ProjectCommandContext{} + + for _, cmd := range cmds { + + // specifically use the command name in the context instead of the arg + // since we can return multiple commands worth of contexts for a given command name arg + // to effectively pipeline them. + cmd.SetScope(cmd.CommandName.String()) + projectCmds = append(projectCmds, cmd) + } + + return +} + type DefaultProjectCommandContextBuilder struct { CommentBuilder CommentBuilder } @@ -86,6 +127,7 @@ func (cb *DefaultProjectCommandContextBuilder) BuildProjectContext( parallelApply, parallelPlan, verbose, + ctx.Scope, ctx.PullRequestStatus, ) @@ -147,6 +189,7 @@ func (cb *PolicyCheckProjectCommandContextBuilder) BuildProjectContext( parallelApply, parallelPlan, verbose, + ctx.Scope, ctx.PullRequestStatus, )) } @@ -170,6 +213,7 @@ func newProjectCommandContext(ctx *CommandContext, parallelApplyEnabled bool, parallelPlanEnabled bool, verbose bool, + scope tally.Scope, pullStatus models.PullReqStatus, ) models.ProjectCommandContext { @@ -205,6 +249,7 @@ func newProjectCommandContext(ctx *CommandContext, Steps: steps, HeadRepo: ctx.HeadRepo, Log: ctx.Log, + Scope: scope, ProjectPlanStatus: projectPlanStatus, Pull: ctx.Pull, ProjectName: projCfg.Name, diff --git a/server/events/vcs/instrumented_client.go b/server/events/vcs/instrumented_client.go new file mode 100644 index 000000000..18d5025b9 --- /dev/null +++ b/server/events/vcs/instrumented_client.go @@ -0,0 +1,237 @@ +package vcs + +import ( + "fmt" + "strconv" + + "github.com/google/go-github/v31/github" + "github.com/runatlantis/atlantis/server/events/models" + "github.com/runatlantis/atlantis/server/logging" + "github.com/runatlantis/atlantis/server/metrics" + "github.com/uber-go/tally" +) + +// NewInstrumentedGithubClient creates a client proxy responsible for gathering stats and logging +func NewInstrumentedGithubClient(client *GithubClient, statsScope tally.Scope, logger logging.SimpleLogging) IGithubClient { + scope := statsScope.SubScope("github") + + instrumentedGHClient := &InstrumentedClient{ + Client: client, + StatsScope: scope, + Logger: logger, + } + + return &InstrumentedGithubClient{ + InstrumentedClient: instrumentedGHClient, + PullRequestGetter: client, + StatsScope: scope, + Logger: logger, + } +} + +//go:generate pegomock generate -m --use-experimental-model-gen --package mocks -o mocks/mock_github_pull_request_getter.go GithubPullRequestGetter + +type GithubPullRequestGetter interface { + GetPullRequest(repo models.Repo, pullNum int) (*github.PullRequest, error) +} + +// IGithubClient exists to bridge the gap between GithubPullRequestGetter and Client interface to allow +// for a single instrumented client +type IGithubClient interface { + Client + GithubPullRequestGetter +} + +// InstrumentedGithubClient should delegate to the underlying InstrumentedClient for vcs provider-agnostic +// methods and implement soley any github specific interfaces. +type InstrumentedGithubClient struct { + *InstrumentedClient + PullRequestGetter GithubPullRequestGetter + StatsScope tally.Scope + Logger logging.SimpleLogging +} + +func (c *InstrumentedGithubClient) GetPullRequest(repo models.Repo, pullNum int) (*github.PullRequest, error) { + scope := c.StatsScope.SubScope("get_pull_request") + logger := c.Logger.WithHistory([]interface{}{ + "repository", fmt.Sprintf("%s/%s", repo.Owner, repo.Name), + "pull-num", strconv.Itoa(pullNum), + }...) + + executionTime := scope.Timer(metrics.ExecutionTimeMetric).Start() + defer executionTime.Stop() + + executionSuccess := scope.Counter(metrics.ExecutionSuccessMetric) + executionError := scope.Counter(metrics.ExecutionErrorMetric) + + pull, err := c.PullRequestGetter.GetPullRequest(repo, pullNum) + + if err != nil { + executionError.Inc(1) + logger.Err("Unable to get pull number for repo, error: %s", err.Error()) + } else { + executionSuccess.Inc(1) + } + + return pull, err + +} + +type InstrumentedClient struct { + Client + StatsScope tally.Scope + Logger logging.SimpleLogging +} + +func (c *InstrumentedClient) GetModifiedFiles(repo models.Repo, pull models.PullRequest) ([]string, error) { + scope := c.StatsScope.SubScope("get_modified_files") + logger := c.Logger.WithHistory(fmtLogSrc(repo, pull.Num)...) + + executionTime := scope.Timer(metrics.ExecutionTimeMetric).Start() + defer executionTime.Stop() + + executionSuccess := scope.Counter(metrics.ExecutionSuccessMetric) + executionError := scope.Counter(metrics.ExecutionErrorMetric) + + files, err := c.Client.GetModifiedFiles(repo, pull) + + if err != nil { + executionError.Inc(1) + logger.Err("Unable to get modified files, error: %s", err.Error()) + } else { + executionSuccess.Inc(1) + } + + return files, err + +} +func (c *InstrumentedClient) CreateComment(repo models.Repo, pullNum int, comment string, command string) error { + scope := c.StatsScope.SubScope("create_comment") + logger := c.Logger.WithHistory(fmtLogSrc(repo, pullNum)...) + + executionTime := scope.Timer(metrics.ExecutionTimeMetric).Start() + defer executionTime.Stop() + + executionSuccess := scope.Counter(metrics.ExecutionSuccessMetric) + executionError := scope.Counter(metrics.ExecutionErrorMetric) + + if err := c.Client.CreateComment(repo, pullNum, comment, command); err != nil { + executionError.Inc(1) + logger.Err("Unable to create comment for command %s, error: %s", command, err.Error()) + return err + } + + executionSuccess.Inc(1) + return nil +} +func (c *InstrumentedClient) HidePrevCommandComments(repo models.Repo, pullNum int, command string) error { + scope := c.StatsScope.SubScope("hide_prev_plan_comments") + logger := c.Logger.WithHistory(fmtLogSrc(repo, pullNum)...) + + executionTime := scope.Timer(metrics.ExecutionTimeMetric).Start() + defer executionTime.Stop() + + executionSuccess := scope.Counter(metrics.ExecutionSuccessMetric) + executionError := scope.Counter(metrics.ExecutionErrorMetric) + + if err := c.Client.HidePrevCommandComments(repo, pullNum, command); err != nil { + executionError.Inc(1) + logger.Err("Unable to hide previous %s comments, error: %s", command, err.Error()) + return err + } + + executionSuccess.Inc(1) + return nil + +} +func (c *InstrumentedClient) PullIsApproved(repo models.Repo, pull models.PullRequest) (models.ApprovalStatus, error) { + scope := c.StatsScope.SubScope("pull_is_approved") + logger := c.Logger.WithHistory(fmtLogSrc(repo, pull.Num)...) + + executionTime := scope.Timer(metrics.ExecutionTimeMetric).Start() + defer executionTime.Stop() + + executionSuccess := scope.Counter(metrics.ExecutionSuccessMetric) + executionError := scope.Counter(metrics.ExecutionErrorMetric) + + approved, err := c.Client.PullIsApproved(repo, pull) + + if err != nil { + executionError.Inc(1) + logger.Err("Unable to check pull approval status, error: %s", err.Error()) + } else { + executionSuccess.Inc(1) + } + + return approved, err + +} +func (c *InstrumentedClient) PullIsMergeable(repo models.Repo, pull models.PullRequest) (bool, error) { + scope := c.StatsScope.SubScope("pull_is_mergeable") + logger := c.Logger.WithHistory(fmtLogSrc(repo, pull.Num)...) + + executionTime := scope.Timer(metrics.ExecutionTimeMetric).Start() + defer executionTime.Stop() + + executionSuccess := scope.Counter(metrics.ExecutionSuccessMetric) + executionError := scope.Counter(metrics.ExecutionErrorMetric) + + mergeable, err := c.Client.PullIsMergeable(repo, pull) + + if err != nil { + executionError.Inc(1) + logger.Err("Unable to check pull mergeable status, error: %s", err.Error()) + } else { + executionSuccess.Inc(1) + } + + return mergeable, err +} + +func (c *InstrumentedClient) UpdateStatus(repo models.Repo, pull models.PullRequest, state models.CommitStatus, src string, description string, url string) error { + scope := c.StatsScope.SubScope("update_status") + logger := c.Logger.WithHistory(fmtLogSrc(repo, pull.Num)...) + + executionTime := scope.Timer(metrics.ExecutionTimeMetric).Start() + defer executionTime.Stop() + + executionSuccess := scope.Counter(metrics.ExecutionSuccessMetric) + executionError := scope.Counter(metrics.ExecutionErrorMetric) + + if err := c.Client.UpdateStatus(repo, pull, state, src, description, url); err != nil { + executionError.Inc(1) + logger.Err("Unable to update status at url: %s, error: %s", url, err.Error()) + return err + } + + executionSuccess.Inc(1) + return nil + +} +func (c *InstrumentedClient) MergePull(pull models.PullRequest, pullOptions models.PullRequestOptions) error { + scope := c.StatsScope.SubScope("merge_pull") + logger := c.Logger.WithHistory("pull-num", pull.Num) + + executionTime := scope.Timer(metrics.ExecutionTimeMetric).Start() + defer executionTime.Stop() + + executionSuccess := scope.Counter(metrics.ExecutionSuccessMetric) + executionError := scope.Counter(metrics.ExecutionErrorMetric) + + if err := c.Client.MergePull(pull, pullOptions); err != nil { + executionError.Inc(1) + logger.Err("Unable to merge pull, error: %s", err.Error()) + } + + executionSuccess.Inc(1) + return nil + +} + +// taken from other parts of the code, would be great to have this in a shared spot +func fmtLogSrc(repo models.Repo, pullNum int) []interface{} { + return []interface{}{ + "repository", repo.FullName, + "pull-num", strconv.Itoa(pullNum), + } +} diff --git a/server/metrics/common.go b/server/metrics/common.go new file mode 100644 index 000000000..e32d59a4c --- /dev/null +++ b/server/metrics/common.go @@ -0,0 +1,8 @@ +package metrics + +const ( + ExecutionTimeMetric = "execution_time" + ExecutionSuccessMetric = "execution_success" + ExecutionErrorMetric = "execution_error" + ExecutionFailureMetric = "execution_failure" +) diff --git a/server/metrics/debug.go b/server/metrics/debug.go new file mode 100644 index 000000000..10560755e --- /dev/null +++ b/server/metrics/debug.go @@ -0,0 +1,93 @@ +package metrics + +import ( + "time" + + "github.com/runatlantis/atlantis/server/logging" + "github.com/uber-go/tally" +) + +// newLoggingReporter returns a tally reporter that logs to the provided logger at debug level. This is useful for +// local development where the usual sinks are not available. +func newLoggingReporter(logger logging.SimpleLogging) tally.StatsReporter { + return &debugReporter{log: logger} +} + +type debugReporter struct { + log logging.SimpleLogging +} + +// Capabilities interface. + +func (r *debugReporter) Reporting() bool { + return true +} + +func (r *debugReporter) Tagging() bool { + return true +} + +func (r *debugReporter) Capabilities() tally.Capabilities { + return r +} + +// Reporter interface. + +func (r *debugReporter) Flush() { + // Silence. +} + +func (r *debugReporter) ReportCounter(name string, tags map[string]string, value int64) { + log := r.log.With("name", name, "value", value, "tags", tags, "type", "counter") + log.Debug("counter") +} + +func (r *debugReporter) ReportGauge(name string, tags map[string]string, value float64) { + log := r.log.With("name", name, "value", value, "tags", tags, "type", "gauge") + log.Debug("gauge") +} + +func (r *debugReporter) ReportTimer(name string, tags map[string]string, interval time.Duration) { + log := r.log.With("name", name, "value", interval, "tags", tags, "type", "timer") + log.Debug("timer") +} + +func (r *debugReporter) ReportHistogramValueSamples( + name string, + tags map[string]string, + buckets tally.Buckets, + bucketLowerBound, + bucketUpperBound float64, + samples int64, +) { + log := r.log.With( + "name", name, + "buckets", buckets.AsValues(), + "bucketLowerBound", bucketLowerBound, + "bucketUpperBound", bucketUpperBound, + "samples", samples, + "tags", tags, + "type", "valueHistogram", + ) + log.Debug("histogram") +} + +func (r *debugReporter) ReportHistogramDurationSamples( + name string, + tags map[string]string, + buckets tally.Buckets, + bucketLowerBound, + bucketUpperBound time.Duration, + samples int64, +) { + log := r.log.With( + "name", name, + "buckets", buckets.AsValues(), + "bucketLowerBound", bucketLowerBound, + "bucketUpperBound", bucketUpperBound, + "samples", samples, + "tags", tags, + "type", "durationHistogram", + ) + log.Debug("histogram") +} diff --git a/server/metrics/scope.go b/server/metrics/scope.go new file mode 100644 index 000000000..023ca6417 --- /dev/null +++ b/server/metrics/scope.go @@ -0,0 +1,63 @@ +package metrics + +import ( + "io" + "strings" + "time" + + "github.com/cactus/go-statsd-client/statsd" + "github.com/pkg/errors" + "github.com/runatlantis/atlantis/server/core/config/valid" + "github.com/runatlantis/atlantis/server/logging" + "github.com/uber-go/tally" + tallystatsd "github.com/uber-go/tally/statsd" +) + +func NewLoggingScope(logger logging.SimpleLogging, statsNamespace string) (tally.Scope, io.Closer, error) { + reporter, err := newReporter(valid.Metrics{}, logger) + + if err != nil { + return nil, nil, errors.Wrap(err, "initializing stats reporter") + } + + scope, closer := tally.NewRootScope(tally.ScopeOptions{ + Prefix: statsNamespace, + Reporter: reporter, + }, time.Second) + + return scope, closer, nil +} + +func NewScope(cfg valid.Metrics, logger logging.SimpleLogging, statsNamespace string) (tally.Scope, io.Closer, error) { + reporter, err := newReporter(cfg, logger) + + if err != nil { + return nil, nil, errors.Wrap(err, "initializing stats reporter") + } + + scope, closer := tally.NewRootScope(tally.ScopeOptions{ + Prefix: statsNamespace, + Reporter: reporter, + }, time.Second) + + return scope, closer, nil +} + +func newReporter(cfg valid.Metrics, logger logging.SimpleLogging) (tally.StatsReporter, error) { + if cfg.Statsd == nil { + // return logging reporter and proceed + return newLoggingReporter(logger), nil + } + + statsdCfg := cfg.Statsd + + client, err := statsd.NewClientWithConfig(&statsd.ClientConfig{ + Address: strings.Join([]string{statsdCfg.Host, statsdCfg.Port}, ":"), + }) + + if err != nil { + return nil, errors.Wrap(err, "initializing statsd client") + } + + return tallystatsd.NewReporter(client, tallystatsd.Options{}), nil +} diff --git a/server/scheduled/executor_service.go b/server/scheduled/executor_service.go new file mode 100644 index 000000000..e4dab1c3e --- /dev/null +++ b/server/scheduled/executor_service.go @@ -0,0 +1,100 @@ +package scheduled + +import ( + "context" + "github.com/runatlantis/atlantis/server/logging" + "github.com/uber-go/tally" + "os" + "os/signal" + "sync" + "syscall" + "time" +) + +type ExecutorService struct { + log logging.SimpleLogging + + // jobs + runtimeStatsPublisher JobDefinition +} + +func NewExecutorService( + statsScope tally.Scope, + log logging.SimpleLogging, +) *ExecutorService { + + scheduledScope := statsScope.SubScope("scheduled") + runtimeStatsPublisher := NewRuntimeStats(scheduledScope) + + runtimeStatsPublisherJob := JobDefinition{ + Job: runtimeStatsPublisher, + Period: 10 * time.Second, + } + + return &ExecutorService{ + log: log, + runtimeStatsPublisher: runtimeStatsPublisherJob, + } +} + +type JobDefinition struct { + Job Job + Period time.Duration +} + +func (s *ExecutorService) Run() { + s.log.Info("Scheduled Executor Service started") + + ctx, cancel := context.WithCancel(context.Background()) + + var wg sync.WaitGroup + + s.runScheduledJob(ctx, &wg, s.runtimeStatsPublisher) + + interrupt := make(chan os.Signal, 1) + + // Stop on SIGINTs and SIGTERMs. + signal.Notify(interrupt, os.Interrupt, syscall.SIGTERM) + + <-interrupt + + s.log.Warn("Received interrupt. Attempting to Shut down scheduled executor service") + + cancel() + wg.Wait() + + s.log.Warn("All jobs completed, exiting.") +} + +func (s *ExecutorService) runScheduledJob(ctx context.Context, wg *sync.WaitGroup, jd JobDefinition) { + ticker := time.NewTicker(jd.Period) + wg.Add(1) + + go func() { + defer wg.Done() + defer ticker.Stop() + + // Ensure we recover from any panics to keep the jobs isolated. + // Keep the recovery outside the select to ensure that we don't infinitely panic. + defer func() { + if r := recover(); r != nil { + s.log.Err("Recovered from panic: %v", r) + } + }() + + for { + select { + case <-ctx.Done(): + s.log.Warn("Received interrupt, cancelling job") + return + case <-ticker.C: + jd.Job.Run() + } + } + }() + +} + +type Job interface { + Run() +} diff --git a/server/scheduled/runtime_stats.go b/server/scheduled/runtime_stats.go new file mode 100644 index 000000000..78836267e --- /dev/null +++ b/server/scheduled/runtime_stats.go @@ -0,0 +1,128 @@ +package scheduled + +import ( + "runtime" + + "github.com/uber-go/tally" +) + +type RuntimeStatCollector struct { + runtimeMetrics runtimeMetrics +} + +type runtimeMetrics struct { + cpuGoroutines tally.Gauge + cpuCgoCalls tally.Gauge + + memoryAlloc tally.Gauge + memoryTotal tally.Gauge + memorySys tally.Gauge + memoryLookups tally.Gauge + memoryMalloc tally.Gauge + memoryFrees tally.Gauge + + memoryHeapAlloc tally.Gauge + memoryHeapSys tally.Gauge + memoryHeapIdle tally.Gauge + memoryHeapInuse tally.Gauge + memoryHeapReleased tally.Gauge + memoryHeapObjects tally.Gauge + + memoryStackInuse tally.Gauge + memoryStackSys tally.Gauge + memoryStackMSpanInuse tally.Gauge + memoryStackMSpanSys tally.Gauge + memoryStackMCacheInuse tally.Gauge + memoryStackMCacheSys tally.Gauge + + memoryOtherSys tally.Gauge + + memoryGCSys tally.Gauge + memoryGCNext tally.Gauge + memoryGCLast tally.Gauge + memoryGCPauseTotal tally.Gauge + memoryGCCount tally.Gauge +} + +func NewRuntimeStats(scope tally.Scope) *RuntimeStatCollector { + runtimeScope := scope.SubScope("runtime") + runtimeMetrics := runtimeMetrics{ + // cpu + cpuGoroutines: runtimeScope.Gauge("cpu.goroutines"), + cpuCgoCalls: runtimeScope.Gauge("cpu.cgo_calls"), + // memory + memoryAlloc: runtimeScope.Gauge("memory.alloc"), + memoryTotal: runtimeScope.Gauge("memory.total"), + memorySys: runtimeScope.Gauge("memory.sys"), + memoryLookups: runtimeScope.Gauge("memory.lookups"), + memoryMalloc: runtimeScope.Gauge("memory.malloc"), + memoryFrees: runtimeScope.Gauge("memory.frees"), + // heap + memoryHeapAlloc: runtimeScope.Gauge("memory.heap.alloc"), + memoryHeapSys: runtimeScope.Gauge("memory.heap.sys"), + memoryHeapIdle: runtimeScope.Gauge("memory.heap.idle"), + memoryHeapInuse: runtimeScope.Gauge("memory.heap.inuse"), + memoryHeapReleased: runtimeScope.Gauge("memory.heap.released"), + memoryHeapObjects: runtimeScope.Gauge("memory.heap.objects"), + // stack + memoryStackInuse: runtimeScope.Gauge("memory.stack.inuse"), + memoryStackSys: runtimeScope.Gauge("memory.stack.sys"), + memoryStackMSpanInuse: runtimeScope.Gauge("memory.stack.mspan_inuse"), + memoryStackMSpanSys: runtimeScope.Gauge("memory.stack.sys"), + memoryStackMCacheInuse: runtimeScope.Gauge("memory.stack.mcache_inuse"), + memoryStackMCacheSys: runtimeScope.Gauge("memory.stack.mcache_sys"), + memoryOtherSys: runtimeScope.Gauge("memory.othersys"), + // GC + memoryGCSys: runtimeScope.Gauge("memory.gc.sys"), + memoryGCNext: runtimeScope.Gauge("memory.gc.next"), + memoryGCLast: runtimeScope.Gauge("memory.gc.last"), + memoryGCPauseTotal: runtimeScope.Gauge("memory.gc.pause_total"), + memoryGCCount: runtimeScope.Gauge("memory.gc.count"), + } + + return &RuntimeStatCollector{ + runtimeMetrics: runtimeMetrics, + } +} + +func (r *RuntimeStatCollector) Run() { + // cpu stats + r.runtimeMetrics.cpuGoroutines.Update(float64(runtime.NumGoroutine())) + r.runtimeMetrics.cpuCgoCalls.Update(float64(runtime.NumCgoCall())) + + var memStats runtime.MemStats + runtime.ReadMemStats(&memStats) + + // general + r.runtimeMetrics.memoryAlloc.Update(float64(memStats.Alloc)) + r.runtimeMetrics.memoryTotal.Update(float64(memStats.TotalAlloc)) + r.runtimeMetrics.memorySys.Update(float64(memStats.Sys)) + r.runtimeMetrics.memoryLookups.Update(float64(memStats.Lookups)) + r.runtimeMetrics.memoryMalloc.Update(float64(memStats.Mallocs)) + r.runtimeMetrics.memoryFrees.Update(float64(memStats.Frees)) + + // heap + r.runtimeMetrics.memoryHeapAlloc.Update(float64(memStats.HeapAlloc)) + r.runtimeMetrics.memoryHeapSys.Update(float64(memStats.HeapSys)) + r.runtimeMetrics.memoryHeapIdle.Update(float64(memStats.HeapIdle)) + r.runtimeMetrics.memoryHeapInuse.Update(float64(memStats.HeapInuse)) + r.runtimeMetrics.memoryHeapReleased.Update(float64(memStats.HeapReleased)) + r.runtimeMetrics.memoryHeapObjects.Update(float64(memStats.HeapObjects)) + + // stack + r.runtimeMetrics.memoryStackInuse.Update(float64(memStats.StackInuse)) + r.runtimeMetrics.memoryStackSys.Update(float64(memStats.StackSys)) + r.runtimeMetrics.memoryStackMSpanInuse.Update(float64(memStats.MSpanInuse)) + r.runtimeMetrics.memoryStackMSpanSys.Update(float64(memStats.MSpanSys)) + r.runtimeMetrics.memoryStackMCacheInuse.Update(float64(memStats.MCacheInuse)) + r.runtimeMetrics.memoryStackMCacheSys.Update(float64(memStats.MCacheSys)) + r.runtimeMetrics.memoryOtherSys.Update(float64(memStats.OtherSys)) + + // GC + r.runtimeMetrics.memoryGCSys.Update(float64(memStats.GCSys)) + r.runtimeMetrics.memoryGCNext.Update(float64(memStats.NextGC)) + r.runtimeMetrics.memoryGCLast.Update(float64(memStats.LastGC)) + r.runtimeMetrics.memoryGCPauseTotal.Update(float64(memStats.PauseTotalNs)) + r.runtimeMetrics.memoryGCCount.Update(float64(memStats.NumGC)) + +} diff --git a/server/server.go b/server/server.go index acccf559c..5045a9530 100644 --- a/server/server.go +++ b/server/server.go @@ -19,6 +19,7 @@ import ( "context" "flag" "fmt" + "io" "log" "net/http" "net/url" @@ -31,9 +32,13 @@ import ( "time" "github.com/mitchellh/go-homedir" + cfg "github.com/runatlantis/atlantis/server/core/config" "github.com/runatlantis/atlantis/server/core/config/valid" "github.com/runatlantis/atlantis/server/core/db" "github.com/runatlantis/atlantis/server/jobs" + "github.com/runatlantis/atlantis/server/metrics" + "github.com/runatlantis/atlantis/server/scheduled" + "github.com/uber-go/tally" assetfs "github.com/elazarl/go-bindata-assetfs" "github.com/gorilla/mux" @@ -42,7 +47,6 @@ import ( events_controllers "github.com/runatlantis/atlantis/server/controllers/events" "github.com/runatlantis/atlantis/server/controllers/templates" "github.com/runatlantis/atlantis/server/controllers/websocket" - cfgParser "github.com/runatlantis/atlantis/server/core/config" "github.com/runatlantis/atlantis/server/core/locking" "github.com/runatlantis/atlantis/server/core/runtime" "github.com/runatlantis/atlantis/server/core/runtime/policy" @@ -88,6 +92,8 @@ type Server struct { PreWorkflowHooksCommandRunner *events.DefaultPreWorkflowHooksCommandRunner CommandRunner *events.DefaultCommandRunner Logger logging.SimpleLogging + StatsScope tally.Scope + StatsCloser io.Closer Locker locking.Locker ApplyLocker locking.ApplyLocker VCSEventsController *events_controllers.VCSEventsController @@ -106,6 +112,7 @@ type Server struct { WebUsername string WebPassword string ProjectCmdOutputHandler jobs.ProjectCommandOutputHandler + ScheduledExecutorService *scheduled.ExecutorService } // Config holds config for server that isn't passed in by the user. @@ -144,7 +151,7 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) { } var supportedVCSHosts []models.VCSHostType - var githubClient *vcs.GithubClient + var githubClient vcs.IGithubClient var githubAppEnabled bool var githubCredentials vcs.GithubCredentials var gitlabClient *vcs.GitlabClient @@ -158,6 +165,34 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) { policyChecksEnabled = true } + validator := &cfg.ParserValidator{} + + globalCfg := valid.NewGlobalCfgFromArgs( + valid.GlobalCfgArgs{ + AllowRepoCfg: userConfig.AllowRepoConfig, + MergeableReq: userConfig.RequireMergeable, + ApprovedReq: userConfig.RequireApproval, + UnDivergedReq: userConfig.RequireUnDiverged, + PolicyCheckEnabled: userConfig.EnablePolicyChecksFlag, + }) + if userConfig.RepoConfig != "" { + globalCfg, err = validator.ParseGlobalCfg(userConfig.RepoConfig, globalCfg) + if err != nil { + return nil, errors.Wrapf(err, "parsing %s file", userConfig.RepoConfig) + } + } else if userConfig.RepoConfigJSON != "" { + globalCfg, err = validator.ParseGlobalCfgJSON(userConfig.RepoConfigJSON, globalCfg) + if err != nil { + return nil, errors.Wrapf(err, "parsing --%s", config.RepoConfigJSONFlag) + } + } + + statsScope, closer, err := metrics.NewScope(globalCfg.Metrics, logger, userConfig.StatsNamespace) + + if err != nil { + return nil, errors.Wrapf(err, "instantiating metrics scope") + } + if userConfig.GithubUser != "" || userConfig.GithubAppID != 0 { supportedVCSHosts = append(supportedVCSHosts, models.Github) if userConfig.GithubUser != "" { @@ -188,10 +223,12 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) { } var err error - githubClient, err = vcs.NewGithubClient(userConfig.GithubHostname, githubCredentials, logger) + rawGithubClient, err := vcs.NewGithubClient(userConfig.GithubHostname, githubCredentials, logger) if err != nil { return nil, err } + + githubClient = vcs.NewInstrumentedGithubClient(rawGithubClient, statsScope, logger) } if userConfig.GitlabUser != "" { supportedVCSHosts = append(supportedVCSHosts, models.Gitlab) @@ -392,37 +429,19 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) { DB: boltdb, } - validator := &cfgParser.ParserValidator{} - - globalCfg := valid.NewGlobalCfgFromArgs( - valid.GlobalCfgArgs{ - AllowRepoCfg: userConfig.AllowRepoConfig, - MergeableReq: userConfig.RequireMergeable, - ApprovedReq: userConfig.RequireApproval, - UnDivergedReq: userConfig.RequireUnDiverged, - PolicyCheckEnabled: userConfig.EnablePolicyChecksFlag, - }) - if userConfig.RepoConfig != "" { - globalCfg, err = validator.ParseGlobalCfg(userConfig.RepoConfig, globalCfg) - if err != nil { - return nil, errors.Wrapf(err, "parsing %s file", userConfig.RepoConfig) - } - } else if userConfig.RepoConfigJSON != "" { - globalCfg, err = validator.ParseGlobalCfgJSON(userConfig.RepoConfigJSON, globalCfg) - if err != nil { - return nil, errors.Wrapf(err, "parsing --%s", config.RepoConfigJSONFlag) - } - } - - pullClosedExecutor := &events.PullClosedExecutor{ - VCSClient: vcsClient, - Locker: lockingClient, - WorkingDir: workingDir, - Logger: logger, - DB: boltdb, - LogStreamResourceCleaner: projectCmdOutputHandler, - PullClosedTemplate: &events.PullClosedEventTemplate{}, - } + pullClosedExecutor := events.NewInstrumentedPullClosedExecutor( + statsScope, + logger, + &events.PullClosedExecutor{ + Locker: lockingClient, + WorkingDir: workingDir, + Logger: logger, + DB: boltdb, + PullClosedTemplate: &events.PullClosedEventTemplate{}, + LogStreamResourceCleaner: projectCmdOutputHandler, + VCSClient: vcsClient, + }, + ) eventParser := &events.EventParser{ GithubUser: userConfig.GithubUser, GithubToken: userConfig.GithubToken, @@ -468,7 +487,7 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) { WorkingDir: workingDir, PostWorkflowHookRunner: runtime.DefaultPostWorkflowHookRunner{}, } - projectCommandBuilder := events.NewProjectCommandBuilder( + projectCommandBuilder := events.NewInstrumentedProjectCommandBuilder( policyChecksEnabled, validator, &events.DefaultProjectFinder{}, @@ -481,6 +500,8 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) { userConfig.SkipCloneNoChanges, userConfig.EnableRegExpCmd, userConfig.AutoplanFileList, + statsScope, + logger, ) showStepRunner, err := runtime.NewShowStepRunner(terraformClient, defaultTfVersion) @@ -557,12 +578,15 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) { ProjectCommandRunner: projectCommandRunner, JobURLSetter: jobs.NewJobURLSetter(router, commitStatusUpdater), } + instrumentedProjectCmdRunner := &events.InstrumentedProjectCommandRunner{ + ProjectCommandRunner: projectOutputWrapper, + } policyCheckCommandRunner := events.NewPolicyCheckCommandRunner( dbUpdater, pullUpdater, commitStatusUpdater, - projectOutputWrapper, + instrumentedProjectCmdRunner, userConfig.ParallelPoolSize, userConfig.SilenceVCSStatusNoProjects, ) @@ -575,7 +599,7 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) { workingDir, commitStatusUpdater, projectCommandBuilder, - projectOutputWrapper, + instrumentedProjectCmdRunner, dbUpdater, pullUpdater, policyCheckCommandRunner, @@ -592,7 +616,7 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) { applyLockingClient, commitStatusUpdater, projectCommandBuilder, - projectOutputWrapper, + instrumentedProjectCmdRunner, autoMerger, pullUpdater, dbUpdater, @@ -606,7 +630,7 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) { approvePoliciesCommandRunner := events.NewApprovePoliciesCommandRunner( commitStatusUpdater, projectCommandBuilder, - projectOutputWrapper, + instrumentedProjectCmdRunner, pullUpdater, dbUpdater, userConfig.SilenceNoProjects, @@ -649,6 +673,7 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) { EventParser: eventParser, Logger: logger, GlobalCfg: globalCfg, + StatsScope: statsScope.SubScope("cmd"), AllowForkPRs: userConfig.AllowForkPRs, AllowForkPRsFlag: config.AllowForkPRsFlag, SilenceForkPRErrors: userConfig.SilenceForkPRErrors, @@ -693,6 +718,7 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) { Db: boltdb, WsMux: wsMux, KeyGenerator: controllers.JobIDKeyGenerator{}, + StatsScope: statsScope.SubScope("api"), } eventsController := &events_controllers.VCSEventsController{ @@ -701,6 +727,7 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) { Parser: eventParser, CommentParser: commentParser, Logger: logger, + Scope: statsScope, ApplyDisabled: userConfig.DisableApply, GithubWebhookSecret: []byte(userConfig.GithubWebhookSecret), GithubRequestValidator: &events_controllers.DefaultGithubRequestValidator{}, @@ -722,6 +749,11 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) { GithubHostname: userConfig.GithubHostname, GithubOrg: userConfig.GithubOrg, } + scheduledExecutorService := scheduled.NewExecutorService( + statsScope, + logger, + ) + return &Server{ AtlantisVersion: config.AtlantisVersion, AtlantisURL: parsedURL, @@ -731,6 +763,8 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) { PreWorkflowHooksCommandRunner: preWorkflowHooksCommandRunner, CommandRunner: commandRunner, Logger: logger, + StatsScope: statsScope, + StatsCloser: closer, Locker: lockingClient, ApplyLocker: applyLockingClient, VCSEventsController: eventsController, @@ -749,6 +783,7 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) { WebAuthentication: userConfig.WebBasicAuth, WebUsername: userConfig.WebUsername, WebPassword: userConfig.WebPassword, + ScheduledExecutorService: scheduledExecutorService, }, nil } @@ -786,6 +821,8 @@ func (s *Server) Start() error { // Stop on SIGINTs and SIGTERMs. signal.Notify(stop, os.Interrupt, syscall.SIGTERM) + go s.ScheduledExecutorService.Run() + go func() { s.ProjectCmdOutputHandler.Handle() }() @@ -809,6 +846,12 @@ func (s *Server) Start() error { s.Logger.Warn("Received interrupt. Waiting for in-progress operations to complete") s.waitForDrain() + + // flush stats before shutdown + if err := s.StatsCloser.Close(); err != nil { + s.Logger.Err(err.Error()) + } + ctx, _ := context.WithTimeout(context.Background(), 5*time.Second) // nolint: vet if err := server.Shutdown(ctx); err != nil { return cli.NewExitError(fmt.Sprintf("while shutting down: %s", err), 1) diff --git a/server/user_config.go b/server/user_config.go index d0d1811aa..cab3d8d9f 100644 --- a/server/user_config.go +++ b/server/user_config.go @@ -49,6 +49,7 @@ type UserConfig struct { HidePrevPlanComments bool `mapstructure:"hide-prev-plan-comments"` LogLevel string `mapstructure:"log-level"` ParallelPoolSize int `mapstructure:"parallel-pool-size"` + StatsNamespace string `mapstructure:"stats-namespace"` PlanDrafts bool `mapstructure:"allow-draft-prs"` Port int `mapstructure:"port"` RepoConfig string `mapstructure:"repo-config"`