public/atlantis
1
0
Fork 0
mirror of https://git.vectorsigma.ru/public/atlantis.git synced 2026-07-29 00:38:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: Prometheus metrics support (#2204)

Browse Source 
* support prometheus metrics

* remove redundant error checking

* load prometheus metrics config

* dynamic separator for subscopes

* removing unnecessary config file

* adding a few more metrics test cases

* remove testing handler

* fix upstream merge error
This commit is contained in:
Steve Yoon
2022-06-28 11:58:15 -04:00
committed by GitHub
parent dfd2a23017
commit e90aa575e0
8 changed files with 153 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
go.mod
View File

@@ -134,6 +134,13 @@ require (
)
require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/m3db/prometheus_client_golang v0.8.1 // indirect
github.com/m3db/prometheus_client_model v0.1.0 // indirect
github.com/m3db/prometheus_common v0.1.0 // indirect
github.com/m3db/prometheus_procfs v0.8.1 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
github.com/onsi/ginkgo v1.14.0 // indirect
cloud.google.com/go/compute v1.6.1 // indirect
cloud.google.com/go/iam v0.3.0 // indirect
github.com/onsi/ginkgo v1.14.0 // indirect

11
go.sum
View File

@@ -100,6 +100,7 @@ github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLj
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1UJrqV3uuy861HCTo708pDMbjHHdCas=
github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4=
@@ -372,6 +373,15 @@ github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
github.com/lusis/slack-test v0.0.0-20190426140909-c40012f20018 h1:MNApn+Z+fIT4NPZopPfCc1obT6aY3SVM6DOctz1A9ZU=
github.com/lusis/slack-test v0.0.0-20190426140909-c40012f20018/go.mod h1:sFlOUpQL1YcjhFVXhg1CG8ZASEs/Mf1oVb6H75JL/zg=
github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w=
github.com/m3db/prometheus_client_golang v0.8.1 h1:t7w/tcFws81JL1j5sqmpqcOyQOpH4RDOmIe3A3fdN3w=
github.com/m3db/prometheus_client_golang v0.8.1/go.mod h1:8R/f1xYhXWq59KD/mbRqoBulXejss7vYtYzWmruNUwI=
github.com/m3db/prometheus_client_model v0.1.0 h1:cg1+DiuyT6x8h9voibtarkH1KT6CmsewBSaBhe8wzLo=
github.com/m3db/prometheus_client_model v0.1.0/go.mod h1:Qfsxn+LypxzF+lNhak7cF7k0zxK7uB/ynGYoj80zcD4=
github.com/m3db/prometheus_common v0.1.0 h1:YJu6eCIV6MQlcwND24cRG/aRkZDX1jvYbsNNs1ZYr0w=
github.com/m3db/prometheus_common v0.1.0/go.mod h1:EBmDQaMAy4B8i+qsg1wMXAelLNVbp49i/JOeVszQ/rs=
github.com/m3db/prometheus_procfs v0.8.1 h1:LsxWzVELhDU9sLsZTaFLCeAwCn7bC7qecZcK4zobs/g=
github.com/m3db/prometheus_procfs v0.8.1/go.mod h1:N8lv8fLh3U3koZx1Bnisj60GYUMDpWb09x1R+dmMOJo=
github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo=
github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
@@ -389,6 +399,7 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky
github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/mcdafydd/go-azuredevops v0.12.1 h1:WxwLVyGuJ8oL7uWQp1/J6GefX1wMQQZUHWRGsrm+uE8=
github.com/mcdafydd/go-azuredevops v0.12.1/go.mod h1:B4UDyn7WEj1/97f45j3VnzEfkWKe05+/dCcAPdOET4A=

25
server/core/config/raw/metrics.go
View File

@@ -7,7 +7,16 @@ import (
)
type Metrics struct {
Statsd *Statsd `yaml:"statsd" json:"statsd"`
Statsd *Statsd `yaml:"statsd" json:"statsd"`
Prometheus *Prometheus `yaml:"prometheus" json:"prometheus"`
}
type Prometheus struct {
Endpoint string `yaml:"endpoint" json:"endpoint"`
}
func (p *Prometheus) Validate() error {
return validation.ValidateStruct(p, validation.Field(&p.Endpoint, validation.Required))
}
type Statsd struct {
@@ -24,9 +33,11 @@ func (s *Statsd) Validate() error {
}
func (m Metrics) Validate() error {
return validation.ValidateStruct(&m,
validation.Field(&m.Statsd),
res := validation.ValidateStruct(&m,
validation.Field(&m.Statsd, validation.NilOrNotEmpty),
validation.Field(&m.Prometheus, validation.NilOrNotEmpty),
)
return res
}
func (m Metrics) ToValid() valid.Metrics {
@@ -39,6 +50,12 @@ func (m Metrics) ToValid() valid.Metrics {
},
}
}
if m.Prometheus != nil {
return valid.Metrics{
Prometheus: &valid.Prometheus{
Endpoint: m.Prometheus.Endpoint,
},
}
}
return valid.Metrics{}
}

35
server/core/config/raw/metrics_test.go
View File

@@ -16,6 +16,8 @@ func TestMetrics_Unmarshal(t *testing.T) {
statsd:
host: 127.0.0.1
port: 8125
prometheus:
endpoint: /metrics
`
var result raw.Metrics
@@ -30,6 +32,9 @@ statsd:
"statsd": {
"host": "127.0.0.1",
"port": "8125"
},
"prometheus": {
"endpoint": "/metrics"
}
}
`
@@ -47,7 +52,7 @@ func TestMetrics_Validate_Success(t *testing.T) {
subject raw.Metrics
}{
{
description: "success",
description: "success with stats config",
subject: raw.Metrics{
Statsd: &raw.Statsd{
Host: "127.0.0.1",
@@ -58,6 +63,26 @@ func TestMetrics_Validate_Success(t *testing.T) {
{
description: "missing stats",
},
{
description: "success with prometheus config",
subject: raw.Metrics{
Prometheus: &raw.Prometheus{
Endpoint: "/metrics",
},
},
},
{
description: "success with both configs",
subject: raw.Metrics{
Statsd: &raw.Statsd{
Host: "127.0.0.1",
Port: "8125",
},
Prometheus: &raw.Prometheus{
Endpoint: "/metrics",
},
},
},
}
for _, c := range cases {
@@ -106,6 +131,14 @@ func TestMetrics_Validate_Error(t *testing.T) {
},
},
},
{
description: "invalid endpoint",
subject: raw.Metrics{
Prometheus: &raw.Prometheus{
Endpoint: "",
},
},
},
}
for _, c := range cases {

7
server/core/config/valid/global_cfg.go
View File

@@ -39,7 +39,8 @@ type GlobalCfg struct {
}
type Metrics struct {
Statsd *Statsd
Statsd *Statsd
Prometheus *Prometheus
}
type Statsd struct {
@@ -47,6 +48,10 @@ type Statsd struct {
Host string
}
type Prometheus struct {
Endpoint string
}
// Repo is the final parsed version of server-side repo config.
type Repo struct {
// ID is the exact match id of this config.

57
server/metrics/scope.go
View File

@@ -10,45 +10,56 @@ import (
"github.com/runatlantis/atlantis/server/core/config/valid"
"github.com/runatlantis/atlantis/server/logging"
"github.com/uber-go/tally"
tallyprom "github.com/uber-go/tally/prometheus"
tallystatsd "github.com/uber-go/tally/statsd"
)
func NewLoggingScope(logger logging.SimpleLogging, statsNamespace string) (tally.Scope, io.Closer, error) {
reporter, err := newReporter(valid.Metrics{}, logger)
if err != nil {
return nil, nil, errors.Wrap(err, "initializing stats reporter")
}
scope, closer := tally.NewRootScope(tally.ScopeOptions{
Prefix: statsNamespace,
Reporter: reporter,
}, time.Second)
return scope, closer, nil
scope, _, closer, err := NewScope(valid.Metrics{}, logger, statsNamespace)
return scope, closer, err
}
func NewScope(cfg valid.Metrics, logger logging.SimpleLogging, statsNamespace string) (tally.Scope, io.Closer, error) {
func NewScope(cfg valid.Metrics, logger logging.SimpleLogging, statsNamespace string) (tally.Scope, tally.BaseStatsReporter, io.Closer, error) {
reporter, err := newReporter(cfg, logger)
if err != nil {
return nil, nil, errors.Wrap(err, "initializing stats reporter")
return nil, nil, nil, errors.Wrap(err, "initializing stats reporter")
}
scope, closer := tally.NewRootScope(tally.ScopeOptions{
Prefix: statsNamespace,
Reporter: reporter,
}, time.Second)
scopeOpts := tally.ScopeOptions{
Prefix: statsNamespace,
}
return scope, closer, nil
if r, ok := reporter.(tally.StatsReporter); ok {
scopeOpts.Reporter = r
} else if r, ok := reporter.(tally.CachedStatsReporter); ok {
scopeOpts.CachedReporter = r
scopeOpts.Separator = tallyprom.DefaultSeparator
}
scope, closer := tally.NewRootScope(scopeOpts, time.Second)
return scope, reporter, closer, nil
}
func newReporter(cfg valid.Metrics, logger logging.SimpleLogging) (tally.StatsReporter, error) {
if cfg.Statsd == nil {
// return logging reporter and proceed
return newLoggingReporter(logger), nil
func newReporter(cfg valid.Metrics, logger logging.SimpleLogging) (tally.BaseStatsReporter, error) {
// return statsd metrics if configured
if cfg.Statsd != nil {
return newStatsReporter(cfg, logger)
}
// return prometheus metrics if configured
if cfg.Prometheus != nil {
return tallyprom.NewReporter(tallyprom.Options{}), nil
}
// return logging reporter and proceed
return newLoggingReporter(logger), nil
}
func newStatsReporter(cfg valid.Metrics, logger logging.SimpleLogging) (tally.StatsReporter, error) {
statsdCfg := cfg.Statsd
client, err := statsd.NewClientWithConfig(&statsd.ClientConfig{

57
server/scheduled/runtime_stats.go
View File

@@ -46,38 +46,43 @@ type runtimeMetrics struct {
func NewRuntimeStats(scope tally.Scope) *RuntimeStatCollector {
runtimeScope := scope.SubScope("runtime")
cpuScope := runtimeScope.SubScope("cpu")
memoryScope := runtimeScope.SubScope("memory")
heapScope := memoryScope.SubScope("heap")
stackScope := memoryScope.SubScope("stack")
gcScope := memoryScope.SubScope("gc")
runtimeMetrics := runtimeMetrics{
// cpu
cpuGoroutines: runtimeScope.Gauge("cpu.goroutines"),
cpuCgoCalls: runtimeScope.Gauge("cpu.cgo_calls"),
cpuGoroutines: cpuScope.Gauge("goroutines"),
cpuCgoCalls: cpuScope.Gauge("cgo_calls"),
// memory
memoryAlloc: runtimeScope.Gauge("memory.alloc"),
memoryTotal: runtimeScope.Gauge("memory.total"),
memorySys: runtimeScope.Gauge("memory.sys"),
memoryLookups: runtimeScope.Gauge("memory.lookups"),
memoryMalloc: runtimeScope.Gauge("memory.malloc"),
memoryFrees: runtimeScope.Gauge("memory.frees"),
memoryAlloc: memoryScope.Gauge("alloc"),
memoryTotal: memoryScope.Gauge("total"),
memorySys: memoryScope.Gauge("sys"),
memoryLookups: memoryScope.Gauge("lookups"),
memoryMalloc: memoryScope.Gauge("malloc"),
memoryFrees: memoryScope.Gauge("frees"),
// heap
memoryHeapAlloc: runtimeScope.Gauge("memory.heap.alloc"),
memoryHeapSys: runtimeScope.Gauge("memory.heap.sys"),
memoryHeapIdle: runtimeScope.Gauge("memory.heap.idle"),
memoryHeapInuse: runtimeScope.Gauge("memory.heap.inuse"),
memoryHeapReleased: runtimeScope.Gauge("memory.heap.released"),
memoryHeapObjects: runtimeScope.Gauge("memory.heap.objects"),
memoryHeapAlloc: heapScope.Gauge("alloc"),
memoryHeapSys: heapScope.Gauge("sys"),
memoryHeapIdle: heapScope.Gauge("idle"),
memoryHeapInuse: heapScope.Gauge("inuse"),
memoryHeapReleased: heapScope.Gauge("released"),
memoryHeapObjects: heapScope.Gauge("objects"),
// stack
memoryStackInuse: runtimeScope.Gauge("memory.stack.inuse"),
memoryStackSys: runtimeScope.Gauge("memory.stack.sys"),
memoryStackMSpanInuse: runtimeScope.Gauge("memory.stack.mspan_inuse"),
memoryStackMSpanSys: runtimeScope.Gauge("memory.stack.sys"),
memoryStackMCacheInuse: runtimeScope.Gauge("memory.stack.mcache_inuse"),
memoryStackMCacheSys: runtimeScope.Gauge("memory.stack.mcache_sys"),
memoryOtherSys: runtimeScope.Gauge("memory.othersys"),
memoryStackInuse: stackScope.Gauge("inuse"),
memoryStackSys: stackScope.Gauge("sys"),
memoryStackMSpanInuse: stackScope.Gauge("mspan_inuse"),
memoryStackMSpanSys: stackScope.Gauge("sys"),
memoryStackMCacheInuse: stackScope.Gauge("mcache_inuse"),
memoryStackMCacheSys: stackScope.Gauge("mcache_sys"),
memoryOtherSys: memoryScope.Gauge("othersys"),
// GC
memoryGCSys: runtimeScope.Gauge("memory.gc.sys"),
memoryGCNext: runtimeScope.Gauge("memory.gc.next"),
memoryGCLast: runtimeScope.Gauge("memory.gc.last"),
memoryGCPauseTotal: runtimeScope.Gauge("memory.gc.pause_total"),
memoryGCCount: runtimeScope.Gauge("memory.gc.count"),
memoryGCSys: gcScope.Gauge("sys"),
memoryGCNext: gcScope.Gauge("next"),
memoryGCLast: gcScope.Gauge("last"),
memoryGCPauseTotal: gcScope.Gauge("pause_total"),
memoryGCCount: gcScope.Gauge("count"),
}
return &RuntimeStatCollector{

10
server/server.go
View File

@@ -39,6 +39,7 @@ import (
"github.com/runatlantis/atlantis/server/metrics"
"github.com/runatlantis/atlantis/server/scheduled"
"github.com/uber-go/tally"
"github.com/uber-go/tally/prometheus"
assetfs "github.com/elazarl/go-bindata-assetfs"
"github.com/gorilla/mux"
@@ -94,6 +95,7 @@ type Server struct {
CommandRunner *events.DefaultCommandRunner
Logger logging.SimpleLogging
StatsScope tally.Scope
StatsReporter tally.BaseStatsReporter
StatsCloser io.Closer
Locker locking.Locker
ApplyLocker locking.ApplyLocker
@@ -188,7 +190,7 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) {
}
}
statsScope, closer, err := metrics.NewScope(globalCfg.Metrics, logger, userConfig.StatsNamespace)
statsScope, statsReporter, closer, err := metrics.NewScope(globalCfg.Metrics, logger, userConfig.StatsNamespace)
if err != nil {
return nil, errors.Wrapf(err, "instantiating metrics scope")
@@ -773,6 +775,7 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) {
CommandRunner: commandRunner,
Logger: logger,
StatsScope: statsScope,
StatsReporter: statsReporter,
StatsCloser: closer,
Locker: lockingClient,
ApplyLocker: applyLockingClient,
@@ -815,6 +818,11 @@ func (s *Server) Start() error {
s.Router.HandleFunc("/jobs/{job-id}", s.JobsController.GetProjectJobs).Methods("GET").Name(ProjectJobsViewRouteName)
s.Router.HandleFunc("/jobs/{job-id}/ws", s.JobsController.GetProjectJobsWS).Methods("GET")
r, ok := s.StatsReporter.(prometheus.Reporter)
if ok {
s.Router.Handle(s.CommandRunner.GlobalCfg.Metrics.Prometheus.Endpoint, r.HTTPHandler())
}
n := negroni.New(&negroni.Recovery{
Logger: log.New(os.Stdout, "", log.LstdFlags),
PrintStack: false,