From e2431bdf9d698f13a77342f0bd27ffe60163f4b3 Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Wed, 30 Oct 2019 19:05:40 -0700 Subject: [PATCH 01/12] Add dqlite support --- pkg/cli/cmds/dqlite.go | 7 + pkg/cluster/cluster.go | 4 + pkg/cluster/dqlite.go | 138 +++++++++++++ pkg/cluster/nocluster.go | 4 + pkg/dqlite/controller/client/controller.go | 149 ++++++++++++++ pkg/dqlite/dialer/dial.go | 36 ++++ pkg/dqlite/join.go | 51 +++++ pkg/dqlite/log.go | 21 ++ pkg/dqlite/pipe/http.go | 56 ++++++ pkg/dqlite/pipe/pipe.go | 46 +++++ pkg/dqlite/proxy.go | 53 +++++ pkg/dqlite/reset.go | 40 ++++ pkg/dqlite/router.go | 80 ++++++++ pkg/dqlite/server.go | 213 +++++++++++++++++++++ pkg/server/router.go | 1 + 15 files changed, 899 insertions(+) create mode 100644 pkg/cli/cmds/dqlite.go create mode 100644 pkg/cluster/dqlite.go create mode 100644 pkg/dqlite/controller/client/controller.go create mode 100644 pkg/dqlite/dialer/dial.go create mode 100644 pkg/dqlite/join.go create mode 100644 pkg/dqlite/log.go create mode 100644 pkg/dqlite/pipe/http.go create mode 100644 pkg/dqlite/pipe/pipe.go create mode 100644 pkg/dqlite/proxy.go create mode 100644 pkg/dqlite/reset.go create mode 100644 pkg/dqlite/router.go create mode 100644 pkg/dqlite/server.go diff --git a/pkg/cli/cmds/dqlite.go b/pkg/cli/cmds/dqlite.go new file mode 100644 index 0000000000..b7f77e36cd --- /dev/null +++ b/pkg/cli/cmds/dqlite.go @@ -0,0 +1,7 @@ +// +build dqlite + +package cmds + +const ( + hideDqlite = false +) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index d5372dae0c..73eeac70ab 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -37,6 +37,10 @@ func (c *Cluster) Start(ctx context.Context) error { } } + if err := c.testClusterDB(ctx); err != nil { + return err + } + return c.joined() } diff --git a/pkg/cluster/dqlite.go b/pkg/cluster/dqlite.go new file mode 100644 index 0000000000..ab2d05cb0a --- /dev/null +++ b/pkg/cluster/dqlite.go @@ -0,0 +1,138 @@ +// +build dqlite + +package cluster + +import ( + "context" + "crypto/tls" + "encoding/json" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "github.com/canonical/go-dqlite/client" + "github.com/rancher/dynamiclistener/factory" + "github.com/rancher/k3s/pkg/clientaccess" + "github.com/rancher/k3s/pkg/daemons/config" + "github.com/rancher/k3s/pkg/dqlite" + "github.com/rancher/kine/pkg/endpoint" + v1 "github.com/rancher/wrangler-api/pkg/generated/controllers/core/v1" + "github.com/sirupsen/logrus" +) + +func (c *Cluster) testClusterDB(ctx context.Context) error { + if !c.enabled() { + return nil + } + + dqlite := c.db.(*dqlite.DQLite) + for { + if err := dqlite.Test(ctx); err != nil { + logrus.Infof("Failed to test dqlite connection: %v", err) + } else { + return nil + } + + select { + case <-time.After(2 * time.Second): + case <-ctx.Done(): + return ctx.Err() + } + } +} + +func (c *Cluster) initClusterDB(ctx context.Context, l net.Listener, handler http.Handler) (net.Listener, http.Handler, error) { + if !c.enabled() { + return l, handler, nil + } + + dqlite := dqlite.New(c.config.DataDir, c.config.AdvertiseIP, c.config.AdvertisePort, func() v1.NodeController { + if c.runtime.Core == nil { + return nil + } + return c.runtime.Core.Core().V1().Node() + }) + + certs, err := toGetCerts(c.runtime) + if err != nil { + return nil, nil, err + } + + handler, err = dqlite.Start(ctx, c.config.ClusterInit, certs, handler) + if err != nil { + return nil, nil, err + } + + if c.config.ClusterReset { + if err := dqlite.Reset(ctx); err == nil { + logrus.Info("Cluster reset") + os.Exit(0) + } else { + logrus.Fatal("Cluster reset failed: %v", err) + } + } + + c.db = dqlite + if !strings.HasPrefix(c.config.Storage.Endpoint, "dqlite://") { + c.config.Storage = endpoint.Config{ + Endpoint: dqlite.StorageEndpoint, + } + } + + return l, handler, err +} + +func (c *Cluster) enabled() bool { + stamp := filepath.Join(c.config.DataDir, "db", "state.dqlite") + if _, err := os.Stat(stamp); err == nil { + return true + } + + return c.config.Storage.Endpoint == "" && (c.config.ClusterInit || c.runtime.Cluster.Join) +} + +func (c *Cluster) postJoin(ctx context.Context) error { + if !c.enabled() { + return nil + } + + resp, err := clientaccess.Get("/db/info", c.clientAccessInfo) + if err != nil { + return err + } + + dqlite := c.db.(*dqlite.DQLite) + var nodes []client.NodeInfo + + if err := json.Unmarshal(resp, &nodes); err != nil { + return err + } + + return dqlite.Join(ctx, nodes) +} + +func toGetCerts(runtime *config.ControlRuntime) (*dqlite.Certs, error) { + clientCA, _, err := factory.LoadCerts(runtime.ClientCA, runtime.ClientCAKey) + if err != nil { + return nil, err + } + + ca, _, err := factory.LoadCerts(runtime.ServerCA, runtime.ServerCAKey) + if err != nil { + return nil, err + } + + clientCert, err := tls.LoadX509KeyPair(runtime.ClientKubeAPICert, runtime.ClientKubeAPIKey) + if err != nil { + return nil, err + } + + return &dqlite.Certs{ + ServerTrust: ca, + ClientTrust: clientCA, + ClientCert: clientCert, + }, nil +} diff --git a/pkg/cluster/nocluster.go b/pkg/cluster/nocluster.go index ce5029156a..a7ed7be3d4 100644 --- a/pkg/cluster/nocluster.go +++ b/pkg/cluster/nocluster.go @@ -8,6 +8,10 @@ import ( "net/http" ) +func (c *Cluster) testClusterDB(ctx context.Context) error { + return nil +} + func (c *Cluster) initClusterDB(ctx context.Context, l net.Listener, handler http.Handler) (net.Listener, http.Handler, error) { return l, handler, nil } diff --git a/pkg/dqlite/controller/client/controller.go b/pkg/dqlite/controller/client/controller.go new file mode 100644 index 0000000000..9df5780457 --- /dev/null +++ b/pkg/dqlite/controller/client/controller.go @@ -0,0 +1,149 @@ +package client + +import ( + "context" + "fmt" + "strconv" + + "github.com/canonical/go-dqlite/client" + controllerv1 "github.com/rancher/wrangler-api/pkg/generated/controllers/core/v1" + "github.com/sirupsen/logrus" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" +) + +const ( + allKey = "_all_" + nodeID = "cluster.k3s.cattle.io/node-id" + nodeAddress = "cluster.k3s.cattle.io/node-address" + master = "node-role.kubernetes.io/master" +) + +func Register(ctx context.Context, nodeName string, nodeInfo client.NodeInfo, + nodeStore client.NodeStore, nodes controllerv1.NodeController, opts []client.Option) { + h := &handler{ + nodeStore: nodeStore, + nodeController: nodes, + nodeName: nodeName, + id: strconv.FormatUint(nodeInfo.ID, 10), + address: nodeInfo.Address, + ctx: ctx, + opts: opts, + } + nodes.OnChange(ctx, "dqlite-client", h.sync) + nodes.OnRemove(ctx, "dqlite-client", h.onRemove) +} + +type handler struct { + nodeStore client.NodeStore + nodeController controllerv1.NodeController + nodeName string + id string + address string + ctx context.Context + opts []client.Option +} + +func (h *handler) sync(key string, node *v1.Node) (*v1.Node, error) { + if key == allKey { + return nil, h.updateNodeStore() + } + + if node == nil { + return nil, nil + } + + if key == h.nodeName { + return h.handleSelf(node) + } + + if node.Labels[master] == "true" { + h.nodeController.Enqueue(allKey) + } + + return node, nil +} + +func (h *handler) handleSelf(node *v1.Node) (*v1.Node, error) { + if node.Annotations[nodeID] == h.id && node.Annotations[nodeAddress] == h.address { + return node, nil + } + + node = node.DeepCopy() + if node.Annotations == nil { + node.Annotations = map[string]string{} + } + node.Annotations[nodeID] = h.id + node.Annotations[nodeAddress] = h.address + + return h.nodeController.Update(node) +} + +func (h *handler) onRemove(key string, node *v1.Node) (*v1.Node, error) { + address := node.Annotations[nodeAddress] + if address == "" { + return node, nil + } + return node, h.delete(address) +} + +func (h *handler) delete(address string) error { + c, err := client.FindLeader(h.ctx, h.nodeStore, h.opts...) + if err != nil { + return err + } + defer c.Close() + + members, err := c.Cluster(h.ctx) + if err != nil { + return err + } + + for _, member := range members { + if member.Address == address { + logrus.Infof("Removing %s %d from dqlite", member.Address, member.ID) + return c.Remove(h.ctx, member.ID) + } + } + + return nil +} + +func (h *handler) updateNodeStore() error { + nodes, err := h.nodeController.Cache().List(labels.SelectorFromSet(labels.Set{ + master: "true", + })) + if err != nil { + return err + } + + var nodeInfos []client.NodeInfo + for _, node := range nodes { + address, ok := node.Annotations[nodeAddress] + if !ok { + continue + } + + nodeIDStr, ok := node.Annotations[nodeID] + if !ok { + continue + } + + id, err := strconv.ParseUint(nodeIDStr, 10, 64) + if err != nil { + logrus.Errorf("invalid %s=%s, must be a number: %v", nodeID, nodeIDStr, err) + continue + } + + nodeInfos = append(nodeInfos, client.NodeInfo{ + ID: id, + Address: address, + }) + } + + if len(nodeInfos) == 0 { + return fmt.Errorf("not setting dqlient NodeStore len to 0") + } + + return h.nodeStore.Set(h.ctx, nodeInfos) +} diff --git a/pkg/dqlite/dialer/dial.go b/pkg/dqlite/dialer/dial.go new file mode 100644 index 0000000000..87b801a7dc --- /dev/null +++ b/pkg/dqlite/dialer/dial.go @@ -0,0 +1,36 @@ +package dialer + +import ( + "context" + "crypto/tls" + "fmt" + "net" + + "github.com/canonical/go-dqlite/client" + "github.com/rancher/k3s/pkg/dqlite/pipe" +) + +func NewHTTPDialer(advertiseAddress, bindAddress string, tls *tls.Config) (client.DialFunc, error) { + d := &dialer{ + advertiseAddress: advertiseAddress, + bindAddress: bindAddress, + tls: tls, + } + + return d.dial, nil +} + +type dialer struct { + advertiseAddress string + bindAddress string + tls *tls.Config +} + +func (d *dialer) dial(ctx context.Context, address string) (net.Conn, error) { + if address == d.advertiseAddress { + return net.Dial("unix", d.bindAddress) + } + + url := fmt.Sprintf("https://%s/db/connect", address) + return pipe.ToHTTP(ctx, url, d.tls) +} diff --git a/pkg/dqlite/join.go b/pkg/dqlite/join.go new file mode 100644 index 0000000000..27cfd76ecf --- /dev/null +++ b/pkg/dqlite/join.go @@ -0,0 +1,51 @@ +package dqlite + +import ( + "context" + + "github.com/canonical/go-dqlite/client" + "github.com/sirupsen/logrus" +) + +func (d *DQLite) Test(ctx context.Context) error { + var ips []string + peers, err := d.NodeStore.Get(ctx) + if err != nil { + return err + } + + for _, peer := range peers { + ips = append(ips, peer.Address) + } + + logrus.Infof("Testing connection to peers %v", ips) + return d.Join(ctx, nil) +} + +func (d *DQLite) Join(ctx context.Context, nodes []client.NodeInfo) error { + if len(nodes) > 0 { + if err := d.NodeStore.Set(ctx, nodes); err != nil { + return err + } + } + + client, err := client.FindLeader(ctx, d.NodeStore, d.clientOpts...) + if err != nil { + return err + } + defer client.Close() + + current, err := client.Cluster(ctx) + if err != nil { + return err + } + + for _, testNode := range current { + if testNode.Address == d.NodeInfo.Address { + return nil + } + } + + logrus.Infof("Joining dqlite cluster as address=%s, id=%d") + return client.Add(ctx, d.NodeInfo) +} diff --git a/pkg/dqlite/log.go b/pkg/dqlite/log.go new file mode 100644 index 0000000000..e7185b7797 --- /dev/null +++ b/pkg/dqlite/log.go @@ -0,0 +1,21 @@ +package dqlite + +import ( + "github.com/canonical/go-dqlite/client" + "github.com/sirupsen/logrus" +) + +func log() client.LogFunc { + return func(level client.LogLevel, s string, i ...interface{}) { + switch level { + case client.LogDebug: + logrus.Debugf(s, i...) + case client.LogError: + logrus.Errorf(s, i...) + case client.LogInfo: + logrus.Infof(s, i...) + case client.LogWarn: + logrus.Warnf(s, i...) + } + } +} diff --git a/pkg/dqlite/pipe/http.go b/pkg/dqlite/pipe/http.go new file mode 100644 index 0000000000..036d4c69a2 --- /dev/null +++ b/pkg/dqlite/pipe/http.go @@ -0,0 +1,56 @@ +package pipe + +import ( + "bufio" + "context" + "crypto/tls" + "fmt" + "net" + "net/http" + + "github.com/pkg/errors" +) + +func ToHTTP(ctx context.Context, url string, tlsConfig *tls.Config) (net.Conn, error) { + request, err := http.NewRequest(http.MethodPost, url, nil) + if err != nil { + return nil, err + } + + request = request.WithContext(ctx) + netDial := &net.Dialer{} + + if deadline, ok := ctx.Deadline(); ok { + netDial.Deadline = deadline + } + + conn, err := tls.DialWithDialer(netDial, "tcp", request.URL.Host, tlsConfig) + if err != nil { + return nil, errors.Wrap(err, "tls dial") + } + + err = request.Write(conn) + if err != nil { + return nil, errors.Wrap(err, "request write") + } + + response, err := http.ReadResponse(bufio.NewReader(conn), request) + if err != nil { + return nil, errors.Wrap(err, "read request") + } + if response.StatusCode != http.StatusSwitchingProtocols { + return nil, fmt.Errorf("expected 101 response, got: %d", response.StatusCode) + } + + listener, err := net.Listen("unix", "") + if err != nil { + return nil, errors.Wrap(err, "Failed to create unix listener") + } + defer listener.Close() + + if err := Unix(conn, listener.Addr().String()); err != nil { + return nil, err + } + + return listener.Accept() +} diff --git a/pkg/dqlite/pipe/pipe.go b/pkg/dqlite/pipe/pipe.go new file mode 100644 index 0000000000..cf8d6113c0 --- /dev/null +++ b/pkg/dqlite/pipe/pipe.go @@ -0,0 +1,46 @@ +package pipe + +import ( + "io" + "net" + + "github.com/lxc/lxd/shared/eagain" + "github.com/sirupsen/logrus" +) + +func UnixPiper(srcs <-chan net.Conn, bindAddress string) { + for src := range srcs { + go Unix(src, bindAddress) + } +} + +func Unix(src net.Conn, target string) error { + dst, err := net.Dial("unix", target) + if err != nil { + src.Close() + return err + } + + Connect(src, dst) + return nil +} + +func Connect(src net.Conn, dst net.Conn) { + go func() { + _, err := io.Copy(eagain.Writer{Writer: dst}, eagain.Reader{Reader: src}) + if err != nil && err != io.EOF { + logrus.Warnf("copy pipe src->dst closed: %v", err) + } + src.Close() + dst.Close() + }() + + go func() { + _, err := io.Copy(eagain.Writer{Writer: src}, eagain.Reader{Reader: dst}) + if err != nil { + logrus.Warnf("copy pipe dst->src closed: %v", err) + } + src.Close() + dst.Close() + }() +} diff --git a/pkg/dqlite/proxy.go b/pkg/dqlite/proxy.go new file mode 100644 index 0000000000..0387ad023e --- /dev/null +++ b/pkg/dqlite/proxy.go @@ -0,0 +1,53 @@ +package dqlite + +import ( + "context" + "net" + "net/http" + + "github.com/pkg/errors" + "github.com/rancher/k3s/pkg/dqlite/pipe" +) + +var ( + upgradeResponse = []byte("HTTP/1.1 101 Switching Protocols\r\nUpgrade: dqlite\r\n\r\n") +) + +type proxy struct { + conns chan net.Conn +} + +func newProxy(ctx context.Context, bindAddress string) http.Handler { + p := &proxy{ + conns: make(chan net.Conn, 100), + } + go func() { + <-ctx.Done() + close(p.conns) + }() + go pipe.UnixPiper(p.conns, bindAddress) + + return p +} + +func (h *proxy) ServeHTTP(rw http.ResponseWriter, r *http.Request) { + hijacker, ok := rw.(http.Hijacker) + if !ok { + http.Error(rw, "failed to hijack", http.StatusInternalServerError) + return + } + + conn, _, err := hijacker.Hijack() + if err != nil { + err := errors.Wrap(err, "Hijack connection") + http.Error(rw, err.Error(), http.StatusInternalServerError) + return + } + + if n, err := conn.Write(upgradeResponse); err != nil || n != len(upgradeResponse) { + conn.Close() + return + } + + h.conns <- conn +} diff --git a/pkg/dqlite/reset.go b/pkg/dqlite/reset.go new file mode 100644 index 0000000000..95b263e55d --- /dev/null +++ b/pkg/dqlite/reset.go @@ -0,0 +1,40 @@ +package dqlite + +import ( + "context" + "fmt" + + "github.com/canonical/go-dqlite/client" + "github.com/sirupsen/logrus" +) + +func (d *DQLite) Reset(ctx context.Context) error { + dqClient, err := client.New(ctx, d.getBindAddress(), client.WithLogFunc(log())) + if err != nil { + return err + } + + current, err := dqClient.Cluster(ctx) + if err != nil { + return err + } + + // There's a chance our ID and the ID the server has doesn't match so find the ID + var surviving []client.NodeInfo + for _, testNode := range current { + if testNode.Address == d.NodeInfo.Address && testNode.ID == d.NodeInfo.ID { + surviving = append(surviving, testNode) + continue + } + if err := dqClient.Remove(ctx, testNode.ID); err != nil { + return err + } + } + + if len(surviving) != 1 { + return fmt.Errorf("failed to find %s in the current node, can not reset", d.NodeInfo.Address) + } + + logrus.Infof("Resetting cluster to single master, please rejoin members") + return d.node.Recover(surviving) +} diff --git a/pkg/dqlite/router.go b/pkg/dqlite/router.go new file mode 100644 index 0000000000..a88ce07e4a --- /dev/null +++ b/pkg/dqlite/router.go @@ -0,0 +1,80 @@ +package dqlite + +import ( + "context" + "crypto/x509" + "encoding/json" + "net/http" + + "github.com/canonical/go-dqlite" + "github.com/canonical/go-dqlite/client" + "github.com/gorilla/mux" +) + +func router(ctx context.Context, next http.Handler, nodeInfo dqlite.NodeInfo, clientCA *x509.Certificate, clientCN string, bindAddress string) http.Handler { + mux := mux.NewRouter() + mux.Handle("/db/connect", newChecker(newProxy(ctx, bindAddress), clientCA, clientCN)) + mux.Handle("/db/info", infoHandler(ctx, nodeInfo, bindAddress)) + mux.NotFoundHandler = next + return mux +} + +func infoHandler(ctx context.Context, nodeInfo dqlite.NodeInfo, bindAddress string) http.Handler { + return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + client, err := client.New(ctx, bindAddress, client.WithLogFunc(log())) + if err != nil { + http.Error(rw, err.Error(), http.StatusInternalServerError) + return + } + defer client.Close() + + info, err := client.Cluster(ctx) + if err != nil { + http.Error(rw, err.Error(), http.StatusInternalServerError) + return + } + + rw.Header().Set("Content-Type", "application/json") + json.NewEncoder(rw).Encode(info) + }) +} + +type checker struct { + next http.Handler + verify x509.VerifyOptions + cn string +} + +func newChecker(next http.Handler, ca *x509.Certificate, cn string) http.Handler { + pool := x509.NewCertPool() + pool.AddCert(ca) + return &checker{ + next: next, + verify: x509.VerifyOptions{ + Roots: pool, + KeyUsages: []x509.ExtKeyUsage{ + x509.ExtKeyUsageClientAuth, + }, + DNSName: cn, + }, + cn: cn, + } +} + +func (c *checker) ServeHTTP(rw http.ResponseWriter, req *http.Request) { + if !c.check(req) { + http.Error(rw, "unauthorized", http.StatusUnauthorized) + return + } + c.next.ServeHTTP(rw, req) +} + +func (c *checker) check(r *http.Request) bool { + for _, cert := range r.TLS.PeerCertificates { + _, err := cert.Verify(c.verify) + if err == nil { + return cert.Subject.CommonName == c.cn + } + } + return false +} diff --git a/pkg/dqlite/server.go b/pkg/dqlite/server.go new file mode 100644 index 0000000000..b5e6b04a4a --- /dev/null +++ b/pkg/dqlite/server.go @@ -0,0 +1,213 @@ +package dqlite + +import ( + "context" + "crypto/tls" + "crypto/x509" + "fmt" + "io/ioutil" + "math/rand" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/canonical/go-dqlite" + "github.com/canonical/go-dqlite/client" + "github.com/pkg/errors" + controllerclient "github.com/rancher/k3s/pkg/dqlite/controller/client" + "github.com/rancher/k3s/pkg/dqlite/dialer" + dqlitedriver "github.com/rancher/kine/pkg/drivers/dqlite" + v1 "github.com/rancher/wrangler-api/pkg/generated/controllers/core/v1" + "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/util/net" +) + +type Certs struct { + ServerTrust *x509.Certificate + ClientTrust *x509.Certificate + ClientCert tls.Certificate +} + +type DQLite struct { + ClientCA string + ClientCAKey string + ClientCert string + ClientCertKey string + ServerCA string + ServerCAKey string + AdvertiseIP string + AdvertisePort int + DataDir string + NodeStore client.NodeStore + NodeInfo client.NodeInfo + node *dqlite.Node + StorageEndpoint string + NodeControllerGetter NodeControllerGetter + clientOpts []client.Option +} + +type NodeControllerGetter func() v1.NodeController + +func New(dataDir, advertiseIP string, advertisePort int, getter NodeControllerGetter) *DQLite { + return &DQLite{ + AdvertiseIP: advertiseIP, + AdvertisePort: advertisePort, + DataDir: dataDir, + NodeControllerGetter: getter, + } +} + +func (d *DQLite) Start(ctx context.Context, initCluster bool, certs *Certs, next http.Handler) (http.Handler, error) { + bindAddress := d.getBindAddress() + + clientTLSConfig, err := getClientTLSConfig(certs.ClientCert, certs.ServerTrust) + if err != nil { + return nil, err + } + + advertise, err := getAdvertiseAddress(d.AdvertiseIP, d.AdvertisePort) + if err != nil { + return nil, errors.Wrap(err, "get advertise address") + } + + dial, err := getDialer(advertise, bindAddress, clientTLSConfig) + if err != nil { + return nil, err + } + + dqlitedriver.Dialer = dial + dqlitedriver.Logger = log() + + d.clientOpts = append(d.clientOpts, client.WithDialFunc(dial), client.WithLogFunc(log())) + + nodeInfo, node, err := getNode(d.DataDir, advertise, bindAddress, initCluster, dial) + if err != nil { + return nil, err + } + + d.NodeInfo = nodeInfo + + go func() { + <-ctx.Done() + node.Close() + }() + + if err := d.nodeStore(ctx, initCluster); err != nil { + return nil, err + } + + go d.startController(ctx) + + return router(ctx, next, nodeInfo, certs.ClientTrust, "kube-apiserver", bindAddress), node.Start() +} + +func (d *DQLite) startController(ctx context.Context) { + for { + if nc := d.NodeControllerGetter(); nc != nil { + if os.Getenv("NODE_NAME") == "" { + logrus.Errorf("--disable-agent is not compatible with dqlite") + } else { + break + } + } + time.Sleep(time.Second) + } + + controllerclient.Register(ctx, os.Getenv("NODE_NAME"), d.NodeInfo, d.NodeStore, d.NodeControllerGetter(), d.clientOpts) +} + +func (d *DQLite) nodeStore(ctx context.Context, initCluster bool) error { + peerDB := filepath.Join(d.DataDir, "db", "state.dqlite", "peers.db") + ns, err := client.DefaultNodeStore(peerDB) + if err != nil { + return err + } + d.NodeStore = ns + d.StorageEndpoint = fmt.Sprintf("dqlite://?peer-file=%s", peerDB) + if initCluster { + if err := dqlitedriver.AddPeers(ctx, d.NodeStore, d.NodeInfo); err != nil { + return err + } + } + return nil +} + +func getAdvertiseAddress(advertiseIP string, advertisePort int) (string, error) { + ip := advertiseIP + if ip == "" { + ipAddr, err := net.ChooseHostInterface() + if err != nil { + return "", err + } + ip = ipAddr.String() + } + + return fmt.Sprintf("%s:%d", ip, advertisePort), nil +} + +func getClientTLSConfig(cert tls.Certificate, ca *x509.Certificate) (*tls.Config, error) { + tlsConfig := &tls.Config{ + RootCAs: x509.NewCertPool(), + Certificates: []tls.Certificate{ + cert, + }, + ServerName: "kubernetes", + } + tlsConfig.RootCAs.AddCert(ca) + + return tlsConfig, nil +} + +func getDialer(advertiseAddress, bindAddress string, tlsConfig *tls.Config) (client.DialFunc, error) { + return dialer.NewHTTPDialer(advertiseAddress, bindAddress, tlsConfig) +} + +func getNode(dataDir string, advertiseAddress, bindAddress string, initCluster bool, dial client.DialFunc) (dqlite.NodeInfo, *dqlite.Node, error) { + id, err := getClusterID(initCluster, dataDir) + if err != nil { + return dqlite.NodeInfo{}, nil, errors.Wrap(err, "reading cluster id") + } + + dbDir := filepath.Join(dataDir, "db", "state.dqlite") + + node, err := dqlite.New(id, advertiseAddress, dbDir, + dqlite.WithBindAddress(bindAddress), + dqlite.WithDialFunc(dial), + dqlite.WithNetworkLatency(20*time.Millisecond)) + return dqlite.NodeInfo{ + ID: id, + Address: advertiseAddress, + }, node, err +} + +func getClusterID(initCluster bool, dataDir string) (uint64, error) { + idFile := filepath.Join(dataDir, "db/state.dqlite/node-id") + content, err := ioutil.ReadFile(idFile) + if os.IsNotExist(err) { + content = nil + } else if err != nil { + return 0, err + } + + idStr := strings.TrimSpace(string(content)) + if idStr == "" { + if err := os.MkdirAll(filepath.Dir(idFile), 0700); err != nil { + return 0, err + } + id := rand.Uint64() + if initCluster { + id = 1 + } + return id, ioutil.WriteFile(idFile, []byte(strconv.FormatUint(id, 10)), 0644) + } + + return strconv.ParseUint(idStr, 10, 64) +} + +func (d *DQLite) getBindAddress() string { + // only anonymous works??? + return "@" + filepath.Join(d.DataDir, "db", "state.dqlite", "dqlite.sock") +} diff --git a/pkg/server/router.go b/pkg/server/router.go index 83ed3e06f0..9909d13288 100644 --- a/pkg/server/router.go +++ b/pkg/server/router.go @@ -49,6 +49,7 @@ func router(serverConfig *config.Control, tunnel http.Handler, ca []byte) http.H serverAuthed := mux.NewRouter() serverAuthed.Use(authMiddleware(serverConfig, "k3s:server")) serverAuthed.NotFoundHandler = nodeAuthed + serverAuthed.Path("/db/info").Handler(nodeAuthed) serverAuthed.Path("/v1-k3s/server-bootstrap").Handler(bootstrap.Handler(&serverConfig.Runtime.ControlRuntimeBootstrap)) staticDir := filepath.Join(serverConfig.DataDir, "static") From b3336f69cf3b2129cbb90537df7e5400d9a7bae3 Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Fri, 8 Nov 2019 21:45:10 +0000 Subject: [PATCH 02/12] Update kine and dynamiclistener --- go.mod | 14 +- go.sum | 57 +- vendor/github.com/Rican7/retry/.travis.yml | 28 + vendor/github.com/Rican7/retry/LICENSE | 19 + vendor/github.com/Rican7/retry/Makefile | 72 ++ vendor/github.com/Rican7/retry/README.md | 98 ++ .../Rican7/retry/backoff/backoff.go | 67 + .../github.com/Rican7/retry/jitter/jitter.go | 89 ++ vendor/github.com/Rican7/retry/retry.go | 36 + .../Rican7/retry/strategy/strategy.go | 85 ++ .../canonical/go-dqlite/.dir-locals.el | 8 + .../github.com/canonical/go-dqlite/.gitignore | 4 + .../canonical/go-dqlite/.travis.yml | 31 + vendor/github.com/canonical/go-dqlite/AUTHORS | 1 + vendor/github.com/canonical/go-dqlite/LICENSE | 201 +++ .../github.com/canonical/go-dqlite/README.md | 91 ++ .../canonical/go-dqlite/client/client.go | 221 ++++ .../canonical/go-dqlite/client/leader.go | 35 + .../canonical/go-dqlite/client/log.go | 30 + .../canonical/go-dqlite/client/store.go | 136 ++ .../canonical/go-dqlite/driver/driver.go | 670 ++++++++++ .../go-dqlite/internal/bindings/build.go | 6 + .../go-dqlite/internal/bindings/errors.go | 19 + .../go-dqlite/internal/bindings/server.go | 252 ++++ .../go-dqlite/internal/logging/func.go | 26 + .../go-dqlite/internal/logging/level.go | 27 + .../go-dqlite/internal/protocol/buffer.go | 11 + .../go-dqlite/internal/protocol/config.go | 14 + .../go-dqlite/internal/protocol/connector.go | 250 ++++ .../go-dqlite/internal/protocol/constants.go | 58 + .../go-dqlite/internal/protocol/dial.go | 20 + .../go-dqlite/internal/protocol/errors.go | 29 + .../go-dqlite/internal/protocol/message.go | 660 ++++++++++ .../go-dqlite/internal/protocol/protocol.go | 340 +++++ .../go-dqlite/internal/protocol/request.go | 131 ++ .../go-dqlite/internal/protocol/response.go | 254 ++++ .../go-dqlite/internal/protocol/schema.go | 33 + .../go-dqlite/internal/protocol/schema.sh | 144 +++ .../go-dqlite/internal/protocol/store.go | 49 + vendor/github.com/canonical/go-dqlite/node.go | 126 ++ vendor/github.com/coreos/pkg/capnslog/init.go | 2 +- .../github.com/flosch/pongo2/.gitattributes | 1 + vendor/github.com/flosch/pongo2/.gitignore | 42 + vendor/github.com/flosch/pongo2/.travis.yml | 8 + vendor/github.com/flosch/pongo2/AUTHORS | 11 + vendor/github.com/flosch/pongo2/LICENSE | 20 + vendor/github.com/flosch/pongo2/README.md | 273 ++++ vendor/github.com/flosch/pongo2/context.go | 136 ++ vendor/github.com/flosch/pongo2/doc.go | 31 + vendor/github.com/flosch/pongo2/error.go | 91 ++ vendor/github.com/flosch/pongo2/filters.go | 143 +++ .../flosch/pongo2/filters_builtin.go | 927 ++++++++++++++ vendor/github.com/flosch/pongo2/go.mod | 13 + vendor/github.com/flosch/pongo2/helpers.go | 15 + vendor/github.com/flosch/pongo2/lexer.go | 432 +++++++ vendor/github.com/flosch/pongo2/nodes.go | 16 + vendor/github.com/flosch/pongo2/nodes_html.go | 23 + .../github.com/flosch/pongo2/nodes_wrapper.go | 16 + vendor/github.com/flosch/pongo2/options.go | 26 + vendor/github.com/flosch/pongo2/parser.go | 309 +++++ .../flosch/pongo2/parser_document.go | 59 + .../flosch/pongo2/parser_expression.go | 503 ++++++++ vendor/github.com/flosch/pongo2/pongo2.go | 14 + vendor/github.com/flosch/pongo2/tags.go | 135 ++ .../flosch/pongo2/tags_autoescape.go | 52 + vendor/github.com/flosch/pongo2/tags_block.go | 129 ++ .../github.com/flosch/pongo2/tags_comment.go | 27 + vendor/github.com/flosch/pongo2/tags_cycle.go | 106 ++ .../github.com/flosch/pongo2/tags_extends.go | 52 + .../github.com/flosch/pongo2/tags_filter.go | 95 ++ .../github.com/flosch/pongo2/tags_firstof.go | 49 + vendor/github.com/flosch/pongo2/tags_for.go | 159 +++ vendor/github.com/flosch/pongo2/tags_if.go | 76 ++ .../flosch/pongo2/tags_ifchanged.go | 116 ++ .../github.com/flosch/pongo2/tags_ifequal.go | 78 ++ .../flosch/pongo2/tags_ifnotequal.go | 78 ++ .../github.com/flosch/pongo2/tags_import.go | 84 ++ .../github.com/flosch/pongo2/tags_include.go | 146 +++ vendor/github.com/flosch/pongo2/tags_lorem.go | 133 ++ vendor/github.com/flosch/pongo2/tags_macro.go | 149 +++ vendor/github.com/flosch/pongo2/tags_now.go | 50 + vendor/github.com/flosch/pongo2/tags_set.go | 50 + .../flosch/pongo2/tags_spaceless.go | 54 + vendor/github.com/flosch/pongo2/tags_ssi.go | 68 + .../flosch/pongo2/tags_templatetag.go | 45 + .../flosch/pongo2/tags_widthratio.go | 83 ++ vendor/github.com/flosch/pongo2/tags_with.go | 88 ++ vendor/github.com/flosch/pongo2/template.go | 277 ++++ .../flosch/pongo2/template_loader.go | 157 +++ .../github.com/flosch/pongo2/template_sets.go | 305 +++++ vendor/github.com/flosch/pongo2/value.go | 520 ++++++++ vendor/github.com/flosch/pongo2/variable.go | 695 ++++++++++ .../github.com/gorilla/websocket/.travis.yml | 19 - vendor/github.com/gorilla/websocket/README.md | 10 +- vendor/github.com/gorilla/websocket/client.go | 4 +- vendor/github.com/gorilla/websocket/conn.go | 112 +- vendor/github.com/gorilla/websocket/doc.go | 47 + vendor/github.com/gorilla/websocket/go.mod | 3 + vendor/github.com/gorilla/websocket/go.sum | 2 + vendor/github.com/gorilla/websocket/join.go | 42 + vendor/github.com/gorilla/websocket/proxy.go | 8 +- vendor/github.com/gorilla/websocket/server.go | 4 +- vendor/github.com/gorilla/websocket/util.go | 132 +- vendor/github.com/lxc/lxd/AUTHORS | 5 + vendor/github.com/lxc/lxd/COPYING | 202 +++ .../lxc/lxd/shared/api/certificate.go | 30 + .../github.com/lxc/lxd/shared/api/cluster.go | 63 + .../lxc/lxd/shared/api/container.go | 141 +++ .../lxc/lxd/shared/api/container_backup.go | 29 + .../lxc/lxd/shared/api/container_console.go | 17 + .../lxc/lxd/shared/api/container_exec.go | 26 + .../lxc/lxd/shared/api/container_snapshot.go | 53 + .../lxc/lxd/shared/api/container_state.go | 70 ++ vendor/github.com/lxc/lxd/shared/api/doc.go | 13 + vendor/github.com/lxc/lxd/shared/api/event.go | 32 + vendor/github.com/lxc/lxd/shared/api/image.go | 132 ++ .../github.com/lxc/lxd/shared/api/instance.go | 137 ++ .../lxc/lxd/shared/api/instance_backup.go | 36 + .../lxc/lxd/shared/api/instance_console.go | 17 + .../lxc/lxd/shared/api/instance_exec.go | 26 + .../lxc/lxd/shared/api/instance_snapshot.go | 60 + .../lxc/lxd/shared/api/instance_state.go | 84 ++ .../github.com/lxc/lxd/shared/api/network.go | 89 ++ .../lxc/lxd/shared/api/operation.go | 23 + .../github.com/lxc/lxd/shared/api/profile.go | 35 + .../github.com/lxc/lxd/shared/api/project.go | 42 + .../github.com/lxc/lxd/shared/api/resource.go | 293 +++++ .../github.com/lxc/lxd/shared/api/response.go | 90 ++ .../github.com/lxc/lxd/shared/api/server.go | 65 + .../lxc/lxd/shared/api/status_code.go | 53 + .../lxc/lxd/shared/api/storage_pool.go | 42 + .../lxc/lxd/shared/api/storage_pool_volume.go | 92 ++ .../api/storage_pool_volume_snapshot.go | 31 + .../lxc/lxd/shared/archive_linux.go | 147 +++ .../lxc/lxd/shared/cancel/canceler.go | 73 ++ vendor/github.com/lxc/lxd/shared/cert.go | 531 ++++++++ vendor/github.com/lxc/lxd/shared/cgo.go | 12 + vendor/github.com/lxc/lxd/shared/container.go | 425 +++++++ .../lxc/lxd/shared/eagain/file_unix.go | 53 + .../lxc/lxd/shared/ioprogress/data.go | 16 + .../lxc/lxd/shared/ioprogress/reader.go | 25 + .../lxc/lxd/shared/ioprogress/tracker.go | 77 ++ .../lxc/lxd/shared/ioprogress/writer.go | 25 + vendor/github.com/lxc/lxd/shared/json.go | 63 + .../lxc/lxd/shared/logger/format.go | 25 + .../github.com/lxc/lxd/shared/logger/log.go | 101 ++ .../lxc/lxd/shared/logger/log_debug.go | 124 ++ vendor/github.com/lxc/lxd/shared/network.go | 568 +++++++++ .../github.com/lxc/lxd/shared/network_unix.go | 26 + .../lxc/lxd/shared/network_windows.go | 60 + vendor/github.com/lxc/lxd/shared/proxy.go | 162 +++ .../github.com/lxc/lxd/shared/units/units.go | 162 +++ vendor/github.com/lxc/lxd/shared/util.go | 1115 +++++++++++++++++ .../github.com/lxc/lxd/shared/util_linux.go | 378 ++++++ .../lxc/lxd/shared/util_linux_cgo.go | 461 +++++++ .../lxc/lxd/shared/util_linux_notcgo.go | 5 + vendor/github.com/lxc/lxd/shared/util_unix.go | 15 + .../github.com/lxc/lxd/shared/util_windows.go | 11 + .../rancher/dynamiclistener/listener.go | 8 +- .../storage/kubernetes/controller.go | 20 +- .../rancher/kine/pkg/drivers/dqlite/dqlite.go | 228 ++++ .../kine/pkg/drivers/dqlite/no_dqlite.go | 14 + .../kine/pkg/drivers/generic/generic.go | 35 +- .../rancher/kine/pkg/drivers/sqlite/sqlite.go | 15 +- .../rancher/kine/pkg/endpoint/endpoint.go | 8 +- .../kine/pkg/logstructured/sqllog/sql.go | 28 +- vendor/go.uber.org/atomic/.travis.yml | 18 +- vendor/go.uber.org/atomic/Makefile | 33 +- vendor/go.uber.org/atomic/README.md | 4 +- vendor/go.uber.org/multierr/.travis.yml | 2 +- vendor/go.uber.org/multierr/error.go | 2 +- vendor/go.uber.org/zap/.travis.yml | 4 +- vendor/go.uber.org/zap/CHANGELOG.md | 22 + vendor/go.uber.org/zap/Makefile | 4 +- vendor/go.uber.org/zap/global.go | 1 - vendor/go.uber.org/zap/global_go112.go | 26 + vendor/go.uber.org/zap/global_prego112.go | 26 + vendor/go.uber.org/zap/zapcore/field.go | 13 +- .../go.uber.org/zap/zapcore/json_encoder.go | 3 + .../go.uber.org/zap/zapcore/memory_encoder.go | 2 +- .../api/annotations/annotations.pb.go | 21 +- .../googleapis/api/annotations/client.pb.go | 25 +- .../api/annotations/field_behavior.pb.go | 27 +- .../googleapis/api/annotations/http.pb.go | 162 +-- .../googleapis/api/annotations/resource.pb.go | 82 +- .../googleapis/rpc/status/status.pb.go | 26 +- vendor/gopkg.in/robfig/cron.v2/.gitignore | 22 + vendor/gopkg.in/robfig/cron.v2/.travis.yml | 1 + vendor/gopkg.in/robfig/cron.v2/LICENSE | 21 + vendor/gopkg.in/robfig/cron.v2/README.md | 1 + .../gopkg.in/robfig/cron.v2/constantdelay.go | 27 + vendor/gopkg.in/robfig/cron.v2/cron.go | 236 ++++ vendor/gopkg.in/robfig/cron.v2/doc.go | 132 ++ vendor/gopkg.in/robfig/cron.v2/parser.go | 246 ++++ vendor/gopkg.in/robfig/cron.v2/spec.go | 165 +++ vendor/modules.txt | 55 +- 196 files changed, 20007 insertions(+), 438 deletions(-) create mode 100644 vendor/github.com/Rican7/retry/.travis.yml create mode 100644 vendor/github.com/Rican7/retry/LICENSE create mode 100644 vendor/github.com/Rican7/retry/Makefile create mode 100644 vendor/github.com/Rican7/retry/README.md create mode 100644 vendor/github.com/Rican7/retry/backoff/backoff.go create mode 100644 vendor/github.com/Rican7/retry/jitter/jitter.go create mode 100644 vendor/github.com/Rican7/retry/retry.go create mode 100644 vendor/github.com/Rican7/retry/strategy/strategy.go create mode 100644 vendor/github.com/canonical/go-dqlite/.dir-locals.el create mode 100644 vendor/github.com/canonical/go-dqlite/.gitignore create mode 100644 vendor/github.com/canonical/go-dqlite/.travis.yml create mode 100644 vendor/github.com/canonical/go-dqlite/AUTHORS create mode 100644 vendor/github.com/canonical/go-dqlite/LICENSE create mode 100644 vendor/github.com/canonical/go-dqlite/README.md create mode 100644 vendor/github.com/canonical/go-dqlite/client/client.go create mode 100644 vendor/github.com/canonical/go-dqlite/client/leader.go create mode 100644 vendor/github.com/canonical/go-dqlite/client/log.go create mode 100644 vendor/github.com/canonical/go-dqlite/client/store.go create mode 100644 vendor/github.com/canonical/go-dqlite/driver/driver.go create mode 100644 vendor/github.com/canonical/go-dqlite/internal/bindings/build.go create mode 100644 vendor/github.com/canonical/go-dqlite/internal/bindings/errors.go create mode 100644 vendor/github.com/canonical/go-dqlite/internal/bindings/server.go create mode 100644 vendor/github.com/canonical/go-dqlite/internal/logging/func.go create mode 100644 vendor/github.com/canonical/go-dqlite/internal/logging/level.go create mode 100644 vendor/github.com/canonical/go-dqlite/internal/protocol/buffer.go create mode 100644 vendor/github.com/canonical/go-dqlite/internal/protocol/config.go create mode 100644 vendor/github.com/canonical/go-dqlite/internal/protocol/connector.go create mode 100644 vendor/github.com/canonical/go-dqlite/internal/protocol/constants.go create mode 100644 vendor/github.com/canonical/go-dqlite/internal/protocol/dial.go create mode 100644 vendor/github.com/canonical/go-dqlite/internal/protocol/errors.go create mode 100644 vendor/github.com/canonical/go-dqlite/internal/protocol/message.go create mode 100644 vendor/github.com/canonical/go-dqlite/internal/protocol/protocol.go create mode 100644 vendor/github.com/canonical/go-dqlite/internal/protocol/request.go create mode 100644 vendor/github.com/canonical/go-dqlite/internal/protocol/response.go create mode 100644 vendor/github.com/canonical/go-dqlite/internal/protocol/schema.go create mode 100644 vendor/github.com/canonical/go-dqlite/internal/protocol/schema.sh create mode 100644 vendor/github.com/canonical/go-dqlite/internal/protocol/store.go create mode 100644 vendor/github.com/canonical/go-dqlite/node.go create mode 100644 vendor/github.com/flosch/pongo2/.gitattributes create mode 100644 vendor/github.com/flosch/pongo2/.gitignore create mode 100644 vendor/github.com/flosch/pongo2/.travis.yml create mode 100644 vendor/github.com/flosch/pongo2/AUTHORS create mode 100644 vendor/github.com/flosch/pongo2/LICENSE create mode 100644 vendor/github.com/flosch/pongo2/README.md create mode 100644 vendor/github.com/flosch/pongo2/context.go create mode 100644 vendor/github.com/flosch/pongo2/doc.go create mode 100644 vendor/github.com/flosch/pongo2/error.go create mode 100644 vendor/github.com/flosch/pongo2/filters.go create mode 100644 vendor/github.com/flosch/pongo2/filters_builtin.go create mode 100644 vendor/github.com/flosch/pongo2/go.mod create mode 100644 vendor/github.com/flosch/pongo2/helpers.go create mode 100644 vendor/github.com/flosch/pongo2/lexer.go create mode 100644 vendor/github.com/flosch/pongo2/nodes.go create mode 100644 vendor/github.com/flosch/pongo2/nodes_html.go create mode 100644 vendor/github.com/flosch/pongo2/nodes_wrapper.go create mode 100644 vendor/github.com/flosch/pongo2/options.go create mode 100644 vendor/github.com/flosch/pongo2/parser.go create mode 100644 vendor/github.com/flosch/pongo2/parser_document.go create mode 100644 vendor/github.com/flosch/pongo2/parser_expression.go create mode 100644 vendor/github.com/flosch/pongo2/pongo2.go create mode 100644 vendor/github.com/flosch/pongo2/tags.go create mode 100644 vendor/github.com/flosch/pongo2/tags_autoescape.go create mode 100644 vendor/github.com/flosch/pongo2/tags_block.go create mode 100644 vendor/github.com/flosch/pongo2/tags_comment.go create mode 100644 vendor/github.com/flosch/pongo2/tags_cycle.go create mode 100644 vendor/github.com/flosch/pongo2/tags_extends.go create mode 100644 vendor/github.com/flosch/pongo2/tags_filter.go create mode 100644 vendor/github.com/flosch/pongo2/tags_firstof.go create mode 100644 vendor/github.com/flosch/pongo2/tags_for.go create mode 100644 vendor/github.com/flosch/pongo2/tags_if.go create mode 100644 vendor/github.com/flosch/pongo2/tags_ifchanged.go create mode 100644 vendor/github.com/flosch/pongo2/tags_ifequal.go create mode 100644 vendor/github.com/flosch/pongo2/tags_ifnotequal.go create mode 100644 vendor/github.com/flosch/pongo2/tags_import.go create mode 100644 vendor/github.com/flosch/pongo2/tags_include.go create mode 100644 vendor/github.com/flosch/pongo2/tags_lorem.go create mode 100644 vendor/github.com/flosch/pongo2/tags_macro.go create mode 100644 vendor/github.com/flosch/pongo2/tags_now.go create mode 100644 vendor/github.com/flosch/pongo2/tags_set.go create mode 100644 vendor/github.com/flosch/pongo2/tags_spaceless.go create mode 100644 vendor/github.com/flosch/pongo2/tags_ssi.go create mode 100644 vendor/github.com/flosch/pongo2/tags_templatetag.go create mode 100644 vendor/github.com/flosch/pongo2/tags_widthratio.go create mode 100644 vendor/github.com/flosch/pongo2/tags_with.go create mode 100644 vendor/github.com/flosch/pongo2/template.go create mode 100644 vendor/github.com/flosch/pongo2/template_loader.go create mode 100644 vendor/github.com/flosch/pongo2/template_sets.go create mode 100644 vendor/github.com/flosch/pongo2/value.go create mode 100644 vendor/github.com/flosch/pongo2/variable.go delete mode 100644 vendor/github.com/gorilla/websocket/.travis.yml create mode 100644 vendor/github.com/gorilla/websocket/go.mod create mode 100644 vendor/github.com/gorilla/websocket/go.sum create mode 100644 vendor/github.com/gorilla/websocket/join.go create mode 100644 vendor/github.com/lxc/lxd/AUTHORS create mode 100644 vendor/github.com/lxc/lxd/COPYING create mode 100644 vendor/github.com/lxc/lxd/shared/api/certificate.go create mode 100644 vendor/github.com/lxc/lxd/shared/api/cluster.go create mode 100644 vendor/github.com/lxc/lxd/shared/api/container.go create mode 100644 vendor/github.com/lxc/lxd/shared/api/container_backup.go create mode 100644 vendor/github.com/lxc/lxd/shared/api/container_console.go create mode 100644 vendor/github.com/lxc/lxd/shared/api/container_exec.go create mode 100644 vendor/github.com/lxc/lxd/shared/api/container_snapshot.go create mode 100644 vendor/github.com/lxc/lxd/shared/api/container_state.go create mode 100644 vendor/github.com/lxc/lxd/shared/api/doc.go create mode 100644 vendor/github.com/lxc/lxd/shared/api/event.go create mode 100644 vendor/github.com/lxc/lxd/shared/api/image.go create mode 100644 vendor/github.com/lxc/lxd/shared/api/instance.go create mode 100644 vendor/github.com/lxc/lxd/shared/api/instance_backup.go create mode 100644 vendor/github.com/lxc/lxd/shared/api/instance_console.go create mode 100644 vendor/github.com/lxc/lxd/shared/api/instance_exec.go create mode 100644 vendor/github.com/lxc/lxd/shared/api/instance_snapshot.go create mode 100644 vendor/github.com/lxc/lxd/shared/api/instance_state.go create mode 100644 vendor/github.com/lxc/lxd/shared/api/network.go create mode 100644 vendor/github.com/lxc/lxd/shared/api/operation.go create mode 100644 vendor/github.com/lxc/lxd/shared/api/profile.go create mode 100644 vendor/github.com/lxc/lxd/shared/api/project.go create mode 100644 vendor/github.com/lxc/lxd/shared/api/resource.go create mode 100644 vendor/github.com/lxc/lxd/shared/api/response.go create mode 100644 vendor/github.com/lxc/lxd/shared/api/server.go create mode 100644 vendor/github.com/lxc/lxd/shared/api/status_code.go create mode 100644 vendor/github.com/lxc/lxd/shared/api/storage_pool.go create mode 100644 vendor/github.com/lxc/lxd/shared/api/storage_pool_volume.go create mode 100644 vendor/github.com/lxc/lxd/shared/api/storage_pool_volume_snapshot.go create mode 100644 vendor/github.com/lxc/lxd/shared/archive_linux.go create mode 100644 vendor/github.com/lxc/lxd/shared/cancel/canceler.go create mode 100644 vendor/github.com/lxc/lxd/shared/cert.go create mode 100644 vendor/github.com/lxc/lxd/shared/cgo.go create mode 100644 vendor/github.com/lxc/lxd/shared/container.go create mode 100644 vendor/github.com/lxc/lxd/shared/eagain/file_unix.go create mode 100644 vendor/github.com/lxc/lxd/shared/ioprogress/data.go create mode 100644 vendor/github.com/lxc/lxd/shared/ioprogress/reader.go create mode 100644 vendor/github.com/lxc/lxd/shared/ioprogress/tracker.go create mode 100644 vendor/github.com/lxc/lxd/shared/ioprogress/writer.go create mode 100644 vendor/github.com/lxc/lxd/shared/json.go create mode 100644 vendor/github.com/lxc/lxd/shared/logger/format.go create mode 100644 vendor/github.com/lxc/lxd/shared/logger/log.go create mode 100644 vendor/github.com/lxc/lxd/shared/logger/log_debug.go create mode 100644 vendor/github.com/lxc/lxd/shared/network.go create mode 100644 vendor/github.com/lxc/lxd/shared/network_unix.go create mode 100644 vendor/github.com/lxc/lxd/shared/network_windows.go create mode 100644 vendor/github.com/lxc/lxd/shared/proxy.go create mode 100644 vendor/github.com/lxc/lxd/shared/units/units.go create mode 100644 vendor/github.com/lxc/lxd/shared/util.go create mode 100644 vendor/github.com/lxc/lxd/shared/util_linux.go create mode 100644 vendor/github.com/lxc/lxd/shared/util_linux_cgo.go create mode 100644 vendor/github.com/lxc/lxd/shared/util_linux_notcgo.go create mode 100644 vendor/github.com/lxc/lxd/shared/util_unix.go create mode 100644 vendor/github.com/lxc/lxd/shared/util_windows.go create mode 100644 vendor/github.com/rancher/kine/pkg/drivers/dqlite/dqlite.go create mode 100644 vendor/github.com/rancher/kine/pkg/drivers/dqlite/no_dqlite.go create mode 100644 vendor/go.uber.org/zap/global_go112.go create mode 100644 vendor/go.uber.org/zap/global_prego112.go create mode 100644 vendor/gopkg.in/robfig/cron.v2/.gitignore create mode 100644 vendor/gopkg.in/robfig/cron.v2/.travis.yml create mode 100644 vendor/gopkg.in/robfig/cron.v2/LICENSE create mode 100644 vendor/gopkg.in/robfig/cron.v2/README.md create mode 100644 vendor/gopkg.in/robfig/cron.v2/constantdelay.go create mode 100644 vendor/gopkg.in/robfig/cron.v2/cron.go create mode 100644 vendor/gopkg.in/robfig/cron.v2/doc.go create mode 100644 vendor/gopkg.in/robfig/cron.v2/parser.go create mode 100644 vendor/gopkg.in/robfig/cron.v2/spec.go diff --git a/go.mod b/go.mod index 45d7edc698..84c5eb744d 100644 --- a/go.mod +++ b/go.mod @@ -31,6 +31,7 @@ replace ( github.com/prometheus/client_model => github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 github.com/prometheus/common => github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 github.com/prometheus/procfs => github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a + github.com/rancher/kine => ../kine k8s.io/api => github.com/rancher/kubernetes/staging/src/k8s.io/api v1.16.2-k3s.1 k8s.io/apiextensions-apiserver => github.com/rancher/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.16.2-k3s.1 k8s.io/apimachinery => github.com/rancher/kubernetes/staging/src/k8s.io/apimachinery v1.16.2-k3s.1 @@ -64,6 +65,7 @@ require ( github.com/bhendo/go-powershell v0.0.0-20190719160123-219e7fb4e41e // indirect github.com/bronze1man/goStrongswanVici v0.0.0-20190828090544-27d02f80ba40 // indirect github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23 // indirect + github.com/canonical/go-dqlite v1.1.0 github.com/containerd/cgroups v0.0.0-20190923161937-abd0b19954a6 // indirect github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69 github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6 // indirect @@ -75,30 +77,32 @@ require ( github.com/containernetworking/plugins v0.8.2 // indirect github.com/coreos/flannel v0.11.0 github.com/coreos/go-iptables v0.4.2 - github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e + github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f github.com/docker/docker v0.7.3-0.20190731001754-589f1dad8dad github.com/docker/go-metrics v0.0.1 // indirect github.com/docker/libnetwork v0.8.0-dev.2.0.20190624125649-f0e46a78ea34 // indirect + github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4 // indirect github.com/go-bindata/go-bindata v3.1.2+incompatible github.com/go-sql-driver/mysql v1.4.1 github.com/gofrs/flock v0.7.1 // indirect github.com/gogo/googleapis v1.3.0 // indirect github.com/google/tcpproxy v0.0.0-20180808230851-dfa16c61dad2 github.com/gorilla/mux v1.7.3 - github.com/gorilla/websocket v1.4.0 + github.com/gorilla/websocket v1.4.1 github.com/juju/errors v0.0.0-20190806202954-0232dcc7464d // indirect github.com/juju/testing v0.0.0-20190723135506-ce30eb24acd2 // indirect github.com/kubernetes-sigs/cri-tools v0.0.0-00010101000000-000000000000 github.com/lib/pq v1.1.1 + github.com/lxc/lxd v0.0.0-20191108214106-60ea15630455 github.com/mattn/go-sqlite3 v1.10.0 github.com/mindprince/gonvml v0.0.0-20190828220739-9ebdce4bb989 // indirect github.com/natefinch/lumberjack v2.0.0+incompatible github.com/opencontainers/runc v1.0.0-rc2.0.20190611121236-6cc515888830 github.com/pkg/errors v0.8.1 github.com/rakelkar/gonetsh v0.0.0-20190719023240-501daadcadf8 // indirect - github.com/rancher/dynamiclistener v0.1.1-0.20191108205817-245f86cc340a + github.com/rancher/dynamiclistener v0.1.1-0.20191110035254-aaa5bc0d2a07 github.com/rancher/helm-controller v0.2.2 - github.com/rancher/kine v0.1.2-0.20191107225357-527576e3452f + github.com/rancher/kine v0.2.0 github.com/rancher/remotedialer v0.2.0 github.com/rancher/wrangler v0.2.0 github.com/rancher/wrangler-api v0.2.0 @@ -108,11 +112,11 @@ require ( github.com/tchap/go-patricia v2.3.0+incompatible // indirect github.com/theckman/go-flock v0.7.1 // indirect github.com/urfave/cli v1.21.0 - go.etcd.io/bbolt v1.3.3 // indirect golang.org/x/net v0.0.0-20190812203447-cdfb69ac37fc golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3 google.golang.org/grpc v1.23.0 gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect + gopkg.in/robfig/cron.v2 v2.0.0-20150107220207-be2e0b0deed5 // indirect gopkg.in/yaml.v2 v2.2.4 k8s.io/api v0.0.0 k8s.io/apimachinery v0.0.0 diff --git a/go.sum b/go.sum index c3e5dd5d1d..2a24578703 100644 --- a/go.sum +++ b/go.sum @@ -80,6 +80,8 @@ github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7 github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23 h1:D21IyuvjDCshj1/qq+pCNd3VZOAEI9jy6Bi131YlXgI= github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= github.com/caddyserver/caddy v1.0.3/go.mod h1:G+ouvOY32gENkJC+jhgl62TyhvqEsFaDiZ4uw0RzP1E= +github.com/canonical/go-dqlite v1.1.0 h1:vXGVhHrql++q038JVZk17/VZDbvbH5ySWcObWjuxiBQ= +github.com/canonical/go-dqlite v1.1.0/go.mod h1:wp00vfMvPYgNCyxcPdHB5XExmDoCGoPUGymloAQT17Y= github.com/cenkalti/backoff v2.1.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cespare/prettybench v0.0.0-20150116022406-03b8cfe5406c/go.mod h1:Xe6ZsFhtM8HrDku0pxJ3/Lr51rwykrzgFwpmTzleatY= github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 h1:7aWHqerlJ41y6FOsEUvknqgXnGmJyJSbjhAWq5pO4F8= @@ -119,6 +121,8 @@ github.com/containernetworking/plugins v0.8.2/go.mod h1:TxALKWZpWL79BC3GOYKJzzXr github.com/coredns/corefile-migration v1.0.2/go.mod h1:OFwBp/Wc9dJt5cAZzHWMNhK1r5L0p0jDwIBc6j8NC8E= github.com/coreos/bbolt v1.3.1-coreos.6 h1:uTXKg9gY70s9jMAKdfljFQcuh4e/BXOM+V+d00KFj3A= github.com/coreos/bbolt v1.3.1-coreos.6/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/bbolt v1.3.3 h1:n6AiVyVRKQFNb6mJlwESEvvLoDyiTzXX7ORAUlkeBdY= +github.com/coreos/bbolt v1.3.3/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.15+incompatible h1:+9RjdC18gMxNQVvSiXvObLu29mOFmkgdsB4cRTlV+EE= @@ -135,6 +139,8 @@ github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7 h1:u9SHYsPQNyt5t github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea h1:n2Ltr3SrfQlf/9nOna1DoGKxLx3qTSI8Ttl6Xrqp6mw= github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/rkt v1.30.0 h1:Kkt6sYeEGKxA3Y7SCrY+nHoXkWed6Jr2BBY42GqMymM= github.com/coreos/rkt v1.30.0/go.mod h1:O634mlH6U7qk87poQifK6M2rsFNt+FyUTWNMnP1hF1U= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= @@ -187,6 +193,8 @@ github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZM github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8= github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= github.com/fatih/color v1.6.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4 h1:GY1+t5Dr9OKADM64SYnQjw/w99HMYvQ0A8/JoUkxVmc= +github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -201,6 +209,8 @@ github.com/go-acme/lego v2.5.0+incompatible/go.mod h1:yzMNe9CasVUhkquNvti5nAtPmG github.com/go-bindata/go-bindata v3.1.1+incompatible/go.mod h1:xK8Dsgwmeed+BBsSy2XTopBn/8uK2HWuGSnA11C3Joo= github.com/go-bindata/go-bindata v3.1.2+incompatible h1:5vjJMVhowQdPzjE1LdxyFF7YFTXg5IgGVW4gBr5IbvE= github.com/go-bindata/go-bindata v3.1.2+incompatible/go.mod h1:xK8Dsgwmeed+BBsSy2XTopBn/8uK2HWuGSnA11C3Joo= +github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJYCFOLkIBwI7xFExG03bbkOkCvUPI= +github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540/go.mod h1:+sE8vrLDS2M0pZkBk0wy6+nLdKexVDrl/jBqQOTDThA= github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= @@ -318,6 +328,8 @@ github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e h1:KhcknUwkWHKZ github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/cadvisor v0.34.0 h1:No7G6U/TasplR9uNqyc5Jj0Bet5VSYsK5xLygOf4pUw= github.com/google/cadvisor v0.34.0/go.mod h1:1nql6U13uTHaLYB8rLS5x9IJc2qT6Xd/Tr1sTX6NE48= github.com/google/certificate-transparency-go v1.0.21 h1:Yf1aXowfZ2nuboBsg7iYGLmwsOARdV86pfH3g95wXmE= @@ -351,15 +363,21 @@ github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM= +github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE= github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7 h1:6TSoaYExHper8PYsJu23GWVNOyYRCSnIFyxKgLSZ54w= github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v0.0.0-20190222133341-cfaf5686ec79 h1:lR9ssWAqp9qL0bALxqEEkuudiP1eweOdv9jsRK3e7lE= github.com/grpc-ecosystem/go-grpc-middleware v0.0.0-20190222133341-cfaf5686ec79/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 h1:THDBEeQ9xZ8JEaCLyLQqXMMdRqNr0QAUJTIkQAUtFjg= +github.com/grpc-ecosystem/go-grpc-middleware v1.1.0/go.mod h1:f5nM7jw/oeRSadq3xCzHAvxcr8HZnzsqU6ILg/0NiiE= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.3.0 h1:HJtP6RRwj2EpPCD/mhAWzSvLL/dFTdPm1UrWwanoFos= github.com/grpc-ecosystem/grpc-gateway v1.3.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= +github.com/grpc-ecosystem/grpc-gateway v1.11.2 h1:bUDfHRK8aKGdya+msYJHffDwNxB8Eileyl7Jf2qqYjI= +github.com/grpc-ecosystem/grpc-gateway v1.11.2/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= @@ -375,8 +393,6 @@ github.com/heketi/tests v0.0.0-20151005000721-f3775cbcefd6/go.mod h1:xGMAM8JLi7U github.com/heketi/utils v0.0.0-20170317161834-435bc5bdfa64/go.mod h1:RYlF4ghFZPPmk2TC5REt5OFwvfb6lzxFWrTWB+qs28s= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/ibuildthecloud/kine v0.1.1 h1:HAqWHrjRDoqQ3+pKuJrk7Sx9nzoW3zotzhxMfAZ4YME= -github.com/ibuildthecloud/kine v0.1.1/go.mod h1:TTWUtUeu7dHQan9BrCtlRbKr9eK7epHqrBFOAae15Bg= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.7 h1:Y+UAYTZ7gDEuOfhxKWy+dvb5dRQ6rJjFSdX2HZY1/gI= github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= @@ -396,10 +412,13 @@ github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/juju/errors v0.0.0-20180806074554-22422dad46e1/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= +github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= github.com/juju/errors v0.0.0-20190806202954-0232dcc7464d h1:hJXjZMxj0SWlMoQkzeZDLi2cmeiWKa7y1B8Rg+qaoEc= github.com/juju/errors v0.0.0-20190806202954-0232dcc7464d/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= +github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8 h1:UUHMLvzt/31azWTN/ifGWef4WUqvXk0iRqdhdy/2uzI= github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= +github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= github.com/juju/testing v0.0.0-20190613124551-e81189438503/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= github.com/juju/testing v0.0.0-20190723135506-ce30eb24acd2 h1:Pp8RxiF4rSoXP9SED26WCfNB28/dwTDpPXS8XMJR8rc= github.com/juju/testing v0.0.0-20190723135506-ce30eb24acd2/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= @@ -438,6 +457,8 @@ github.com/lucas-clemente/aes12 v0.0.0-20171027163421-cd47fb39b79f/go.mod h1:JpH github.com/lucas-clemente/quic-clients v0.1.0/go.mod h1:y5xVIEoObKqULIKivu+gD/LU90pL73bTdtQjPBvtCBk= github.com/lucas-clemente/quic-go v0.10.2/go.mod h1:hvaRS9IHjFLMq76puFJeWNfmn+H70QZ/CXoxqw9bzao= github.com/lucas-clemente/quic-go-certificates v0.0.0-20160823095156-d2f86524cced/go.mod h1:NCcRLrOTZbzhZvixZLlERbJtDtYsmMw8Jc4vS8Z0g58= +github.com/lxc/lxd v0.0.0-20191108214106-60ea15630455 h1:gQQV7It0kjZxMLJkS/+5Mc6w0zM6pKGzl3OS0h2RHrY= +github.com/lxc/lxd v0.0.0-20191108214106-60ea15630455/go.mod h1:2BaZflfwsv8a3uy3/Vw+de4Avn4DSrAiqaHJjCIXMV4= github.com/magiconair/properties v1.7.6/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -527,6 +548,7 @@ github.com/opencontainers/runtime-spec v0.0.0-20180911193056-5684b8af48c1/go.mod github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= github.com/opencontainers/selinux v1.2.2 h1:Kx9J6eDG5/24A6DtUquGSpJQ+m2MUTahn4FtGEe8bFg= github.com/opencontainers/selinux v1.2.2/go.mod h1:+BLncwf63G4dgOzykXAxcmnFlUaOlkDdmw/CqsW6pjs= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.0.1/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= @@ -562,16 +584,12 @@ github.com/rancher/cri v1.3.0-k3s.2 h1:k2XFyD+ZdsGvNfugdvqD38KSMANT3JmTFULFM2CtI github.com/rancher/cri v1.3.0-k3s.2/go.mod h1:Ht5T1dIKzm+4NExmb7wDVG6qR+j0xeXIjjhCv1d9geY= github.com/rancher/cri-tools v1.16.1-k3s.1 h1:iporgQ46noE6dtLzq6fWcIO2qjyPZy2m42d2P+UnGJg= github.com/rancher/cri-tools v1.16.1-k3s.1/go.mod h1:TEKhKv2EJIZp+p9jnEy4C63g8CosJzsI4kyKKkHag+8= -github.com/rancher/dynamiclistener v0.1.1-0.20191031022009-6224794ef3cb h1:bMoA9UHr1QNTWVrf0fSJCba6YDU1xmt2jmeohpiugKg= -github.com/rancher/dynamiclistener v0.1.1-0.20191031022009-6224794ef3cb/go.mod h1:fs/dxyNcB3YT6W9fVz4bDGfhmSQS17QQup6BIcGF++s= -github.com/rancher/dynamiclistener v0.1.1-0.20191108205817-245f86cc340a h1:yIQXTC2BjGQ4Bt5Y7QhnxNWbbq8e6koH+pFrJL2VsIs= -github.com/rancher/dynamiclistener v0.1.1-0.20191108205817-245f86cc340a/go.mod h1:fs/dxyNcB3YT6W9fVz4bDGfhmSQS17QQup6BIcGF++s= +github.com/rancher/dynamiclistener v0.1.1-0.20191110035254-aaa5bc0d2a07 h1:wR1hnAh7d7ZicsAwDyw2nfvGFDOvPojcfClwA8WGy5g= +github.com/rancher/dynamiclistener v0.1.1-0.20191110035254-aaa5bc0d2a07/go.mod h1:fs/dxyNcB3YT6W9fVz4bDGfhmSQS17QQup6BIcGF++s= github.com/rancher/flannel v0.11.0-k3s.1 h1:mIwnfWDafjzQgFkZeJ1AkFrrAT3EdBaA1giE0eLJKo8= github.com/rancher/flannel v0.11.0-k3s.1/go.mod h1:Hn4ZV+eq0LhLZP63xZnxdGwXEoRSxs5sxELxu27M3UA= github.com/rancher/helm-controller v0.2.2 h1:MUqisy53/Ay1EYOF2uTCYBbGpgtZLNKKrI01BdxIbQo= github.com/rancher/helm-controller v0.2.2/go.mod h1:0JkL0UjxddNbT4FmLoESarD4Mz8xzA5YlejqJ/U4g+8= -github.com/rancher/kine v0.1.2-0.20191107225357-527576e3452f h1:tUNKo4xpQfR2Qfg/vyI5/pdBaC730Lu/jqTEqgeo83o= -github.com/rancher/kine v0.1.2-0.20191107225357-527576e3452f/go.mod h1:TTWUtUeu7dHQan9BrCtlRbKr9eK7epHqrBFOAae15Bg= github.com/rancher/kubernetes v1.16.2-k3s.1 h1:+oJEecXgQDkEOD/X8z2YUdYVonbXZtGzXsmtKDPYesg= github.com/rancher/kubernetes v1.16.2-k3s.1/go.mod h1:SmhGgKfQ30imqjFVj8AI+iW+zSyFsswNErKYeTfgoH0= github.com/rancher/kubernetes/staging/src/k8s.io/api v1.16.2-k3s.1 h1:2kK5KD6MU86txBYKG+tM6j5zbey02DaIDtwpG5JsfnI= @@ -629,6 +647,7 @@ github.com/rancher/wrangler-api v0.2.0/go.mod h1:zTPdNLZO07KvRaVOx6XQbKBSV55Fnn4 github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= github.com/robfig/cron v1.1.0 h1:jk4/Hud3TTdcrJgUOBgsqrZBarcxl6ADIjSC2iniwLY= github.com/robfig/cron v1.1.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rootless-containers/rootlesskit v0.6.0 h1:L7DxVAlaNhg4M/+i2GCl24kRkXO5q81C/lu4jlMz3bE= @@ -657,6 +676,8 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1 github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.3 h1:09wy7WZk4AqO03yH85Ex1X+Uo3vDsil3Fa9AgF8Emss= github.com/soheilhy/cmux v0.1.3/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/sourcegraph/go-diff v0.5.1/go.mod h1:j2dHj3m8aZgQO8lMTcTnBcXkRRRqi34cd2MNlA9u1mE= github.com/spf13/afero v1.1.0/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= @@ -685,6 +706,8 @@ github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRci github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/syndtr/gocapability v0.0.0-20160928074757-e7cb7fa329f4/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8 h1:zLV6q4e8Jv9EHjNg/iHfzwDkCve6Ua5jCygptrtXHvI= github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= @@ -697,6 +720,8 @@ github.com/thecodeteam/goscaleio v0.1.0/go.mod h1:68sdkZAsK8bvEwBlbQnlLS+xU+hvLY github.com/timakin/bodyclose v0.0.0-20190407043127-4a873e97b2bb/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8 h1:ndzgwNDnKIqyCvHTXaCqh9KlOWKvBry6nuXMJmonVsE= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= @@ -720,6 +745,8 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1: github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18 h1:MPPkRncZLN9Kh4MEFmbnK4h3BD7AUmskWv2+EeZJCCs= github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1 h1:j2hhcujLRHAg872RWAV5yaUrEjHEObwDv3aImCaNLek= github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1/go.mod h1:QcJo0QPSfTONNIgpN5RA8prR7fF8nkF6cTWTcNerRO8= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= @@ -730,10 +757,16 @@ go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.uber.org/atomic v0.0.0-20181018215023-8dc6146f7569 h1:nSQar3Y0E3VQF/VdZ8PTAilaXpER+d7ypdABCrpwMdg= go.uber.org/atomic v0.0.0-20181018215023-8dc6146f7569/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v0.0.0-20180122172545-ddea229ff1df h1:shvkWr0NAZkg4nPuE3XrKP0VuBPijjk3TfX6Y6acFNg= go.uber.org/multierr v0.0.0-20180122172545-ddea229ff1df/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v0.0.0-20180814183419-67bc79d13d15 h1:Z2sc4+v0JHV6Mn4kX1f2a5nruNjmV+Th32sugE8zwz8= go.uber.org/zap v0.0.0-20180814183419-67bc79d13d15/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180426230345-b49d69b5da94/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -768,6 +801,7 @@ golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181102091132-c10e9556a7bc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -798,6 +832,7 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181004145325-8469e314837c/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190124100055-b90733256f2e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -829,6 +864,7 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190121143147-24cd39ecf745/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -864,6 +900,8 @@ google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873 h1:nfPFGzJkUDX6uBmpN/pSw7MbOAWegH5QDQuoXFHedLg= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.23.0 h1:AzbTB6ux+okLTzP8Ru1Xs41C303zdcfEht7MQnYJt5A= @@ -888,6 +926,9 @@ gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3 gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/robfig/cron.v2 v2.0.0-20150107220207-be2e0b0deed5 h1:E846t8CnR+lv5nE+VuiKTDG/v1U2stad0QzddfJC7kY= +gopkg.in/robfig/cron.v2 v2.0.0-20150107220207-be2e0b0deed5/go.mod h1:hiOFpYm0ZJbusNj2ywpbrXowU3G8U6GIQzqn2mw1UIE= gopkg.in/square/go-jose.v2 v2.2.2 h1:orlkJ3myw8CN1nVQHBFfloD+L3egixIa4FvUP6RosSA= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= diff --git a/vendor/github.com/Rican7/retry/.travis.yml b/vendor/github.com/Rican7/retry/.travis.yml new file mode 100644 index 0000000000..9dfdf4510b --- /dev/null +++ b/vendor/github.com/Rican7/retry/.travis.yml @@ -0,0 +1,28 @@ +language: go + +go: + - 1.6 + - tip + +sudo: false + +install: + # Get all imported packages + - make install-deps install-deps-dev + + # Basic build errors + - make build + +script: + # Lint + - make format-lint + - make import-lint + - make copyright-lint + + # Run tests + - make test + +matrix: + allow_failures: + - go: tip + fast_finish: true diff --git a/vendor/github.com/Rican7/retry/LICENSE b/vendor/github.com/Rican7/retry/LICENSE new file mode 100644 index 0000000000..361507628d --- /dev/null +++ b/vendor/github.com/Rican7/retry/LICENSE @@ -0,0 +1,19 @@ +Copyright (C) 2016 Trevor N. Suarez (Rican7) + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE +OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/Rican7/retry/Makefile b/vendor/github.com/Rican7/retry/Makefile new file mode 100644 index 0000000000..60623a930c --- /dev/null +++ b/vendor/github.com/Rican7/retry/Makefile @@ -0,0 +1,72 @@ +# Define some VCS context +PARENT_BRANCH ?= master + +# Set a default `min_confidence` value for `golint` +GOLINT_MIN_CONFIDENCE ?= 0.3 + +# Set flags for `gofmt` +GOFMT_FLAGS ?= -s + + +all: install-deps build install + +clean: + go clean -i -x ./... + +build: + go build -v ./... + +install: + go install ./... + +install-deps: + go get -d -t ./... + +install-deps-dev: install-deps + go get github.com/golang/lint/golint + go get golang.org/x/tools/cmd/goimports + +update-deps: + go get -d -t -u ./... + +update-deps-dev: update-deps + go get -u github.com/golang/lint/golint + go get -u golang.org/x/tools/cmd/goimports + +test: + go test -v ./... + +test-with-coverage: + go test -cover ./... + +test-with-coverage-formatted: + go test -cover ./... | column -t | sort -r + +format-lint: + errors=$$(gofmt -l ${GOFMT_FLAGS} .); if [ "$${errors}" != "" ]; then echo "$${errors}"; exit 1; fi + +import-lint: + errors=$$(goimports -l .); if [ "$${errors}" != "" ]; then echo "$${errors}"; exit 1; fi + +style-lint: + errors=$$(golint -min_confidence=${GOLINT_MIN_CONFIDENCE} ./...); if [ "$${errors}" != "" ]; then echo "$${errors}"; exit 1; fi + +copyright-lint: + @old_dates=$$(git diff --diff-filter=ACMRTUXB --name-only "${PARENT_BRANCH}" | xargs grep -E '[Cc]opyright(\s+)[©Cc]?(\s+)[0-9]{4}' | grep -E -v "[Cc]opyright(\s+)[©Cc]?(\s+)$$(date '+%Y')"); if [ "$${old_dates}" != "" ]; then printf "The following files contain outdated copyrights:\n$${old_dates}\n\nThis can be fixed with 'make copyright-fix'\n"; exit 1; fi + +lint: install-deps-dev format-lint import-lint style-lint copyright-lint + +format-fix: + gofmt -w ${GOFMT_FLAGS} . + +import-fix: + goimports -w . + +copyright-fix: + @git diff --diff-filter=ACMRTUXB --name-only "${PARENT_BRANCH}" | xargs -I '_FILENAME' -- sh -c 'sed -i.bak "s/\([Cc]opyright\([[:space:]][©Cc]\{0,1\}[[:space:]]*\)\)[0-9]\{4\}/\1"$$(date '+%Y')"/g" _FILENAME && rm _FILENAME.bak' + +vet: + go vet ./... + + +.PHONY: all clean build install install-deps install-deps-dev update-deps update-deps-dev test test-with-coverage test-with-coverage-formatted format-lint import-lint style-lint copyright-lint lint format-fix import-fix copyright-fix vet diff --git a/vendor/github.com/Rican7/retry/README.md b/vendor/github.com/Rican7/retry/README.md new file mode 100644 index 0000000000..3ffe54655e --- /dev/null +++ b/vendor/github.com/Rican7/retry/README.md @@ -0,0 +1,98 @@ +# retry + +[![Build Status](https://travis-ci.org/Rican7/retry.svg?branch=master)](https://travis-ci.org/Rican7/retry) +[![GoDoc](https://godoc.org/github.com/Rican7/retry?status.png)](https://godoc.org/github.com/Rican7/retry) +[![Go Report Card](https://goreportcard.com/badge/Rican7/retry)](http://goreportcard.com/report/Rican7/retry) +[![Latest Stable Version](https://img.shields.io/github/release/Rican7/retry.svg?style=flat)](https://github.com/Rican7/retry/releases) + +A simple, stateless, functional mechanism to perform actions repetitively until successful. + + +## Project Status + +This project is currently in "pre-release". While the code is heavily tested, the API may change. +Vendor (commit or lock) this dependency if you plan on using it. + + +## Install + +`go get github.com/Rican7/retry` + + +## Examples + +### Basic + +```go +Retry(func(attempt uint) error { + return nil // Do something that may or may not cause an error +}) +``` + +### File Open + +```go +const logFilePath = "/var/log/myapp.log" + +var logFile *os.File + +err := Retry(func(attempt uint) error { + var err error + + logFile, err = os.Open(logFilePath) + + return err +}) + +if nil != err { + log.Fatalf("Unable to open file %q with error %q", logFilePath, err) +} +``` + +### HTTP request with strategies and backoff + +```go +var response *http.Response + +action := func(attempt uint) error { + var err error + + response, err = http.Get("https://api.github.com/repos/Rican7/retry") + + if nil == err && nil != response && response.StatusCode > 200 { + err = fmt.Errorf("failed to fetch (attempt #%d) with status code: %d", attempt, response.StatusCode) + } + + return err +} + +err := Retry( + action, + strategy.Limit(5), + strategy.Backoff(backoff.Fibonacci(10*time.Millisecond)), +) + +if nil != err { + log.Fatalf("Failed to fetch repository with error %q", err) +} +``` + +### Retry with backoff jitter + +```go +action := func(attempt uint) error { + return errors.New("something happened") +} + +seed := time.Now().UnixNano() +random := rand.New(rand.NewSource(seed)) + +Retry( + action, + strategy.Limit(5), + strategy.BackoffWithJitter( + backoff.BinaryExponential(10*time.Millisecond), + jitter.Deviation(random, 0.5), + ), +) +``` diff --git a/vendor/github.com/Rican7/retry/backoff/backoff.go b/vendor/github.com/Rican7/retry/backoff/backoff.go new file mode 100644 index 0000000000..5369a75a1c --- /dev/null +++ b/vendor/github.com/Rican7/retry/backoff/backoff.go @@ -0,0 +1,67 @@ +// Package backoff provides stateless methods of calculating durations based on +// a number of attempts made. +// +// Copyright © 2016 Trevor N. Suarez (Rican7) +package backoff + +import ( + "math" + "time" +) + +// Algorithm defines a function that calculates a time.Duration based on +// the given retry attempt number. +type Algorithm func(attempt uint) time.Duration + +// Incremental creates a Algorithm that increments the initial duration +// by the given increment for each attempt. +func Incremental(initial, increment time.Duration) Algorithm { + return func(attempt uint) time.Duration { + return initial + (increment * time.Duration(attempt)) + } +} + +// Linear creates a Algorithm that linearly multiplies the factor +// duration by the attempt number for each attempt. +func Linear(factor time.Duration) Algorithm { + return func(attempt uint) time.Duration { + return (factor * time.Duration(attempt)) + } +} + +// Exponential creates a Algorithm that multiplies the factor duration by +// an exponentially increasing factor for each attempt, where the factor is +// calculated as the given base raised to the attempt number. +func Exponential(factor time.Duration, base float64) Algorithm { + return func(attempt uint) time.Duration { + return (factor * time.Duration(math.Pow(base, float64(attempt)))) + } +} + +// BinaryExponential creates a Algorithm that multiplies the factor +// duration by an exponentially increasing factor for each attempt, where the +// factor is calculated as `2` raised to the attempt number (2^attempt). +func BinaryExponential(factor time.Duration) Algorithm { + return Exponential(factor, 2) +} + +// Fibonacci creates a Algorithm that multiplies the factor duration by +// an increasing factor for each attempt, where the factor is the Nth number in +// the Fibonacci sequence. +func Fibonacci(factor time.Duration) Algorithm { + return func(attempt uint) time.Duration { + return (factor * time.Duration(fibonacciNumber(attempt))) + } +} + +// fibonacciNumber calculates the Fibonacci sequence number for the given +// sequence position. +func fibonacciNumber(n uint) uint { + if 0 == n { + return 0 + } else if 1 == n { + return 1 + } else { + return fibonacciNumber(n-1) + fibonacciNumber(n-2) + } +} diff --git a/vendor/github.com/Rican7/retry/jitter/jitter.go b/vendor/github.com/Rican7/retry/jitter/jitter.go new file mode 100644 index 0000000000..e94ad89279 --- /dev/null +++ b/vendor/github.com/Rican7/retry/jitter/jitter.go @@ -0,0 +1,89 @@ +// Package jitter provides methods of transforming durations. +// +// Copyright © 2016 Trevor N. Suarez (Rican7) +package jitter + +import ( + "math" + "math/rand" + "time" +) + +// Transformation defines a function that calculates a time.Duration based on +// the given duration. +type Transformation func(duration time.Duration) time.Duration + +// Full creates a Transformation that transforms a duration into a result +// duration in [0, n) randomly, where n is the given duration. +// +// The given generator is what is used to determine the random transformation. +// If a nil generator is passed, a default one will be provided. +// +// Inspired by https://www.awsarchitectureblog.com/2015/03/backoff.html +func Full(generator *rand.Rand) Transformation { + random := fallbackNewRandom(generator) + + return func(duration time.Duration) time.Duration { + return time.Duration(random.Int63n(int64(duration))) + } +} + +// Equal creates a Transformation that transforms a duration into a result +// duration in [n/2, n) randomly, where n is the given duration. +// +// The given generator is what is used to determine the random transformation. +// If a nil generator is passed, a default one will be provided. +// +// Inspired by https://www.awsarchitectureblog.com/2015/03/backoff.html +func Equal(generator *rand.Rand) Transformation { + random := fallbackNewRandom(generator) + + return func(duration time.Duration) time.Duration { + return (duration / 2) + time.Duration(random.Int63n(int64(duration))/2) + } +} + +// Deviation creates a Transformation that transforms a duration into a result +// duration that deviates from the input randomly by a given factor. +// +// The given generator is what is used to determine the random transformation. +// If a nil generator is passed, a default one will be provided. +// +// Inspired by https://developers.google.com/api-client-library/java/google-http-java-client/backoff +func Deviation(generator *rand.Rand, factor float64) Transformation { + random := fallbackNewRandom(generator) + + return func(duration time.Duration) time.Duration { + min := int64(math.Floor(float64(duration) * (1 - factor))) + max := int64(math.Ceil(float64(duration) * (1 + factor))) + + return time.Duration(random.Int63n(max-min) + min) + } +} + +// NormalDistribution creates a Transformation that transforms a duration into a +// result duration based on a normal distribution of the input and the given +// standard deviation. +// +// The given generator is what is used to determine the random transformation. +// If a nil generator is passed, a default one will be provided. +func NormalDistribution(generator *rand.Rand, standardDeviation float64) Transformation { + random := fallbackNewRandom(generator) + + return func(duration time.Duration) time.Duration { + return time.Duration(random.NormFloat64()*standardDeviation + float64(duration)) + } +} + +// fallbackNewRandom returns the passed in random instance if it's not nil, +// and otherwise returns a new random instance seeded with the current time. +func fallbackNewRandom(random *rand.Rand) *rand.Rand { + // Return the passed in value if it's already not null + if nil != random { + return random + } + + seed := time.Now().UnixNano() + + return rand.New(rand.NewSource(seed)) +} diff --git a/vendor/github.com/Rican7/retry/retry.go b/vendor/github.com/Rican7/retry/retry.go new file mode 100644 index 0000000000..15015db257 --- /dev/null +++ b/vendor/github.com/Rican7/retry/retry.go @@ -0,0 +1,36 @@ +// Package retry provides a simple, stateless, functional mechanism to perform +// actions repetitively until successful. +// +// Copyright © 2016 Trevor N. Suarez (Rican7) +package retry + +import "github.com/Rican7/retry/strategy" + +// Action defines a callable function that package retry can handle. +type Action func(attempt uint) error + +// Retry takes an action and performs it, repetitively, until successful. +// +// Optionally, strategies may be passed that assess whether or not an attempt +// should be made. +func Retry(action Action, strategies ...strategy.Strategy) error { + var err error + + for attempt := uint(0); (0 == attempt || nil != err) && shouldAttempt(attempt, strategies...); attempt++ { + err = action(attempt) + } + + return err +} + +// shouldAttempt evaluates the provided strategies with the given attempt to +// determine if the Retry loop should make another attempt. +func shouldAttempt(attempt uint, strategies ...strategy.Strategy) bool { + shouldAttempt := true + + for i := 0; shouldAttempt && i < len(strategies); i++ { + shouldAttempt = shouldAttempt && strategies[i](attempt) + } + + return shouldAttempt +} diff --git a/vendor/github.com/Rican7/retry/strategy/strategy.go b/vendor/github.com/Rican7/retry/strategy/strategy.go new file mode 100644 index 0000000000..a315fa02cb --- /dev/null +++ b/vendor/github.com/Rican7/retry/strategy/strategy.go @@ -0,0 +1,85 @@ +// Package strategy provides a way to change the way that retry is performed. +// +// Copyright © 2016 Trevor N. Suarez (Rican7) +package strategy + +import ( + "time" + + "github.com/Rican7/retry/backoff" + "github.com/Rican7/retry/jitter" +) + +// Strategy defines a function that Retry calls before every successive attempt +// to determine whether it should make the next attempt or not. Returning `true` +// allows for the next attempt to be made. Returning `false` halts the retrying +// process and returns the last error returned by the called Action. +// +// The strategy will be passed an "attempt" number on each successive retry +// iteration, starting with a `0` value before the first attempt is actually +// made. This allows for a pre-action delay, etc. +type Strategy func(attempt uint) bool + +// Limit creates a Strategy that limits the number of attempts that Retry will +// make. +func Limit(attemptLimit uint) Strategy { + return func(attempt uint) bool { + return (attempt <= attemptLimit) + } +} + +// Delay creates a Strategy that waits the given duration before the first +// attempt is made. +func Delay(duration time.Duration) Strategy { + return func(attempt uint) bool { + if 0 == attempt { + time.Sleep(duration) + } + + return true + } +} + +// Wait creates a Strategy that waits the given durations for each attempt after +// the first. If the number of attempts is greater than the number of durations +// provided, then the strategy uses the last duration provided. +func Wait(durations ...time.Duration) Strategy { + return func(attempt uint) bool { + if 0 < attempt && 0 < len(durations) { + durationIndex := int(attempt - 1) + + if len(durations) <= durationIndex { + durationIndex = len(durations) - 1 + } + + time.Sleep(durations[durationIndex]) + } + + return true + } +} + +// Backoff creates a Strategy that waits before each attempt, with a duration as +// defined by the given backoff.Algorithm. +func Backoff(algorithm backoff.Algorithm) Strategy { + return BackoffWithJitter(algorithm, noJitter()) +} + +// BackoffWithJitter creates a Strategy that waits before each attempt, with a +// duration as defined by the given backoff.Algorithm and jitter.Transformation. +func BackoffWithJitter(algorithm backoff.Algorithm, transformation jitter.Transformation) Strategy { + return func(attempt uint) bool { + if 0 < attempt { + time.Sleep(transformation(algorithm(attempt))) + } + + return true + } +} + +// noJitter creates a jitter.Transformation that simply returns the input. +func noJitter() jitter.Transformation { + return func(duration time.Duration) time.Duration { + return duration + } +} diff --git a/vendor/github.com/canonical/go-dqlite/.dir-locals.el b/vendor/github.com/canonical/go-dqlite/.dir-locals.el new file mode 100644 index 0000000000..300939c293 --- /dev/null +++ b/vendor/github.com/canonical/go-dqlite/.dir-locals.el @@ -0,0 +1,8 @@ +;;; Directory Local Variables +;;; For more information see (info "(emacs) Directory Variables") +((go-mode + . ((go-test-args . "-tags libsqlite3 -timeout 10s") + (eval + . (set + (make-local-variable 'flycheck-go-build-tags) + '("libsqlite3")))))) diff --git a/vendor/github.com/canonical/go-dqlite/.gitignore b/vendor/github.com/canonical/go-dqlite/.gitignore new file mode 100644 index 0000000000..d3da31a83a --- /dev/null +++ b/vendor/github.com/canonical/go-dqlite/.gitignore @@ -0,0 +1,4 @@ +.sqlite +demo +profile.coverprofile +overalls.coverprofile diff --git a/vendor/github.com/canonical/go-dqlite/.travis.yml b/vendor/github.com/canonical/go-dqlite/.travis.yml new file mode 100644 index 0000000000..5a64007eb4 --- /dev/null +++ b/vendor/github.com/canonical/go-dqlite/.travis.yml @@ -0,0 +1,31 @@ +dist: xenial +language: go + +addons: + apt: + sources: + - sourceline: 'ppa:dqlite/master' + packages: + - golint + - libsqlite3-dev + - libuv1-dev + - libraft-dev + - libco-dev + - libdqlite-dev + +before_install: + - go get github.com/go-playground/overalls + - go get github.com/mattn/goveralls + - go get github.com/tsenart/deadcode + +script: + - go get -t -tags libsqlite3 ./... + - go vet -tags libsqlite3 ./... + - golint + - deadcode + - project=github.com/canonical/go-dqlite + - $GOPATH/bin/overalls -project $project -covermode=count -- -tags libsqlite3 -timeout 240s + - $GOPATH/bin/goveralls -coverprofile overalls.coverprofile -service=travis-ci + +go: + - "1.12" diff --git a/vendor/github.com/canonical/go-dqlite/AUTHORS b/vendor/github.com/canonical/go-dqlite/AUTHORS new file mode 100644 index 0000000000..6e13f86ebb --- /dev/null +++ b/vendor/github.com/canonical/go-dqlite/AUTHORS @@ -0,0 +1 @@ +Free Ekanayaka diff --git a/vendor/github.com/canonical/go-dqlite/LICENSE b/vendor/github.com/canonical/go-dqlite/LICENSE new file mode 100644 index 0000000000..261eeb9e9f --- /dev/null +++ b/vendor/github.com/canonical/go-dqlite/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/canonical/go-dqlite/README.md b/vendor/github.com/canonical/go-dqlite/README.md new file mode 100644 index 0000000000..6fee042e6a --- /dev/null +++ b/vendor/github.com/canonical/go-dqlite/README.md @@ -0,0 +1,91 @@ +go-dqlite [![Build Status](https://travis-ci.org/canonical/go-dqlite.png)](https://travis-ci.org/canonical/go-dqlite) [![Coverage Status](https://coveralls.io/repos/github/canonical/go-dqlite/badge.svg?branch=master)](https://coveralls.io/github/canonical/go-dqlite?branch=master) [![Go Report Card](https://goreportcard.com/badge/github.com/canonical/go-dqlite)](https://goreportcard.com/report/github.com/canonical/go-dqlite) [![GoDoc](https://godoc.org/github.com/canonical/go-dqlite?status.svg)](https://godoc.org/github.com/canonical/go-dqlite) +====== + +This repository provides the `go-dqlite` Go package, containing bindings for the +[dqlite](https://github.com/canonical/dqlite) C library and a pure-Go +client for the dqlite wire [protocol](https://github.com/canonical/dqlite/blob/master/doc/protocol.md). + +Usage +----- + +The best way to understand how to use the ```go-dqlite``` package is probably by +looking at the source code of the [demo +program](https://github.com/canonical/go-dqlite/tree/master/cmd/dqlite-demo) and +use it as example. + +Build +----- + +In order to use the go-dqlite package in your application, you'll need to have +the [dqlite](https://github.com/canonical/dqlite) C library installed on your +system, along with its dependencies. You then need to pass the ```-tags``` +argument to the Go tools when building or testing your packages, for example: + +```bash +go build -tags libsqlite3 +go test -tags libsqlite3 +``` + +Documentation +------------- + +The documentation for this package can be found on [Godoc](http://godoc.org/github.com/canonical/go-dqlite). + +Demo +---- + +To see dqlite in action, either install the Debian package from the PPA: + +```bash +sudo add-apt-repository -y ppa:dqlite/v1 +sudo apt install dqlite +``` + +or build the dqlite C library and its dependencies from source, as described +[here](https://github.com/canonical/dqlite#build), and then run: + +``` +go install -tags libsqlite3 ./cmd/dqlite-demo +``` + +from the top-level directory of this repository. + +Once the ```dqlite-demo``` binary is installed, start three nodes of the demo +application, respectively with IDs ```1```, ```2,``` and ```3```: + +```bash +dqlite-demo start 1 & +dqlite-demo start 2 & +dqlite-demo start 3 & +``` + +The node with ID ```1``` automatically becomes the leader of a single node +cluster, while the nodes with IDs ```2``` and ```3``` are waiting to be notified +what cluster they belong to. Let's make nodes ```2``` and ```3``` join the +cluster: + +```bash +dqlite-demo add 2 +dqlite-demo add 3 +``` + +Now we can start using the cluster. The demo application is just a simple +key/value store that stores data in a SQLite table. Let's insert a key pair: + +```bash +dqlite-demo update my-key my-value +``` + +and then retrive it from the database: + +```bash +dqlite-demo query my-key +``` + +Currently node ```1``` is the leader. If we stop it and then try to query the +key again we'll notice that the ```query``` command hangs for a bit waiting for +the failover to occur and for another node to step up as leader: + +``` +kill -TERM %1; sleep 0.1; dqlite-demo query my-key +``` diff --git a/vendor/github.com/canonical/go-dqlite/client/client.go b/vendor/github.com/canonical/go-dqlite/client/client.go new file mode 100644 index 0000000000..85bae496a6 --- /dev/null +++ b/vendor/github.com/canonical/go-dqlite/client/client.go @@ -0,0 +1,221 @@ +package client + +import ( + "context" + "encoding/binary" + "io" + "net" + "strings" + + "github.com/canonical/go-dqlite/internal/protocol" + "github.com/pkg/errors" +) + +// DialFunc is a function that can be used to establish a network connection. +type DialFunc = protocol.DialFunc + +// Client speaks the dqlite wire protocol. +type Client struct { + protocol *protocol.Protocol +} + +// Option that can be used to tweak client parameters. +type Option func(*options) + +type options struct { + DialFunc DialFunc + LogFunc LogFunc +} + +// WithDialFunc sets a custom dial function for creating the client network +// connection. +func WithDialFunc(dial DialFunc) Option { + return func(options *options) { + options.DialFunc = dial + } +} + +// WithLogFunc sets a custom log function. +// connection. +func WithLogFunc(log LogFunc) Option { + return func(options *options) { + options.LogFunc = log + } +} + +// New creates a new client connected to the dqlite node with the given +// address. +func New(ctx context.Context, address string, options ...Option) (*Client, error) { + o := defaultOptions() + + for _, option := range options { + option(o) + } + // Establish the connection. + conn, err := o.DialFunc(ctx, address) + if err != nil { + return nil, errors.Wrap(err, "failed to establish network connection") + } + + // Latest protocol version. + proto := make([]byte, 8) + binary.LittleEndian.PutUint64(proto, protocol.VersionOne) + + // Perform the protocol handshake. + n, err := conn.Write(proto) + if err != nil { + conn.Close() + return nil, errors.Wrap(err, "failed to send handshake") + } + if n != 8 { + conn.Close() + return nil, errors.Wrap(io.ErrShortWrite, "failed to send handshake") + } + + client := &Client{protocol: protocol.NewProtocol(protocol.VersionOne, conn)} + + return client, nil +} + +// Leader returns information about the current leader, if any. +func (c *Client) Leader(ctx context.Context) (*NodeInfo, error) { + request := protocol.Message{} + request.Init(16) + response := protocol.Message{} + response.Init(512) + + protocol.EncodeLeader(&request) + + if err := c.protocol.Call(ctx, &request, &response); err != nil { + return nil, errors.Wrap(err, "failed to send Leader request") + } + + id, address, err := protocol.DecodeNode(&response) + if err != nil { + return nil, errors.Wrap(err, "failed to parse Node response") + } + + info := &NodeInfo{ID: id, Address: address} + + return info, nil +} + +// Cluster returns information about all nodes in the cluster. +func (c *Client) Cluster(ctx context.Context) ([]NodeInfo, error) { + request := protocol.Message{} + request.Init(16) + response := protocol.Message{} + response.Init(512) + + protocol.EncodeCluster(&request) + + if err := c.protocol.Call(ctx, &request, &response); err != nil { + return nil, errors.Wrap(err, "failed to send Cluster request") + } + + servers, err := protocol.DecodeNodes(&response) + if err != nil { + return nil, errors.Wrap(err, "failed to parse Node response") + } + + return servers, nil +} + +// File holds the content of a single database file. +type File struct { + Name string + Data []byte +} + +// Dump the content of the database with the given name. Two files will be +// returned, the first is the main database file (which has the same name as +// the database), the second is the WAL file (which has the same name as the +// database plus the suffix "-wal"). +func (c *Client) Dump(ctx context.Context, dbname string) ([]File, error) { + request := protocol.Message{} + request.Init(16) + response := protocol.Message{} + response.Init(512) + + protocol.EncodeDump(&request, dbname) + + if err := c.protocol.Call(ctx, &request, &response); err != nil { + return nil, errors.Wrap(err, "failed to send dump request") + } + + files, err := protocol.DecodeFiles(&response) + if err != nil { + return nil, errors.Wrap(err, "failed to parse files response") + } + defer files.Close() + + dump := make([]File, 0) + + for { + name, data := files.Next() + if name == "" { + break + } + dump = append(dump, File{Name: name, Data: data}) + } + + return dump, nil +} + +// Add a node to a cluster. +func (c *Client) Add(ctx context.Context, node NodeInfo) error { + request := protocol.Message{} + request.Init(4096) + response := protocol.Message{} + response.Init(4096) + + protocol.EncodeJoin(&request, node.ID, node.Address) + + if err := c.protocol.Call(ctx, &request, &response); err != nil { + return err + } + + protocol.EncodePromote(&request, node.ID) + + if err := c.protocol.Call(ctx, &request, &response); err != nil { + return err + } + + return nil +} + +// Remove a node from the cluster. +func (c *Client) Remove(ctx context.Context, id uint64) error { + request := protocol.Message{} + request.Init(4096) + response := protocol.Message{} + response.Init(4096) + + protocol.EncodeRemove(&request, id) + + if err := c.protocol.Call(ctx, &request, &response); err != nil { + return err + } + + return nil +} + +// Close the client. +func (c *Client) Close() error { + return c.protocol.Close() +} + +// Create a client options object with sane defaults. +func defaultOptions() *options { + return &options{ + DialFunc: DefaultDialFunc, + LogFunc: DefaultLogFunc, + } +} + +func DefaultDialFunc(ctx context.Context, address string) (net.Conn, error) { + if strings.HasPrefix(address, "@") { + return protocol.UnixDial(ctx, address) + } + return protocol.TCPDial(ctx, address) +} diff --git a/vendor/github.com/canonical/go-dqlite/client/leader.go b/vendor/github.com/canonical/go-dqlite/client/leader.go new file mode 100644 index 0000000000..4cc305fa76 --- /dev/null +++ b/vendor/github.com/canonical/go-dqlite/client/leader.go @@ -0,0 +1,35 @@ +package client + +import ( + "context" + "time" + + "github.com/Rican7/retry/backoff" + "github.com/Rican7/retry/strategy" + "github.com/canonical/go-dqlite/internal/protocol" +) + +// FindLeader returns a Client connected to the current cluster leader, if any. +func FindLeader(ctx context.Context, store NodeStore, options ...Option) (*Client, error) { + o := defaultOptions() + + for _, option := range options { + option(o) + } + + config := protocol.Config{ + Dial: o.DialFunc, + AttemptTimeout: time.Second, + RetryStrategies: []strategy.Strategy{ + strategy.Backoff(backoff.BinaryExponential(time.Millisecond))}, + } + connector := protocol.NewConnector(0, store, config, o.LogFunc) + protocol, err := connector.Connect(ctx) + if err != nil { + return nil, err + } + + client := &Client{protocol: protocol} + + return client, nil +} diff --git a/vendor/github.com/canonical/go-dqlite/client/log.go b/vendor/github.com/canonical/go-dqlite/client/log.go new file mode 100644 index 0000000000..5c2a00de3e --- /dev/null +++ b/vendor/github.com/canonical/go-dqlite/client/log.go @@ -0,0 +1,30 @@ +package client + +import ( + "fmt" + "log" + "os" + + "github.com/canonical/go-dqlite/internal/logging" +) + +// LogFunc is a function that can be used for logging. +type LogFunc = logging.Func + +// LogLevel defines the logging level. +type LogLevel = logging.Level + +// Available logging levels. +const ( + LogDebug = logging.Debug + LogInfo = logging.Info + LogWarn = logging.Warn + LogError = logging.Error +) + +// DefaultLogFunc emits messages using the stdlib's logger. +func DefaultLogFunc(l LogLevel, format string, a ...interface{}) { + logger := log.New(os.Stdout, "", log.LstdFlags|log.Lmicroseconds) + format = fmt.Sprintf("[%s]: %s", l.String(), format) + logger.Printf(format, a...) +} diff --git a/vendor/github.com/canonical/go-dqlite/client/store.go b/vendor/github.com/canonical/go-dqlite/client/store.go new file mode 100644 index 0000000000..04cae8c9a9 --- /dev/null +++ b/vendor/github.com/canonical/go-dqlite/client/store.go @@ -0,0 +1,136 @@ +package client + +import ( + "context" + "database/sql" + "fmt" + + "github.com/pkg/errors" + + "github.com/canonical/go-dqlite/internal/protocol" + _ "github.com/mattn/go-sqlite3" // Go SQLite bindings +) + +// NodeStore is used by a dqlite client to get an initial list of candidate +// dqlite nodes that it can dial in order to find a leader dqlite node to use. +type NodeStore = protocol.NodeStore + +// NodeInfo holds information about a single server. +type NodeInfo = protocol.NodeInfo + +// InmemNodeStore keeps the list of target dqlite nodes in memory. +type InmemNodeStore = protocol.InmemNodeStore + +// NewInmemNodeStore creates NodeStore which stores its data in-memory. +var NewInmemNodeStore = protocol.NewInmemNodeStore + +// DatabaseNodeStore persists a list addresses of dqlite nodes in a SQL table. +type DatabaseNodeStore struct { + db *sql.DB // Database handle to use. + schema string // Name of the schema holding the servers table. + table string // Name of the servers table. + column string // Column name in the servers table holding the server address. +} + +// DefaultNodeStore creates a new NodeStore using the given filename to +// open a SQLite database, with default names for the schema, table and column +// parameters. +// +// It also creates the table if it doesn't exist yet. +func DefaultNodeStore(filename string) (*DatabaseNodeStore, error) { + // Open the database. + db, err := sql.Open("sqlite3", filename) + if err != nil { + return nil, errors.Wrap(err, "failed to open database") + } + + // Since we're setting SQLite single-thread mode, we need to have one + // connection at most. + db.SetMaxOpenConns(1) + + // Create the servers table if it does not exist yet. + _, err = db.Exec("CREATE TABLE IF NOT EXISTS servers (address TEXT, UNIQUE(address))") + if err != nil { + return nil, errors.Wrap(err, "failed to create servers table") + } + + store := NewNodeStore(db, "main", "servers", "address") + + return store, nil +} + +// NewNodeStore creates a new NodeStore. +func NewNodeStore(db *sql.DB, schema, table, column string) *DatabaseNodeStore { + return &DatabaseNodeStore{ + db: db, + schema: schema, + table: table, + column: column, + } +} + +// Get the current servers. +func (d *DatabaseNodeStore) Get(ctx context.Context) ([]NodeInfo, error) { + tx, err := d.db.Begin() + if err != nil { + return nil, errors.Wrap(err, "failed to begin transaction") + } + defer tx.Rollback() + + query := fmt.Sprintf("SELECT %s FROM %s.%s", d.column, d.schema, d.table) + rows, err := tx.QueryContext(ctx, query) + if err != nil { + return nil, errors.Wrap(err, "failed to query servers table") + } + defer rows.Close() + + servers := make([]NodeInfo, 0) + for rows.Next() { + var address string + err := rows.Scan(&address) + if err != nil { + return nil, errors.Wrap(err, "failed to fetch server address") + } + servers = append(servers, NodeInfo{ID: 1, Address: address}) + } + if err := rows.Err(); err != nil { + return nil, errors.Wrap(err, "result set failure") + } + + return servers, nil +} + +// Set the servers addresses. +func (d *DatabaseNodeStore) Set(ctx context.Context, servers []NodeInfo) error { + tx, err := d.db.Begin() + if err != nil { + return errors.Wrap(err, "failed to begin transaction") + } + + query := fmt.Sprintf("DELETE FROM %s.%s", d.schema, d.table) + if _, err := tx.ExecContext(ctx, query); err != nil { + tx.Rollback() + return errors.Wrap(err, "failed to delete existing servers rows") + } + + query = fmt.Sprintf("INSERT INTO %s.%s(%s) VALUES (?)", d.schema, d.table, d.column) + stmt, err := tx.PrepareContext(ctx, query) + if err != nil { + tx.Rollback() + return errors.Wrap(err, "failed to prepare insert statement") + } + defer stmt.Close() + + for _, server := range servers { + if _, err := stmt.ExecContext(ctx, server.Address); err != nil { + tx.Rollback() + return errors.Wrapf(err, "failed to insert server %s", server.Address) + } + } + + if err := tx.Commit(); err != nil { + return errors.Wrap(err, "failed to commit transaction") + } + + return nil +} diff --git a/vendor/github.com/canonical/go-dqlite/driver/driver.go b/vendor/github.com/canonical/go-dqlite/driver/driver.go new file mode 100644 index 0000000000..cfe742b19b --- /dev/null +++ b/vendor/github.com/canonical/go-dqlite/driver/driver.go @@ -0,0 +1,670 @@ +// Copyright 2017 Canonical Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package driver + +import ( + "context" + "database/sql/driver" + "io" + "net" + "reflect" + "time" + + "github.com/Rican7/retry/backoff" + "github.com/Rican7/retry/strategy" + "github.com/pkg/errors" + + "github.com/canonical/go-dqlite/client" + "github.com/canonical/go-dqlite/internal/bindings" + "github.com/canonical/go-dqlite/internal/protocol" +) + +// Driver perform queries against a dqlite server. +type Driver struct { + log client.LogFunc // Log function to use + store client.NodeStore // Holds addresses of dqlite servers + context context.Context // Global cancellation context + connectionTimeout time.Duration // Max time to wait for a new connection + contextTimeout time.Duration // Default client context timeout. + clientConfig protocol.Config // Configuration for dqlite client instances +} + +// Error is returned in case of database errors. +type Error = bindings.Error + +// Error codes. Values here mostly overlap with native SQLite codes. +const ( + ErrBusy = 5 + errIoErr = 10 + errIoErrNotLeader = errIoErr | 32<<8 + errIoErrLeadershipLost = errIoErr | (33 << 8) +) + +// Option can be used to tweak driver parameters. +type Option func(*options) + +// NodeStore is a convenience alias of client.NodeStore. +type NodeStore = client.NodeStore + +// NodeInfo is a convenience alias of client.NodeInfo. +type NodeInfo = client.NodeInfo + +// DefaultNodeStore is a convenience alias of client.DefaultNodeStore. +var DefaultNodeStore = client.DefaultNodeStore + +// WithLogFunc sets a custom logging function. +func WithLogFunc(log client.LogFunc) Option { + return func(options *options) { + options.Log = log + } +} + +// DialFunc is a function that can be used to establish a network connection +// with a dqlite node. +type DialFunc = protocol.DialFunc + +// WithDialFunc sets a custom dial function. +func WithDialFunc(dial DialFunc) Option { + return func(options *options) { + options.Dial = protocol.DialFunc(dial) + } +} + +// WithConnectionTimeout sets the connection timeout. +// +// If not used, the default is 5 seconds. +func WithConnectionTimeout(timeout time.Duration) Option { + return func(options *options) { + options.ConnectionTimeout = timeout + } +} + +// WithConnectionBackoffFactor sets the exponential backoff factor for retrying +// failed connection attempts. +// +// If not used, the default is 50 milliseconds. +func WithConnectionBackoffFactor(factor time.Duration) Option { + return func(options *options) { + options.ConnectionBackoffFactor = factor + } +} + +// WithConnectionBackoffCap sets the maximum connection retry backoff value, +// (regardless of the backoff factor) for retrying failed connection attempts. +// +// If not used, the default is 1 second. +func WithConnectionBackoffCap(cap time.Duration) Option { + return func(options *options) { + options.ConnectionBackoffCap = cap + } +} + +// WithContext sets a global cancellation context. +func WithContext(context context.Context) Option { + return func(options *options) { + options.Context = context + } +} + +// WithContextTimeout sets the default client context timeout when no context +// deadline is provided. +// +// If not used, the default is 5 seconds. +func WithContextTimeout(timeout time.Duration) Option { + return func(options *options) { + options.ContextTimeout = timeout + } +} + +// NewDriver creates a new dqlite driver, which also implements the +// driver.Driver interface. +func New(store client.NodeStore, options ...Option) (*Driver, error) { + o := defaultOptions() + + for _, option := range options { + option(o) + } + + driver := &Driver{ + log: o.Log, + store: store, + context: o.Context, + connectionTimeout: o.ConnectionTimeout, + contextTimeout: o.ContextTimeout, + } + + driver.clientConfig.Dial = o.Dial + driver.clientConfig.AttemptTimeout = 5 * time.Second + driver.clientConfig.RetryStrategies = []strategy.Strategy{ + driverConnectionRetryStrategy( + o.ConnectionBackoffFactor, + o.ConnectionBackoffCap, + ), + } + + return driver, nil +} + +// Hold configuration options for a dqlite driver. +type options struct { + Log client.LogFunc + Dial protocol.DialFunc + ConnectionTimeout time.Duration + ContextTimeout time.Duration + ConnectionBackoffFactor time.Duration + ConnectionBackoffCap time.Duration + Context context.Context +} + +// Create a options object with sane defaults. +func defaultOptions() *options { + return &options{ + Log: client.DefaultLogFunc, + Dial: client.DefaultDialFunc, + ConnectionTimeout: 15 * time.Second, + ContextTimeout: 2 * time.Second, + ConnectionBackoffFactor: 50 * time.Millisecond, + ConnectionBackoffCap: time.Second, + Context: context.Background(), + } +} + +// Return a retry strategy with jittered exponential backoff, capped at the +// given amount of time. +func driverConnectionRetryStrategy(factor, cap time.Duration) strategy.Strategy { + backoff := backoff.BinaryExponential(factor) + + return func(attempt uint) bool { + if attempt > 0 { + duration := backoff(attempt) + if duration > cap { + duration = cap + } + time.Sleep(duration) + } + + return true + } +} + +// Open establishes a new connection to a SQLite database on the dqlite server. +// +// The given name must be a pure file name without any directory segment, +// dqlite will connect to a database with that name in its data directory. +// +// Query parameters are always valid except for "mode=memory". +// +// If this node is not the leader, or the leader is unknown an ErrNotLeader +// error is returned. +func (d *Driver) Open(uri string) (driver.Conn, error) { + ctx, cancel := context.WithTimeout(d.context, d.connectionTimeout) + defer cancel() + + // TODO: generate a client ID. + connector := protocol.NewConnector(0, d.store, d.clientConfig, d.log) + + conn := &Conn{ + log: d.log, + contextTimeout: d.contextTimeout, + } + + var err error + conn.protocol, err = connector.Connect(ctx) + if err != nil { + return nil, errors.Wrap(err, "failed to create dqlite connection") + } + conn.protocol.SetContextTimeout(d.contextTimeout) + + conn.request.Init(4096) + conn.response.Init(4096) + + defer conn.request.Reset() + defer conn.response.Reset() + + protocol.EncodeOpen(&conn.request, uri, 0, "volatile") + + if err := conn.protocol.Call(ctx, &conn.request, &conn.response); err != nil { + conn.protocol.Close() + return nil, errors.Wrap(err, "failed to open database") + } + + conn.id, err = protocol.DecodeDb(&conn.response) + if err != nil { + conn.protocol.Close() + return nil, errors.Wrap(err, "failed to open database") + } + + return conn, nil +} + +// SetContextTimeout sets the default client timeout when no context deadline +// is provided. +func (d *Driver) SetContextTimeout(timeout time.Duration) { + d.contextTimeout = timeout +} + +// ErrNoAvailableLeader is returned as root cause of Open() if there's no +// leader available in the cluster. +var ErrNoAvailableLeader = protocol.ErrNoAvailableLeader + +// Conn implements the sql.Conn interface. +type Conn struct { + log client.LogFunc + protocol *protocol.Protocol + request protocol.Message + response protocol.Message + id uint32 // Database ID. + contextTimeout time.Duration +} + +// PrepareContext returns a prepared statement, bound to this connection. +// context is for the preparation of the statement, it must not store the +// context within the statement itself. +func (c *Conn) PrepareContext(ctx context.Context, query string) (driver.Stmt, error) { + defer c.request.Reset() + defer c.response.Reset() + + stmt := &Stmt{ + protocol: c.protocol, + request: &c.request, + response: &c.response, + } + + protocol.EncodePrepare(&c.request, uint64(c.id), query) + + if err := c.protocol.Call(ctx, &c.request, &c.response); err != nil { + return nil, driverError(err) + } + + var err error + stmt.db, stmt.id, stmt.params, err = protocol.DecodeStmt(&c.response) + if err != nil { + return nil, driverError(err) + } + + return stmt, nil +} + +// Prepare returns a prepared statement, bound to this connection. +func (c *Conn) Prepare(query string) (driver.Stmt, error) { + return c.PrepareContext(context.Background(), query) +} + +// ExecContext is an optional interface that may be implemented by a Conn. +func (c *Conn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { + defer c.request.Reset() + defer c.response.Reset() + + protocol.EncodeExecSQL(&c.request, uint64(c.id), query, args) + + if err := c.protocol.Call(ctx, &c.request, &c.response); err != nil { + return nil, driverError(err) + } + + result, err := protocol.DecodeResult(&c.response) + if err != nil { + return nil, driverError(err) + } + + return &Result{result: result}, nil +} + +// Query is an optional interface that may be implemented by a Conn. +func (c *Conn) Query(query string, args []driver.Value) (driver.Rows, error) { + return c.QueryContext(context.Background(), query, valuesToNamedValues(args)) +} + +// QueryContext is an optional interface that may be implemented by a Conn. +func (c *Conn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { + defer c.request.Reset() + + protocol.EncodeQuerySQL(&c.request, uint64(c.id), query, args) + + if err := c.protocol.Call(ctx, &c.request, &c.response); err != nil { + c.response.Reset() + return nil, driverError(err) + } + + rows, err := protocol.DecodeRows(&c.response) + if err != nil { + c.response.Reset() + return nil, driverError(err) + } + + return &Rows{ctx: ctx, request: &c.request, response: &c.response, protocol: c.protocol, rows: rows}, nil +} + +// Exec is an optional interface that may be implemented by a Conn. +func (c *Conn) Exec(query string, args []driver.Value) (driver.Result, error) { + return c.ExecContext(context.Background(), query, valuesToNamedValues(args)) +} + +// Close invalidates and potentially stops any current prepared statements and +// transactions, marking this connection as no longer in use. +// +// Because the sql package maintains a free pool of connections and only calls +// Close when there's a surplus of idle connections, it shouldn't be necessary +// for drivers to do their own connection caching. +func (c *Conn) Close() error { + return c.protocol.Close() +} + +// BeginTx starts and returns a new transaction. If the context is canceled by +// the user the sql package will call Tx.Rollback before discarding and closing +// the connection. +// +// This must check opts.Isolation to determine if there is a set isolation +// level. If the driver does not support a non-default level and one is set or +// if there is a non-default isolation level that is not supported, an error +// must be returned. +// +// This must also check opts.ReadOnly to determine if the read-only value is +// true to either set the read-only transaction property if supported or return +// an error if it is not supported. +func (c *Conn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) { + if _, err := c.ExecContext(ctx, "BEGIN", nil); err != nil { + return nil, driverError(err) + } + + tx := &Tx{ + conn: c, + } + + return tx, nil +} + +// Begin starts and returns a new transaction. +// +// Deprecated: Drivers should implement ConnBeginTx instead (or additionally). +func (c *Conn) Begin() (driver.Tx, error) { + return c.BeginTx(context.Background(), driver.TxOptions{}) +} + +// Tx is a transaction. +type Tx struct { + conn *Conn +} + +// Commit the transaction. +func (tx *Tx) Commit() error { + ctx, cancel := context.WithTimeout(context.Background(), tx.conn.contextTimeout) + defer cancel() + + if _, err := tx.conn.ExecContext(ctx, "COMMIT", nil); err != nil { + return driverError(err) + } + + return nil +} + +// Rollback the transaction. +func (tx *Tx) Rollback() error { + ctx, cancel := context.WithTimeout(context.Background(), tx.conn.contextTimeout) + defer cancel() + + if _, err := tx.conn.ExecContext(ctx, "ROLLBACK", nil); err != nil { + return driverError(err) + } + + return nil +} + +// Stmt is a prepared statement. It is bound to a Conn and not +// used by multiple goroutines concurrently. +type Stmt struct { + protocol *protocol.Protocol + request *protocol.Message + response *protocol.Message + db uint32 + id uint32 + params uint64 +} + +// Close closes the statement. +func (s *Stmt) Close() error { + defer s.request.Reset() + defer s.response.Reset() + + protocol.EncodeFinalize(s.request, s.db, s.id) + + ctx := context.Background() + + if err := s.protocol.Call(ctx, s.request, s.response); err != nil { + return driverError(err) + } + + if err := protocol.DecodeEmpty(s.response); err != nil { + return driverError(err) + } + + return nil +} + +// NumInput returns the number of placeholder parameters. +func (s *Stmt) NumInput() int { + return int(s.params) +} + +// ExecContext executes a query that doesn't return rows, such +// as an INSERT or UPDATE. +// +// ExecContext must honor the context timeout and return when it is canceled. +func (s *Stmt) ExecContext(ctx context.Context, args []driver.NamedValue) (driver.Result, error) { + defer s.request.Reset() + defer s.response.Reset() + + protocol.EncodeExec(s.request, s.db, s.id, args) + + if err := s.protocol.Call(ctx, s.request, s.response); err != nil { + return nil, driverError(err) + } + + result, err := protocol.DecodeResult(s.response) + if err != nil { + return nil, driverError(err) + } + + return &Result{result: result}, nil +} + +// Exec executes a query that doesn't return rows, such +func (s *Stmt) Exec(args []driver.Value) (driver.Result, error) { + return s.ExecContext(context.Background(), valuesToNamedValues(args)) +} + +// QueryContext executes a query that may return rows, such as a +// SELECT. +// +// QueryContext must honor the context timeout and return when it is canceled. +func (s *Stmt) QueryContext(ctx context.Context, args []driver.NamedValue) (driver.Rows, error) { + defer s.request.Reset() + + // FIXME: this shouldn't be needed but we have hit a few panics + // probably due to the response object not being fully reset. + s.response.Reset() + + protocol.EncodeQuery(s.request, s.db, s.id, args) + + if err := s.protocol.Call(ctx, s.request, s.response); err != nil { + s.response.Reset() + return nil, driverError(err) + } + + rows, err := protocol.DecodeRows(s.response) + if err != nil { + s.response.Reset() + return nil, driverError(err) + } + + return &Rows{ctx: ctx, request: s.request, response: s.response, protocol: s.protocol, rows: rows}, nil +} + +// Query executes a query that may return rows, such as a +func (s *Stmt) Query(args []driver.Value) (driver.Rows, error) { + return s.QueryContext(context.Background(), valuesToNamedValues(args)) +} + +// Result is the result of a query execution. +type Result struct { + result protocol.Result +} + +// LastInsertId returns the database's auto-generated ID +// after, for example, an INSERT into a table with primary +// key. +func (r *Result) LastInsertId() (int64, error) { + return int64(r.result.LastInsertID), nil +} + +// RowsAffected returns the number of rows affected by the +// query. +func (r *Result) RowsAffected() (int64, error) { + return int64(r.result.RowsAffected), nil +} + +// Rows is an iterator over an executed query's results. +type Rows struct { + ctx context.Context + protocol *protocol.Protocol + request *protocol.Message + response *protocol.Message + rows protocol.Rows + consumed bool +} + +// Columns returns the names of the columns. The number of +// columns of the result is inferred from the length of the +// slice. If a particular column name isn't known, an empty +// string should be returned for that entry. +func (r *Rows) Columns() []string { + return r.rows.Columns +} + +// Close closes the rows iterator. +func (r *Rows) Close() error { + err := r.rows.Close() + + // If we consumed the whole result set, there's nothing to do as + // there's no pending response from the server. + if r.consumed { + return nil + } + + // If there is was a single-response result set, we're done. + if err == io.EOF { + return nil + } + + // Let's issue an interrupt request and wait until we get an empty + // response, signalling that the query was interrupted. + if err := r.protocol.Interrupt(r.ctx, r.request, r.response); err != nil { + return driverError(err) + } + + return nil +} + +// Next is called to populate the next row of data into +// the provided slice. The provided slice will be the same +// size as the Columns() are wide. +// +// Next should return io.EOF when there are no more rows. +func (r *Rows) Next(dest []driver.Value) error { + err := r.rows.Next(dest) + + if err == protocol.ErrRowsPart { + r.rows.Close() + if err := r.protocol.More(r.ctx, r.response); err != nil { + return driverError(err) + } + rows, err := protocol.DecodeRows(r.response) + if err != nil { + return driverError(err) + } + r.rows = rows + return r.rows.Next(dest) + } + + if err == io.EOF { + r.consumed = true + } + + return err +} + +// ColumnTypeScanType implements RowsColumnTypeScanType. +func (r *Rows) ColumnTypeScanType(i int) reflect.Type { + // column := sql.NewColumn(r.rows, i) + + // typ, err := r.protocol.ColumnTypeScanType(context.Background(), column) + // if err != nil { + // return nil + // } + + // return typ.DriverType() + return nil +} + +// ColumnTypeDatabaseTypeName implements RowsColumnTypeDatabaseTypeName. +func (r *Rows) ColumnTypeDatabaseTypeName(i int) string { + // column := sql.NewColumn(r.rows, i) + + // typeName, err := r.protocol.ColumnTypeDatabaseTypeName(context.Background(), column) + // if err != nil { + // return "" + // } + + // return typeName.Value + return "" +} + +// Convert a driver.Value slice into a driver.NamedValue slice. +func valuesToNamedValues(args []driver.Value) []driver.NamedValue { + namedValues := make([]driver.NamedValue, len(args)) + for i, value := range args { + namedValues[i] = driver.NamedValue{ + Ordinal: i + 1, + Value: value, + } + } + return namedValues +} + +func driverError(err error) error { + switch err := errors.Cause(err).(type) { + case *net.OpError: + return driver.ErrBadConn + case protocol.ErrRequest: + switch err.Code { + case errIoErrNotLeader: + fallthrough + case errIoErrLeadershipLost: + return driver.ErrBadConn + default: + return Error{ + Code: int(err.Code), + Message: err.Description, + } + } + } + return err +} + +func init() { + err := bindings.Init() + if err != nil { + panic(errors.Wrap(err, "failed to initialize dqlite")) + } +} diff --git a/vendor/github.com/canonical/go-dqlite/internal/bindings/build.go b/vendor/github.com/canonical/go-dqlite/internal/bindings/build.go new file mode 100644 index 0000000000..bc64ff209d --- /dev/null +++ b/vendor/github.com/canonical/go-dqlite/internal/bindings/build.go @@ -0,0 +1,6 @@ +package bindings + +/* +#cgo linux LDFLAGS: -lsqlite3 -lraft -lco -ldqlite +*/ +import "C" diff --git a/vendor/github.com/canonical/go-dqlite/internal/bindings/errors.go b/vendor/github.com/canonical/go-dqlite/internal/bindings/errors.go new file mode 100644 index 0000000000..1fe3caacb6 --- /dev/null +++ b/vendor/github.com/canonical/go-dqlite/internal/bindings/errors.go @@ -0,0 +1,19 @@ +package bindings + +/* +#include +*/ +import "C" + +// Error holds information about a SQLite error. +type Error struct { + Code int + Message string +} + +func (e Error) Error() string { + if e.Message != "" { + return e.Message + } + return C.GoString(C.sqlite3_errstr(C.int(e.Code))) +} diff --git a/vendor/github.com/canonical/go-dqlite/internal/bindings/server.go b/vendor/github.com/canonical/go-dqlite/internal/bindings/server.go new file mode 100644 index 0000000000..1703518177 --- /dev/null +++ b/vendor/github.com/canonical/go-dqlite/internal/bindings/server.go @@ -0,0 +1,252 @@ +package bindings + +/* +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#define EMIT_BUF_LEN 1024 + +typedef unsigned long long nanoseconds_t; + +// Duplicate a file descriptor and prevent it from being cloned into child processes. +static int dupCloexec(int oldfd) { + int newfd = -1; + + newfd = dup(oldfd); + if (newfd < 0) { + return -1; + } + + if (fcntl(newfd, F_SETFD, FD_CLOEXEC) < 0) { + return -1; + } + + return newfd; +} + +// C to Go trampoline for custom connect function. +int connectWithDial(uintptr_t handle, char *address, int *fd); + +// Wrapper to call the Go trampoline. +static int connectTrampoline(void *data, const char *address, int *fd) { + uintptr_t handle = (uintptr_t)(data); + return connectWithDial(handle, (char*)address, fd); +} + +// Configure a custom connect function. +static int configConnectFunc(dqlite_node *t, uintptr_t handle) { + return dqlite_node_set_connect_func(t, connectTrampoline, (void*)handle); +} + +static int initializeSQLite() +{ + int rc; + + // Configure SQLite for single-thread mode. This is a global config. + rc = sqlite3_config(SQLITE_CONFIG_SINGLETHREAD); + if (rc != SQLITE_OK) { + assert(rc == SQLITE_MISUSE); + return DQLITE_MISUSE; + } + return 0; +} + +static dqlite_node_info *makeInfos(int n) { + return calloc(n, sizeof(dqlite_node_info)); +} + +static void setInfo(dqlite_node_info *infos, unsigned i, unsigned id, const char *address) { + dqlite_node_info *info = &infos[i]; + info->id = id; + info->address = address; +} + +*/ +import "C" +import ( + "context" + "fmt" + "net" + "os" + "sync" + "time" + "unsafe" + + "github.com/canonical/go-dqlite/internal/protocol" +) + +type Node C.dqlite_node + +// Init initializes dqlite global state. +func Init() error { + // FIXME: ignore SIGPIPE, see https://github.com/joyent/libuv/issues/1254 + C.signal(C.SIGPIPE, C.SIG_IGN) + // Don't enable single thread mode when running tests. TODO: find a + // better way to expose this functionality. + if os.Getenv("GO_DQLITE_MULTITHREAD") == "1" { + return nil + } + if rc := C.initializeSQLite(); rc != 0 { + return fmt.Errorf("%d", rc) + } + return nil +} + +// NewNode creates a new Node instance. +func NewNode(id uint64, address string, dir string) (*Node, error) { + var server *C.dqlite_node + cid := C.unsigned(id) + + caddress := C.CString(address) + defer C.free(unsafe.Pointer(caddress)) + + cdir := C.CString(dir) + defer C.free(unsafe.Pointer(cdir)) + + if rc := C.dqlite_node_create(cid, caddress, cdir, &server); rc != 0 { + return nil, fmt.Errorf("failed to create task object") + } + + return (*Node)(unsafe.Pointer(server)), nil +} + +func (s *Node) SetDialFunc(dial protocol.DialFunc) error { + server := (*C.dqlite_node)(unsafe.Pointer(s)) + connectLock.Lock() + defer connectLock.Unlock() + connectIndex++ + connectRegistry[connectIndex] = dial + if rc := C.configConnectFunc(server, connectIndex); rc != 0 { + return fmt.Errorf("failed to set connect func") + } + return nil +} + +func (s *Node) SetBindAddress(address string) error { + server := (*C.dqlite_node)(unsafe.Pointer(s)) + caddress := C.CString(address) + defer C.free(unsafe.Pointer(caddress)) + if rc := C.dqlite_node_set_bind_address(server, caddress); rc != 0 { + return fmt.Errorf("failed to set bind address") + } + return nil +} + +func (s *Node) SetNetworkLatency(nanoseconds uint64) error { + server := (*C.dqlite_node)(unsafe.Pointer(s)) + cnanoseconds := C.nanoseconds_t(nanoseconds) + if rc := C.dqlite_node_set_network_latency(server, cnanoseconds); rc != 0 { + return fmt.Errorf("failed to set network latency") + } + return nil +} + +func (s *Node) GetBindAddress() string { + server := (*C.dqlite_node)(unsafe.Pointer(s)) + return C.GoString(C.dqlite_node_get_bind_address(server)) +} + +func (s *Node) Start() error { + server := (*C.dqlite_node)(unsafe.Pointer(s)) + if rc := C.dqlite_node_start(server); rc != 0 { + return fmt.Errorf("failed to start task") + } + return nil +} + +func (s *Node) Stop() error { + server := (*C.dqlite_node)(unsafe.Pointer(s)) + if rc := C.dqlite_node_stop(server); rc != 0 { + return fmt.Errorf("task stopped with error code %d", rc) + } + return nil +} + +// Close the server releasing all used resources. +func (s *Node) Close() { + server := (*C.dqlite_node)(unsafe.Pointer(s)) + C.dqlite_node_destroy(server) +} + +func (s *Node) Recover(cluster []protocol.NodeInfo) error { + server := (*C.dqlite_node)(unsafe.Pointer(s)) + n := C.int(len(cluster)) + infos := C.makeInfos(n) + defer C.free(unsafe.Pointer(infos)) + for i, info := range cluster { + cid := C.unsigned(info.ID) + caddress := C.CString(info.Address) + defer C.free(unsafe.Pointer(caddress)) + C.setInfo(infos, C.unsigned(i), cid, caddress) + } + if rc := C.dqlite_node_recover(server, infos, n); rc != 0 { + return fmt.Errorf("recover failed with error code %d", rc) + } + return nil +} + +// Extract the underlying socket from a connection. +func connToSocket(conn net.Conn) (C.int, error) { + file, err := conn.(fileConn).File() + if err != nil { + return C.int(-1), err + } + + fd1 := C.int(file.Fd()) + + // Duplicate the file descriptor, in order to prevent Go's finalizer to + // close it. + fd2 := C.dupCloexec(fd1) + if fd2 < 0 { + return C.int(-1), fmt.Errorf("failed to dup socket fd") + } + + conn.Close() + + return fd2, nil +} + +// Interface that net.Conn must implement in order to extract the underlying +// file descriptor. +type fileConn interface { + File() (*os.File, error) +} + +//export connectWithDial +func connectWithDial(handle C.uintptr_t, address *C.char, fd *C.int) C.int { + connectLock.Lock() + defer connectLock.Unlock() + dial := connectRegistry[handle] + // TODO: make timeout customizable. + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + conn, err := dial(ctx, C.GoString(address)) + if err != nil { + return C.RAFT_NOCONNECTION + } + socket, err := connToSocket(conn) + if err != nil { + return C.RAFT_NOCONNECTION + } + *fd = socket + return C.int(0) +} + +// Use handles to avoid passing Go pointers to C. +var connectRegistry = make(map[C.uintptr_t]protocol.DialFunc) +var connectIndex C.uintptr_t = 100 +var connectLock = sync.Mutex{} + +// ErrNodeStopped is returned by Node.Handle() is the server was stopped. +var ErrNodeStopped = fmt.Errorf("server was stopped") + +// To compare bool values. +var cfalse C.bool diff --git a/vendor/github.com/canonical/go-dqlite/internal/logging/func.go b/vendor/github.com/canonical/go-dqlite/internal/logging/func.go new file mode 100644 index 0000000000..57e0525fa8 --- /dev/null +++ b/vendor/github.com/canonical/go-dqlite/internal/logging/func.go @@ -0,0 +1,26 @@ +package logging + +import ( + "fmt" + "testing" +) + +// Func is a function that can be used for logging. +type Func func(Level, string, ...interface{}) + +// Test returns a logging function that forwards messages to the test logger. +func Test(t *testing.T) Func { + return func(l Level, format string, a ...interface{}) { + format = fmt.Sprintf("%s: %s", l.String(), format) + t.Logf(format, a...) + } +} + +// Stdout returns a logging function that prints log messages on standard +// output. +func Stdout() Func { + return func(l Level, format string, a ...interface{}) { + format = fmt.Sprintf("%s: %s\n", l.String(), format) + fmt.Printf(format, a...) + } +} diff --git a/vendor/github.com/canonical/go-dqlite/internal/logging/level.go b/vendor/github.com/canonical/go-dqlite/internal/logging/level.go new file mode 100644 index 0000000000..0f412c2970 --- /dev/null +++ b/vendor/github.com/canonical/go-dqlite/internal/logging/level.go @@ -0,0 +1,27 @@ +package logging + +// Level defines the logging level. +type Level int + +// Available logging levels. +const ( + Debug Level = iota + Info + Warn + Error +) + +func (l Level) String() string { + switch l { + case Debug: + return "DEBUG" + case Info: + return "INFO" + case Warn: + return "WARN" + case Error: + return "ERROR" + default: + return "UNKNOWN" + } +} diff --git a/vendor/github.com/canonical/go-dqlite/internal/protocol/buffer.go b/vendor/github.com/canonical/go-dqlite/internal/protocol/buffer.go new file mode 100644 index 0000000000..356b96f4d2 --- /dev/null +++ b/vendor/github.com/canonical/go-dqlite/internal/protocol/buffer.go @@ -0,0 +1,11 @@ +package protocol + +// Buffer for reading responses or writing requests. +type buffer struct { + Bytes []byte + Offset int +} + +func (b *buffer) Advance(amount int) { + b.Offset += amount +} diff --git a/vendor/github.com/canonical/go-dqlite/internal/protocol/config.go b/vendor/github.com/canonical/go-dqlite/internal/protocol/config.go new file mode 100644 index 0000000000..9b89d57f63 --- /dev/null +++ b/vendor/github.com/canonical/go-dqlite/internal/protocol/config.go @@ -0,0 +1,14 @@ +package protocol + +import ( + "time" + + "github.com/Rican7/retry/strategy" +) + +// Config holds various configuration parameters for a dqlite client. +type Config struct { + Dial DialFunc // Network dialer. + AttemptTimeout time.Duration // Timeout for each individual Dial attempt. + RetryStrategies []strategy.Strategy // Strategies used for retrying to connect to a leader. +} diff --git a/vendor/github.com/canonical/go-dqlite/internal/protocol/connector.go b/vendor/github.com/canonical/go-dqlite/internal/protocol/connector.go new file mode 100644 index 0000000000..583f819051 --- /dev/null +++ b/vendor/github.com/canonical/go-dqlite/internal/protocol/connector.go @@ -0,0 +1,250 @@ +package protocol + +import ( + "context" + "encoding/binary" + "fmt" + "io" + "net" + + "github.com/Rican7/retry" + "github.com/canonical/go-dqlite/internal/logging" + "github.com/pkg/errors" +) + +// DialFunc is a function that can be used to establish a network connection. +type DialFunc func(context.Context, string) (net.Conn, error) + +// Connector is in charge of creating a dqlite SQL client connected to the +// current leader of a cluster. +type Connector struct { + id uint64 // Conn ID to use when registering against the server. + store NodeStore // Used to get and update current cluster servers. + config Config // Connection parameters. + log logging.Func // Logging function. +} + +// NewConnector returns a new connector that can be used by a dqlite driver to +// create new clients connected to a leader dqlite server. +func NewConnector(id uint64, store NodeStore, config Config, log logging.Func) *Connector { + connector := &Connector{ + id: id, + store: store, + config: config, + log: log, + } + + return connector +} + +// Connect finds the leader server and returns a connection to it. +// +// If the connector is stopped before a leader is found, nil is returned. +func (c *Connector) Connect(ctx context.Context) (*Protocol, error) { + var protocol *Protocol + + // The retry strategy should be configured to retry indefinitely, until + // the given context is done. + err := retry.Retry(func(attempt uint) error { + log := func(l logging.Level, format string, a ...interface{}) { + format += fmt.Sprintf(" attempt=%d", attempt) + c.log(l, fmt.Sprintf(format, a...)) + } + + select { + case <-ctx.Done(): + // Stop retrying + return nil + default: + } + + var err error + protocol, err = c.connectAttemptAll(ctx, log) + if err != nil { + log(logging.Debug, "connection failed err=%v", err) + return err + } + + return nil + }, c.config.RetryStrategies...) + + if err != nil { + // The retry strategy should never give up until success or + // context expiration. + panic("connect retry aborted unexpectedly") + } + + if ctx.Err() != nil { + return nil, ErrNoAvailableLeader + } + + return protocol, nil +} + +// Make a single attempt to establish a connection to the leader server trying +// all addresses available in the store. +func (c *Connector) connectAttemptAll(ctx context.Context, log logging.Func) (*Protocol, error) { + servers, err := c.store.Get(ctx) + if err != nil { + return nil, errors.Wrap(err, "failed to get cluster servers") + } + + // Make an attempt for each address until we find the leader. + for _, server := range servers { + log := func(l logging.Level, format string, a ...interface{}) { + format += fmt.Sprintf(" address=%s", server.Address) + log(l, fmt.Sprintf(format, a...)) + } + + ctx, cancel := context.WithTimeout(ctx, c.config.AttemptTimeout) + defer cancel() + + version := VersionOne + protocol, leader, err := c.connectAttemptOne(ctx, server.Address, version) + if err == errBadProtocol { + version = VersionLegacy + protocol, leader, err = c.connectAttemptOne(ctx, server.Address, version) + } + if err != nil { + // This server is unavailable, try with the next target. + log(logging.Debug, "server connection failed err=%v", err) + continue + } + if protocol != nil { + // We found the leader + log(logging.Info, "connected") + return protocol, nil + } + if leader == "" { + // This server does not know who the current leader is, + // try with the next target. + continue + } + + // If we get here, it means this server reported that another + // server is the leader, let's close the connection to this + // server and try with the suggested one. + //logger = logger.With(zap.String("leader", leader)) + protocol, leader, err = c.connectAttemptOne(ctx, leader, version) + if err != nil { + // The leader reported by the previous server is + // unavailable, try with the next target. + //logger.Info("leader server connection failed", zap.String("err", err.Error())) + continue + } + if protocol == nil { + // The leader reported by the target server does not consider itself + // the leader, try with the next target. + //logger.Info("reported leader server is not the leader") + continue + } + log(logging.Info, "connected") + return protocol, nil + } + + return nil, ErrNoAvailableLeader +} + +// Connect establishes a connection with a dqlite node. +func Connect(ctx context.Context, dial DialFunc, address string, version uint64) (*Protocol, error) { + // Establish the connection. + conn, err := dial(ctx, address) + if err != nil { + return nil, errors.Wrap(err, "failed to establish network connection") + } + + // Latest protocol version. + protocol := make([]byte, 8) + binary.LittleEndian.PutUint64(protocol, version) + + // Perform the protocol handshake. + n, err := conn.Write(protocol) + if err != nil { + conn.Close() + return nil, errors.Wrap(err, "failed to send handshake") + } + if n != 8 { + conn.Close() + return nil, errors.Wrap(io.ErrShortWrite, "failed to send handshake") + } + + return NewProtocol(version, conn), nil +} + +// Connect to the given dqlite server and check if it's the leader. +// +// Return values: +// +// - Any failure is hit: -> nil, "", err +// - Target not leader and no leader known: -> nil, "", nil +// - Target not leader and leader known: -> nil, leader, nil +// - Target is the leader: -> server, "", nil +// +func (c *Connector) connectAttemptOne(ctx context.Context, address string, version uint64) (*Protocol, string, error) { + protocol, err := Connect(ctx, c.config.Dial, address, version) + if err != nil { + return nil, "", err + } + + // Send the initial Leader request. + request := Message{} + request.Init(16) + response := Message{} + response.Init(512) + + EncodeLeader(&request) + + if err := protocol.Call(ctx, &request, &response); err != nil { + protocol.Close() + cause := errors.Cause(err) + // Best-effort detection of a pre-1.0 dqlite node: when sent + // version 1 it should close the connection immediately. + if _, ok := cause.(*net.OpError); ok || cause == io.EOF { + return nil, "", errBadProtocol + } + + return nil, "", errors.Wrap(err, "failed to send Leader request") + } + + _, leader, err := DecodeNodeCompat(protocol, &response) + if err != nil { + protocol.Close() + return nil, "", errors.Wrap(err, "failed to parse Node response") + } + + switch leader { + case "": + // Currently this server does not know about any leader. + protocol.Close() + return nil, "", nil + case address: + // This server is the leader, register ourselves and return. + request.Reset() + response.Reset() + + EncodeClient(&request, c.id) + + if err := protocol.Call(ctx, &request, &response); err != nil { + protocol.Close() + return nil, "", errors.Wrap(err, "failed to send Conn request") + } + + _, err := DecodeWelcome(&response) + if err != nil { + protocol.Close() + return nil, "", errors.Wrap(err, "failed to parse Welcome response") + } + + // TODO: enable heartbeat + // protocol.heartbeatTimeout = time.Duration(heartbeatTimeout) * time.Millisecond + //go protocol.heartbeat() + + return protocol, "", nil + default: + // This server claims to know who the current leader is. + protocol.Close() + return nil, leader, nil + } +} + +var errBadProtocol = fmt.Errorf("bad protocol") diff --git a/vendor/github.com/canonical/go-dqlite/internal/protocol/constants.go b/vendor/github.com/canonical/go-dqlite/internal/protocol/constants.go new file mode 100644 index 0000000000..3f7b8994a2 --- /dev/null +++ b/vendor/github.com/canonical/go-dqlite/internal/protocol/constants.go @@ -0,0 +1,58 @@ +package protocol + +// VersionOne is version 1 of the server protocol. +const VersionOne = uint64(1) + +// VersionLegacy is the pre 1.0 dqlite server protocol version. +const VersionLegacy = uint64(0x86104dd760433fe5) + +// SQLite datatype codes +const ( + Integer = 1 + Float = 2 + Text = 3 + Blob = 4 + Null = 5 +) + +// Special data types for time values. +const ( + UnixTime = 9 + ISO8601 = 10 + Boolean = 11 +) + +// Request types. +const ( + RequestLeader = 0 + RequestClient = 1 + RequestHeartbeat = 2 + RequestOpen = 3 + RequestPrepare = 4 + RequestExec = 5 + RequestQuery = 6 + RequestFinalize = 7 + RequestExecSQL = 8 + RequestQuerySQL = 9 + RequestInterrupt = 10 + RequestJoin = 12 + RequestPromote = 13 + RequestRemove = 14 + RequestDump = 15 + RequestCluster = 16 +) + +// Response types. +const ( + ResponseFailure = 0 + ResponseNode = 1 + ResponseNodeLegacy = 1 + ResponseWelcome = 2 + ResponseNodes = 3 + ResponseDb = 4 + ResponseStmt = 5 + ResponseResult = 6 + ResponseRows = 7 + ResponseEmpty = 8 + ResponseFiles = 9 +) diff --git a/vendor/github.com/canonical/go-dqlite/internal/protocol/dial.go b/vendor/github.com/canonical/go-dqlite/internal/protocol/dial.go new file mode 100644 index 0000000000..1f1e9b89aa --- /dev/null +++ b/vendor/github.com/canonical/go-dqlite/internal/protocol/dial.go @@ -0,0 +1,20 @@ +package protocol + +import ( + "context" + "net" +) + +// TCPDial is a dial function using plain TCP to establish the network +// connection. +func TCPDial(ctx context.Context, address string) (net.Conn, error) { + dialer := net.Dialer{} + return dialer.DialContext(ctx, "tcp", address) +} + +// UnixDial is a dial function using Unix sockets to establish the network +// connection. +func UnixDial(ctx context.Context, address string) (net.Conn, error) { + dialer := net.Dialer{} + return dialer.DialContext(ctx, "unix", address) +} diff --git a/vendor/github.com/canonical/go-dqlite/internal/protocol/errors.go b/vendor/github.com/canonical/go-dqlite/internal/protocol/errors.go new file mode 100644 index 0000000000..4e5875aede --- /dev/null +++ b/vendor/github.com/canonical/go-dqlite/internal/protocol/errors.go @@ -0,0 +1,29 @@ +package protocol + +import ( + "fmt" +) + +// Client errors. +var ( + ErrNoAvailableLeader = fmt.Errorf("no available dqlite leader server found") + errStop = fmt.Errorf("connector was stopped") + errStaleLeader = fmt.Errorf("server has lost leadership") + errNotClustered = fmt.Errorf("server is not clustered") + errNegativeRead = fmt.Errorf("reader returned negative count from Read") + errMessageEOF = fmt.Errorf("message eof") +) + +// ErrRequest is returned in case of request failure. +type ErrRequest struct { + Code uint64 + Description string +} + +func (e ErrRequest) Error() string { + return fmt.Sprintf("%s (%d)", e.Description, e.Code) +} + +// ErrRowsPart is returned when the first batch of a multi-response result +// batch is done. +var ErrRowsPart = fmt.Errorf("not all rows were returned in this response") diff --git a/vendor/github.com/canonical/go-dqlite/internal/protocol/message.go b/vendor/github.com/canonical/go-dqlite/internal/protocol/message.go new file mode 100644 index 0000000000..6eb31c01e6 --- /dev/null +++ b/vendor/github.com/canonical/go-dqlite/internal/protocol/message.go @@ -0,0 +1,660 @@ +package protocol + +import ( + "bytes" + "database/sql/driver" + "encoding/binary" + "fmt" + "io" + "math" + "strings" + "time" +) + +// NamedValues is a type alias of a slice of driver.NamedValue. It's used by +// schema.sh to generate encoding logic for statement parameters. +type NamedValues = []driver.NamedValue + +// Nodes is a type alias of a slice of NodeInfo. It's used by schema.sh to +// generate decoding logic for the heartbeat response. +type Nodes []NodeInfo + +// Message holds data about a single request or response. +type Message struct { + words uint32 + mtype uint8 + flags uint8 + extra uint16 + header []byte // Statically allocated header buffer + body1 buffer // Statically allocated body data, using bytes + body2 buffer // Dynamically allocated body data +} + +// Init initializes the message using the given size of the statically +// allocated buffer (i.e. a buffer which is re-used across requests or +// responses encoded or decoded using this message object). +func (m *Message) Init(staticSize int) { + if (staticSize % messageWordSize) != 0 { + panic("static size is not aligned to word boundary") + } + m.header = make([]byte, messageHeaderSize) + m.body1.Bytes = make([]byte, staticSize) + m.Reset() +} + +// Reset the state of the message so it can be used to encode or decode again. +func (m *Message) Reset() { + m.words = 0 + m.mtype = 0 + m.flags = 0 + m.extra = 0 + for i := 0; i < messageHeaderSize; i++ { + m.header[i] = 0 + } + m.body1.Offset = 0 + m.body2.Bytes = nil + m.body2.Offset = 0 +} + +// Append a byte slice to the message. +func (m *Message) putBlob(v []byte) { + size := len(v) + m.putUint64(uint64(size)) + + pad := 0 + if (size % messageWordSize) != 0 { + // Account for padding + pad = messageWordSize - (size % messageWordSize) + size += pad + } + + b := m.bufferForPut(size) + defer b.Advance(size) + + // Copy the bytes into the buffer. + offset := b.Offset + copy(b.Bytes[offset:], v) + offset += len(v) + + // Add padding + for i := 0; i < pad; i++ { + b.Bytes[offset] = 0 + offset++ + } +} + +// Append a string to the message. +func (m *Message) putString(v string) { + size := len(v) + 1 + pad := 0 + if (size % messageWordSize) != 0 { + // Account for padding + pad = messageWordSize - (size % messageWordSize) + size += pad + } + + b := m.bufferForPut(size) + defer b.Advance(size) + + // Copy the string bytes into the buffer. + offset := b.Offset + copy(b.Bytes[offset:], v) + offset += len(v) + + // Add a nul byte + b.Bytes[offset] = 0 + offset++ + + // Add padding + for i := 0; i < pad; i++ { + b.Bytes[offset] = 0 + offset++ + } +} + +// Append a byte to the message. +func (m *Message) putUint8(v uint8) { + b := m.bufferForPut(1) + defer b.Advance(1) + + b.Bytes[b.Offset] = v +} + +// Append a 2-byte word to the message. +func (m *Message) putUint16(v uint16) { + b := m.bufferForPut(2) + defer b.Advance(2) + + binary.LittleEndian.PutUint16(b.Bytes[b.Offset:], v) +} + +// Append a 4-byte word to the message. +func (m *Message) putUint32(v uint32) { + b := m.bufferForPut(4) + defer b.Advance(4) + + binary.LittleEndian.PutUint32(b.Bytes[b.Offset:], v) +} + +// Append an 8-byte word to the message. +func (m *Message) putUint64(v uint64) { + b := m.bufferForPut(8) + defer b.Advance(8) + + binary.LittleEndian.PutUint64(b.Bytes[b.Offset:], v) +} + +// Append a signed 8-byte word to the message. +func (m *Message) putInt64(v int64) { + b := m.bufferForPut(8) + defer b.Advance(8) + + binary.LittleEndian.PutUint64(b.Bytes[b.Offset:], uint64(v)) +} + +// Append a floating point number to the message. +func (m *Message) putFloat64(v float64) { + b := m.bufferForPut(8) + defer b.Advance(8) + + binary.LittleEndian.PutUint64(b.Bytes[b.Offset:], math.Float64bits(v)) +} + +// Encode the given driver values as binding parameters. +func (m *Message) putNamedValues(values NamedValues) { + n := uint8(len(values)) // N of params + if n == 0 { + return + } + + m.putUint8(n) + + for i := range values { + if values[i].Ordinal != i+1 { + panic("unexpected ordinal") + } + + switch values[i].Value.(type) { + case int64: + m.putUint8(Integer) + case float64: + m.putUint8(Float) + case bool: + m.putUint8(Boolean) + case []byte: + m.putUint8(Blob) + case string: + m.putUint8(Text) + case nil: + m.putUint8(Null) + case time.Time: + m.putUint8(ISO8601) + default: + panic("unsupported value type") + } + } + + b := m.bufferForPut(1) + + if trailing := b.Offset % messageWordSize; trailing != 0 { + // Skip padding bytes + b.Advance(messageWordSize - trailing) + } + + for i := range values { + switch v := values[i].Value.(type) { + case int64: + m.putInt64(v) + case float64: + m.putFloat64(v) + case bool: + if v { + m.putUint64(1) + } else { + m.putUint64(0) + } + case []byte: + m.putBlob(v) + case string: + m.putString(v) + case nil: + m.putInt64(0) + case time.Time: + timestamp := v.Format(iso8601Formats[0]) + m.putString(timestamp) + default: + panic("unsupported value type") + } + } + +} + +// Finalize the message by setting the message type and the number +// of words in the body (calculated from the body size). +func (m *Message) putHeader(mtype uint8) { + if m.body1.Offset <= 0 { + panic("static offset is not positive") + } + + if (m.body1.Offset % messageWordSize) != 0 { + panic("static body is not aligned") + } + + m.mtype = mtype + m.flags = 0 + m.extra = 0 + + m.words = uint32(m.body1.Offset) / messageWordSize + + if m.body2.Bytes == nil { + m.finalize() + return + } + + if m.body2.Offset <= 0 { + panic("dynamic offset is not positive") + } + + if (m.body2.Offset % messageWordSize) != 0 { + panic("dynamic body is not aligned") + } + + m.words += uint32(m.body2.Offset) / messageWordSize + + m.finalize() +} + +func (m *Message) finalize() { + if m.words == 0 { + panic("empty message body") + } + + binary.LittleEndian.PutUint32(m.header[0:], m.words) + m.header[4] = m.mtype + m.header[5] = m.flags + binary.LittleEndian.PutUint16(m.header[6:], m.extra) +} + +func (m *Message) bufferForPut(size int) *buffer { + if m.body2.Bytes != nil { + if (m.body2.Offset + size) > len(m.body2.Bytes) { + // Grow body2. + // + // TODO: find a good grow strategy. + bytes := make([]byte, m.body2.Offset+size) + copy(bytes, m.body2.Bytes) + m.body2.Bytes = bytes + } + + return &m.body2 + } + + if (m.body1.Offset + size) > len(m.body1.Bytes) { + m.body2.Bytes = make([]byte, size) + m.body2.Offset = 0 + + return &m.body2 + } + + return &m.body1 +} + +// Return the message type and its flags. +func (m *Message) getHeader() (uint8, uint8) { + return m.mtype, m.flags +} + +// Read a string from the message body. +func (m *Message) getString() string { + b := m.bufferForGet() + + index := bytes.IndexByte(b.Bytes[b.Offset:], 0) + if index == -1 { + // Check if the string overflows in the dynamic buffer. + if b == &m.body1 && m.body2.Bytes != nil { + // Assert that this is the first read of the dynamic buffer. + if m.body2.Offset != 0 { + panic("static buffer read after dynamic buffer one") + } + index = bytes.IndexByte(m.body2.Bytes[0:], 0) + if index != -1 { + // We found the trailing part of the string. + data := b.Bytes[b.Offset:] + data = append(data, m.body2.Bytes[0:index]...) + + index++ + + if trailing := index % messageWordSize; trailing != 0 { + // Account for padding, moving index to the next word boundary. + index += messageWordSize - trailing + } + + m.body1.Offset = len(m.body1.Bytes) + m.body2.Advance(index) + + return string(data) + } + } + panic("no string found") + } + s := string(b.Bytes[b.Offset : b.Offset+index]) + + index++ + + if trailing := index % messageWordSize; trailing != 0 { + // Account for padding, moving index to the next word boundary. + index += messageWordSize - trailing + } + + b.Advance(index) + + return s +} + +func (m *Message) getBlob() []byte { + size := m.getUint64() + data := make([]byte, size) + for i := range data { + data[i] = m.getUint8() + } + pad := 0 + if (size % messageWordSize) != 0 { + // Account for padding + pad = int(messageWordSize - (size % messageWordSize)) + } + // Consume padding + for i := 0; i < pad; i++ { + m.getUint8() + } + return data +} + +// Read a byte from the message body. +func (m *Message) getUint8() uint8 { + b := m.bufferForGet() + defer b.Advance(1) + + return b.Bytes[b.Offset] +} + +// Read a 2-byte word from the message body. +func (m *Message) getUint16() uint16 { + b := m.bufferForGet() + defer b.Advance(2) + + return binary.LittleEndian.Uint16(b.Bytes[b.Offset:]) +} + +// Read a 4-byte word from the message body. +func (m *Message) getUint32() uint32 { + b := m.bufferForGet() + defer b.Advance(4) + + return binary.LittleEndian.Uint32(b.Bytes[b.Offset:]) +} + +// Read reads an 8-byte word from the message body. +func (m *Message) getUint64() uint64 { + b := m.bufferForGet() + defer b.Advance(8) + + return binary.LittleEndian.Uint64(b.Bytes[b.Offset:]) +} + +// Read a signed 8-byte word from the message body. +func (m *Message) getInt64() int64 { + b := m.bufferForGet() + defer b.Advance(8) + + return int64(binary.LittleEndian.Uint64(b.Bytes[b.Offset:])) +} + +// Read a floating point number from the message body. +func (m *Message) getFloat64() float64 { + b := m.bufferForGet() + defer b.Advance(8) + + return math.Float64frombits(binary.LittleEndian.Uint64(b.Bytes[b.Offset:])) +} + +// Decode a list of server objects from the message body. +func (m *Message) getNodes() Nodes { + n := m.getUint64() + servers := make(Nodes, n) + + for i := 0; i < int(n); i++ { + servers[i].ID = m.getUint64() + servers[i].Address = m.getString() + } + + return servers +} + +// Decode a statement result object from the message body. +func (m *Message) getResult() Result { + return Result{ + LastInsertID: m.getUint64(), + RowsAffected: m.getUint64(), + } +} + +// Decode a query result set object from the message body. +func (m *Message) getRows() Rows { + // Read the column count and column names. + columns := make([]string, m.getUint64()) + + for i := range columns { + columns[i] = m.getString() + } + + rows := Rows{ + Columns: columns, + message: m, + } + return rows +} + +func (m *Message) getFiles() Files { + files := Files{ + n: m.getUint64(), + message: m, + } + return files +} + +func (m *Message) hasBeenConsumed() bool { + size := int(m.words * messageWordSize) + return (m.body1.Offset == size || m.body1.Offset == len(m.body1.Bytes)) && + m.body1.Offset+m.body2.Offset == size +} + +func (m *Message) lastByte() byte { + size := int(m.words * messageWordSize) + if size > len(m.body1.Bytes) { + size = size - m.body1.Offset + return m.body2.Bytes[size-1] + } + return m.body1.Bytes[size-1] +} + +func (m *Message) bufferForGet() *buffer { + size := int(m.words * messageWordSize) + if m.body1.Offset == size || m.body1.Offset == len(m.body1.Bytes) { + // The static body has been exahusted, use the dynamic one. + if m.body1.Offset+m.body2.Offset == size { + err := fmt.Errorf("short message: type=%d words=%d off=%d", m.mtype, m.words, m.body1.Offset) + panic(err) + } + return &m.body2 + } + + return &m.body1 +} + +// Result holds the result of a statement. +type Result struct { + LastInsertID uint64 + RowsAffected uint64 +} + +// Rows holds a result set encoded in a message body. +type Rows struct { + Columns []string + message *Message +} + +// Next returns the next row in the result set. +func (r *Rows) Next(dest []driver.Value) error { + types := make([]uint8, len(r.Columns)) + + // Each column needs a 4 byte slot to store the column type. The row + // header must be padded to reach word boundary. + headerBits := len(types) * 4 + padBits := 0 + if trailingBits := (headerBits % messageWordBits); trailingBits != 0 { + padBits = (messageWordBits - trailingBits) + } + + headerSize := (headerBits + padBits) / messageWordBits * messageWordSize + + for i := 0; i < headerSize; i++ { + slot := r.message.getUint8() + + if slot == 0xee { + // More rows are available. + return ErrRowsPart + } + + if slot == 0xff { + // Rows EOF marker + return io.EOF + } + + index := i * 2 + + if index >= len(types) { + continue // This is padding. + } + + types[index] = slot & 0x0f + + index++ + + if index >= len(types) { + continue // This is padding byte. + } + + types[index] = slot >> 4 + } + + for i := range types { + switch types[i] { + case Integer: + dest[i] = r.message.getInt64() + case Float: + dest[i] = r.message.getFloat64() + case Blob: + dest[i] = r.message.getBlob() + case Text: + dest[i] = r.message.getString() + case Null: + r.message.getUint64() + dest[i] = nil + case UnixTime: + timestamp := time.Unix(r.message.getInt64(), 0) + dest[i] = timestamp + case ISO8601: + value := r.message.getString() + if value == "" { + dest[i] = time.Time{} + break + } + var t time.Time + var timeVal time.Time + var err error + value = strings.TrimSuffix(value, "Z") + for _, format := range iso8601Formats { + if timeVal, err = time.ParseInLocation(format, value, time.UTC); err == nil { + t = timeVal + break + } + } + if err != nil { + return err + } + t = t.In(time.Local) + dest[i] = t + case Boolean: + dest[i] = r.message.getInt64() != 0 + default: + panic("unknown data type") + } + } + + return nil +} + +// Close the result set and reset the underlying message. +func (r *Rows) Close() error { + // If we didn't go through all rows, let's look at the last byte. + var err error + if !r.message.hasBeenConsumed() { + slot := r.message.lastByte() + if slot == 0xee { + // More rows are available. + err = ErrRowsPart + } else if slot == 0xff { + // Rows EOF marker + err = io.EOF + } else { + err = fmt.Errorf("unexpected end of message") + } + } + r.message.Reset() + return err +} + +// Files holds a set of files encoded in a message body. +type Files struct { + n uint64 + message *Message +} + +func (f *Files) Next() (string, []byte) { + if f.n == 0 { + return "", nil + } + f.n-- + name := f.message.getString() + length := f.message.getUint64() + data := make([]byte, length) + for i := 0; i < int(length); i++ { + data[i] = f.message.getUint8() + } + return name, data +} + +func (f *Files) Close() { + f.message.Reset() +} + +const ( + messageWordSize = 8 + messageWordBits = messageWordSize * 8 + messageHeaderSize = messageWordSize + messageMaxConsecutiveEmptyReads = 100 +) + +var iso8601Formats = []string{ + // By default, store timestamps with whatever timezone they come with. + // When parsed, they will be returned with the same timezone. + "2006-01-02 15:04:05.999999999-07:00", + "2006-01-02T15:04:05.999999999-07:00", + "2006-01-02 15:04:05.999999999", + "2006-01-02T15:04:05.999999999", + "2006-01-02 15:04:05", + "2006-01-02T15:04:05", + "2006-01-02 15:04", + "2006-01-02T15:04", + "2006-01-02", +} diff --git a/vendor/github.com/canonical/go-dqlite/internal/protocol/protocol.go b/vendor/github.com/canonical/go-dqlite/internal/protocol/protocol.go new file mode 100644 index 0000000000..47774813f4 --- /dev/null +++ b/vendor/github.com/canonical/go-dqlite/internal/protocol/protocol.go @@ -0,0 +1,340 @@ +package protocol + +import ( + "context" + "encoding/binary" + "io" + "net" + "sync" + "time" + + "github.com/pkg/errors" +) + +// Protocol sends and receive the dqlite message on the wire. +type Protocol struct { + version uint64 // Protocol version + conn net.Conn // Underlying network connection. + contextTimeout time.Duration // Default context timeout. + closeCh chan struct{} // Stops the heartbeat when the connection gets closed + mu sync.Mutex // Serialize requests + netErr error // A network error occurred +} + +func NewProtocol(version uint64, conn net.Conn) *Protocol { + protocol := &Protocol{ + version: version, + conn: conn, + closeCh: make(chan struct{}), + contextTimeout: 5 * time.Second, + } + + return protocol +} + +// SetContextTimeout sets the default context timeout when no deadline is +// provided. +func (p *Protocol) SetContextTimeout(timeout time.Duration) { + p.contextTimeout = timeout +} + +// Call invokes a dqlite RPC, sending a request message and receiving a +// response message. +func (p *Protocol) Call(ctx context.Context, request, response *Message) (err error) { + // We need to take a lock since the dqlite server currently does not + // support concurrent requests. + p.mu.Lock() + defer p.mu.Unlock() + + if p.netErr != nil { + return p.netErr + } + + // Honor the ctx deadline, if present, or use a default. + deadline, ok := ctx.Deadline() + if !ok { + deadline = time.Now().Add(p.contextTimeout) + } + + p.conn.SetDeadline(deadline) + + if err = p.send(request); err != nil { + err = errors.Wrap(err, "failed to send request") + goto err + } + + if err = p.recv(response); err != nil { + err = errors.Wrap(err, "failed to receive response") + goto err + } + + return + +err: + switch errors.Cause(err).(type) { + case *net.OpError: + p.netErr = err + } + return +} + +// More is used when a request maps to multiple responses. +func (p *Protocol) More(ctx context.Context, response *Message) error { + return p.recv(response) +} + +// Interrupt sends an interrupt request and awaits for the server's empty +// response. +func (p *Protocol) Interrupt(ctx context.Context, request *Message, response *Message) error { + // We need to take a lock since the dqlite server currently does not + // support concurrent requests. + p.mu.Lock() + defer p.mu.Unlock() + + // Honor the ctx deadline, if present, or use a default. + deadline, ok := ctx.Deadline() + if !ok { + deadline = time.Now().Add(2 * time.Second) + } + p.conn.SetDeadline(deadline) + + defer request.Reset() + + EncodeInterrupt(request, 0) + + if err := p.send(request); err != nil { + return errors.Wrap(err, "failed to send interrupt request") + } + + for { + if err := p.recv(response); err != nil { + response.Reset() + return errors.Wrap(err, "failed to receive response") + } + + mtype, _ := response.getHeader() + response.Reset() + + if mtype == ResponseEmpty { + break + } + } + + return nil +} + +// Close the client connection. +func (p *Protocol) Close() error { + close(p.closeCh) + return p.conn.Close() +} + +func (p *Protocol) send(req *Message) error { + if err := p.sendHeader(req); err != nil { + return errors.Wrap(err, "failed to send header") + } + + if err := p.sendBody(req); err != nil { + return errors.Wrap(err, "failed to send body") + } + + return nil +} + +func (p *Protocol) sendHeader(req *Message) error { + n, err := p.conn.Write(req.header[:]) + if err != nil { + return errors.Wrap(err, "failed to send header") + } + + if n != messageHeaderSize { + return errors.Wrap(io.ErrShortWrite, "failed to send header") + } + + return nil +} + +func (p *Protocol) sendBody(req *Message) error { + buf := req.body1.Bytes[:req.body1.Offset] + n, err := p.conn.Write(buf) + if err != nil { + return errors.Wrap(err, "failed to send static body") + } + + if n != len(buf) { + return errors.Wrap(io.ErrShortWrite, "failed to write body") + } + + if req.body2.Bytes == nil { + return nil + } + + buf = req.body2.Bytes[:req.body2.Offset] + n, err = p.conn.Write(buf) + if err != nil { + return errors.Wrap(err, "failed to send dynamic body") + } + + if n != len(buf) { + return errors.Wrap(io.ErrShortWrite, "failed to write body") + } + + return nil +} + +func (p *Protocol) recv(res *Message) error { + if err := p.recvHeader(res); err != nil { + return errors.Wrap(err, "failed to receive header") + } + + if err := p.recvBody(res); err != nil { + return errors.Wrap(err, "failed to receive body") + } + + return nil +} + +func (p *Protocol) recvHeader(res *Message) error { + if err := p.recvPeek(res.header); err != nil { + return errors.Wrap(err, "failed to receive header") + } + + res.words = binary.LittleEndian.Uint32(res.header[0:]) + res.mtype = res.header[4] + res.flags = res.header[5] + res.extra = binary.LittleEndian.Uint16(res.header[6:]) + + return nil +} + +func (p *Protocol) recvBody(res *Message) error { + n := int(res.words) * messageWordSize + n1 := n + n2 := 0 + + if n1 > len(res.body1.Bytes) { + // We need to allocate the dynamic buffer. + n1 = len(res.body1.Bytes) + n2 = n - n1 + } + + buf := res.body1.Bytes[:n1] + + if err := p.recvPeek(buf); err != nil { + return errors.Wrap(err, "failed to read body") + } + + if n2 > 0 { + res.body2.Bytes = make([]byte, n2) + res.body2.Offset = 0 + buf = res.body2.Bytes + if err := p.recvPeek(buf); err != nil { + return errors.Wrap(err, "failed to read body") + } + } + + return nil +} + +// Read until buf is full. +func (p *Protocol) recvPeek(buf []byte) error { + for offset := 0; offset < len(buf); { + n, err := p.recvFill(buf[offset:]) + if err != nil { + return err + } + offset += n + } + + return nil +} + +// Try to fill buf, but perform at most one read. +func (p *Protocol) recvFill(buf []byte) (int, error) { + // Read new data: try a limited number of times. + // + // This technique is copied from bufio.Reader. + for i := messageMaxConsecutiveEmptyReads; i > 0; i-- { + n, err := p.conn.Read(buf) + if n < 0 { + panic(errNegativeRead) + } + if err != nil { + return -1, err + } + if n > 0 { + return n, nil + } + } + return -1, io.ErrNoProgress +} + +/* +func (p *Protocol) heartbeat() { + request := Message{} + request.Init(16) + response := Message{} + response.Init(512) + + for { + delay := c.heartbeatTimeout / 3 + + //c.logger.Debug("sending heartbeat", zap.Duration("delay", delay)) + time.Sleep(delay) + + // Check if we've been closed. + select { + case <-c.closeCh: + return + default: + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + + EncodeHeartbeat(&request, uint64(time.Now().Unix())) + + err := c.Call(ctx, &request, &response) + + // We bail out upon failures. + // + // TODO: make the client survive temporary disconnections. + if err != nil { + cancel() + //c.logger.Error("heartbeat failed", zap.Error(err)) + return + } + + //addresses, err := DecodeNodes(&response) + _, err = DecodeNodes(&response) + if err != nil { + cancel() + //c.logger.Error("invalid heartbeat response", zap.Error(err)) + return + } + + // if err := c.store.Set(ctx, addresses); err != nil { + // cancel() + // c.logger.Error("failed to update servers", zap.Error(err)) + // return + // } + + cancel() + + request.Reset() + response.Reset() + } +} +*/ + +// DecodeNodeCompat handles also pre-1.0 legacy server messages. +func DecodeNodeCompat(protocol *Protocol, response *Message) (uint64, string, error) { + if protocol.version == VersionLegacy { + address, err := DecodeNodeLegacy(response) + if err != nil { + return 0, "", err + } + return 0, address, nil + + } + return DecodeNode(response) +} diff --git a/vendor/github.com/canonical/go-dqlite/internal/protocol/request.go b/vendor/github.com/canonical/go-dqlite/internal/protocol/request.go new file mode 100644 index 0000000000..cd3f57d3b2 --- /dev/null +++ b/vendor/github.com/canonical/go-dqlite/internal/protocol/request.go @@ -0,0 +1,131 @@ +package protocol + +// DO NOT EDIT +// +// This file was generated by ./schema.sh + + +// EncodeLeader encodes a Leader request. +func EncodeLeader(request *Message) { + request.putUint64(0) + + request.putHeader(RequestLeader) +} + +// EncodeClient encodes a Client request. +func EncodeClient(request *Message, id uint64) { + request.putUint64(id) + + request.putHeader(RequestClient) +} + +// EncodeHeartbeat encodes a Heartbeat request. +func EncodeHeartbeat(request *Message, timestamp uint64) { + request.putUint64(timestamp) + + request.putHeader(RequestHeartbeat) +} + +// EncodeOpen encodes a Open request. +func EncodeOpen(request *Message, name string, flags uint64, vfs string) { + request.putString(name) + request.putUint64(flags) + request.putString(vfs) + + request.putHeader(RequestOpen) +} + +// EncodePrepare encodes a Prepare request. +func EncodePrepare(request *Message, db uint64, sql string) { + request.putUint64(db) + request.putString(sql) + + request.putHeader(RequestPrepare) +} + +// EncodeExec encodes a Exec request. +func EncodeExec(request *Message, db uint32, stmt uint32, values NamedValues) { + request.putUint32(db) + request.putUint32(stmt) + request.putNamedValues(values) + + request.putHeader(RequestExec) +} + +// EncodeQuery encodes a Query request. +func EncodeQuery(request *Message, db uint32, stmt uint32, values NamedValues) { + request.putUint32(db) + request.putUint32(stmt) + request.putNamedValues(values) + + request.putHeader(RequestQuery) +} + +// EncodeFinalize encodes a Finalize request. +func EncodeFinalize(request *Message, db uint32, stmt uint32) { + request.putUint32(db) + request.putUint32(stmt) + + request.putHeader(RequestFinalize) +} + +// EncodeExecSQL encodes a ExecSQL request. +func EncodeExecSQL(request *Message, db uint64, sql string, values NamedValues) { + request.putUint64(db) + request.putString(sql) + request.putNamedValues(values) + + request.putHeader(RequestExecSQL) +} + +// EncodeQuerySQL encodes a QuerySQL request. +func EncodeQuerySQL(request *Message, db uint64, sql string, values NamedValues) { + request.putUint64(db) + request.putString(sql) + request.putNamedValues(values) + + request.putHeader(RequestQuerySQL) +} + +// EncodeInterrupt encodes a Interrupt request. +func EncodeInterrupt(request *Message, db uint64) { + request.putUint64(db) + + request.putHeader(RequestInterrupt) +} + +// EncodeJoin encodes a Join request. +func EncodeJoin(request *Message, id uint64, address string) { + request.putUint64(id) + request.putString(address) + + request.putHeader(RequestJoin) +} + +// EncodePromote encodes a Promote request. +func EncodePromote(request *Message, id uint64) { + request.putUint64(id) + + request.putHeader(RequestPromote) +} + +// EncodeRemove encodes a Remove request. +func EncodeRemove(request *Message, id uint64) { + request.putUint64(id) + + request.putHeader(RequestRemove) +} + +// EncodeDump encodes a Dump request. +func EncodeDump(request *Message, name string) { + request.putString(name) + + request.putHeader(RequestDump) +} + +// EncodeCluster encodes a Cluster request. +func EncodeCluster(request *Message) { + request.putUint64(0) + + request.putHeader(RequestCluster) +} diff --git a/vendor/github.com/canonical/go-dqlite/internal/protocol/response.go b/vendor/github.com/canonical/go-dqlite/internal/protocol/response.go new file mode 100644 index 0000000000..c942720749 --- /dev/null +++ b/vendor/github.com/canonical/go-dqlite/internal/protocol/response.go @@ -0,0 +1,254 @@ +package protocol + +// DO NOT EDIT +// +// This file was generated by ./schema.sh + +import "fmt" + +// DecodeFailure decodes a Failure response. +func DecodeFailure(response *Message) (code uint64, message string, err error) { + mtype, _ := response.getHeader() + + if mtype == ResponseFailure { + e := ErrRequest{} + e.Code = response.getUint64() + e.Description = response.getString() + err = e + return + } + + if mtype != ResponseFailure { + err = fmt.Errorf("unexpected response type %d", mtype) + return + } + + code = response.getUint64() + message = response.getString() + + return +} + +// DecodeWelcome decodes a Welcome response. +func DecodeWelcome(response *Message) (heartbeatTimeout uint64, err error) { + mtype, _ := response.getHeader() + + if mtype == ResponseFailure { + e := ErrRequest{} + e.Code = response.getUint64() + e.Description = response.getString() + err = e + return + } + + if mtype != ResponseWelcome { + err = fmt.Errorf("unexpected response type %d", mtype) + return + } + + heartbeatTimeout = response.getUint64() + + return +} + +// DecodeNodeLegacy decodes a NodeLegacy response. +func DecodeNodeLegacy(response *Message) (address string, err error) { + mtype, _ := response.getHeader() + + if mtype == ResponseFailure { + e := ErrRequest{} + e.Code = response.getUint64() + e.Description = response.getString() + err = e + return + } + + if mtype != ResponseNodeLegacy { + err = fmt.Errorf("unexpected response type %d", mtype) + return + } + + address = response.getString() + + return +} + +// DecodeNode decodes a Node response. +func DecodeNode(response *Message) (id uint64, address string, err error) { + mtype, _ := response.getHeader() + + if mtype == ResponseFailure { + e := ErrRequest{} + e.Code = response.getUint64() + e.Description = response.getString() + err = e + return + } + + if mtype != ResponseNode { + err = fmt.Errorf("unexpected response type %d", mtype) + return + } + + id = response.getUint64() + address = response.getString() + + return +} + +// DecodeNodes decodes a Nodes response. +func DecodeNodes(response *Message) (servers Nodes, err error) { + mtype, _ := response.getHeader() + + if mtype == ResponseFailure { + e := ErrRequest{} + e.Code = response.getUint64() + e.Description = response.getString() + err = e + return + } + + if mtype != ResponseNodes { + err = fmt.Errorf("unexpected response type %d", mtype) + return + } + + servers = response.getNodes() + + return +} + +// DecodeDb decodes a Db response. +func DecodeDb(response *Message) (id uint32, err error) { + mtype, _ := response.getHeader() + + if mtype == ResponseFailure { + e := ErrRequest{} + e.Code = response.getUint64() + e.Description = response.getString() + err = e + return + } + + if mtype != ResponseDb { + err = fmt.Errorf("unexpected response type %d", mtype) + return + } + + id = response.getUint32() + response.getUint32() + + return +} + +// DecodeStmt decodes a Stmt response. +func DecodeStmt(response *Message) (db uint32, id uint32, params uint64, err error) { + mtype, _ := response.getHeader() + + if mtype == ResponseFailure { + e := ErrRequest{} + e.Code = response.getUint64() + e.Description = response.getString() + err = e + return + } + + if mtype != ResponseStmt { + err = fmt.Errorf("unexpected response type %d", mtype) + return + } + + db = response.getUint32() + id = response.getUint32() + params = response.getUint64() + + return +} + +// DecodeEmpty decodes a Empty response. +func DecodeEmpty(response *Message) (err error) { + mtype, _ := response.getHeader() + + if mtype == ResponseFailure { + e := ErrRequest{} + e.Code = response.getUint64() + e.Description = response.getString() + err = e + return + } + + if mtype != ResponseEmpty { + err = fmt.Errorf("unexpected response type %d", mtype) + return + } + + response.getUint64() + + return +} + +// DecodeResult decodes a Result response. +func DecodeResult(response *Message) (result Result, err error) { + mtype, _ := response.getHeader() + + if mtype == ResponseFailure { + e := ErrRequest{} + e.Code = response.getUint64() + e.Description = response.getString() + err = e + return + } + + if mtype != ResponseResult { + err = fmt.Errorf("unexpected response type %d", mtype) + return + } + + result = response.getResult() + + return +} + +// DecodeRows decodes a Rows response. +func DecodeRows(response *Message) (rows Rows, err error) { + mtype, _ := response.getHeader() + + if mtype == ResponseFailure { + e := ErrRequest{} + e.Code = response.getUint64() + e.Description = response.getString() + err = e + return + } + + if mtype != ResponseRows { + err = fmt.Errorf("unexpected response type %d", mtype) + return + } + + rows = response.getRows() + + return +} + +// DecodeFiles decodes a Files response. +func DecodeFiles(response *Message) (files Files, err error) { + mtype, _ := response.getHeader() + + if mtype == ResponseFailure { + e := ErrRequest{} + e.Code = response.getUint64() + e.Description = response.getString() + err = e + return + } + + if mtype != ResponseFiles { + err = fmt.Errorf("unexpected response type %d", mtype) + return + } + + files = response.getFiles() + + return +} diff --git a/vendor/github.com/canonical/go-dqlite/internal/protocol/schema.go b/vendor/github.com/canonical/go-dqlite/internal/protocol/schema.go new file mode 100644 index 0000000000..cf02f8c3f4 --- /dev/null +++ b/vendor/github.com/canonical/go-dqlite/internal/protocol/schema.go @@ -0,0 +1,33 @@ +package protocol + +//go:generate ./schema.sh --request init + +//go:generate ./schema.sh --request Leader unused:uint64 +//go:generate ./schema.sh --request Client id:uint64 +//go:generate ./schema.sh --request Heartbeat timestamp:uint64 +//go:generate ./schema.sh --request Open name:string flags:uint64 vfs:string +//go:generate ./schema.sh --request Prepare db:uint64 sql:string +//go:generate ./schema.sh --request Exec db:uint32 stmt:uint32 values:NamedValues +//go:generate ./schema.sh --request Query db:uint32 stmt:uint32 values:NamedValues +//go:generate ./schema.sh --request Finalize db:uint32 stmt:uint32 +//go:generate ./schema.sh --request ExecSQL db:uint64 sql:string values:NamedValues +//go:generate ./schema.sh --request QuerySQL db:uint64 sql:string values:NamedValues +//go:generate ./schema.sh --request Interrupt db:uint64 +//go:generate ./schema.sh --request Join id:uint64 address:string +//go:generate ./schema.sh --request Promote id:uint64 +//go:generate ./schema.sh --request Remove id:uint64 +//go:generate ./schema.sh --request Dump name:string +//go:generate ./schema.sh --request Cluster unused:uint64 + +//go:generate ./schema.sh --response init +//go:generate ./schema.sh --response Failure code:uint64 message:string +//go:generate ./schema.sh --response Welcome heartbeatTimeout:uint64 +//go:generate ./schema.sh --response NodeLegacy address:string +//go:generate ./schema.sh --response Node id:uint64 address:string +//go:generate ./schema.sh --response Nodes servers:Nodes +//go:generate ./schema.sh --response Db id:uint32 unused:uint32 +//go:generate ./schema.sh --response Stmt db:uint32 id:uint32 params:uint64 +//go:generate ./schema.sh --response Empty unused:uint64 +//go:generate ./schema.sh --response Result result:Result +//go:generate ./schema.sh --response Rows rows:Rows +//go:generate ./schema.sh --response Files files:Files diff --git a/vendor/github.com/canonical/go-dqlite/internal/protocol/schema.sh b/vendor/github.com/canonical/go-dqlite/internal/protocol/schema.sh new file mode 100644 index 0000000000..1e7b53f4c6 --- /dev/null +++ b/vendor/github.com/canonical/go-dqlite/internal/protocol/schema.sh @@ -0,0 +1,144 @@ +#!/bin/bash + +request_init() { + cat > request.go < response.go <> request.go <> request.go <> request.go <> response.go <> response.go <> response.go < + +Contributors (in no specific order): + +* @romanoaugusto88 +* @vitalbh +* @blaubaer + +Feel free to add yourself to the list or to modify your entry if you did a contribution. diff --git a/vendor/github.com/flosch/pongo2/LICENSE b/vendor/github.com/flosch/pongo2/LICENSE new file mode 100644 index 0000000000..e876f86905 --- /dev/null +++ b/vendor/github.com/flosch/pongo2/LICENSE @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2013-2014 Florian Schlachter + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/flosch/pongo2/README.md b/vendor/github.com/flosch/pongo2/README.md new file mode 100644 index 0000000000..f70f502547 --- /dev/null +++ b/vendor/github.com/flosch/pongo2/README.md @@ -0,0 +1,273 @@ +# [pongo](https://en.wikipedia.org/wiki/Pongo_%28genus%29)2 + +[![Join the chat at https://gitter.im/flosch/pongo2](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/flosch/pongo2) +[![GoDoc](https://godoc.org/github.com/flosch/pongo2?status.svg)](https://godoc.org/github.com/flosch/pongo2) +[![Build Status](https://travis-ci.org/flosch/pongo2.svg?branch=master)](https://travis-ci.org/flosch/pongo2) +[![Backers on Open Collective](https://opencollective.com/pongo2/backers/badge.svg)](#backers) +[![Sponsors on Open Collective](https://opencollective.com/pongo2/sponsors/badge.svg)](#sponsors) + +pongo2 is the successor of [pongo](https://github.com/flosch/pongo), a Django-syntax like templating-language. + +Install/update using `go get` (no dependencies required by pongo2): +``` +go get -u github.com/flosch/pongo2 +``` + +Please use the [issue tracker](https://github.com/flosch/pongo2/issues) if you're encountering any problems with pongo2 or if you need help with implementing tags or filters ([create a ticket!](https://github.com/flosch/pongo2/issues/new)). + +## First impression of a template + +```HTML+Django +Our admins and users +{# This is a short example to give you a quick overview of pongo2's syntax. #} + +{% macro user_details(user, is_admin=false) %} +
+ +

= 40) || (user.karma > calc_avg_karma(userlist)+5) %} + class="karma-good"{% endif %}> + + + {{ user }} +

+ + +

This user registered {{ user.register_date|naturaltime }}.

+ + +

The user's biography:

+

{{ user.biography|markdown|truncatewords_html:15 }} + read more

+ + {% if is_admin %}

This user is an admin!

{% endif %} +
+{% endmacro %} + + + + +

Our admins

+ {% for admin in adminlist %} + {{ user_details(admin, true) }} + {% endfor %} + +

Our members

+ {% for user in userlist %} + {{ user_details(user) }} + {% endfor %} + + +``` + +## Development status + +**Latest stable release**: v3.0 (`go get -u gopkg.in/flosch/pongo2.v3` / [`v3`](https://github.com/flosch/pongo2/tree/v3)-branch) + +**Current development**: v4 (`master`-branch) + +*Note*: With the release of pongo v4 the branch v2 will be deprecated. + +**Deprecated versions** (not supported anymore): v1 + +| Topic | Status | +| ------------------------------------ | -------------------------------------------------------------------------------------- | +| Django version compatibility: | [1.7](https://docs.djangoproject.com/en/1.7/ref/templates/builtins/) | +| *Missing* (planned) **filters**: | none ([hints](https://github.com/flosch/pongo2/blob/master/filters_builtin.go#L3)) | +| *Missing* (planned) **tags**: | none ([hints](https://github.com/flosch/pongo2/blob/master/tags.go#L3)) | + +Please also have a look on the [caveats](https://github.com/flosch/pongo2#caveats) and on the [official add-ons](https://github.com/flosch/pongo2#official). + +## Features (and new in pongo2) + + * Entirely rewritten from the ground-up. + * [Advanced C-like expressions](https://github.com/flosch/pongo2/blob/master/template_tests/expressions.tpl). + * [Complex function calls within expressions](https://github.com/flosch/pongo2/blob/master/template_tests/function_calls_wrapper.tpl). + * [Easy API to create new filters and tags](http://godoc.org/github.com/flosch/pongo2#RegisterFilter) ([including parsing arguments](http://godoc.org/github.com/flosch/pongo2#Parser)) + * Additional features: + * Macros including importing macros from other files (see [template_tests/macro.tpl](https://github.com/flosch/pongo2/blob/master/template_tests/macro.tpl)) + * [Template sandboxing](https://godoc.org/github.com/flosch/pongo2#TemplateSet) ([directory patterns](http://golang.org/pkg/path/filepath/#Match), banned tags/filters) + +## Recent API changes within pongo2 + +If you're using the `master`-branch of pongo2, you might be interested in this section. Since pongo2 is still in development (even though there is a first stable release!), there could be (backwards-incompatible) API changes over time. To keep track of these and therefore make it painless for you to adapt your codebase, I'll list them here. + + * Function signature for tag execution changed: not taking a `bytes.Buffer` anymore; instead `Execute()`-functions are now taking a `TemplateWriter` interface. + * Function signature for tag and filter parsing/execution changed (`error` return type changed to `*Error`). + * `INodeEvaluator` has been removed and got replaced by `IEvaluator`. You can change your existing tags/filters by simply replacing the interface. + * Two new helper functions: [`RenderTemplateFile()`](https://godoc.org/github.com/flosch/pongo2#RenderTemplateFile) and [`RenderTemplateString()`](https://godoc.org/github.com/flosch/pongo2#RenderTemplateString). + * `Template.ExecuteRW()` is now [`Template.ExecuteWriter()`](https://godoc.org/github.com/flosch/pongo2#Template.ExecuteWriter) + * `Template.Execute*()` functions do now take a `pongo2.Context` directly (no pointer anymore). + +## How you can help + + * Write [filters](https://github.com/flosch/pongo2/blob/master/filters_builtin.go#L3) / [tags] by forking pongo2 and sending pull requests + * Write/improve code tests (use the following command to see what tests are missing: `go test -v -cover -covermode=count -coverprofile=cover.out && go tool cover -html=cover.out` or have a look on [gocover.io/github.com/flosch/pongo2](http://gocover.io/github.com/flosch/pongo2)) + * Write/improve template tests (see the `template_tests/` directory) + * Write middleware, libraries and websites using pongo2. :-) + +# Documentation + +For a documentation on how the templating language works you can [head over to the Django documentation](https://docs.djangoproject.com/en/dev/topics/templates/). pongo2 aims to be compatible with it. + +You can access pongo2's API documentation on [godoc](https://godoc.org/github.com/flosch/pongo2). + +## Caveats + +### Filters + + * **date** / **time**: The `date` and `time` filter are taking the Golang specific time- and date-format (not Django's one) currently. [Take a look on the format here](http://golang.org/pkg/time/#Time.Format). + * **stringformat**: `stringformat` does **not** take Python's string format syntax as a parameter, instead it takes Go's. Essentially `{{ 3.14|stringformat:"pi is %.2f" }}` is `fmt.Sprintf("pi is %.2f", 3.14)`. + * **escape** / **force_escape**: Unlike Django's behaviour, the `escape`-filter is applied immediately. Therefore there is no need for a `force_escape`-filter yet. + +### Tags + + * **for**: All the `forloop` fields (like `forloop.counter`) are written with a capital letter at the beginning. For example, the `counter` can be accessed by `forloop.Counter` and the parentloop by `forloop.Parentloop`. + * **now**: takes Go's time format (see **date** and **time**-filter). + +### Misc + + * **not in-operator**: You can check whether a map/struct/string contains a key/field/substring by using the in-operator (or the negation of it): + `{% if key in map %}Key is in map{% else %}Key not in map{% endif %}` or `{% if !(key in map) %}Key is NOT in map{% else %}Key is in map{% endif %}`. + +# Add-ons, libraries and helpers + +## Official + + * [ponginae](https://github.com/flosch/ponginae) - A web-framework for Go (using pongo2). + * [pongo2-tools](https://github.com/flosch/pongo2-tools) - Official tools and helpers for pongo2 + * [pongo2-addons](https://github.com/flosch/pongo2-addons) - Official additional filters/tags for pongo2 (for example a **markdown**-filter). They are in their own repository because they're relying on 3rd-party-libraries. + +## 3rd-party + + * [beego-pongo2](https://github.com/oal/beego-pongo2) - A tiny little helper for using Pongo2 with [Beego](https://github.com/astaxie/beego). + * [beego-pongo2.v2](https://github.com/ipfans/beego-pongo2.v2) - Same as `beego-pongo2`, but for pongo2 v2. + * [macaron-pongo2](https://github.com/macaron-contrib/pongo2) - pongo2 support for [Macaron](https://github.com/Unknwon/macaron), a modular web framework. + * [ginpongo2](https://github.com/ngerakines/ginpongo2) - middleware for [gin](github.com/gin-gonic/gin) to use pongo2 templates + * [Build'n support for Iris' template engine](https://github.com/kataras/iris) + * [pongo2gin](https://gitlab.com/go-box/pongo2gin) - alternative renderer for [gin](github.com/gin-gonic/gin) to use pongo2 templates + * [pongo2-trans](https://github.com/digitalcrab/pongo2trans) - `trans`-tag implementation for internationalization + * [tpongo2](https://github.com/tango-contrib/tpongo2) - pongo2 support for [Tango](https://github.com/lunny/tango), a micro-kernel & pluggable web framework. + * [p2cli](https://github.com/wrouesnel/p2cli) - command line templating utility based on pongo2 + +Please add your project to this list and send me a pull request when you've developed something nice for pongo2. + +# API-usage examples + +Please see the documentation for a full list of provided API methods. + +## A tiny example (template string) + +```Go +// Compile the template first (i. e. creating the AST) +tpl, err := pongo2.FromString("Hello {{ name|capfirst }}!") +if err != nil { + panic(err) +} +// Now you can render the template with the given +// pongo2.Context how often you want to. +out, err := tpl.Execute(pongo2.Context{"name": "florian"}) +if err != nil { + panic(err) +} +fmt.Println(out) // Output: Hello Florian! +``` + +## Example server-usage (template file) + +```Go +package main + +import ( + "github.com/flosch/pongo2" + "net/http" +) + +// Pre-compiling the templates at application startup using the +// little Must()-helper function (Must() will panic if FromFile() +// or FromString() will return with an error - that's it). +// It's faster to pre-compile it anywhere at startup and only +// execute the template later. +var tplExample = pongo2.Must(pongo2.FromFile("example.html")) + +func examplePage(w http.ResponseWriter, r *http.Request) { + // Execute the template per HTTP request + err := tplExample.ExecuteWriter(pongo2.Context{"query": r.FormValue("query")}, w) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } +} + +func main() { + http.HandleFunc("/", examplePage) + http.ListenAndServe(":8080", nil) +} +``` + +# Benchmark + +The benchmarks have been run on the my machine (`Intel(R) Core(TM) i7-2600 CPU @ 3.40GHz`) using the command: + + go test -bench . -cpu 1,2,4,8 + +All benchmarks are compiling (depends on the benchmark) and executing the `template_tests/complex.tpl` template. + +The results are: + + BenchmarkExecuteComplexWithSandboxActive 50000 60450 ns/op + BenchmarkExecuteComplexWithSandboxActive-2 50000 56998 ns/op + BenchmarkExecuteComplexWithSandboxActive-4 50000 60343 ns/op + BenchmarkExecuteComplexWithSandboxActive-8 50000 64229 ns/op + BenchmarkCompileAndExecuteComplexWithSandboxActive 10000 164410 ns/op + BenchmarkCompileAndExecuteComplexWithSandboxActive-2 10000 156682 ns/op + BenchmarkCompileAndExecuteComplexWithSandboxActive-4 10000 164821 ns/op + BenchmarkCompileAndExecuteComplexWithSandboxActive-8 10000 171806 ns/op + BenchmarkParallelExecuteComplexWithSandboxActive 50000 60428 ns/op + BenchmarkParallelExecuteComplexWithSandboxActive-2 50000 31887 ns/op + BenchmarkParallelExecuteComplexWithSandboxActive-4 100000 22810 ns/op + BenchmarkParallelExecuteComplexWithSandboxActive-8 100000 18820 ns/op + BenchmarkExecuteComplexWithoutSandbox 50000 56942 ns/op + BenchmarkExecuteComplexWithoutSandbox-2 50000 56168 ns/op + BenchmarkExecuteComplexWithoutSandbox-4 50000 57838 ns/op + BenchmarkExecuteComplexWithoutSandbox-8 50000 60539 ns/op + BenchmarkCompileAndExecuteComplexWithoutSandbox 10000 162086 ns/op + BenchmarkCompileAndExecuteComplexWithoutSandbox-2 10000 159771 ns/op + BenchmarkCompileAndExecuteComplexWithoutSandbox-4 10000 163826 ns/op + BenchmarkCompileAndExecuteComplexWithoutSandbox-8 10000 169062 ns/op + BenchmarkParallelExecuteComplexWithoutSandbox 50000 57152 ns/op + BenchmarkParallelExecuteComplexWithoutSandbox-2 50000 30276 ns/op + BenchmarkParallelExecuteComplexWithoutSandbox-4 100000 22065 ns/op + BenchmarkParallelExecuteComplexWithoutSandbox-8 100000 18034 ns/op + +Benchmarked on October 2nd 2014. + +## Contributors + +This project exists thanks to all the people who contribute. + + + +## Backers + +Thank you to all our backers! 🙏 [[Become a backer](https://opencollective.com/pongo2#backer)] + + + + +## Sponsors + +Support this project by becoming a sponsor. Your logo will show up here with a link to your website. [[Become a sponsor](https://opencollective.com/pongo2#sponsor)] + + + + + + + + + + + + + diff --git a/vendor/github.com/flosch/pongo2/context.go b/vendor/github.com/flosch/pongo2/context.go new file mode 100644 index 0000000000..2934d70a49 --- /dev/null +++ b/vendor/github.com/flosch/pongo2/context.go @@ -0,0 +1,136 @@ +package pongo2 + +import ( + "regexp" + + "github.com/juju/errors" +) + +var reIdentifiers = regexp.MustCompile("^[a-zA-Z0-9_]+$") + +var autoescape = true + +func SetAutoescape(newValue bool) { + autoescape = newValue +} + +// A Context type provides constants, variables, instances or functions to a template. +// +// pongo2 automatically provides meta-information or functions through the "pongo2"-key. +// Currently, context["pongo2"] contains the following keys: +// 1. version: returns the version string +// +// Template examples for accessing items from your context: +// {{ myconstant }} +// {{ myfunc("test", 42) }} +// {{ user.name }} +// {{ pongo2.version }} +type Context map[string]interface{} + +func (c Context) checkForValidIdentifiers() *Error { + for k, v := range c { + if !reIdentifiers.MatchString(k) { + return &Error{ + Sender: "checkForValidIdentifiers", + OrigError: errors.Errorf("context-key '%s' (value: '%+v') is not a valid identifier", k, v), + } + } + } + return nil +} + +// Update updates this context with the key/value-pairs from another context. +func (c Context) Update(other Context) Context { + for k, v := range other { + c[k] = v + } + return c +} + +// ExecutionContext contains all data important for the current rendering state. +// +// If you're writing a custom tag, your tag's Execute()-function will +// have access to the ExecutionContext. This struct stores anything +// about the current rendering process's Context including +// the Context provided by the user (field Public). +// You can safely use the Private context to provide data to the user's +// template (like a 'forloop'-information). The Shared-context is used +// to share data between tags. All ExecutionContexts share this context. +// +// Please be careful when accessing the Public data. +// PLEASE DO NOT MODIFY THE PUBLIC CONTEXT (read-only). +// +// To create your own execution context within tags, use the +// NewChildExecutionContext(parent) function. +type ExecutionContext struct { + template *Template + + Autoescape bool + Public Context + Private Context + Shared Context +} + +var pongo2MetaContext = Context{ + "version": Version, +} + +func newExecutionContext(tpl *Template, ctx Context) *ExecutionContext { + privateCtx := make(Context) + + // Make the pongo2-related funcs/vars available to the context + privateCtx["pongo2"] = pongo2MetaContext + + return &ExecutionContext{ + template: tpl, + + Public: ctx, + Private: privateCtx, + Autoescape: autoescape, + } +} + +func NewChildExecutionContext(parent *ExecutionContext) *ExecutionContext { + newctx := &ExecutionContext{ + template: parent.template, + + Public: parent.Public, + Private: make(Context), + Autoescape: parent.Autoescape, + } + newctx.Shared = parent.Shared + + // Copy all existing private items + newctx.Private.Update(parent.Private) + + return newctx +} + +func (ctx *ExecutionContext) Error(msg string, token *Token) *Error { + return ctx.OrigError(errors.New(msg), token) +} + +func (ctx *ExecutionContext) OrigError(err error, token *Token) *Error { + filename := ctx.template.name + var line, col int + if token != nil { + // No tokens available + // TODO: Add location (from where?) + filename = token.Filename + line = token.Line + col = token.Col + } + return &Error{ + Template: ctx.template, + Filename: filename, + Line: line, + Column: col, + Token: token, + Sender: "execution", + OrigError: err, + } +} + +func (ctx *ExecutionContext) Logf(format string, args ...interface{}) { + ctx.template.set.logf(format, args...) +} diff --git a/vendor/github.com/flosch/pongo2/doc.go b/vendor/github.com/flosch/pongo2/doc.go new file mode 100644 index 0000000000..5a23e2b2d8 --- /dev/null +++ b/vendor/github.com/flosch/pongo2/doc.go @@ -0,0 +1,31 @@ +// A Django-syntax like template-engine +// +// Blog posts about pongo2 (including introduction and migration): +// https://www.florian-schlachter.de/?tag=pongo2 +// +// Complete documentation on the template language: +// https://docs.djangoproject.com/en/dev/topics/templates/ +// +// Try out pongo2 live in the pongo2 playground: +// https://www.florian-schlachter.de/pongo2/ +// +// Make sure to read README.md in the repository as well. +// +// A tiny example with template strings: +// +// (Snippet on playground: https://www.florian-schlachter.de/pongo2/?id=1206546277) +// +// // Compile the template first (i. e. creating the AST) +// tpl, err := pongo2.FromString("Hello {{ name|capfirst }}!") +// if err != nil { +// panic(err) +// } +// // Now you can render the template with the given +// // pongo2.Context how often you want to. +// out, err := tpl.Execute(pongo2.Context{"name": "fred"}) +// if err != nil { +// panic(err) +// } +// fmt.Println(out) // Output: Hello Fred! +// +package pongo2 diff --git a/vendor/github.com/flosch/pongo2/error.go b/vendor/github.com/flosch/pongo2/error.go new file mode 100644 index 0000000000..8aec8c1003 --- /dev/null +++ b/vendor/github.com/flosch/pongo2/error.go @@ -0,0 +1,91 @@ +package pongo2 + +import ( + "bufio" + "fmt" + "os" +) + +// The Error type is being used to address an error during lexing, parsing or +// execution. If you want to return an error object (for example in your own +// tag or filter) fill this object with as much information as you have. +// Make sure "Sender" is always given (if you're returning an error within +// a filter, make Sender equals 'filter:yourfilter'; same goes for tags: 'tag:mytag'). +// It's okay if you only fill in ErrorMsg if you don't have any other details at hand. +type Error struct { + Template *Template + Filename string + Line int + Column int + Token *Token + Sender string + OrigError error +} + +func (e *Error) updateFromTokenIfNeeded(template *Template, t *Token) *Error { + if e.Template == nil { + e.Template = template + } + + if e.Token == nil { + e.Token = t + if e.Line <= 0 { + e.Line = t.Line + e.Column = t.Col + } + } + + return e +} + +// Returns a nice formatted error string. +func (e *Error) Error() string { + s := "[Error" + if e.Sender != "" { + s += " (where: " + e.Sender + ")" + } + if e.Filename != "" { + s += " in " + e.Filename + } + if e.Line > 0 { + s += fmt.Sprintf(" | Line %d Col %d", e.Line, e.Column) + if e.Token != nil { + s += fmt.Sprintf(" near '%s'", e.Token.Val) + } + } + s += "] " + s += e.OrigError.Error() + return s +} + +// RawLine returns the affected line from the original template, if available. +func (e *Error) RawLine() (line string, available bool, outErr error) { + if e.Line <= 0 || e.Filename == "" { + return "", false, nil + } + + filename := e.Filename + if e.Template != nil { + filename = e.Template.set.resolveFilename(e.Template, e.Filename) + } + file, err := os.Open(filename) + if err != nil { + return "", false, err + } + defer func() { + err := file.Close() + if err != nil && outErr == nil { + outErr = err + } + }() + + scanner := bufio.NewScanner(file) + l := 0 + for scanner.Scan() { + l++ + if l == e.Line { + return scanner.Text(), true, nil + } + } + return "", false, nil +} diff --git a/vendor/github.com/flosch/pongo2/filters.go b/vendor/github.com/flosch/pongo2/filters.go new file mode 100644 index 0000000000..1092705b0b --- /dev/null +++ b/vendor/github.com/flosch/pongo2/filters.go @@ -0,0 +1,143 @@ +package pongo2 + +import ( + "fmt" + + "github.com/juju/errors" +) + +// FilterFunction is the type filter functions must fulfil +type FilterFunction func(in *Value, param *Value) (out *Value, err *Error) + +var filters map[string]FilterFunction + +func init() { + filters = make(map[string]FilterFunction) +} + +// FilterExists returns true if the given filter is already registered +func FilterExists(name string) bool { + _, existing := filters[name] + return existing +} + +// RegisterFilter registers a new filter. If there's already a filter with the same +// name, RegisterFilter will panic. You usually want to call this +// function in the filter's init() function: +// http://golang.org/doc/effective_go.html#init +// +// See http://www.florian-schlachter.de/post/pongo2/ for more about +// writing filters and tags. +func RegisterFilter(name string, fn FilterFunction) error { + if FilterExists(name) { + return errors.Errorf("filter with name '%s' is already registered", name) + } + filters[name] = fn + return nil +} + +// ReplaceFilter replaces an already registered filter with a new implementation. Use this +// function with caution since it allows you to change existing filter behaviour. +func ReplaceFilter(name string, fn FilterFunction) error { + if !FilterExists(name) { + return errors.Errorf("filter with name '%s' does not exist (therefore cannot be overridden)", name) + } + filters[name] = fn + return nil +} + +// MustApplyFilter behaves like ApplyFilter, but panics on an error. +func MustApplyFilter(name string, value *Value, param *Value) *Value { + val, err := ApplyFilter(name, value, param) + if err != nil { + panic(err) + } + return val +} + +// ApplyFilter applies a filter to a given value using the given parameters. +// Returns a *pongo2.Value or an error. +func ApplyFilter(name string, value *Value, param *Value) (*Value, *Error) { + fn, existing := filters[name] + if !existing { + return nil, &Error{ + Sender: "applyfilter", + OrigError: errors.Errorf("Filter with name '%s' not found.", name), + } + } + + // Make sure param is a *Value + if param == nil { + param = AsValue(nil) + } + + return fn(value, param) +} + +type filterCall struct { + token *Token + + name string + parameter IEvaluator + + filterFunc FilterFunction +} + +func (fc *filterCall) Execute(v *Value, ctx *ExecutionContext) (*Value, *Error) { + var param *Value + var err *Error + + if fc.parameter != nil { + param, err = fc.parameter.Evaluate(ctx) + if err != nil { + return nil, err + } + } else { + param = AsValue(nil) + } + + filteredValue, err := fc.filterFunc(v, param) + if err != nil { + return nil, err.updateFromTokenIfNeeded(ctx.template, fc.token) + } + return filteredValue, nil +} + +// Filter = IDENT | IDENT ":" FilterArg | IDENT "|" Filter +func (p *Parser) parseFilter() (*filterCall, *Error) { + identToken := p.MatchType(TokenIdentifier) + + // Check filter ident + if identToken == nil { + return nil, p.Error("Filter name must be an identifier.", nil) + } + + filter := &filterCall{ + token: identToken, + name: identToken.Val, + } + + // Get the appropriate filter function and bind it + filterFn, exists := filters[identToken.Val] + if !exists { + return nil, p.Error(fmt.Sprintf("Filter '%s' does not exist.", identToken.Val), identToken) + } + + filter.filterFunc = filterFn + + // Check for filter-argument (2 tokens needed: ':' ARG) + if p.Match(TokenSymbol, ":") != nil { + if p.Peek(TokenSymbol, "}}") != nil { + return nil, p.Error("Filter parameter required after ':'.", nil) + } + + // Get filter argument expression + v, err := p.parseVariableOrLiteral() + if err != nil { + return nil, err + } + filter.parameter = v + } + + return filter, nil +} diff --git a/vendor/github.com/flosch/pongo2/filters_builtin.go b/vendor/github.com/flosch/pongo2/filters_builtin.go new file mode 100644 index 0000000000..f02b4918ad --- /dev/null +++ b/vendor/github.com/flosch/pongo2/filters_builtin.go @@ -0,0 +1,927 @@ +package pongo2 + +/* Filters that are provided through github.com/flosch/pongo2-addons: + ------------------------------------------------------------------ + + filesizeformat + slugify + timesince + timeuntil + + Filters that won't be added: + ---------------------------- + + get_static_prefix (reason: web-framework specific) + pprint (reason: python-specific) + static (reason: web-framework specific) + + Reconsideration (not implemented yet): + -------------------------------------- + + force_escape (reason: not yet needed since this is the behaviour of pongo2's escape filter) + safeseq (reason: same reason as `force_escape`) + unordered_list (python-specific; not sure whether needed or not) + dictsort (python-specific; maybe one could add a filter to sort a list of structs by a specific field name) + dictsortreversed (see dictsort) +*/ + +import ( + "bytes" + "fmt" + "math/rand" + "net/url" + "regexp" + "strconv" + "strings" + "time" + "unicode/utf8" + + "github.com/juju/errors" +) + +func init() { + rand.Seed(time.Now().Unix()) + + RegisterFilter("escape", filterEscape) + RegisterFilter("safe", filterSafe) + RegisterFilter("escapejs", filterEscapejs) + + RegisterFilter("add", filterAdd) + RegisterFilter("addslashes", filterAddslashes) + RegisterFilter("capfirst", filterCapfirst) + RegisterFilter("center", filterCenter) + RegisterFilter("cut", filterCut) + RegisterFilter("date", filterDate) + RegisterFilter("default", filterDefault) + RegisterFilter("default_if_none", filterDefaultIfNone) + RegisterFilter("divisibleby", filterDivisibleby) + RegisterFilter("first", filterFirst) + RegisterFilter("floatformat", filterFloatformat) + RegisterFilter("get_digit", filterGetdigit) + RegisterFilter("iriencode", filterIriencode) + RegisterFilter("join", filterJoin) + RegisterFilter("last", filterLast) + RegisterFilter("length", filterLength) + RegisterFilter("length_is", filterLengthis) + RegisterFilter("linebreaks", filterLinebreaks) + RegisterFilter("linebreaksbr", filterLinebreaksbr) + RegisterFilter("linenumbers", filterLinenumbers) + RegisterFilter("ljust", filterLjust) + RegisterFilter("lower", filterLower) + RegisterFilter("make_list", filterMakelist) + RegisterFilter("phone2numeric", filterPhone2numeric) + RegisterFilter("pluralize", filterPluralize) + RegisterFilter("random", filterRandom) + RegisterFilter("removetags", filterRemovetags) + RegisterFilter("rjust", filterRjust) + RegisterFilter("slice", filterSlice) + RegisterFilter("split", filterSplit) + RegisterFilter("stringformat", filterStringformat) + RegisterFilter("striptags", filterStriptags) + RegisterFilter("time", filterDate) // time uses filterDate (same golang-format) + RegisterFilter("title", filterTitle) + RegisterFilter("truncatechars", filterTruncatechars) + RegisterFilter("truncatechars_html", filterTruncatecharsHTML) + RegisterFilter("truncatewords", filterTruncatewords) + RegisterFilter("truncatewords_html", filterTruncatewordsHTML) + RegisterFilter("upper", filterUpper) + RegisterFilter("urlencode", filterUrlencode) + RegisterFilter("urlize", filterUrlize) + RegisterFilter("urlizetrunc", filterUrlizetrunc) + RegisterFilter("wordcount", filterWordcount) + RegisterFilter("wordwrap", filterWordwrap) + RegisterFilter("yesno", filterYesno) + + RegisterFilter("float", filterFloat) // pongo-specific + RegisterFilter("integer", filterInteger) // pongo-specific +} + +func filterTruncatecharsHelper(s string, newLen int) string { + runes := []rune(s) + if newLen < len(runes) { + if newLen >= 3 { + return fmt.Sprintf("%s...", string(runes[:newLen-3])) + } + // Not enough space for the ellipsis + return string(runes[:newLen]) + } + return string(runes) +} + +func filterTruncateHTMLHelper(value string, newOutput *bytes.Buffer, cond func() bool, fn func(c rune, s int, idx int) int, finalize func()) { + vLen := len(value) + var tagStack []string + idx := 0 + + for idx < vLen && !cond() { + c, s := utf8.DecodeRuneInString(value[idx:]) + if c == utf8.RuneError { + idx += s + continue + } + + if c == '<' { + newOutput.WriteRune(c) + idx += s // consume "<" + + if idx+1 < vLen { + if value[idx] == '/' { + // Close tag + + newOutput.WriteString("/") + + tag := "" + idx++ // consume "/" + + for idx < vLen { + c2, size2 := utf8.DecodeRuneInString(value[idx:]) + if c2 == utf8.RuneError { + idx += size2 + continue + } + + // End of tag found + if c2 == '>' { + idx++ // consume ">" + break + } + tag += string(c2) + idx += size2 + } + + if len(tagStack) > 0 { + // Ideally, the close tag is TOP of tag stack + // In malformed HTML, it must not be, so iterate through the stack and remove the tag + for i := len(tagStack) - 1; i >= 0; i-- { + if tagStack[i] == tag { + // Found the tag + tagStack[i] = tagStack[len(tagStack)-1] + tagStack = tagStack[:len(tagStack)-1] + break + } + } + } + + newOutput.WriteString(tag) + newOutput.WriteString(">") + } else { + // Open tag + + tag := "" + + params := false + for idx < vLen { + c2, size2 := utf8.DecodeRuneInString(value[idx:]) + if c2 == utf8.RuneError { + idx += size2 + continue + } + + newOutput.WriteRune(c2) + + // End of tag found + if c2 == '>' { + idx++ // consume ">" + break + } + + if !params { + if c2 == ' ' { + params = true + } else { + tag += string(c2) + } + } + + idx += size2 + } + + // Add tag to stack + tagStack = append(tagStack, tag) + } + } + } else { + idx = fn(c, s, idx) + } + } + + finalize() + + for i := len(tagStack) - 1; i >= 0; i-- { + tag := tagStack[i] + // Close everything from the regular tag stack + newOutput.WriteString(fmt.Sprintf("", tag)) + } +} + +func filterTruncatechars(in *Value, param *Value) (*Value, *Error) { + s := in.String() + newLen := param.Integer() + return AsValue(filterTruncatecharsHelper(s, newLen)), nil +} + +func filterTruncatecharsHTML(in *Value, param *Value) (*Value, *Error) { + value := in.String() + newLen := max(param.Integer()-3, 0) + + newOutput := bytes.NewBuffer(nil) + + textcounter := 0 + + filterTruncateHTMLHelper(value, newOutput, func() bool { + return textcounter >= newLen + }, func(c rune, s int, idx int) int { + textcounter++ + newOutput.WriteRune(c) + + return idx + s + }, func() { + if textcounter >= newLen && textcounter < len(value) { + newOutput.WriteString("...") + } + }) + + return AsSafeValue(newOutput.String()), nil +} + +func filterTruncatewords(in *Value, param *Value) (*Value, *Error) { + words := strings.Fields(in.String()) + n := param.Integer() + if n <= 0 { + return AsValue(""), nil + } + nlen := min(len(words), n) + out := make([]string, 0, nlen) + for i := 0; i < nlen; i++ { + out = append(out, words[i]) + } + + if n < len(words) { + out = append(out, "...") + } + + return AsValue(strings.Join(out, " ")), nil +} + +func filterTruncatewordsHTML(in *Value, param *Value) (*Value, *Error) { + value := in.String() + newLen := max(param.Integer(), 0) + + newOutput := bytes.NewBuffer(nil) + + wordcounter := 0 + + filterTruncateHTMLHelper(value, newOutput, func() bool { + return wordcounter >= newLen + }, func(_ rune, _ int, idx int) int { + // Get next word + wordFound := false + + for idx < len(value) { + c2, size2 := utf8.DecodeRuneInString(value[idx:]) + if c2 == utf8.RuneError { + idx += size2 + continue + } + + if c2 == '<' { + // HTML tag start, don't consume it + return idx + } + + newOutput.WriteRune(c2) + idx += size2 + + if c2 == ' ' || c2 == '.' || c2 == ',' || c2 == ';' { + // Word ends here, stop capturing it now + break + } else { + wordFound = true + } + } + + if wordFound { + wordcounter++ + } + + return idx + }, func() { + if wordcounter >= newLen { + newOutput.WriteString("...") + } + }) + + return AsSafeValue(newOutput.String()), nil +} + +func filterEscape(in *Value, param *Value) (*Value, *Error) { + output := strings.Replace(in.String(), "&", "&", -1) + output = strings.Replace(output, ">", ">", -1) + output = strings.Replace(output, "<", "<", -1) + output = strings.Replace(output, "\"", """, -1) + output = strings.Replace(output, "'", "'", -1) + return AsValue(output), nil +} + +func filterSafe(in *Value, param *Value) (*Value, *Error) { + return in, nil // nothing to do here, just to keep track of the safe application +} + +func filterEscapejs(in *Value, param *Value) (*Value, *Error) { + sin := in.String() + + var b bytes.Buffer + + idx := 0 + for idx < len(sin) { + c, size := utf8.DecodeRuneInString(sin[idx:]) + if c == utf8.RuneError { + idx += size + continue + } + + if c == '\\' { + // Escape seq? + if idx+1 < len(sin) { + switch sin[idx+1] { + case 'r': + b.WriteString(fmt.Sprintf(`\u%04X`, '\r')) + idx += 2 + continue + case 'n': + b.WriteString(fmt.Sprintf(`\u%04X`, '\n')) + idx += 2 + continue + /*case '\'': + b.WriteString(fmt.Sprintf(`\u%04X`, '\'')) + idx += 2 + continue + case '"': + b.WriteString(fmt.Sprintf(`\u%04X`, '"')) + idx += 2 + continue*/ + } + } + } + + if (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == ' ' || c == '/' { + b.WriteRune(c) + } else { + b.WriteString(fmt.Sprintf(`\u%04X`, c)) + } + + idx += size + } + + return AsValue(b.String()), nil +} + +func filterAdd(in *Value, param *Value) (*Value, *Error) { + if in.IsNumber() && param.IsNumber() { + if in.IsFloat() || param.IsFloat() { + return AsValue(in.Float() + param.Float()), nil + } + return AsValue(in.Integer() + param.Integer()), nil + } + // If in/param is not a number, we're relying on the + // Value's String() conversion and just add them both together + return AsValue(in.String() + param.String()), nil +} + +func filterAddslashes(in *Value, param *Value) (*Value, *Error) { + output := strings.Replace(in.String(), "\\", "\\\\", -1) + output = strings.Replace(output, "\"", "\\\"", -1) + output = strings.Replace(output, "'", "\\'", -1) + return AsValue(output), nil +} + +func filterCut(in *Value, param *Value) (*Value, *Error) { + return AsValue(strings.Replace(in.String(), param.String(), "", -1)), nil +} + +func filterLength(in *Value, param *Value) (*Value, *Error) { + return AsValue(in.Len()), nil +} + +func filterLengthis(in *Value, param *Value) (*Value, *Error) { + return AsValue(in.Len() == param.Integer()), nil +} + +func filterDefault(in *Value, param *Value) (*Value, *Error) { + if !in.IsTrue() { + return param, nil + } + return in, nil +} + +func filterDefaultIfNone(in *Value, param *Value) (*Value, *Error) { + if in.IsNil() { + return param, nil + } + return in, nil +} + +func filterDivisibleby(in *Value, param *Value) (*Value, *Error) { + if param.Integer() == 0 { + return AsValue(false), nil + } + return AsValue(in.Integer()%param.Integer() == 0), nil +} + +func filterFirst(in *Value, param *Value) (*Value, *Error) { + if in.CanSlice() && in.Len() > 0 { + return in.Index(0), nil + } + return AsValue(""), nil +} + +func filterFloatformat(in *Value, param *Value) (*Value, *Error) { + val := in.Float() + + decimals := -1 + if !param.IsNil() { + // Any argument provided? + decimals = param.Integer() + } + + // if the argument is not a number (e. g. empty), the default + // behaviour is trim the result + trim := !param.IsNumber() + + if decimals <= 0 { + // argument is negative or zero, so we + // want the output being trimmed + decimals = -decimals + trim = true + } + + if trim { + // Remove zeroes + if float64(int(val)) == val { + return AsValue(in.Integer()), nil + } + } + + return AsValue(strconv.FormatFloat(val, 'f', decimals, 64)), nil +} + +func filterGetdigit(in *Value, param *Value) (*Value, *Error) { + i := param.Integer() + l := len(in.String()) // do NOT use in.Len() here! + if i <= 0 || i > l { + return in, nil + } + return AsValue(in.String()[l-i] - 48), nil +} + +const filterIRIChars = "/#%[]=:;$&()+,!?*@'~" + +func filterIriencode(in *Value, param *Value) (*Value, *Error) { + var b bytes.Buffer + + sin := in.String() + for _, r := range sin { + if strings.IndexRune(filterIRIChars, r) >= 0 { + b.WriteRune(r) + } else { + b.WriteString(url.QueryEscape(string(r))) + } + } + + return AsValue(b.String()), nil +} + +func filterJoin(in *Value, param *Value) (*Value, *Error) { + if !in.CanSlice() { + return in, nil + } + sep := param.String() + sl := make([]string, 0, in.Len()) + for i := 0; i < in.Len(); i++ { + sl = append(sl, in.Index(i).String()) + } + return AsValue(strings.Join(sl, sep)), nil +} + +func filterLast(in *Value, param *Value) (*Value, *Error) { + if in.CanSlice() && in.Len() > 0 { + return in.Index(in.Len() - 1), nil + } + return AsValue(""), nil +} + +func filterUpper(in *Value, param *Value) (*Value, *Error) { + return AsValue(strings.ToUpper(in.String())), nil +} + +func filterLower(in *Value, param *Value) (*Value, *Error) { + return AsValue(strings.ToLower(in.String())), nil +} + +func filterMakelist(in *Value, param *Value) (*Value, *Error) { + s := in.String() + result := make([]string, 0, len(s)) + for _, c := range s { + result = append(result, string(c)) + } + return AsValue(result), nil +} + +func filterCapfirst(in *Value, param *Value) (*Value, *Error) { + if in.Len() <= 0 { + return AsValue(""), nil + } + t := in.String() + r, size := utf8.DecodeRuneInString(t) + return AsValue(strings.ToUpper(string(r)) + t[size:]), nil +} + +func filterCenter(in *Value, param *Value) (*Value, *Error) { + width := param.Integer() + slen := in.Len() + if width <= slen { + return in, nil + } + + spaces := width - slen + left := spaces/2 + spaces%2 + right := spaces / 2 + + return AsValue(fmt.Sprintf("%s%s%s", strings.Repeat(" ", left), + in.String(), strings.Repeat(" ", right))), nil +} + +func filterDate(in *Value, param *Value) (*Value, *Error) { + t, isTime := in.Interface().(time.Time) + if !isTime { + return nil, &Error{ + Sender: "filter:date", + OrigError: errors.New("filter input argument must be of type 'time.Time'"), + } + } + return AsValue(t.Format(param.String())), nil +} + +func filterFloat(in *Value, param *Value) (*Value, *Error) { + return AsValue(in.Float()), nil +} + +func filterInteger(in *Value, param *Value) (*Value, *Error) { + return AsValue(in.Integer()), nil +} + +func filterLinebreaks(in *Value, param *Value) (*Value, *Error) { + if in.Len() == 0 { + return in, nil + } + + var b bytes.Buffer + + // Newline =
+ // Double newline =

...

+ lines := strings.Split(in.String(), "\n") + lenlines := len(lines) + + opened := false + + for idx, line := range lines { + + if !opened { + b.WriteString("

") + opened = true + } + + b.WriteString(line) + + if idx < lenlines-1 && strings.TrimSpace(lines[idx]) != "" { + // We've not reached the end + if strings.TrimSpace(lines[idx+1]) == "" { + // Next line is empty + if opened { + b.WriteString("

") + opened = false + } + } else { + b.WriteString("
") + } + } + } + + if opened { + b.WriteString("

") + } + + return AsValue(b.String()), nil +} + +func filterSplit(in *Value, param *Value) (*Value, *Error) { + chunks := strings.Split(in.String(), param.String()) + + return AsValue(chunks), nil +} + +func filterLinebreaksbr(in *Value, param *Value) (*Value, *Error) { + return AsValue(strings.Replace(in.String(), "\n", "
", -1)), nil +} + +func filterLinenumbers(in *Value, param *Value) (*Value, *Error) { + lines := strings.Split(in.String(), "\n") + output := make([]string, 0, len(lines)) + for idx, line := range lines { + output = append(output, fmt.Sprintf("%d. %s", idx+1, line)) + } + return AsValue(strings.Join(output, "\n")), nil +} + +func filterLjust(in *Value, param *Value) (*Value, *Error) { + times := param.Integer() - in.Len() + if times < 0 { + times = 0 + } + return AsValue(fmt.Sprintf("%s%s", in.String(), strings.Repeat(" ", times))), nil +} + +func filterUrlencode(in *Value, param *Value) (*Value, *Error) { + return AsValue(url.QueryEscape(in.String())), nil +} + +// TODO: This regexp could do some work +var filterUrlizeURLRegexp = regexp.MustCompile(`((((http|https)://)|www\.|((^|[ ])[0-9A-Za-z_\-]+(\.com|\.net|\.org|\.info|\.biz|\.de))))(?U:.*)([ ]+|$)`) +var filterUrlizeEmailRegexp = regexp.MustCompile(`(\w+@\w+\.\w{2,4})`) + +func filterUrlizeHelper(input string, autoescape bool, trunc int) (string, error) { + var soutErr error + sout := filterUrlizeURLRegexp.ReplaceAllStringFunc(input, func(raw_url string) string { + var prefix string + var suffix string + if strings.HasPrefix(raw_url, " ") { + prefix = " " + } + if strings.HasSuffix(raw_url, " ") { + suffix = " " + } + + raw_url = strings.TrimSpace(raw_url) + + t, err := ApplyFilter("iriencode", AsValue(raw_url), nil) + if err != nil { + soutErr = err + return "" + } + url := t.String() + + if !strings.HasPrefix(url, "http") { + url = fmt.Sprintf("http://%s", url) + } + + title := raw_url + + if trunc > 3 && len(title) > trunc { + title = fmt.Sprintf("%s...", title[:trunc-3]) + } + + if autoescape { + t, err := ApplyFilter("escape", AsValue(title), nil) + if err != nil { + soutErr = err + return "" + } + title = t.String() + } + + return fmt.Sprintf(`%s%s%s`, prefix, url, title, suffix) + }) + if soutErr != nil { + return "", soutErr + } + + sout = filterUrlizeEmailRegexp.ReplaceAllStringFunc(sout, func(mail string) string { + title := mail + + if trunc > 3 && len(title) > trunc { + title = fmt.Sprintf("%s...", title[:trunc-3]) + } + + return fmt.Sprintf(`%s`, mail, title) + }) + + return sout, nil +} + +func filterUrlize(in *Value, param *Value) (*Value, *Error) { + autoescape := true + if param.IsBool() { + autoescape = param.Bool() + } + + s, err := filterUrlizeHelper(in.String(), autoescape, -1) + if err != nil { + + } + + return AsValue(s), nil +} + +func filterUrlizetrunc(in *Value, param *Value) (*Value, *Error) { + s, err := filterUrlizeHelper(in.String(), true, param.Integer()) + if err != nil { + return nil, &Error{ + Sender: "filter:urlizetrunc", + OrigError: errors.New("you cannot pass more than 2 arguments to filter 'pluralize'"), + } + } + return AsValue(s), nil +} + +func filterStringformat(in *Value, param *Value) (*Value, *Error) { + return AsValue(fmt.Sprintf(param.String(), in.Interface())), nil +} + +var reStriptags = regexp.MustCompile("<[^>]*?>") + +func filterStriptags(in *Value, param *Value) (*Value, *Error) { + s := in.String() + + // Strip all tags + s = reStriptags.ReplaceAllString(s, "") + + return AsValue(strings.TrimSpace(s)), nil +} + +// https://en.wikipedia.org/wiki/Phoneword +var filterPhone2numericMap = map[string]string{ + "a": "2", "b": "2", "c": "2", "d": "3", "e": "3", "f": "3", "g": "4", "h": "4", "i": "4", "j": "5", "k": "5", + "l": "5", "m": "6", "n": "6", "o": "6", "p": "7", "q": "7", "r": "7", "s": "7", "t": "8", "u": "8", "v": "8", + "w": "9", "x": "9", "y": "9", "z": "9", +} + +func filterPhone2numeric(in *Value, param *Value) (*Value, *Error) { + sin := in.String() + for k, v := range filterPhone2numericMap { + sin = strings.Replace(sin, k, v, -1) + sin = strings.Replace(sin, strings.ToUpper(k), v, -1) + } + return AsValue(sin), nil +} + +func filterPluralize(in *Value, param *Value) (*Value, *Error) { + if in.IsNumber() { + // Works only on numbers + if param.Len() > 0 { + endings := strings.Split(param.String(), ",") + if len(endings) > 2 { + return nil, &Error{ + Sender: "filter:pluralize", + OrigError: errors.New("you cannot pass more than 2 arguments to filter 'pluralize'"), + } + } + if len(endings) == 1 { + // 1 argument + if in.Integer() != 1 { + return AsValue(endings[0]), nil + } + } else { + if in.Integer() != 1 { + // 2 arguments + return AsValue(endings[1]), nil + } + return AsValue(endings[0]), nil + } + } else { + if in.Integer() != 1 { + // return default 's' + return AsValue("s"), nil + } + } + + return AsValue(""), nil + } + return nil, &Error{ + Sender: "filter:pluralize", + OrigError: errors.New("filter 'pluralize' does only work on numbers"), + } +} + +func filterRandom(in *Value, param *Value) (*Value, *Error) { + if !in.CanSlice() || in.Len() <= 0 { + return in, nil + } + i := rand.Intn(in.Len()) + return in.Index(i), nil +} + +func filterRemovetags(in *Value, param *Value) (*Value, *Error) { + s := in.String() + tags := strings.Split(param.String(), ",") + + // Strip only specific tags + for _, tag := range tags { + re := regexp.MustCompile(fmt.Sprintf("", tag)) + s = re.ReplaceAllString(s, "") + } + + return AsValue(strings.TrimSpace(s)), nil +} + +func filterRjust(in *Value, param *Value) (*Value, *Error) { + return AsValue(fmt.Sprintf(fmt.Sprintf("%%%ds", param.Integer()), in.String())), nil +} + +func filterSlice(in *Value, param *Value) (*Value, *Error) { + comp := strings.Split(param.String(), ":") + if len(comp) != 2 { + return nil, &Error{ + Sender: "filter:slice", + OrigError: errors.New("Slice string must have the format 'from:to' [from/to can be omitted, but the ':' is required]"), + } + } + + if !in.CanSlice() { + return in, nil + } + + from := AsValue(comp[0]).Integer() + to := in.Len() + + if from > to { + from = to + } + + vto := AsValue(comp[1]).Integer() + if vto >= from && vto <= in.Len() { + to = vto + } + + return in.Slice(from, to), nil +} + +func filterTitle(in *Value, param *Value) (*Value, *Error) { + if !in.IsString() { + return AsValue(""), nil + } + return AsValue(strings.Title(strings.ToLower(in.String()))), nil +} + +func filterWordcount(in *Value, param *Value) (*Value, *Error) { + return AsValue(len(strings.Fields(in.String()))), nil +} + +func filterWordwrap(in *Value, param *Value) (*Value, *Error) { + words := strings.Fields(in.String()) + wordsLen := len(words) + wrapAt := param.Integer() + if wrapAt <= 0 { + return in, nil + } + + linecount := wordsLen/wrapAt + wordsLen%wrapAt + lines := make([]string, 0, linecount) + for i := 0; i < linecount; i++ { + lines = append(lines, strings.Join(words[wrapAt*i:min(wrapAt*(i+1), wordsLen)], " ")) + } + return AsValue(strings.Join(lines, "\n")), nil +} + +func filterYesno(in *Value, param *Value) (*Value, *Error) { + choices := map[int]string{ + 0: "yes", + 1: "no", + 2: "maybe", + } + paramString := param.String() + customChoices := strings.Split(paramString, ",") + if len(paramString) > 0 { + if len(customChoices) > 3 { + return nil, &Error{ + Sender: "filter:yesno", + OrigError: errors.Errorf("You cannot pass more than 3 options to the 'yesno'-filter (got: '%s').", paramString), + } + } + if len(customChoices) < 2 { + return nil, &Error{ + Sender: "filter:yesno", + OrigError: errors.Errorf("You must pass either no or at least 2 arguments to the 'yesno'-filter (got: '%s').", paramString), + } + } + + // Map to the options now + choices[0] = customChoices[0] + choices[1] = customChoices[1] + if len(customChoices) == 3 { + choices[2] = customChoices[2] + } + } + + // maybe + if in.IsNil() { + return AsValue(choices[2]), nil + } + + // yes + if in.IsTrue() { + return AsValue(choices[0]), nil + } + + // no + return AsValue(choices[1]), nil +} diff --git a/vendor/github.com/flosch/pongo2/go.mod b/vendor/github.com/flosch/pongo2/go.mod new file mode 100644 index 0000000000..06b6c2566f --- /dev/null +++ b/vendor/github.com/flosch/pongo2/go.mod @@ -0,0 +1,13 @@ +module github.com/flosch/pongo2 + +require ( + github.com/go-check/check v0.0.0-20180628173108-788fd7840127 + github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5 + github.com/juju/loggo v0.0.0-20180524022052-584905176618 // indirect + github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073 // indirect + github.com/kr/pretty v0.1.0 // indirect + github.com/mattn/goveralls v0.0.2 // indirect + golang.org/x/tools v0.0.0-20181221001348-537d06c36207 // indirect + gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce // indirect + gopkg.in/yaml.v2 v2.2.2 // indirect +) diff --git a/vendor/github.com/flosch/pongo2/helpers.go b/vendor/github.com/flosch/pongo2/helpers.go new file mode 100644 index 0000000000..880dbc0444 --- /dev/null +++ b/vendor/github.com/flosch/pongo2/helpers.go @@ -0,0 +1,15 @@ +package pongo2 + +func max(a, b int) int { + if a > b { + return a + } + return b +} + +func min(a, b int) int { + if a < b { + return a + } + return b +} diff --git a/vendor/github.com/flosch/pongo2/lexer.go b/vendor/github.com/flosch/pongo2/lexer.go new file mode 100644 index 0000000000..67b0b95023 --- /dev/null +++ b/vendor/github.com/flosch/pongo2/lexer.go @@ -0,0 +1,432 @@ +package pongo2 + +import ( + "fmt" + "strings" + "unicode/utf8" + + "github.com/juju/errors" +) + +const ( + TokenError = iota + EOF + + TokenHTML + + TokenKeyword + TokenIdentifier + TokenString + TokenNumber + TokenSymbol +) + +var ( + tokenSpaceChars = " \n\r\t" + tokenIdentifierChars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_" + tokenIdentifierCharsWithDigits = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_0123456789" + tokenDigits = "0123456789" + + // Available symbols in pongo2 (within filters/tag) + TokenSymbols = []string{ + // 3-Char symbols + "{{-", "-}}", "{%-", "-%}", + + // 2-Char symbols + "==", ">=", "<=", "&&", "||", "{{", "}}", "{%", "%}", "!=", "<>", + + // 1-Char symbol + "(", ")", "+", "-", "*", "<", ">", "/", "^", ",", ".", "!", "|", ":", "=", "%", + } + + // Available keywords in pongo2 + TokenKeywords = []string{"in", "and", "or", "not", "true", "false", "as", "export"} +) + +type TokenType int +type Token struct { + Filename string + Typ TokenType + Val string + Line int + Col int + TrimWhitespaces bool +} + +type lexerStateFn func() lexerStateFn +type lexer struct { + name string + input string + start int // start pos of the item + pos int // current pos + width int // width of last rune + tokens []*Token + errored bool + startline int + startcol int + line int + col int + + inVerbatim bool + verbatimName string +} + +func (t *Token) String() string { + val := t.Val + if len(val) > 1000 { + val = fmt.Sprintf("%s...%s", val[:10], val[len(val)-5:len(val)]) + } + + typ := "" + switch t.Typ { + case TokenHTML: + typ = "HTML" + case TokenError: + typ = "Error" + case TokenIdentifier: + typ = "Identifier" + case TokenKeyword: + typ = "Keyword" + case TokenNumber: + typ = "Number" + case TokenString: + typ = "String" + case TokenSymbol: + typ = "Symbol" + default: + typ = "Unknown" + } + + return fmt.Sprintf("", + typ, t.Typ, val, t.Line, t.Col, t.TrimWhitespaces) +} + +func lex(name string, input string) ([]*Token, *Error) { + l := &lexer{ + name: name, + input: input, + tokens: make([]*Token, 0, 100), + line: 1, + col: 1, + startline: 1, + startcol: 1, + } + l.run() + if l.errored { + errtoken := l.tokens[len(l.tokens)-1] + return nil, &Error{ + Filename: name, + Line: errtoken.Line, + Column: errtoken.Col, + Sender: "lexer", + OrigError: errors.New(errtoken.Val), + } + } + return l.tokens, nil +} + +func (l *lexer) value() string { + return l.input[l.start:l.pos] +} + +func (l *lexer) length() int { + return l.pos - l.start +} + +func (l *lexer) emit(t TokenType) { + tok := &Token{ + Filename: l.name, + Typ: t, + Val: l.value(), + Line: l.startline, + Col: l.startcol, + } + + if t == TokenString { + // Escape sequence \" in strings + tok.Val = strings.Replace(tok.Val, `\"`, `"`, -1) + tok.Val = strings.Replace(tok.Val, `\\`, `\`, -1) + } + + if t == TokenSymbol && len(tok.Val) == 3 && (strings.HasSuffix(tok.Val, "-") || strings.HasPrefix(tok.Val, "-")) { + tok.TrimWhitespaces = true + tok.Val = strings.Replace(tok.Val, "-", "", -1) + } + + l.tokens = append(l.tokens, tok) + l.start = l.pos + l.startline = l.line + l.startcol = l.col +} + +func (l *lexer) next() rune { + if l.pos >= len(l.input) { + l.width = 0 + return EOF + } + r, w := utf8.DecodeRuneInString(l.input[l.pos:]) + l.width = w + l.pos += l.width + l.col += l.width + return r +} + +func (l *lexer) backup() { + l.pos -= l.width + l.col -= l.width +} + +func (l *lexer) peek() rune { + r := l.next() + l.backup() + return r +} + +func (l *lexer) ignore() { + l.start = l.pos + l.startline = l.line + l.startcol = l.col +} + +func (l *lexer) accept(what string) bool { + if strings.IndexRune(what, l.next()) >= 0 { + return true + } + l.backup() + return false +} + +func (l *lexer) acceptRun(what string) { + for strings.IndexRune(what, l.next()) >= 0 { + } + l.backup() +} + +func (l *lexer) errorf(format string, args ...interface{}) lexerStateFn { + t := &Token{ + Filename: l.name, + Typ: TokenError, + Val: fmt.Sprintf(format, args...), + Line: l.startline, + Col: l.startcol, + } + l.tokens = append(l.tokens, t) + l.errored = true + l.startline = l.line + l.startcol = l.col + return nil +} + +func (l *lexer) eof() bool { + return l.start >= len(l.input)-1 +} + +func (l *lexer) run() { + for { + // TODO: Support verbatim tag names + // https://docs.djangoproject.com/en/dev/ref/templates/builtins/#verbatim + if l.inVerbatim { + name := l.verbatimName + if name != "" { + name += " " + } + if strings.HasPrefix(l.input[l.pos:], fmt.Sprintf("{%% endverbatim %s%%}", name)) { // end verbatim + if l.pos > l.start { + l.emit(TokenHTML) + } + w := len("{% endverbatim %}") + l.pos += w + l.col += w + l.ignore() + l.inVerbatim = false + } + } else if strings.HasPrefix(l.input[l.pos:], "{% verbatim %}") { // tag + if l.pos > l.start { + l.emit(TokenHTML) + } + l.inVerbatim = true + w := len("{% verbatim %}") + l.pos += w + l.col += w + l.ignore() + } + + if !l.inVerbatim { + // Ignore single-line comments {# ... #} + if strings.HasPrefix(l.input[l.pos:], "{#") { + if l.pos > l.start { + l.emit(TokenHTML) + } + + l.pos += 2 // pass '{#' + l.col += 2 + + for { + switch l.peek() { + case EOF: + l.errorf("Single-line comment not closed.") + return + case '\n': + l.errorf("Newline not permitted in a single-line comment.") + return + } + + if strings.HasPrefix(l.input[l.pos:], "#}") { + l.pos += 2 // pass '#}' + l.col += 2 + break + } + + l.next() + } + l.ignore() // ignore whole comment + + // Comment skipped + continue // next token + } + + if strings.HasPrefix(l.input[l.pos:], "{{") || // variable + strings.HasPrefix(l.input[l.pos:], "{%") { // tag + if l.pos > l.start { + l.emit(TokenHTML) + } + l.tokenize() + if l.errored { + return + } + continue + } + } + + switch l.peek() { + case '\n': + l.line++ + l.col = 0 + } + if l.next() == EOF { + break + } + } + + if l.pos > l.start { + l.emit(TokenHTML) + } + + if l.inVerbatim { + l.errorf("verbatim-tag not closed, got EOF.") + } +} + +func (l *lexer) tokenize() { + for state := l.stateCode; state != nil; { + state = state() + } +} + +func (l *lexer) stateCode() lexerStateFn { +outer_loop: + for { + switch { + case l.accept(tokenSpaceChars): + if l.value() == "\n" { + return l.errorf("Newline not allowed within tag/variable.") + } + l.ignore() + continue + case l.accept(tokenIdentifierChars): + return l.stateIdentifier + case l.accept(tokenDigits): + return l.stateNumber + case l.accept(`"'`): + return l.stateString + } + + // Check for symbol + for _, sym := range TokenSymbols { + if strings.HasPrefix(l.input[l.start:], sym) { + l.pos += len(sym) + l.col += l.length() + l.emit(TokenSymbol) + + if sym == "%}" || sym == "-%}" || sym == "}}" || sym == "-}}" { + // Tag/variable end, return after emit + return nil + } + + continue outer_loop + } + } + + break + } + + // Normal shut down + return nil +} + +func (l *lexer) stateIdentifier() lexerStateFn { + l.acceptRun(tokenIdentifierChars) + l.acceptRun(tokenIdentifierCharsWithDigits) + for _, kw := range TokenKeywords { + if kw == l.value() { + l.emit(TokenKeyword) + return l.stateCode + } + } + l.emit(TokenIdentifier) + return l.stateCode +} + +func (l *lexer) stateNumber() lexerStateFn { + l.acceptRun(tokenDigits) + if l.accept(tokenIdentifierCharsWithDigits) { + // This seems to be an identifier starting with a number. + // See https://github.com/flosch/pongo2/issues/151 + return l.stateIdentifier() + } + /* + Maybe context-sensitive number lexing? + * comments.0.Text // first comment + * usercomments.1.0 // second user, first comment + * if (score >= 8.5) // 8.5 as a number + + if l.peek() == '.' { + l.accept(".") + if !l.accept(tokenDigits) { + return l.errorf("Malformed number.") + } + l.acceptRun(tokenDigits) + } + */ + l.emit(TokenNumber) + return l.stateCode +} + +func (l *lexer) stateString() lexerStateFn { + quotationMark := l.value() + l.ignore() + l.startcol-- // we're starting the position at the first " + for !l.accept(quotationMark) { + switch l.next() { + case '\\': + // escape sequence + switch l.peek() { + case '"', '\\': + l.next() + default: + return l.errorf("Unknown escape sequence: \\%c", l.peek()) + } + case EOF: + return l.errorf("Unexpected EOF, string not closed.") + case '\n': + return l.errorf("Newline in string is not allowed.") + } + } + l.backup() + l.emit(TokenString) + + l.next() + l.ignore() + + return l.stateCode +} diff --git a/vendor/github.com/flosch/pongo2/nodes.go b/vendor/github.com/flosch/pongo2/nodes.go new file mode 100644 index 0000000000..5b039cdf40 --- /dev/null +++ b/vendor/github.com/flosch/pongo2/nodes.go @@ -0,0 +1,16 @@ +package pongo2 + +// The root document +type nodeDocument struct { + Nodes []INode +} + +func (doc *nodeDocument) Execute(ctx *ExecutionContext, writer TemplateWriter) *Error { + for _, n := range doc.Nodes { + err := n.Execute(ctx, writer) + if err != nil { + return err + } + } + return nil +} diff --git a/vendor/github.com/flosch/pongo2/nodes_html.go b/vendor/github.com/flosch/pongo2/nodes_html.go new file mode 100644 index 0000000000..c735defeb2 --- /dev/null +++ b/vendor/github.com/flosch/pongo2/nodes_html.go @@ -0,0 +1,23 @@ +package pongo2 + +import ( + "strings" +) + +type nodeHTML struct { + token *Token + trimLeft bool + trimRight bool +} + +func (n *nodeHTML) Execute(ctx *ExecutionContext, writer TemplateWriter) *Error { + res := n.token.Val + if n.trimLeft { + res = strings.TrimLeft(res, tokenSpaceChars) + } + if n.trimRight { + res = strings.TrimRight(res, tokenSpaceChars) + } + writer.WriteString(res) + return nil +} diff --git a/vendor/github.com/flosch/pongo2/nodes_wrapper.go b/vendor/github.com/flosch/pongo2/nodes_wrapper.go new file mode 100644 index 0000000000..d1bcb8d851 --- /dev/null +++ b/vendor/github.com/flosch/pongo2/nodes_wrapper.go @@ -0,0 +1,16 @@ +package pongo2 + +type NodeWrapper struct { + Endtag string + nodes []INode +} + +func (wrapper *NodeWrapper) Execute(ctx *ExecutionContext, writer TemplateWriter) *Error { + for _, n := range wrapper.nodes { + err := n.Execute(ctx, writer) + if err != nil { + return err + } + } + return nil +} diff --git a/vendor/github.com/flosch/pongo2/options.go b/vendor/github.com/flosch/pongo2/options.go new file mode 100644 index 0000000000..9c39e467ef --- /dev/null +++ b/vendor/github.com/flosch/pongo2/options.go @@ -0,0 +1,26 @@ +package pongo2 + +// Options allow you to change the behavior of template-engine. +// You can change the options before calling the Execute method. +type Options struct { + // If this is set to true the first newline after a block is removed (block, not variable tag!). Defaults to false. + TrimBlocks bool + + // If this is set to true leading spaces and tabs are stripped from the start of a line to a block. Defaults to false + LStripBlocks bool +} + +func newOptions() *Options { + return &Options{ + TrimBlocks: false, + LStripBlocks: false, + } +} + +// Update updates this options from another options. +func (opt *Options) Update(other *Options) *Options { + opt.TrimBlocks = other.TrimBlocks + opt.LStripBlocks = other.LStripBlocks + + return opt +} diff --git a/vendor/github.com/flosch/pongo2/parser.go b/vendor/github.com/flosch/pongo2/parser.go new file mode 100644 index 0000000000..2279e3c496 --- /dev/null +++ b/vendor/github.com/flosch/pongo2/parser.go @@ -0,0 +1,309 @@ +package pongo2 + +import ( + "fmt" + "strings" + + "github.com/juju/errors" +) + +type INode interface { + Execute(*ExecutionContext, TemplateWriter) *Error +} + +type IEvaluator interface { + INode + GetPositionToken() *Token + Evaluate(*ExecutionContext) (*Value, *Error) + FilterApplied(name string) bool +} + +// The parser provides you a comprehensive and easy tool to +// work with the template document and arguments provided by +// the user for your custom tag. +// +// The parser works on a token list which will be provided by pongo2. +// A token is a unit you can work with. Tokens are either of type identifier, +// string, number, keyword, HTML or symbol. +// +// (See Token's documentation for more about tokens) +type Parser struct { + name string + idx int + tokens []*Token + lastToken *Token + + // if the parser parses a template document, here will be + // a reference to it (needed to access the template through Tags) + template *Template +} + +// Creates a new parser to parse tokens. +// Used inside pongo2 to parse documents and to provide an easy-to-use +// parser for tag authors +func newParser(name string, tokens []*Token, template *Template) *Parser { + p := &Parser{ + name: name, + tokens: tokens, + template: template, + } + if len(tokens) > 0 { + p.lastToken = tokens[len(tokens)-1] + } + return p +} + +// Consume one token. It will be gone forever. +func (p *Parser) Consume() { + p.ConsumeN(1) +} + +// Consume N tokens. They will be gone forever. +func (p *Parser) ConsumeN(count int) { + p.idx += count +} + +// Returns the current token. +func (p *Parser) Current() *Token { + return p.Get(p.idx) +} + +// Returns the CURRENT token if the given type matches. +// Consumes this token on success. +func (p *Parser) MatchType(typ TokenType) *Token { + if t := p.PeekType(typ); t != nil { + p.Consume() + return t + } + return nil +} + +// Returns the CURRENT token if the given type AND value matches. +// Consumes this token on success. +func (p *Parser) Match(typ TokenType, val string) *Token { + if t := p.Peek(typ, val); t != nil { + p.Consume() + return t + } + return nil +} + +// Returns the CURRENT token if the given type AND *one* of +// the given values matches. +// Consumes this token on success. +func (p *Parser) MatchOne(typ TokenType, vals ...string) *Token { + for _, val := range vals { + if t := p.Peek(typ, val); t != nil { + p.Consume() + return t + } + } + return nil +} + +// Returns the CURRENT token if the given type matches. +// It DOES NOT consume the token. +func (p *Parser) PeekType(typ TokenType) *Token { + return p.PeekTypeN(0, typ) +} + +// Returns the CURRENT token if the given type AND value matches. +// It DOES NOT consume the token. +func (p *Parser) Peek(typ TokenType, val string) *Token { + return p.PeekN(0, typ, val) +} + +// Returns the CURRENT token if the given type AND *one* of +// the given values matches. +// It DOES NOT consume the token. +func (p *Parser) PeekOne(typ TokenType, vals ...string) *Token { + for _, v := range vals { + t := p.PeekN(0, typ, v) + if t != nil { + return t + } + } + return nil +} + +// Returns the tokens[current position + shift] token if the +// given type AND value matches for that token. +// DOES NOT consume the token. +func (p *Parser) PeekN(shift int, typ TokenType, val string) *Token { + t := p.Get(p.idx + shift) + if t != nil { + if t.Typ == typ && t.Val == val { + return t + } + } + return nil +} + +// Returns the tokens[current position + shift] token if the given type matches. +// DOES NOT consume the token for that token. +func (p *Parser) PeekTypeN(shift int, typ TokenType) *Token { + t := p.Get(p.idx + shift) + if t != nil { + if t.Typ == typ { + return t + } + } + return nil +} + +// Returns the UNCONSUMED token count. +func (p *Parser) Remaining() int { + return len(p.tokens) - p.idx +} + +// Returns the total token count. +func (p *Parser) Count() int { + return len(p.tokens) +} + +// Returns tokens[i] or NIL (if i >= len(tokens)) +func (p *Parser) Get(i int) *Token { + if i < len(p.tokens) && i >= 0 { + return p.tokens[i] + } + return nil +} + +// Returns tokens[current-position + shift] or NIL +// (if (current-position + i) >= len(tokens)) +func (p *Parser) GetR(shift int) *Token { + i := p.idx + shift + return p.Get(i) +} + +// Error produces a nice error message and returns an error-object. +// The 'token'-argument is optional. If provided, it will take +// the token's position information. If not provided, it will +// automatically use the CURRENT token's position information. +func (p *Parser) Error(msg string, token *Token) *Error { + if token == nil { + // Set current token + token = p.Current() + if token == nil { + // Set to last token + if len(p.tokens) > 0 { + token = p.tokens[len(p.tokens)-1] + } + } + } + var line, col int + if token != nil { + line = token.Line + col = token.Col + } + return &Error{ + Template: p.template, + Filename: p.name, + Sender: "parser", + Line: line, + Column: col, + Token: token, + OrigError: errors.New(msg), + } +} + +// Wraps all nodes between starting tag and "{% endtag %}" and provides +// one simple interface to execute the wrapped nodes. +// It returns a parser to process provided arguments to the tag. +func (p *Parser) WrapUntilTag(names ...string) (*NodeWrapper, *Parser, *Error) { + wrapper := &NodeWrapper{} + + var tagArgs []*Token + + for p.Remaining() > 0 { + // New tag, check whether we have to stop wrapping here + if p.Peek(TokenSymbol, "{%") != nil { + tagIdent := p.PeekTypeN(1, TokenIdentifier) + + if tagIdent != nil { + // We've found a (!) end-tag + + found := false + for _, n := range names { + if tagIdent.Val == n { + found = true + break + } + } + + // We only process the tag if we've found an end tag + if found { + // Okay, endtag found. + p.ConsumeN(2) // '{%' tagname + + for { + if p.Match(TokenSymbol, "%}") != nil { + // Okay, end the wrapping here + wrapper.Endtag = tagIdent.Val + return wrapper, newParser(p.template.name, tagArgs, p.template), nil + } + t := p.Current() + p.Consume() + if t == nil { + return nil, nil, p.Error("Unexpected EOF.", p.lastToken) + } + tagArgs = append(tagArgs, t) + } + } + } + + } + + // Otherwise process next element to be wrapped + node, err := p.parseDocElement() + if err != nil { + return nil, nil, err + } + wrapper.nodes = append(wrapper.nodes, node) + } + + return nil, nil, p.Error(fmt.Sprintf("Unexpected EOF, expected tag %s.", strings.Join(names, " or ")), + p.lastToken) +} + +// Skips all nodes between starting tag and "{% endtag %}" +func (p *Parser) SkipUntilTag(names ...string) *Error { + for p.Remaining() > 0 { + // New tag, check whether we have to stop wrapping here + if p.Peek(TokenSymbol, "{%") != nil { + tagIdent := p.PeekTypeN(1, TokenIdentifier) + + if tagIdent != nil { + // We've found a (!) end-tag + + found := false + for _, n := range names { + if tagIdent.Val == n { + found = true + break + } + } + + // We only process the tag if we've found an end tag + if found { + // Okay, endtag found. + p.ConsumeN(2) // '{%' tagname + + for { + if p.Match(TokenSymbol, "%}") != nil { + // Done skipping, exit. + return nil + } + } + } + } + } + t := p.Current() + p.Consume() + if t == nil { + return p.Error("Unexpected EOF.", p.lastToken) + } + } + + return p.Error(fmt.Sprintf("Unexpected EOF, expected tag %s.", strings.Join(names, " or ")), p.lastToken) +} diff --git a/vendor/github.com/flosch/pongo2/parser_document.go b/vendor/github.com/flosch/pongo2/parser_document.go new file mode 100644 index 0000000000..e3ac2c8e9d --- /dev/null +++ b/vendor/github.com/flosch/pongo2/parser_document.go @@ -0,0 +1,59 @@ +package pongo2 + +// Doc = { ( Filter | Tag | HTML ) } +func (p *Parser) parseDocElement() (INode, *Error) { + t := p.Current() + + switch t.Typ { + case TokenHTML: + n := &nodeHTML{token: t} + left := p.PeekTypeN(-1, TokenSymbol) + right := p.PeekTypeN(1, TokenSymbol) + n.trimLeft = left != nil && left.TrimWhitespaces + n.trimRight = right != nil && right.TrimWhitespaces + p.Consume() // consume HTML element + return n, nil + case TokenSymbol: + switch t.Val { + case "{{": + // parse variable + variable, err := p.parseVariableElement() + if err != nil { + return nil, err + } + return variable, nil + case "{%": + // parse tag + tag, err := p.parseTagElement() + if err != nil { + return nil, err + } + return tag, nil + } + } + return nil, p.Error("Unexpected token (only HTML/tags/filters in templates allowed)", t) +} + +func (tpl *Template) parse() *Error { + tpl.parser = newParser(tpl.name, tpl.tokens, tpl) + doc, err := tpl.parser.parseDocument() + if err != nil { + return err + } + tpl.root = doc + return nil +} + +func (p *Parser) parseDocument() (*nodeDocument, *Error) { + doc := &nodeDocument{} + + for p.Remaining() > 0 { + node, err := p.parseDocElement() + if err != nil { + return nil, err + } + doc.Nodes = append(doc.Nodes, node) + } + + return doc, nil +} diff --git a/vendor/github.com/flosch/pongo2/parser_expression.go b/vendor/github.com/flosch/pongo2/parser_expression.go new file mode 100644 index 0000000000..1663ec4612 --- /dev/null +++ b/vendor/github.com/flosch/pongo2/parser_expression.go @@ -0,0 +1,503 @@ +package pongo2 + +import ( + "fmt" + "math" +) + +type Expression struct { + // TODO: Add location token? + expr1 IEvaluator + expr2 IEvaluator + opToken *Token +} + +type relationalExpression struct { + // TODO: Add location token? + expr1 IEvaluator + expr2 IEvaluator + opToken *Token +} + +type simpleExpression struct { + negate bool + negativeSign bool + term1 IEvaluator + term2 IEvaluator + opToken *Token +} + +type term struct { + // TODO: Add location token? + factor1 IEvaluator + factor2 IEvaluator + opToken *Token +} + +type power struct { + // TODO: Add location token? + power1 IEvaluator + power2 IEvaluator +} + +func (expr *Expression) FilterApplied(name string) bool { + return expr.expr1.FilterApplied(name) && (expr.expr2 == nil || + (expr.expr2 != nil && expr.expr2.FilterApplied(name))) +} + +func (expr *relationalExpression) FilterApplied(name string) bool { + return expr.expr1.FilterApplied(name) && (expr.expr2 == nil || + (expr.expr2 != nil && expr.expr2.FilterApplied(name))) +} + +func (expr *simpleExpression) FilterApplied(name string) bool { + return expr.term1.FilterApplied(name) && (expr.term2 == nil || + (expr.term2 != nil && expr.term2.FilterApplied(name))) +} + +func (expr *term) FilterApplied(name string) bool { + return expr.factor1.FilterApplied(name) && (expr.factor2 == nil || + (expr.factor2 != nil && expr.factor2.FilterApplied(name))) +} + +func (expr *power) FilterApplied(name string) bool { + return expr.power1.FilterApplied(name) && (expr.power2 == nil || + (expr.power2 != nil && expr.power2.FilterApplied(name))) +} + +func (expr *Expression) GetPositionToken() *Token { + return expr.expr1.GetPositionToken() +} + +func (expr *relationalExpression) GetPositionToken() *Token { + return expr.expr1.GetPositionToken() +} + +func (expr *simpleExpression) GetPositionToken() *Token { + return expr.term1.GetPositionToken() +} + +func (expr *term) GetPositionToken() *Token { + return expr.factor1.GetPositionToken() +} + +func (expr *power) GetPositionToken() *Token { + return expr.power1.GetPositionToken() +} + +func (expr *Expression) Execute(ctx *ExecutionContext, writer TemplateWriter) *Error { + value, err := expr.Evaluate(ctx) + if err != nil { + return err + } + writer.WriteString(value.String()) + return nil +} + +func (expr *relationalExpression) Execute(ctx *ExecutionContext, writer TemplateWriter) *Error { + value, err := expr.Evaluate(ctx) + if err != nil { + return err + } + writer.WriteString(value.String()) + return nil +} + +func (expr *simpleExpression) Execute(ctx *ExecutionContext, writer TemplateWriter) *Error { + value, err := expr.Evaluate(ctx) + if err != nil { + return err + } + writer.WriteString(value.String()) + return nil +} + +func (expr *term) Execute(ctx *ExecutionContext, writer TemplateWriter) *Error { + value, err := expr.Evaluate(ctx) + if err != nil { + return err + } + writer.WriteString(value.String()) + return nil +} + +func (expr *power) Execute(ctx *ExecutionContext, writer TemplateWriter) *Error { + value, err := expr.Evaluate(ctx) + if err != nil { + return err + } + writer.WriteString(value.String()) + return nil +} + +func (expr *Expression) Evaluate(ctx *ExecutionContext) (*Value, *Error) { + v1, err := expr.expr1.Evaluate(ctx) + if err != nil { + return nil, err + } + if expr.expr2 != nil { + switch expr.opToken.Val { + case "and", "&&": + if !v1.IsTrue() { + return AsValue(false), nil + } else { + v2, err := expr.expr2.Evaluate(ctx) + if err != nil { + return nil, err + } + return AsValue(v2.IsTrue()), nil + } + case "or", "||": + if v1.IsTrue() { + return AsValue(true), nil + } else { + v2, err := expr.expr2.Evaluate(ctx) + if err != nil { + return nil, err + } + return AsValue(v2.IsTrue()), nil + } + default: + return nil, ctx.Error(fmt.Sprintf("unimplemented: %s", expr.opToken.Val), expr.opToken) + } + } else { + return v1, nil + } +} + +func (expr *relationalExpression) Evaluate(ctx *ExecutionContext) (*Value, *Error) { + v1, err := expr.expr1.Evaluate(ctx) + if err != nil { + return nil, err + } + if expr.expr2 != nil { + v2, err := expr.expr2.Evaluate(ctx) + if err != nil { + return nil, err + } + switch expr.opToken.Val { + case "<=": + if v1.IsFloat() || v2.IsFloat() { + return AsValue(v1.Float() <= v2.Float()), nil + } + return AsValue(v1.Integer() <= v2.Integer()), nil + case ">=": + if v1.IsFloat() || v2.IsFloat() { + return AsValue(v1.Float() >= v2.Float()), nil + } + return AsValue(v1.Integer() >= v2.Integer()), nil + case "==": + return AsValue(v1.EqualValueTo(v2)), nil + case ">": + if v1.IsFloat() || v2.IsFloat() { + return AsValue(v1.Float() > v2.Float()), nil + } + return AsValue(v1.Integer() > v2.Integer()), nil + case "<": + if v1.IsFloat() || v2.IsFloat() { + return AsValue(v1.Float() < v2.Float()), nil + } + return AsValue(v1.Integer() < v2.Integer()), nil + case "!=", "<>": + return AsValue(!v1.EqualValueTo(v2)), nil + case "in": + return AsValue(v2.Contains(v1)), nil + default: + return nil, ctx.Error(fmt.Sprintf("unimplemented: %s", expr.opToken.Val), expr.opToken) + } + } else { + return v1, nil + } +} + +func (expr *simpleExpression) Evaluate(ctx *ExecutionContext) (*Value, *Error) { + t1, err := expr.term1.Evaluate(ctx) + if err != nil { + return nil, err + } + result := t1 + + if expr.negate { + result = result.Negate() + } + + if expr.negativeSign { + if result.IsNumber() { + switch { + case result.IsFloat(): + result = AsValue(-1 * result.Float()) + case result.IsInteger(): + result = AsValue(-1 * result.Integer()) + default: + return nil, ctx.Error("Operation between a number and a non-(float/integer) is not possible", nil) + } + } else { + return nil, ctx.Error("Negative sign on a non-number expression", expr.GetPositionToken()) + } + } + + if expr.term2 != nil { + t2, err := expr.term2.Evaluate(ctx) + if err != nil { + return nil, err + } + switch expr.opToken.Val { + case "+": + if result.IsFloat() || t2.IsFloat() { + // Result will be a float + return AsValue(result.Float() + t2.Float()), nil + } + // Result will be an integer + return AsValue(result.Integer() + t2.Integer()), nil + case "-": + if result.IsFloat() || t2.IsFloat() { + // Result will be a float + return AsValue(result.Float() - t2.Float()), nil + } + // Result will be an integer + return AsValue(result.Integer() - t2.Integer()), nil + default: + return nil, ctx.Error("Unimplemented", expr.GetPositionToken()) + } + } + + return result, nil +} + +func (expr *term) Evaluate(ctx *ExecutionContext) (*Value, *Error) { + f1, err := expr.factor1.Evaluate(ctx) + if err != nil { + return nil, err + } + if expr.factor2 != nil { + f2, err := expr.factor2.Evaluate(ctx) + if err != nil { + return nil, err + } + switch expr.opToken.Val { + case "*": + if f1.IsFloat() || f2.IsFloat() { + // Result will be float + return AsValue(f1.Float() * f2.Float()), nil + } + // Result will be int + return AsValue(f1.Integer() * f2.Integer()), nil + case "/": + if f1.IsFloat() || f2.IsFloat() { + // Result will be float + return AsValue(f1.Float() / f2.Float()), nil + } + // Result will be int + return AsValue(f1.Integer() / f2.Integer()), nil + case "%": + // Result will be int + return AsValue(f1.Integer() % f2.Integer()), nil + default: + return nil, ctx.Error("unimplemented", expr.opToken) + } + } else { + return f1, nil + } +} + +func (expr *power) Evaluate(ctx *ExecutionContext) (*Value, *Error) { + p1, err := expr.power1.Evaluate(ctx) + if err != nil { + return nil, err + } + if expr.power2 != nil { + p2, err := expr.power2.Evaluate(ctx) + if err != nil { + return nil, err + } + return AsValue(math.Pow(p1.Float(), p2.Float())), nil + } + return p1, nil +} + +func (p *Parser) parseFactor() (IEvaluator, *Error) { + if p.Match(TokenSymbol, "(") != nil { + expr, err := p.ParseExpression() + if err != nil { + return nil, err + } + if p.Match(TokenSymbol, ")") == nil { + return nil, p.Error("Closing bracket expected after expression", nil) + } + return expr, nil + } + + return p.parseVariableOrLiteralWithFilter() +} + +func (p *Parser) parsePower() (IEvaluator, *Error) { + pw := new(power) + + power1, err := p.parseFactor() + if err != nil { + return nil, err + } + pw.power1 = power1 + + if p.Match(TokenSymbol, "^") != nil { + power2, err := p.parsePower() + if err != nil { + return nil, err + } + pw.power2 = power2 + } + + if pw.power2 == nil { + // Shortcut for faster evaluation + return pw.power1, nil + } + + return pw, nil +} + +func (p *Parser) parseTerm() (IEvaluator, *Error) { + returnTerm := new(term) + + factor1, err := p.parsePower() + if err != nil { + return nil, err + } + returnTerm.factor1 = factor1 + + for p.PeekOne(TokenSymbol, "*", "/", "%") != nil { + if returnTerm.opToken != nil { + // Create new sub-term + returnTerm = &term{ + factor1: returnTerm, + } + } + + op := p.Current() + p.Consume() + + factor2, err := p.parsePower() + if err != nil { + return nil, err + } + + returnTerm.opToken = op + returnTerm.factor2 = factor2 + } + + if returnTerm.opToken == nil { + // Shortcut for faster evaluation + return returnTerm.factor1, nil + } + + return returnTerm, nil +} + +func (p *Parser) parseSimpleExpression() (IEvaluator, *Error) { + expr := new(simpleExpression) + + if sign := p.MatchOne(TokenSymbol, "+", "-"); sign != nil { + if sign.Val == "-" { + expr.negativeSign = true + } + } + + if p.Match(TokenSymbol, "!") != nil || p.Match(TokenKeyword, "not") != nil { + expr.negate = true + } + + term1, err := p.parseTerm() + if err != nil { + return nil, err + } + expr.term1 = term1 + + for p.PeekOne(TokenSymbol, "+", "-") != nil { + if expr.opToken != nil { + // New sub expr + expr = &simpleExpression{ + term1: expr, + } + } + + op := p.Current() + p.Consume() + + term2, err := p.parseTerm() + if err != nil { + return nil, err + } + + expr.term2 = term2 + expr.opToken = op + } + + if expr.negate == false && expr.negativeSign == false && expr.term2 == nil { + // Shortcut for faster evaluation + return expr.term1, nil + } + + return expr, nil +} + +func (p *Parser) parseRelationalExpression() (IEvaluator, *Error) { + expr1, err := p.parseSimpleExpression() + if err != nil { + return nil, err + } + + expr := &relationalExpression{ + expr1: expr1, + } + + if t := p.MatchOne(TokenSymbol, "==", "<=", ">=", "!=", "<>", ">", "<"); t != nil { + expr2, err := p.parseRelationalExpression() + if err != nil { + return nil, err + } + expr.opToken = t + expr.expr2 = expr2 + } else if t := p.MatchOne(TokenKeyword, "in"); t != nil { + expr2, err := p.parseSimpleExpression() + if err != nil { + return nil, err + } + expr.opToken = t + expr.expr2 = expr2 + } + + if expr.expr2 == nil { + // Shortcut for faster evaluation + return expr.expr1, nil + } + + return expr, nil +} + +func (p *Parser) ParseExpression() (IEvaluator, *Error) { + rexpr1, err := p.parseRelationalExpression() + if err != nil { + return nil, err + } + + exp := &Expression{ + expr1: rexpr1, + } + + if p.PeekOne(TokenSymbol, "&&", "||") != nil || p.PeekOne(TokenKeyword, "and", "or") != nil { + op := p.Current() + p.Consume() + expr2, err := p.ParseExpression() + if err != nil { + return nil, err + } + exp.expr2 = expr2 + exp.opToken = op + } + + if exp.expr2 == nil { + // Shortcut for faster evaluation + return exp.expr1, nil + } + + return exp, nil +} diff --git a/vendor/github.com/flosch/pongo2/pongo2.go b/vendor/github.com/flosch/pongo2/pongo2.go new file mode 100644 index 0000000000..eda3aa07cb --- /dev/null +++ b/vendor/github.com/flosch/pongo2/pongo2.go @@ -0,0 +1,14 @@ +package pongo2 + +// Version string +const Version = "dev" + +// Must panics, if a Template couldn't successfully parsed. This is how you +// would use it: +// var baseTemplate = pongo2.Must(pongo2.FromFile("templates/base.html")) +func Must(tpl *Template, err error) *Template { + if err != nil { + panic(err) + } + return tpl +} diff --git a/vendor/github.com/flosch/pongo2/tags.go b/vendor/github.com/flosch/pongo2/tags.go new file mode 100644 index 0000000000..3668b06a22 --- /dev/null +++ b/vendor/github.com/flosch/pongo2/tags.go @@ -0,0 +1,135 @@ +package pongo2 + +/* Incomplete: + ----------- + + verbatim (only the "name" argument is missing for verbatim) + + Reconsideration: + ---------------- + + debug (reason: not sure what to output yet) + regroup / Grouping on other properties (reason: maybe too python-specific; not sure how useful this would be in Go) + + Following built-in tags wont be added: + -------------------------------------- + + csrf_token (reason: web-framework specific) + load (reason: python-specific) + url (reason: web-framework specific) +*/ + +import ( + "fmt" + + "github.com/juju/errors" +) + +type INodeTag interface { + INode +} + +// This is the function signature of the tag's parser you will have +// to implement in order to create a new tag. +// +// 'doc' is providing access to the whole document while 'arguments' +// is providing access to the user's arguments to the tag: +// +// {% your_tag_name some "arguments" 123 %} +// +// start_token will be the *Token with the tag's name in it (here: your_tag_name). +// +// Please see the Parser documentation on how to use the parser. +// See RegisterTag()'s documentation for more information about +// writing a tag as well. +type TagParser func(doc *Parser, start *Token, arguments *Parser) (INodeTag, *Error) + +type tag struct { + name string + parser TagParser +} + +var tags map[string]*tag + +func init() { + tags = make(map[string]*tag) +} + +// Registers a new tag. You usually want to call this +// function in the tag's init() function: +// http://golang.org/doc/effective_go.html#init +// +// See http://www.florian-schlachter.de/post/pongo2/ for more about +// writing filters and tags. +func RegisterTag(name string, parserFn TagParser) error { + _, existing := tags[name] + if existing { + return errors.Errorf("tag with name '%s' is already registered", name) + } + tags[name] = &tag{ + name: name, + parser: parserFn, + } + return nil +} + +// Replaces an already registered tag with a new implementation. Use this +// function with caution since it allows you to change existing tag behaviour. +func ReplaceTag(name string, parserFn TagParser) error { + _, existing := tags[name] + if !existing { + return errors.Errorf("tag with name '%s' does not exist (therefore cannot be overridden)", name) + } + tags[name] = &tag{ + name: name, + parser: parserFn, + } + return nil +} + +// Tag = "{%" IDENT ARGS "%}" +func (p *Parser) parseTagElement() (INodeTag, *Error) { + p.Consume() // consume "{%" + tokenName := p.MatchType(TokenIdentifier) + + // Check for identifier + if tokenName == nil { + return nil, p.Error("Tag name must be an identifier.", nil) + } + + // Check for the existing tag + tag, exists := tags[tokenName.Val] + if !exists { + // Does not exists + return nil, p.Error(fmt.Sprintf("Tag '%s' not found (or beginning tag not provided)", tokenName.Val), tokenName) + } + + // Check sandbox tag restriction + if _, isBanned := p.template.set.bannedTags[tokenName.Val]; isBanned { + return nil, p.Error(fmt.Sprintf("Usage of tag '%s' is not allowed (sandbox restriction active).", tokenName.Val), tokenName) + } + + var argsToken []*Token + for p.Peek(TokenSymbol, "%}") == nil && p.Remaining() > 0 { + // Add token to args + argsToken = append(argsToken, p.Current()) + p.Consume() // next token + } + + // EOF? + if p.Remaining() == 0 { + return nil, p.Error("Unexpectedly reached EOF, no tag end found.", p.lastToken) + } + + p.Match(TokenSymbol, "%}") + + argParser := newParser(p.name, argsToken, p.template) + if len(argsToken) == 0 { + // This is done to have nice EOF error messages + argParser.lastToken = tokenName + } + + p.template.level++ + defer func() { p.template.level-- }() + return tag.parser(p, tokenName, argParser) +} diff --git a/vendor/github.com/flosch/pongo2/tags_autoescape.go b/vendor/github.com/flosch/pongo2/tags_autoescape.go new file mode 100644 index 0000000000..590a1db350 --- /dev/null +++ b/vendor/github.com/flosch/pongo2/tags_autoescape.go @@ -0,0 +1,52 @@ +package pongo2 + +type tagAutoescapeNode struct { + wrapper *NodeWrapper + autoescape bool +} + +func (node *tagAutoescapeNode) Execute(ctx *ExecutionContext, writer TemplateWriter) *Error { + old := ctx.Autoescape + ctx.Autoescape = node.autoescape + + err := node.wrapper.Execute(ctx, writer) + if err != nil { + return err + } + + ctx.Autoescape = old + + return nil +} + +func tagAutoescapeParser(doc *Parser, start *Token, arguments *Parser) (INodeTag, *Error) { + autoescapeNode := &tagAutoescapeNode{} + + wrapper, _, err := doc.WrapUntilTag("endautoescape") + if err != nil { + return nil, err + } + autoescapeNode.wrapper = wrapper + + modeToken := arguments.MatchType(TokenIdentifier) + if modeToken == nil { + return nil, arguments.Error("A mode is required for autoescape-tag.", nil) + } + if modeToken.Val == "on" { + autoescapeNode.autoescape = true + } else if modeToken.Val == "off" { + autoescapeNode.autoescape = false + } else { + return nil, arguments.Error("Only 'on' or 'off' is valid as an autoescape-mode.", nil) + } + + if arguments.Remaining() > 0 { + return nil, arguments.Error("Malformed autoescape-tag arguments.", nil) + } + + return autoescapeNode, nil +} + +func init() { + RegisterTag("autoescape", tagAutoescapeParser) +} diff --git a/vendor/github.com/flosch/pongo2/tags_block.go b/vendor/github.com/flosch/pongo2/tags_block.go new file mode 100644 index 0000000000..86145f329a --- /dev/null +++ b/vendor/github.com/flosch/pongo2/tags_block.go @@ -0,0 +1,129 @@ +package pongo2 + +import ( + "bytes" + "fmt" +) + +type tagBlockNode struct { + name string +} + +func (node *tagBlockNode) getBlockWrappers(tpl *Template) []*NodeWrapper { + nodeWrappers := make([]*NodeWrapper, 0) + var t *NodeWrapper + + for tpl != nil { + t = tpl.blocks[node.name] + if t != nil { + nodeWrappers = append(nodeWrappers, t) + } + tpl = tpl.child + } + + return nodeWrappers +} + +func (node *tagBlockNode) Execute(ctx *ExecutionContext, writer TemplateWriter) *Error { + tpl := ctx.template + if tpl == nil { + panic("internal error: tpl == nil") + } + + // Determine the block to execute + blockWrappers := node.getBlockWrappers(tpl) + lenBlockWrappers := len(blockWrappers) + + if lenBlockWrappers == 0 { + return ctx.Error("internal error: len(block_wrappers) == 0 in tagBlockNode.Execute()", nil) + } + + blockWrapper := blockWrappers[lenBlockWrappers-1] + ctx.Private["block"] = tagBlockInformation{ + ctx: ctx, + wrappers: blockWrappers[0 : lenBlockWrappers-1], + } + err := blockWrapper.Execute(ctx, writer) + if err != nil { + return err + } + + return nil +} + +type tagBlockInformation struct { + ctx *ExecutionContext + wrappers []*NodeWrapper +} + +func (t tagBlockInformation) Super() string { + lenWrappers := len(t.wrappers) + + if lenWrappers == 0 { + return "" + } + + superCtx := NewChildExecutionContext(t.ctx) + superCtx.Private["block"] = tagBlockInformation{ + ctx: t.ctx, + wrappers: t.wrappers[0 : lenWrappers-1], + } + + blockWrapper := t.wrappers[lenWrappers-1] + buf := bytes.NewBufferString("") + err := blockWrapper.Execute(superCtx, &templateWriter{buf}) + if err != nil { + return "" + } + return buf.String() +} + +func tagBlockParser(doc *Parser, start *Token, arguments *Parser) (INodeTag, *Error) { + if arguments.Count() == 0 { + return nil, arguments.Error("Tag 'block' requires an identifier.", nil) + } + + nameToken := arguments.MatchType(TokenIdentifier) + if nameToken == nil { + return nil, arguments.Error("First argument for tag 'block' must be an identifier.", nil) + } + + if arguments.Remaining() != 0 { + return nil, arguments.Error("Tag 'block' takes exactly 1 argument (an identifier).", nil) + } + + wrapper, endtagargs, err := doc.WrapUntilTag("endblock") + if err != nil { + return nil, err + } + if endtagargs.Remaining() > 0 { + endtagnameToken := endtagargs.MatchType(TokenIdentifier) + if endtagnameToken != nil { + if endtagnameToken.Val != nameToken.Val { + return nil, endtagargs.Error(fmt.Sprintf("Name for 'endblock' must equal to 'block'-tag's name ('%s' != '%s').", + nameToken.Val, endtagnameToken.Val), nil) + } + } + + if endtagnameToken == nil || endtagargs.Remaining() > 0 { + return nil, endtagargs.Error("Either no or only one argument (identifier) allowed for 'endblock'.", nil) + } + } + + tpl := doc.template + if tpl == nil { + panic("internal error: tpl == nil") + } + _, hasBlock := tpl.blocks[nameToken.Val] + if !hasBlock { + tpl.blocks[nameToken.Val] = wrapper + } else { + return nil, arguments.Error(fmt.Sprintf("Block named '%s' already defined", nameToken.Val), nil) + } + + return &tagBlockNode{name: nameToken.Val}, nil +} + +func init() { + RegisterTag("block", tagBlockParser) +} diff --git a/vendor/github.com/flosch/pongo2/tags_comment.go b/vendor/github.com/flosch/pongo2/tags_comment.go new file mode 100644 index 0000000000..56a02ed99d --- /dev/null +++ b/vendor/github.com/flosch/pongo2/tags_comment.go @@ -0,0 +1,27 @@ +package pongo2 + +type tagCommentNode struct{} + +func (node *tagCommentNode) Execute(ctx *ExecutionContext, writer TemplateWriter) *Error { + return nil +} + +func tagCommentParser(doc *Parser, start *Token, arguments *Parser) (INodeTag, *Error) { + commentNode := &tagCommentNode{} + + // TODO: Process the endtag's arguments (see django 'comment'-tag documentation) + err := doc.SkipUntilTag("endcomment") + if err != nil { + return nil, err + } + + if arguments.Count() != 0 { + return nil, arguments.Error("Tag 'comment' does not take any argument.", nil) + } + + return commentNode, nil +} + +func init() { + RegisterTag("comment", tagCommentParser) +} diff --git a/vendor/github.com/flosch/pongo2/tags_cycle.go b/vendor/github.com/flosch/pongo2/tags_cycle.go new file mode 100644 index 0000000000..ffbd254eea --- /dev/null +++ b/vendor/github.com/flosch/pongo2/tags_cycle.go @@ -0,0 +1,106 @@ +package pongo2 + +type tagCycleValue struct { + node *tagCycleNode + value *Value +} + +type tagCycleNode struct { + position *Token + args []IEvaluator + idx int + asName string + silent bool +} + +func (cv *tagCycleValue) String() string { + return cv.value.String() +} + +func (node *tagCycleNode) Execute(ctx *ExecutionContext, writer TemplateWriter) *Error { + item := node.args[node.idx%len(node.args)] + node.idx++ + + val, err := item.Evaluate(ctx) + if err != nil { + return err + } + + if t, ok := val.Interface().(*tagCycleValue); ok { + // {% cycle "test1" "test2" + // {% cycle cycleitem %} + + // Update the cycle value with next value + item := t.node.args[t.node.idx%len(t.node.args)] + t.node.idx++ + + val, err := item.Evaluate(ctx) + if err != nil { + return err + } + + t.value = val + + if !t.node.silent { + writer.WriteString(val.String()) + } + } else { + // Regular call + + cycleValue := &tagCycleValue{ + node: node, + value: val, + } + + if node.asName != "" { + ctx.Private[node.asName] = cycleValue + } + if !node.silent { + writer.WriteString(val.String()) + } + } + + return nil +} + +// HINT: We're not supporting the old comma-separated list of expressions argument-style +func tagCycleParser(doc *Parser, start *Token, arguments *Parser) (INodeTag, *Error) { + cycleNode := &tagCycleNode{ + position: start, + } + + for arguments.Remaining() > 0 { + node, err := arguments.ParseExpression() + if err != nil { + return nil, err + } + cycleNode.args = append(cycleNode.args, node) + + if arguments.MatchOne(TokenKeyword, "as") != nil { + // as + + nameToken := arguments.MatchType(TokenIdentifier) + if nameToken == nil { + return nil, arguments.Error("Name (identifier) expected after 'as'.", nil) + } + cycleNode.asName = nameToken.Val + + if arguments.MatchOne(TokenIdentifier, "silent") != nil { + cycleNode.silent = true + } + + // Now we're finished + break + } + } + + if arguments.Remaining() > 0 { + return nil, arguments.Error("Malformed cycle-tag.", nil) + } + + return cycleNode, nil +} + +func init() { + RegisterTag("cycle", tagCycleParser) +} diff --git a/vendor/github.com/flosch/pongo2/tags_extends.go b/vendor/github.com/flosch/pongo2/tags_extends.go new file mode 100644 index 0000000000..5771020a06 --- /dev/null +++ b/vendor/github.com/flosch/pongo2/tags_extends.go @@ -0,0 +1,52 @@ +package pongo2 + +type tagExtendsNode struct { + filename string +} + +func (node *tagExtendsNode) Execute(ctx *ExecutionContext, writer TemplateWriter) *Error { + return nil +} + +func tagExtendsParser(doc *Parser, start *Token, arguments *Parser) (INodeTag, *Error) { + extendsNode := &tagExtendsNode{} + + if doc.template.level > 1 { + return nil, arguments.Error("The 'extends' tag can only defined on root level.", start) + } + + if doc.template.parent != nil { + // Already one parent + return nil, arguments.Error("This template has already one parent.", start) + } + + if filenameToken := arguments.MatchType(TokenString); filenameToken != nil { + // prepared, static template + + // Get parent's filename + parentFilename := doc.template.set.resolveFilename(doc.template, filenameToken.Val) + + // Parse the parent + parentTemplate, err := doc.template.set.FromFile(parentFilename) + if err != nil { + return nil, err.(*Error) + } + + // Keep track of things + parentTemplate.child = doc.template + doc.template.parent = parentTemplate + extendsNode.filename = parentFilename + } else { + return nil, arguments.Error("Tag 'extends' requires a template filename as string.", nil) + } + + if arguments.Remaining() > 0 { + return nil, arguments.Error("Tag 'extends' does only take 1 argument.", nil) + } + + return extendsNode, nil +} + +func init() { + RegisterTag("extends", tagExtendsParser) +} diff --git a/vendor/github.com/flosch/pongo2/tags_filter.go b/vendor/github.com/flosch/pongo2/tags_filter.go new file mode 100644 index 0000000000..b38fd92982 --- /dev/null +++ b/vendor/github.com/flosch/pongo2/tags_filter.go @@ -0,0 +1,95 @@ +package pongo2 + +import ( + "bytes" +) + +type nodeFilterCall struct { + name string + paramExpr IEvaluator +} + +type tagFilterNode struct { + position *Token + bodyWrapper *NodeWrapper + filterChain []*nodeFilterCall +} + +func (node *tagFilterNode) Execute(ctx *ExecutionContext, writer TemplateWriter) *Error { + temp := bytes.NewBuffer(make([]byte, 0, 1024)) // 1 KiB size + + err := node.bodyWrapper.Execute(ctx, temp) + if err != nil { + return err + } + + value := AsValue(temp.String()) + + for _, call := range node.filterChain { + var param *Value + if call.paramExpr != nil { + param, err = call.paramExpr.Evaluate(ctx) + if err != nil { + return err + } + } else { + param = AsValue(nil) + } + value, err = ApplyFilter(call.name, value, param) + if err != nil { + return ctx.Error(err.Error(), node.position) + } + } + + writer.WriteString(value.String()) + + return nil +} + +func tagFilterParser(doc *Parser, start *Token, arguments *Parser) (INodeTag, *Error) { + filterNode := &tagFilterNode{ + position: start, + } + + wrapper, _, err := doc.WrapUntilTag("endfilter") + if err != nil { + return nil, err + } + filterNode.bodyWrapper = wrapper + + for arguments.Remaining() > 0 { + filterCall := &nodeFilterCall{} + + nameToken := arguments.MatchType(TokenIdentifier) + if nameToken == nil { + return nil, arguments.Error("Expected a filter name (identifier).", nil) + } + filterCall.name = nameToken.Val + + if arguments.MatchOne(TokenSymbol, ":") != nil { + // Filter parameter + // NOTICE: we can't use ParseExpression() here, because it would parse the next filter "|..." as well in the argument list + expr, err := arguments.parseVariableOrLiteral() + if err != nil { + return nil, err + } + filterCall.paramExpr = expr + } + + filterNode.filterChain = append(filterNode.filterChain, filterCall) + + if arguments.MatchOne(TokenSymbol, "|") == nil { + break + } + } + + if arguments.Remaining() > 0 { + return nil, arguments.Error("Malformed filter-tag arguments.", nil) + } + + return filterNode, nil +} + +func init() { + RegisterTag("filter", tagFilterParser) +} diff --git a/vendor/github.com/flosch/pongo2/tags_firstof.go b/vendor/github.com/flosch/pongo2/tags_firstof.go new file mode 100644 index 0000000000..5b2888e2be --- /dev/null +++ b/vendor/github.com/flosch/pongo2/tags_firstof.go @@ -0,0 +1,49 @@ +package pongo2 + +type tagFirstofNode struct { + position *Token + args []IEvaluator +} + +func (node *tagFirstofNode) Execute(ctx *ExecutionContext, writer TemplateWriter) *Error { + for _, arg := range node.args { + val, err := arg.Evaluate(ctx) + if err != nil { + return err + } + + if val.IsTrue() { + if ctx.Autoescape && !arg.FilterApplied("safe") { + val, err = ApplyFilter("escape", val, nil) + if err != nil { + return err + } + } + + writer.WriteString(val.String()) + return nil + } + } + + return nil +} + +func tagFirstofParser(doc *Parser, start *Token, arguments *Parser) (INodeTag, *Error) { + firstofNode := &tagFirstofNode{ + position: start, + } + + for arguments.Remaining() > 0 { + node, err := arguments.ParseExpression() + if err != nil { + return nil, err + } + firstofNode.args = append(firstofNode.args, node) + } + + return firstofNode, nil +} + +func init() { + RegisterTag("firstof", tagFirstofParser) +} diff --git a/vendor/github.com/flosch/pongo2/tags_for.go b/vendor/github.com/flosch/pongo2/tags_for.go new file mode 100644 index 0000000000..5b0b5554c8 --- /dev/null +++ b/vendor/github.com/flosch/pongo2/tags_for.go @@ -0,0 +1,159 @@ +package pongo2 + +type tagForNode struct { + key string + value string // only for maps: for key, value in map + objectEvaluator IEvaluator + reversed bool + sorted bool + + bodyWrapper *NodeWrapper + emptyWrapper *NodeWrapper +} + +type tagForLoopInformation struct { + Counter int + Counter0 int + Revcounter int + Revcounter0 int + First bool + Last bool + Parentloop *tagForLoopInformation +} + +func (node *tagForNode) Execute(ctx *ExecutionContext, writer TemplateWriter) (forError *Error) { + // Backup forloop (as parentloop in public context), key-name and value-name + forCtx := NewChildExecutionContext(ctx) + parentloop := forCtx.Private["forloop"] + + // Create loop struct + loopInfo := &tagForLoopInformation{ + First: true, + } + + // Is it a loop in a loop? + if parentloop != nil { + loopInfo.Parentloop = parentloop.(*tagForLoopInformation) + } + + // Register loopInfo in public context + forCtx.Private["forloop"] = loopInfo + + obj, err := node.objectEvaluator.Evaluate(forCtx) + if err != nil { + return err + } + + obj.IterateOrder(func(idx, count int, key, value *Value) bool { + // There's something to iterate over (correct type and at least 1 item) + + // Update loop infos and public context + forCtx.Private[node.key] = key + if value != nil { + forCtx.Private[node.value] = value + } + loopInfo.Counter = idx + 1 + loopInfo.Counter0 = idx + if idx == 1 { + loopInfo.First = false + } + if idx+1 == count { + loopInfo.Last = true + } + loopInfo.Revcounter = count - idx // TODO: Not sure about this, have to look it up + loopInfo.Revcounter0 = count - (idx + 1) // TODO: Not sure about this, have to look it up + + // Render elements with updated context + err := node.bodyWrapper.Execute(forCtx, writer) + if err != nil { + forError = err + return false + } + return true + }, func() { + // Nothing to iterate over (maybe wrong type or no items) + if node.emptyWrapper != nil { + err := node.emptyWrapper.Execute(forCtx, writer) + if err != nil { + forError = err + } + } + }, node.reversed, node.sorted) + + return forError +} + +func tagForParser(doc *Parser, start *Token, arguments *Parser) (INodeTag, *Error) { + forNode := &tagForNode{} + + // Arguments parsing + var valueToken *Token + keyToken := arguments.MatchType(TokenIdentifier) + if keyToken == nil { + return nil, arguments.Error("Expected an key identifier as first argument for 'for'-tag", nil) + } + + if arguments.Match(TokenSymbol, ",") != nil { + // Value name is provided + valueToken = arguments.MatchType(TokenIdentifier) + if valueToken == nil { + return nil, arguments.Error("Value name must be an identifier.", nil) + } + } + + if arguments.Match(TokenKeyword, "in") == nil { + return nil, arguments.Error("Expected keyword 'in'.", nil) + } + + objectEvaluator, err := arguments.ParseExpression() + if err != nil { + return nil, err + } + forNode.objectEvaluator = objectEvaluator + forNode.key = keyToken.Val + if valueToken != nil { + forNode.value = valueToken.Val + } + + if arguments.MatchOne(TokenIdentifier, "reversed") != nil { + forNode.reversed = true + } + + if arguments.MatchOne(TokenIdentifier, "sorted") != nil { + forNode.sorted = true + } + + if arguments.Remaining() > 0 { + return nil, arguments.Error("Malformed for-loop arguments.", nil) + } + + // Body wrapping + wrapper, endargs, err := doc.WrapUntilTag("empty", "endfor") + if err != nil { + return nil, err + } + forNode.bodyWrapper = wrapper + + if endargs.Count() > 0 { + return nil, endargs.Error("Arguments not allowed here.", nil) + } + + if wrapper.Endtag == "empty" { + // if there's an else in the if-statement, we need the else-Block as well + wrapper, endargs, err = doc.WrapUntilTag("endfor") + if err != nil { + return nil, err + } + forNode.emptyWrapper = wrapper + + if endargs.Count() > 0 { + return nil, endargs.Error("Arguments not allowed here.", nil) + } + } + + return forNode, nil +} + +func init() { + RegisterTag("for", tagForParser) +} diff --git a/vendor/github.com/flosch/pongo2/tags_if.go b/vendor/github.com/flosch/pongo2/tags_if.go new file mode 100644 index 0000000000..3eeaf3b499 --- /dev/null +++ b/vendor/github.com/flosch/pongo2/tags_if.go @@ -0,0 +1,76 @@ +package pongo2 + +type tagIfNode struct { + conditions []IEvaluator + wrappers []*NodeWrapper +} + +func (node *tagIfNode) Execute(ctx *ExecutionContext, writer TemplateWriter) *Error { + for i, condition := range node.conditions { + result, err := condition.Evaluate(ctx) + if err != nil { + return err + } + + if result.IsTrue() { + return node.wrappers[i].Execute(ctx, writer) + } + // Last condition? + if len(node.conditions) == i+1 && len(node.wrappers) > i+1 { + return node.wrappers[i+1].Execute(ctx, writer) + } + } + return nil +} + +func tagIfParser(doc *Parser, start *Token, arguments *Parser) (INodeTag, *Error) { + ifNode := &tagIfNode{} + + // Parse first and main IF condition + condition, err := arguments.ParseExpression() + if err != nil { + return nil, err + } + ifNode.conditions = append(ifNode.conditions, condition) + + if arguments.Remaining() > 0 { + return nil, arguments.Error("If-condition is malformed.", nil) + } + + // Check the rest + for { + wrapper, tagArgs, err := doc.WrapUntilTag("elif", "else", "endif") + if err != nil { + return nil, err + } + ifNode.wrappers = append(ifNode.wrappers, wrapper) + + if wrapper.Endtag == "elif" { + // elif can take a condition + condition, err = tagArgs.ParseExpression() + if err != nil { + return nil, err + } + ifNode.conditions = append(ifNode.conditions, condition) + + if tagArgs.Remaining() > 0 { + return nil, tagArgs.Error("Elif-condition is malformed.", nil) + } + } else { + if tagArgs.Count() > 0 { + // else/endif can't take any conditions + return nil, tagArgs.Error("Arguments not allowed here.", nil) + } + } + + if wrapper.Endtag == "endif" { + break + } + } + + return ifNode, nil +} + +func init() { + RegisterTag("if", tagIfParser) +} diff --git a/vendor/github.com/flosch/pongo2/tags_ifchanged.go b/vendor/github.com/flosch/pongo2/tags_ifchanged.go new file mode 100644 index 0000000000..45296a0a34 --- /dev/null +++ b/vendor/github.com/flosch/pongo2/tags_ifchanged.go @@ -0,0 +1,116 @@ +package pongo2 + +import ( + "bytes" +) + +type tagIfchangedNode struct { + watchedExpr []IEvaluator + lastValues []*Value + lastContent []byte + thenWrapper *NodeWrapper + elseWrapper *NodeWrapper +} + +func (node *tagIfchangedNode) Execute(ctx *ExecutionContext, writer TemplateWriter) *Error { + if len(node.watchedExpr) == 0 { + // Check against own rendered body + + buf := bytes.NewBuffer(make([]byte, 0, 1024)) // 1 KiB + err := node.thenWrapper.Execute(ctx, buf) + if err != nil { + return err + } + + bufBytes := buf.Bytes() + if !bytes.Equal(node.lastContent, bufBytes) { + // Rendered content changed, output it + writer.Write(bufBytes) + node.lastContent = bufBytes + } + } else { + nowValues := make([]*Value, 0, len(node.watchedExpr)) + for _, expr := range node.watchedExpr { + val, err := expr.Evaluate(ctx) + if err != nil { + return err + } + nowValues = append(nowValues, val) + } + + // Compare old to new values now + changed := len(node.lastValues) == 0 + + for idx, oldVal := range node.lastValues { + if !oldVal.EqualValueTo(nowValues[idx]) { + changed = true + break // we can stop here because ONE value changed + } + } + + node.lastValues = nowValues + + if changed { + // Render thenWrapper + err := node.thenWrapper.Execute(ctx, writer) + if err != nil { + return err + } + } else { + // Render elseWrapper + err := node.elseWrapper.Execute(ctx, writer) + if err != nil { + return err + } + } + } + + return nil +} + +func tagIfchangedParser(doc *Parser, start *Token, arguments *Parser) (INodeTag, *Error) { + ifchangedNode := &tagIfchangedNode{} + + for arguments.Remaining() > 0 { + // Parse condition + expr, err := arguments.ParseExpression() + if err != nil { + return nil, err + } + ifchangedNode.watchedExpr = append(ifchangedNode.watchedExpr, expr) + } + + if arguments.Remaining() > 0 { + return nil, arguments.Error("Ifchanged-arguments are malformed.", nil) + } + + // Wrap then/else-blocks + wrapper, endargs, err := doc.WrapUntilTag("else", "endifchanged") + if err != nil { + return nil, err + } + ifchangedNode.thenWrapper = wrapper + + if endargs.Count() > 0 { + return nil, endargs.Error("Arguments not allowed here.", nil) + } + + if wrapper.Endtag == "else" { + // if there's an else in the if-statement, we need the else-Block as well + wrapper, endargs, err = doc.WrapUntilTag("endifchanged") + if err != nil { + return nil, err + } + ifchangedNode.elseWrapper = wrapper + + if endargs.Count() > 0 { + return nil, endargs.Error("Arguments not allowed here.", nil) + } + } + + return ifchangedNode, nil +} + +func init() { + RegisterTag("ifchanged", tagIfchangedParser) +} diff --git a/vendor/github.com/flosch/pongo2/tags_ifequal.go b/vendor/github.com/flosch/pongo2/tags_ifequal.go new file mode 100644 index 0000000000..103f1c7ba6 --- /dev/null +++ b/vendor/github.com/flosch/pongo2/tags_ifequal.go @@ -0,0 +1,78 @@ +package pongo2 + +type tagIfEqualNode struct { + var1, var2 IEvaluator + thenWrapper *NodeWrapper + elseWrapper *NodeWrapper +} + +func (node *tagIfEqualNode) Execute(ctx *ExecutionContext, writer TemplateWriter) *Error { + r1, err := node.var1.Evaluate(ctx) + if err != nil { + return err + } + r2, err := node.var2.Evaluate(ctx) + if err != nil { + return err + } + + result := r1.EqualValueTo(r2) + + if result { + return node.thenWrapper.Execute(ctx, writer) + } + if node.elseWrapper != nil { + return node.elseWrapper.Execute(ctx, writer) + } + return nil +} + +func tagIfEqualParser(doc *Parser, start *Token, arguments *Parser) (INodeTag, *Error) { + ifequalNode := &tagIfEqualNode{} + + // Parse two expressions + var1, err := arguments.ParseExpression() + if err != nil { + return nil, err + } + var2, err := arguments.ParseExpression() + if err != nil { + return nil, err + } + ifequalNode.var1 = var1 + ifequalNode.var2 = var2 + + if arguments.Remaining() > 0 { + return nil, arguments.Error("ifequal only takes 2 arguments.", nil) + } + + // Wrap then/else-blocks + wrapper, endargs, err := doc.WrapUntilTag("else", "endifequal") + if err != nil { + return nil, err + } + ifequalNode.thenWrapper = wrapper + + if endargs.Count() > 0 { + return nil, endargs.Error("Arguments not allowed here.", nil) + } + + if wrapper.Endtag == "else" { + // if there's an else in the if-statement, we need the else-Block as well + wrapper, endargs, err = doc.WrapUntilTag("endifequal") + if err != nil { + return nil, err + } + ifequalNode.elseWrapper = wrapper + + if endargs.Count() > 0 { + return nil, endargs.Error("Arguments not allowed here.", nil) + } + } + + return ifequalNode, nil +} + +func init() { + RegisterTag("ifequal", tagIfEqualParser) +} diff --git a/vendor/github.com/flosch/pongo2/tags_ifnotequal.go b/vendor/github.com/flosch/pongo2/tags_ifnotequal.go new file mode 100644 index 0000000000..0d287d349d --- /dev/null +++ b/vendor/github.com/flosch/pongo2/tags_ifnotequal.go @@ -0,0 +1,78 @@ +package pongo2 + +type tagIfNotEqualNode struct { + var1, var2 IEvaluator + thenWrapper *NodeWrapper + elseWrapper *NodeWrapper +} + +func (node *tagIfNotEqualNode) Execute(ctx *ExecutionContext, writer TemplateWriter) *Error { + r1, err := node.var1.Evaluate(ctx) + if err != nil { + return err + } + r2, err := node.var2.Evaluate(ctx) + if err != nil { + return err + } + + result := !r1.EqualValueTo(r2) + + if result { + return node.thenWrapper.Execute(ctx, writer) + } + if node.elseWrapper != nil { + return node.elseWrapper.Execute(ctx, writer) + } + return nil +} + +func tagIfNotEqualParser(doc *Parser, start *Token, arguments *Parser) (INodeTag, *Error) { + ifnotequalNode := &tagIfNotEqualNode{} + + // Parse two expressions + var1, err := arguments.ParseExpression() + if err != nil { + return nil, err + } + var2, err := arguments.ParseExpression() + if err != nil { + return nil, err + } + ifnotequalNode.var1 = var1 + ifnotequalNode.var2 = var2 + + if arguments.Remaining() > 0 { + return nil, arguments.Error("ifequal only takes 2 arguments.", nil) + } + + // Wrap then/else-blocks + wrapper, endargs, err := doc.WrapUntilTag("else", "endifnotequal") + if err != nil { + return nil, err + } + ifnotequalNode.thenWrapper = wrapper + + if endargs.Count() > 0 { + return nil, endargs.Error("Arguments not allowed here.", nil) + } + + if wrapper.Endtag == "else" { + // if there's an else in the if-statement, we need the else-Block as well + wrapper, endargs, err = doc.WrapUntilTag("endifnotequal") + if err != nil { + return nil, err + } + ifnotequalNode.elseWrapper = wrapper + + if endargs.Count() > 0 { + return nil, endargs.Error("Arguments not allowed here.", nil) + } + } + + return ifnotequalNode, nil +} + +func init() { + RegisterTag("ifnotequal", tagIfNotEqualParser) +} diff --git a/vendor/github.com/flosch/pongo2/tags_import.go b/vendor/github.com/flosch/pongo2/tags_import.go new file mode 100644 index 0000000000..7e0d6a01a5 --- /dev/null +++ b/vendor/github.com/flosch/pongo2/tags_import.go @@ -0,0 +1,84 @@ +package pongo2 + +import ( + "fmt" +) + +type tagImportNode struct { + position *Token + filename string + macros map[string]*tagMacroNode // alias/name -> macro instance +} + +func (node *tagImportNode) Execute(ctx *ExecutionContext, writer TemplateWriter) *Error { + for name, macro := range node.macros { + func(name string, macro *tagMacroNode) { + ctx.Private[name] = func(args ...*Value) *Value { + return macro.call(ctx, args...) + } + }(name, macro) + } + return nil +} + +func tagImportParser(doc *Parser, start *Token, arguments *Parser) (INodeTag, *Error) { + importNode := &tagImportNode{ + position: start, + macros: make(map[string]*tagMacroNode), + } + + filenameToken := arguments.MatchType(TokenString) + if filenameToken == nil { + return nil, arguments.Error("Import-tag needs a filename as string.", nil) + } + + importNode.filename = doc.template.set.resolveFilename(doc.template, filenameToken.Val) + + if arguments.Remaining() == 0 { + return nil, arguments.Error("You must at least specify one macro to import.", nil) + } + + // Compile the given template + tpl, err := doc.template.set.FromFile(importNode.filename) + if err != nil { + return nil, err.(*Error).updateFromTokenIfNeeded(doc.template, start) + } + + for arguments.Remaining() > 0 { + macroNameToken := arguments.MatchType(TokenIdentifier) + if macroNameToken == nil { + return nil, arguments.Error("Expected macro name (identifier).", nil) + } + + asName := macroNameToken.Val + if arguments.Match(TokenKeyword, "as") != nil { + aliasToken := arguments.MatchType(TokenIdentifier) + if aliasToken == nil { + return nil, arguments.Error("Expected macro alias name (identifier).", nil) + } + asName = aliasToken.Val + } + + macroInstance, has := tpl.exportedMacros[macroNameToken.Val] + if !has { + return nil, arguments.Error(fmt.Sprintf("Macro '%s' not found (or not exported) in '%s'.", macroNameToken.Val, + importNode.filename), macroNameToken) + } + + importNode.macros[asName] = macroInstance + + if arguments.Remaining() == 0 { + break + } + + if arguments.Match(TokenSymbol, ",") == nil { + return nil, arguments.Error("Expected ','.", nil) + } + } + + return importNode, nil +} + +func init() { + RegisterTag("import", tagImportParser) +} diff --git a/vendor/github.com/flosch/pongo2/tags_include.go b/vendor/github.com/flosch/pongo2/tags_include.go new file mode 100644 index 0000000000..6d619fdabe --- /dev/null +++ b/vendor/github.com/flosch/pongo2/tags_include.go @@ -0,0 +1,146 @@ +package pongo2 + +type tagIncludeNode struct { + tpl *Template + filenameEvaluator IEvaluator + lazy bool + only bool + filename string + withPairs map[string]IEvaluator + ifExists bool +} + +func (node *tagIncludeNode) Execute(ctx *ExecutionContext, writer TemplateWriter) *Error { + // Building the context for the template + includeCtx := make(Context) + + // Fill the context with all data from the parent + if !node.only { + includeCtx.Update(ctx.Public) + includeCtx.Update(ctx.Private) + } + + // Put all custom with-pairs into the context + for key, value := range node.withPairs { + val, err := value.Evaluate(ctx) + if err != nil { + return err + } + includeCtx[key] = val + } + + // Execute the template + if node.lazy { + // Evaluate the filename + filename, err := node.filenameEvaluator.Evaluate(ctx) + if err != nil { + return err + } + + if filename.String() == "" { + return ctx.Error("Filename for 'include'-tag evaluated to an empty string.", nil) + } + + // Get include-filename + includedFilename := ctx.template.set.resolveFilename(ctx.template, filename.String()) + + includedTpl, err2 := ctx.template.set.FromFile(includedFilename) + if err2 != nil { + // if this is ReadFile error, and "if_exists" flag is enabled + if node.ifExists && err2.(*Error).Sender == "fromfile" { + return nil + } + return err2.(*Error) + } + err2 = includedTpl.ExecuteWriter(includeCtx, writer) + if err2 != nil { + return err2.(*Error) + } + return nil + } + // Template is already parsed with static filename + err := node.tpl.ExecuteWriter(includeCtx, writer) + if err != nil { + return err.(*Error) + } + return nil +} + +type tagIncludeEmptyNode struct{} + +func (node *tagIncludeEmptyNode) Execute(ctx *ExecutionContext, writer TemplateWriter) *Error { + return nil +} + +func tagIncludeParser(doc *Parser, start *Token, arguments *Parser) (INodeTag, *Error) { + includeNode := &tagIncludeNode{ + withPairs: make(map[string]IEvaluator), + } + + if filenameToken := arguments.MatchType(TokenString); filenameToken != nil { + // prepared, static template + + // "if_exists" flag + ifExists := arguments.Match(TokenIdentifier, "if_exists") != nil + + // Get include-filename + includedFilename := doc.template.set.resolveFilename(doc.template, filenameToken.Val) + + // Parse the parent + includeNode.filename = includedFilename + includedTpl, err := doc.template.set.FromFile(includedFilename) + if err != nil { + // if this is ReadFile error, and "if_exists" token presents we should create and empty node + if err.(*Error).Sender == "fromfile" && ifExists { + return &tagIncludeEmptyNode{}, nil + } + return nil, err.(*Error).updateFromTokenIfNeeded(doc.template, filenameToken) + } + includeNode.tpl = includedTpl + } else { + // No String, then the user wants to use lazy-evaluation (slower, but possible) + filenameEvaluator, err := arguments.ParseExpression() + if err != nil { + return nil, err.updateFromTokenIfNeeded(doc.template, filenameToken) + } + includeNode.filenameEvaluator = filenameEvaluator + includeNode.lazy = true + includeNode.ifExists = arguments.Match(TokenIdentifier, "if_exists") != nil // "if_exists" flag + } + + // After having parsed the filename we're gonna parse the with+only options + if arguments.Match(TokenIdentifier, "with") != nil { + for arguments.Remaining() > 0 { + // We have at least one key=expr pair (because of starting "with") + keyToken := arguments.MatchType(TokenIdentifier) + if keyToken == nil { + return nil, arguments.Error("Expected an identifier", nil) + } + if arguments.Match(TokenSymbol, "=") == nil { + return nil, arguments.Error("Expected '='.", nil) + } + valueExpr, err := arguments.ParseExpression() + if err != nil { + return nil, err.updateFromTokenIfNeeded(doc.template, keyToken) + } + + includeNode.withPairs[keyToken.Val] = valueExpr + + // Only? + if arguments.Match(TokenIdentifier, "only") != nil { + includeNode.only = true + break // stop parsing arguments because it's the last option + } + } + } + + if arguments.Remaining() > 0 { + return nil, arguments.Error("Malformed 'include'-tag arguments.", nil) + } + + return includeNode, nil +} + +func init() { + RegisterTag("include", tagIncludeParser) +} diff --git a/vendor/github.com/flosch/pongo2/tags_lorem.go b/vendor/github.com/flosch/pongo2/tags_lorem.go new file mode 100644 index 0000000000..1d353f267d --- /dev/null +++ b/vendor/github.com/flosch/pongo2/tags_lorem.go @@ -0,0 +1,133 @@ +package pongo2 + +import ( + "math/rand" + "strings" + "time" + + "github.com/juju/errors" +) + +var ( + tagLoremParagraphs = strings.Split(tagLoremText, "\n") + tagLoremWords = strings.Fields(tagLoremText) +) + +type tagLoremNode struct { + position *Token + count int // number of paragraphs + method string // w = words, p = HTML paragraphs, b = plain-text (default is b) + random bool // does not use the default paragraph "Lorem ipsum dolor sit amet, ..." +} + +func (node *tagLoremNode) Execute(ctx *ExecutionContext, writer TemplateWriter) *Error { + switch node.method { + case "b": + if node.random { + for i := 0; i < node.count; i++ { + if i > 0 { + writer.WriteString("\n") + } + par := tagLoremParagraphs[rand.Intn(len(tagLoremParagraphs))] + writer.WriteString(par) + } + } else { + for i := 0; i < node.count; i++ { + if i > 0 { + writer.WriteString("\n") + } + par := tagLoremParagraphs[i%len(tagLoremParagraphs)] + writer.WriteString(par) + } + } + case "w": + if node.random { + for i := 0; i < node.count; i++ { + if i > 0 { + writer.WriteString(" ") + } + word := tagLoremWords[rand.Intn(len(tagLoremWords))] + writer.WriteString(word) + } + } else { + for i := 0; i < node.count; i++ { + if i > 0 { + writer.WriteString(" ") + } + word := tagLoremWords[i%len(tagLoremWords)] + writer.WriteString(word) + } + } + case "p": + if node.random { + for i := 0; i < node.count; i++ { + if i > 0 { + writer.WriteString("\n") + } + writer.WriteString("

") + par := tagLoremParagraphs[rand.Intn(len(tagLoremParagraphs))] + writer.WriteString(par) + writer.WriteString("

") + } + } else { + for i := 0; i < node.count; i++ { + if i > 0 { + writer.WriteString("\n") + } + writer.WriteString("

") + par := tagLoremParagraphs[i%len(tagLoremParagraphs)] + writer.WriteString(par) + writer.WriteString("

") + + } + } + default: + return ctx.OrigError(errors.Errorf("unsupported method: %s", node.method), nil) + } + + return nil +} + +func tagLoremParser(doc *Parser, start *Token, arguments *Parser) (INodeTag, *Error) { + loremNode := &tagLoremNode{ + position: start, + count: 1, + method: "b", + } + + if countToken := arguments.MatchType(TokenNumber); countToken != nil { + loremNode.count = AsValue(countToken.Val).Integer() + } + + if methodToken := arguments.MatchType(TokenIdentifier); methodToken != nil { + if methodToken.Val != "w" && methodToken.Val != "p" && methodToken.Val != "b" { + return nil, arguments.Error("lorem-method must be either 'w', 'p' or 'b'.", nil) + } + + loremNode.method = methodToken.Val + } + + if arguments.MatchOne(TokenIdentifier, "random") != nil { + loremNode.random = true + } + + if arguments.Remaining() > 0 { + return nil, arguments.Error("Malformed lorem-tag arguments.", nil) + } + + return loremNode, nil +} + +func init() { + rand.Seed(time.Now().Unix()) + + RegisterTag("lorem", tagLoremParser) +} + +const tagLoremText = `Lorem ipsum dolor sit amet, consectetur adipisici elit, sed eiusmod tempor incidunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquid ex ea commodi consequat. Quis aute iure reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint obcaecat cupiditat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. +Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie consequat, vel illum dolore eu feugiat nulla facilisis at vero eros et accumsan et iusto odio dignissim qui blandit praesent luptatum zzril delenit augue duis dolore te feugait nulla facilisi. Lorem ipsum dolor sit amet, consectetuer adipiscing elit, sed diam nonummy nibh euismod tincidunt ut laoreet dolore magna aliquam erat volutpat. +Ut wisi enim ad minim veniam, quis nostrud exerci tation ullamcorper suscipit lobortis nisl ut aliquip ex ea commodo consequat. Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie consequat, vel illum dolore eu feugiat nulla facilisis at vero eros et accumsan et iusto odio dignissim qui blandit praesent luptatum zzril delenit augue duis dolore te feugait nulla facilisi. +Nam liber tempor cum soluta nobis eleifend option congue nihil imperdiet doming id quod mazim placerat facer possim assum. Lorem ipsum dolor sit amet, consectetuer adipiscing elit, sed diam nonummy nibh euismod tincidunt ut laoreet dolore magna aliquam erat volutpat. Ut wisi enim ad minim veniam, quis nostrud exerci tation ullamcorper suscipit lobortis nisl ut aliquip ex ea commodo consequat. +Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie consequat, vel illum dolore eu feugiat nulla facilisis. +At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, At accusam aliquyam diam diam dolore dolores duo eirmod eos erat, et nonumy sed tempor et et invidunt justo labore Stet clita ea et gubergren, kasd magna no rebum. sanctus sea sed takimata ut vero voluptua. est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat. +Consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.` diff --git a/vendor/github.com/flosch/pongo2/tags_macro.go b/vendor/github.com/flosch/pongo2/tags_macro.go new file mode 100644 index 0000000000..dd3e0bf48a --- /dev/null +++ b/vendor/github.com/flosch/pongo2/tags_macro.go @@ -0,0 +1,149 @@ +package pongo2 + +import ( + "bytes" + "fmt" +) + +type tagMacroNode struct { + position *Token + name string + argsOrder []string + args map[string]IEvaluator + exported bool + + wrapper *NodeWrapper +} + +func (node *tagMacroNode) Execute(ctx *ExecutionContext, writer TemplateWriter) *Error { + ctx.Private[node.name] = func(args ...*Value) *Value { + return node.call(ctx, args...) + } + + return nil +} + +func (node *tagMacroNode) call(ctx *ExecutionContext, args ...*Value) *Value { + argsCtx := make(Context) + + for k, v := range node.args { + if v == nil { + // User did not provided a default value + argsCtx[k] = nil + } else { + // Evaluate the default value + valueExpr, err := v.Evaluate(ctx) + if err != nil { + ctx.Logf(err.Error()) + return AsSafeValue(err.Error()) + } + + argsCtx[k] = valueExpr + } + } + + if len(args) > len(node.argsOrder) { + // Too many arguments, we're ignoring them and just logging into debug mode. + err := ctx.Error(fmt.Sprintf("Macro '%s' called with too many arguments (%d instead of %d).", + node.name, len(args), len(node.argsOrder)), nil).updateFromTokenIfNeeded(ctx.template, node.position) + + ctx.Logf(err.Error()) // TODO: This is a workaround, because the error is not returned yet to the Execution()-methods + return AsSafeValue(err.Error()) + } + + // Make a context for the macro execution + macroCtx := NewChildExecutionContext(ctx) + + // Register all arguments in the private context + macroCtx.Private.Update(argsCtx) + + for idx, argValue := range args { + macroCtx.Private[node.argsOrder[idx]] = argValue.Interface() + } + + var b bytes.Buffer + err := node.wrapper.Execute(macroCtx, &b) + if err != nil { + return AsSafeValue(err.updateFromTokenIfNeeded(ctx.template, node.position).Error()) + } + + return AsSafeValue(b.String()) +} + +func tagMacroParser(doc *Parser, start *Token, arguments *Parser) (INodeTag, *Error) { + macroNode := &tagMacroNode{ + position: start, + args: make(map[string]IEvaluator), + } + + nameToken := arguments.MatchType(TokenIdentifier) + if nameToken == nil { + return nil, arguments.Error("Macro-tag needs at least an identifier as name.", nil) + } + macroNode.name = nameToken.Val + + if arguments.MatchOne(TokenSymbol, "(") == nil { + return nil, arguments.Error("Expected '('.", nil) + } + + for arguments.Match(TokenSymbol, ")") == nil { + argNameToken := arguments.MatchType(TokenIdentifier) + if argNameToken == nil { + return nil, arguments.Error("Expected argument name as identifier.", nil) + } + macroNode.argsOrder = append(macroNode.argsOrder, argNameToken.Val) + + if arguments.Match(TokenSymbol, "=") != nil { + // Default expression follows + argDefaultExpr, err := arguments.ParseExpression() + if err != nil { + return nil, err + } + macroNode.args[argNameToken.Val] = argDefaultExpr + } else { + // No default expression + macroNode.args[argNameToken.Val] = nil + } + + if arguments.Match(TokenSymbol, ")") != nil { + break + } + if arguments.Match(TokenSymbol, ",") == nil { + return nil, arguments.Error("Expected ',' or ')'.", nil) + } + } + + if arguments.Match(TokenKeyword, "export") != nil { + macroNode.exported = true + } + + if arguments.Remaining() > 0 { + return nil, arguments.Error("Malformed macro-tag.", nil) + } + + // Body wrapping + wrapper, endargs, err := doc.WrapUntilTag("endmacro") + if err != nil { + return nil, err + } + macroNode.wrapper = wrapper + + if endargs.Count() > 0 { + return nil, endargs.Error("Arguments not allowed here.", nil) + } + + if macroNode.exported { + // Now register the macro if it wants to be exported + _, has := doc.template.exportedMacros[macroNode.name] + if has { + return nil, doc.Error(fmt.Sprintf("another macro with name '%s' already exported", macroNode.name), start) + } + doc.template.exportedMacros[macroNode.name] = macroNode + } + + return macroNode, nil +} + +func init() { + RegisterTag("macro", tagMacroParser) +} diff --git a/vendor/github.com/flosch/pongo2/tags_now.go b/vendor/github.com/flosch/pongo2/tags_now.go new file mode 100644 index 0000000000..d9fa4a3711 --- /dev/null +++ b/vendor/github.com/flosch/pongo2/tags_now.go @@ -0,0 +1,50 @@ +package pongo2 + +import ( + "time" +) + +type tagNowNode struct { + position *Token + format string + fake bool +} + +func (node *tagNowNode) Execute(ctx *ExecutionContext, writer TemplateWriter) *Error { + var t time.Time + if node.fake { + t = time.Date(2014, time.February, 05, 18, 31, 45, 00, time.UTC) + } else { + t = time.Now() + } + + writer.WriteString(t.Format(node.format)) + + return nil +} + +func tagNowParser(doc *Parser, start *Token, arguments *Parser) (INodeTag, *Error) { + nowNode := &tagNowNode{ + position: start, + } + + formatToken := arguments.MatchType(TokenString) + if formatToken == nil { + return nil, arguments.Error("Expected a format string.", nil) + } + nowNode.format = formatToken.Val + + if arguments.MatchOne(TokenIdentifier, "fake") != nil { + nowNode.fake = true + } + + if arguments.Remaining() > 0 { + return nil, arguments.Error("Malformed now-tag arguments.", nil) + } + + return nowNode, nil +} + +func init() { + RegisterTag("now", tagNowParser) +} diff --git a/vendor/github.com/flosch/pongo2/tags_set.go b/vendor/github.com/flosch/pongo2/tags_set.go new file mode 100644 index 0000000000..be121c12ac --- /dev/null +++ b/vendor/github.com/flosch/pongo2/tags_set.go @@ -0,0 +1,50 @@ +package pongo2 + +type tagSetNode struct { + name string + expression IEvaluator +} + +func (node *tagSetNode) Execute(ctx *ExecutionContext, writer TemplateWriter) *Error { + // Evaluate expression + value, err := node.expression.Evaluate(ctx) + if err != nil { + return err + } + + ctx.Private[node.name] = value + return nil +} + +func tagSetParser(doc *Parser, start *Token, arguments *Parser) (INodeTag, *Error) { + node := &tagSetNode{} + + // Parse variable name + typeToken := arguments.MatchType(TokenIdentifier) + if typeToken == nil { + return nil, arguments.Error("Expected an identifier.", nil) + } + node.name = typeToken.Val + + if arguments.Match(TokenSymbol, "=") == nil { + return nil, arguments.Error("Expected '='.", nil) + } + + // Variable expression + keyExpression, err := arguments.ParseExpression() + if err != nil { + return nil, err + } + node.expression = keyExpression + + // Remaining arguments + if arguments.Remaining() > 0 { + return nil, arguments.Error("Malformed 'set'-tag arguments.", nil) + } + + return node, nil +} + +func init() { + RegisterTag("set", tagSetParser) +} diff --git a/vendor/github.com/flosch/pongo2/tags_spaceless.go b/vendor/github.com/flosch/pongo2/tags_spaceless.go new file mode 100644 index 0000000000..4fa851ba45 --- /dev/null +++ b/vendor/github.com/flosch/pongo2/tags_spaceless.go @@ -0,0 +1,54 @@ +package pongo2 + +import ( + "bytes" + "regexp" +) + +type tagSpacelessNode struct { + wrapper *NodeWrapper +} + +var tagSpacelessRegexp = regexp.MustCompile(`(?U:(<.*>))([\t\n\v\f\r ]+)(?U:(<.*>))`) + +func (node *tagSpacelessNode) Execute(ctx *ExecutionContext, writer TemplateWriter) *Error { + b := bytes.NewBuffer(make([]byte, 0, 1024)) // 1 KiB + + err := node.wrapper.Execute(ctx, b) + if err != nil { + return err + } + + s := b.String() + // Repeat this recursively + changed := true + for changed { + s2 := tagSpacelessRegexp.ReplaceAllString(s, "$1$3") + changed = s != s2 + s = s2 + } + + writer.WriteString(s) + + return nil +} + +func tagSpacelessParser(doc *Parser, start *Token, arguments *Parser) (INodeTag, *Error) { + spacelessNode := &tagSpacelessNode{} + + wrapper, _, err := doc.WrapUntilTag("endspaceless") + if err != nil { + return nil, err + } + spacelessNode.wrapper = wrapper + + if arguments.Remaining() > 0 { + return nil, arguments.Error("Malformed spaceless-tag arguments.", nil) + } + + return spacelessNode, nil +} + +func init() { + RegisterTag("spaceless", tagSpacelessParser) +} diff --git a/vendor/github.com/flosch/pongo2/tags_ssi.go b/vendor/github.com/flosch/pongo2/tags_ssi.go new file mode 100644 index 0000000000..c33858d5f1 --- /dev/null +++ b/vendor/github.com/flosch/pongo2/tags_ssi.go @@ -0,0 +1,68 @@ +package pongo2 + +import ( + "io/ioutil" +) + +type tagSSINode struct { + filename string + content string + template *Template +} + +func (node *tagSSINode) Execute(ctx *ExecutionContext, writer TemplateWriter) *Error { + if node.template != nil { + // Execute the template within the current context + includeCtx := make(Context) + includeCtx.Update(ctx.Public) + includeCtx.Update(ctx.Private) + + err := node.template.execute(includeCtx, writer) + if err != nil { + return err.(*Error) + } + } else { + // Just print out the content + writer.WriteString(node.content) + } + return nil +} + +func tagSSIParser(doc *Parser, start *Token, arguments *Parser) (INodeTag, *Error) { + SSINode := &tagSSINode{} + + if fileToken := arguments.MatchType(TokenString); fileToken != nil { + SSINode.filename = fileToken.Val + + if arguments.Match(TokenIdentifier, "parsed") != nil { + // parsed + temporaryTpl, err := doc.template.set.FromFile(doc.template.set.resolveFilename(doc.template, fileToken.Val)) + if err != nil { + return nil, err.(*Error).updateFromTokenIfNeeded(doc.template, fileToken) + } + SSINode.template = temporaryTpl + } else { + // plaintext + buf, err := ioutil.ReadFile(doc.template.set.resolveFilename(doc.template, fileToken.Val)) + if err != nil { + return nil, (&Error{ + Sender: "tag:ssi", + OrigError: err, + }).updateFromTokenIfNeeded(doc.template, fileToken) + } + SSINode.content = string(buf) + } + } else { + return nil, arguments.Error("First argument must be a string.", nil) + } + + if arguments.Remaining() > 0 { + return nil, arguments.Error("Malformed SSI-tag argument.", nil) + } + + return SSINode, nil +} + +func init() { + RegisterTag("ssi", tagSSIParser) +} diff --git a/vendor/github.com/flosch/pongo2/tags_templatetag.go b/vendor/github.com/flosch/pongo2/tags_templatetag.go new file mode 100644 index 0000000000..164b4dc3d0 --- /dev/null +++ b/vendor/github.com/flosch/pongo2/tags_templatetag.go @@ -0,0 +1,45 @@ +package pongo2 + +type tagTemplateTagNode struct { + content string +} + +var templateTagMapping = map[string]string{ + "openblock": "{%", + "closeblock": "%}", + "openvariable": "{{", + "closevariable": "}}", + "openbrace": "{", + "closebrace": "}", + "opencomment": "{#", + "closecomment": "#}", +} + +func (node *tagTemplateTagNode) Execute(ctx *ExecutionContext, writer TemplateWriter) *Error { + writer.WriteString(node.content) + return nil +} + +func tagTemplateTagParser(doc *Parser, start *Token, arguments *Parser) (INodeTag, *Error) { + ttNode := &tagTemplateTagNode{} + + if argToken := arguments.MatchType(TokenIdentifier); argToken != nil { + output, found := templateTagMapping[argToken.Val] + if !found { + return nil, arguments.Error("Argument not found", argToken) + } + ttNode.content = output + } else { + return nil, arguments.Error("Identifier expected.", nil) + } + + if arguments.Remaining() > 0 { + return nil, arguments.Error("Malformed templatetag-tag argument.", nil) + } + + return ttNode, nil +} + +func init() { + RegisterTag("templatetag", tagTemplateTagParser) +} diff --git a/vendor/github.com/flosch/pongo2/tags_widthratio.go b/vendor/github.com/flosch/pongo2/tags_widthratio.go new file mode 100644 index 0000000000..70c9c3e8af --- /dev/null +++ b/vendor/github.com/flosch/pongo2/tags_widthratio.go @@ -0,0 +1,83 @@ +package pongo2 + +import ( + "fmt" + "math" +) + +type tagWidthratioNode struct { + position *Token + current, max IEvaluator + width IEvaluator + ctxName string +} + +func (node *tagWidthratioNode) Execute(ctx *ExecutionContext, writer TemplateWriter) *Error { + current, err := node.current.Evaluate(ctx) + if err != nil { + return err + } + + max, err := node.max.Evaluate(ctx) + if err != nil { + return err + } + + width, err := node.width.Evaluate(ctx) + if err != nil { + return err + } + + value := int(math.Ceil(current.Float()/max.Float()*width.Float() + 0.5)) + + if node.ctxName == "" { + writer.WriteString(fmt.Sprintf("%d", value)) + } else { + ctx.Private[node.ctxName] = value + } + + return nil +} + +func tagWidthratioParser(doc *Parser, start *Token, arguments *Parser) (INodeTag, *Error) { + widthratioNode := &tagWidthratioNode{ + position: start, + } + + current, err := arguments.ParseExpression() + if err != nil { + return nil, err + } + widthratioNode.current = current + + max, err := arguments.ParseExpression() + if err != nil { + return nil, err + } + widthratioNode.max = max + + width, err := arguments.ParseExpression() + if err != nil { + return nil, err + } + widthratioNode.width = width + + if arguments.MatchOne(TokenKeyword, "as") != nil { + // Name follows + nameToken := arguments.MatchType(TokenIdentifier) + if nameToken == nil { + return nil, arguments.Error("Expected name (identifier).", nil) + } + widthratioNode.ctxName = nameToken.Val + } + + if arguments.Remaining() > 0 { + return nil, arguments.Error("Malformed widthratio-tag arguments.", nil) + } + + return widthratioNode, nil +} + +func init() { + RegisterTag("widthratio", tagWidthratioParser) +} diff --git a/vendor/github.com/flosch/pongo2/tags_with.go b/vendor/github.com/flosch/pongo2/tags_with.go new file mode 100644 index 0000000000..32b3c1c428 --- /dev/null +++ b/vendor/github.com/flosch/pongo2/tags_with.go @@ -0,0 +1,88 @@ +package pongo2 + +type tagWithNode struct { + withPairs map[string]IEvaluator + wrapper *NodeWrapper +} + +func (node *tagWithNode) Execute(ctx *ExecutionContext, writer TemplateWriter) *Error { + //new context for block + withctx := NewChildExecutionContext(ctx) + + // Put all custom with-pairs into the context + for key, value := range node.withPairs { + val, err := value.Evaluate(ctx) + if err != nil { + return err + } + withctx.Private[key] = val + } + + return node.wrapper.Execute(withctx, writer) +} + +func tagWithParser(doc *Parser, start *Token, arguments *Parser) (INodeTag, *Error) { + withNode := &tagWithNode{ + withPairs: make(map[string]IEvaluator), + } + + if arguments.Count() == 0 { + return nil, arguments.Error("Tag 'with' requires at least one argument.", nil) + } + + wrapper, endargs, err := doc.WrapUntilTag("endwith") + if err != nil { + return nil, err + } + withNode.wrapper = wrapper + + if endargs.Count() > 0 { + return nil, endargs.Error("Arguments not allowed here.", nil) + } + + // Scan through all arguments to see which style the user uses (old or new style). + // If we find any "as" keyword we will enforce old style; otherwise we will use new style. + oldStyle := false // by default we're using the new_style + for i := 0; i < arguments.Count(); i++ { + if arguments.PeekN(i, TokenKeyword, "as") != nil { + oldStyle = true + break + } + } + + for arguments.Remaining() > 0 { + if oldStyle { + valueExpr, err := arguments.ParseExpression() + if err != nil { + return nil, err + } + if arguments.Match(TokenKeyword, "as") == nil { + return nil, arguments.Error("Expected 'as' keyword.", nil) + } + keyToken := arguments.MatchType(TokenIdentifier) + if keyToken == nil { + return nil, arguments.Error("Expected an identifier", nil) + } + withNode.withPairs[keyToken.Val] = valueExpr + } else { + keyToken := arguments.MatchType(TokenIdentifier) + if keyToken == nil { + return nil, arguments.Error("Expected an identifier", nil) + } + if arguments.Match(TokenSymbol, "=") == nil { + return nil, arguments.Error("Expected '='.", nil) + } + valueExpr, err := arguments.ParseExpression() + if err != nil { + return nil, err + } + withNode.withPairs[keyToken.Val] = valueExpr + } + } + + return withNode, nil +} + +func init() { + RegisterTag("with", tagWithParser) +} diff --git a/vendor/github.com/flosch/pongo2/template.go b/vendor/github.com/flosch/pongo2/template.go new file mode 100644 index 0000000000..fbe2106ffd --- /dev/null +++ b/vendor/github.com/flosch/pongo2/template.go @@ -0,0 +1,277 @@ +package pongo2 + +import ( + "bytes" + "io" + "strings" + + "github.com/juju/errors" +) + +type TemplateWriter interface { + io.Writer + WriteString(string) (int, error) +} + +type templateWriter struct { + w io.Writer +} + +func (tw *templateWriter) WriteString(s string) (int, error) { + return tw.w.Write([]byte(s)) +} + +func (tw *templateWriter) Write(b []byte) (int, error) { + return tw.w.Write(b) +} + +type Template struct { + set *TemplateSet + + // Input + isTplString bool + name string + tpl string + size int + + // Calculation + tokens []*Token + parser *Parser + + // first come, first serve (it's important to not override existing entries in here) + level int + parent *Template + child *Template + blocks map[string]*NodeWrapper + exportedMacros map[string]*tagMacroNode + + // Output + root *nodeDocument + + // Options allow you to change the behavior of template-engine. + // You can change the options before calling the Execute method. + Options *Options +} + +func newTemplateString(set *TemplateSet, tpl []byte) (*Template, error) { + return newTemplate(set, "", true, tpl) +} + +func newTemplate(set *TemplateSet, name string, isTplString bool, tpl []byte) (*Template, error) { + strTpl := string(tpl) + + // Create the template + t := &Template{ + set: set, + isTplString: isTplString, + name: name, + tpl: strTpl, + size: len(strTpl), + blocks: make(map[string]*NodeWrapper), + exportedMacros: make(map[string]*tagMacroNode), + Options: newOptions(), + } + // Copy all settings from another Options. + t.Options.Update(set.Options) + + // Tokenize it + tokens, err := lex(name, strTpl) + if err != nil { + return nil, err + } + t.tokens = tokens + + // For debugging purposes, show all tokens: + /*for i, t := range tokens { + fmt.Printf("%3d. %s\n", i, t) + }*/ + + // Parse it + err = t.parse() + if err != nil { + return nil, err + } + + return t, nil +} + +func (tpl *Template) newContextForExecution(context Context) (*Template, *ExecutionContext, error) { + if tpl.Options.TrimBlocks || tpl.Options.LStripBlocks { + // Issue #94 https://github.com/flosch/pongo2/issues/94 + // If an application configures pongo2 template to trim_blocks, + // the first newline after a template tag is removed automatically (like in PHP). + prev := &Token{ + Typ: TokenHTML, + Val: "\n", + } + + for _, t := range tpl.tokens { + if tpl.Options.LStripBlocks { + if prev.Typ == TokenHTML && t.Typ != TokenHTML && t.Val == "{%" { + prev.Val = strings.TrimRight(prev.Val, "\t ") + } + } + + if tpl.Options.TrimBlocks { + if prev.Typ != TokenHTML && t.Typ == TokenHTML && prev.Val == "%}" { + if len(t.Val) > 0 && t.Val[0] == '\n' { + t.Val = t.Val[1:len(t.Val)] + } + } + } + + prev = t + } + } + + // Determine the parent to be executed (for template inheritance) + parent := tpl + for parent.parent != nil { + parent = parent.parent + } + + // Create context if none is given + newContext := make(Context) + newContext.Update(tpl.set.Globals) + + if context != nil { + newContext.Update(context) + + if len(newContext) > 0 { + // Check for context name syntax + err := newContext.checkForValidIdentifiers() + if err != nil { + return parent, nil, err + } + + // Check for clashes with macro names + for k := range newContext { + _, has := tpl.exportedMacros[k] + if has { + return parent, nil, &Error{ + Filename: tpl.name, + Sender: "execution", + OrigError: errors.Errorf("context key name '%s' clashes with macro '%s'", k, k), + } + } + } + } + } + + // Create operational context + ctx := newExecutionContext(parent, newContext) + + return parent, ctx, nil +} + +func (tpl *Template) execute(context Context, writer TemplateWriter) error { + parent, ctx, err := tpl.newContextForExecution(context) + if err != nil { + return err + } + + // Run the selected document + if err := parent.root.Execute(ctx, writer); err != nil { + return err + } + + return nil +} + +func (tpl *Template) newTemplateWriterAndExecute(context Context, writer io.Writer) error { + return tpl.execute(context, &templateWriter{w: writer}) +} + +func (tpl *Template) newBufferAndExecute(context Context) (*bytes.Buffer, error) { + // Create output buffer + // We assume that the rendered template will be 30% larger + buffer := bytes.NewBuffer(make([]byte, 0, int(float64(tpl.size)*1.3))) + if err := tpl.execute(context, buffer); err != nil { + return nil, err + } + return buffer, nil +} + +// Executes the template with the given context and writes to writer (io.Writer) +// on success. Context can be nil. Nothing is written on error; instead the error +// is being returned. +func (tpl *Template) ExecuteWriter(context Context, writer io.Writer) error { + buf, err := tpl.newBufferAndExecute(context) + if err != nil { + return err + } + _, err = buf.WriteTo(writer) + if err != nil { + return err + } + return nil +} + +// Same as ExecuteWriter. The only difference between both functions is that +// this function might already have written parts of the generated template in the +// case of an execution error because there's no intermediate buffer involved for +// performance reasons. This is handy if you need high performance template +// generation or if you want to manage your own pool of buffers. +func (tpl *Template) ExecuteWriterUnbuffered(context Context, writer io.Writer) error { + return tpl.newTemplateWriterAndExecute(context, writer) +} + +// Executes the template and returns the rendered template as a []byte +func (tpl *Template) ExecuteBytes(context Context) ([]byte, error) { + // Execute template + buffer, err := tpl.newBufferAndExecute(context) + if err != nil { + return nil, err + } + return buffer.Bytes(), nil +} + +// Executes the template and returns the rendered template as a string +func (tpl *Template) Execute(context Context) (string, error) { + // Execute template + buffer, err := tpl.newBufferAndExecute(context) + if err != nil { + return "", err + } + + return buffer.String(), nil + +} + +func (tpl *Template) ExecuteBlocks(context Context, blocks []string) (map[string]string, error) { + var parents []*Template + result := make(map[string]string) + + parent := tpl + for parent != nil { + parents = append(parents, parent) + parent = parent.parent + } + + for _, t := range parents { + buffer := bytes.NewBuffer(make([]byte, 0, int(float64(t.size)*1.3))) + _, ctx, err := t.newContextForExecution(context) + if err != nil { + return nil, err + } + for _, blockName := range blocks { + if _, ok := result[blockName]; ok { + continue + } + if blockWrapper, ok := t.blocks[blockName]; ok { + bErr := blockWrapper.Execute(ctx, buffer) + if bErr != nil { + return nil, bErr + } + result[blockName] = buffer.String() + buffer.Reset() + } + } + // We have found all blocks + if len(blocks) == len(result) { + break + } + } + + return result, nil +} diff --git a/vendor/github.com/flosch/pongo2/template_loader.go b/vendor/github.com/flosch/pongo2/template_loader.go new file mode 100644 index 0000000000..bc80f4ab72 --- /dev/null +++ b/vendor/github.com/flosch/pongo2/template_loader.go @@ -0,0 +1,157 @@ +package pongo2 + +import ( + "bytes" + "io" + "io/ioutil" + "log" + "os" + "path/filepath" + + "github.com/juju/errors" +) + +// LocalFilesystemLoader represents a local filesystem loader with basic +// BaseDirectory capabilities. The access to the local filesystem is unrestricted. +type LocalFilesystemLoader struct { + baseDir string +} + +// MustNewLocalFileSystemLoader creates a new LocalFilesystemLoader instance +// and panics if there's any error during instantiation. The parameters +// are the same like NewLocalFileSystemLoader. +func MustNewLocalFileSystemLoader(baseDir string) *LocalFilesystemLoader { + fs, err := NewLocalFileSystemLoader(baseDir) + if err != nil { + log.Panic(err) + } + return fs +} + +// NewLocalFileSystemLoader creates a new LocalFilesystemLoader and allows +// templatesto be loaded from disk (unrestricted). If any base directory +// is given (or being set using SetBaseDir), this base directory is being used +// for path calculation in template inclusions/imports. Otherwise the path +// is calculated based relatively to the including template's path. +func NewLocalFileSystemLoader(baseDir string) (*LocalFilesystemLoader, error) { + fs := &LocalFilesystemLoader{} + if baseDir != "" { + if err := fs.SetBaseDir(baseDir); err != nil { + return nil, err + } + } + return fs, nil +} + +// SetBaseDir sets the template's base directory. This directory will +// be used for any relative path in filters, tags and From*-functions to determine +// your template. See the comment for NewLocalFileSystemLoader as well. +func (fs *LocalFilesystemLoader) SetBaseDir(path string) error { + // Make the path absolute + if !filepath.IsAbs(path) { + abs, err := filepath.Abs(path) + if err != nil { + return err + } + path = abs + } + + // Check for existence + fi, err := os.Stat(path) + if err != nil { + return err + } + if !fi.IsDir() { + return errors.Errorf("The given path '%s' is not a directory.", path) + } + + fs.baseDir = path + return nil +} + +// Get reads the path's content from your local filesystem. +func (fs *LocalFilesystemLoader) Get(path string) (io.Reader, error) { + buf, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + return bytes.NewReader(buf), nil +} + +// Abs resolves a filename relative to the base directory. Absolute paths are allowed. +// When there's no base dir set, the absolute path to the filename +// will be calculated based on either the provided base directory (which +// might be a path of a template which includes another template) or +// the current working directory. +func (fs *LocalFilesystemLoader) Abs(base, name string) string { + if filepath.IsAbs(name) { + return name + } + + // Our own base dir has always priority; if there's none + // we use the path provided in base. + var err error + if fs.baseDir == "" { + if base == "" { + base, err = os.Getwd() + if err != nil { + panic(err) + } + return filepath.Join(base, name) + } + + return filepath.Join(filepath.Dir(base), name) + } + + return filepath.Join(fs.baseDir, name) +} + +// SandboxedFilesystemLoader is still WIP. +type SandboxedFilesystemLoader struct { + *LocalFilesystemLoader +} + +// NewSandboxedFilesystemLoader creates a new sandboxed local file system instance. +func NewSandboxedFilesystemLoader(baseDir string) (*SandboxedFilesystemLoader, error) { + fs, err := NewLocalFileSystemLoader(baseDir) + if err != nil { + return nil, err + } + return &SandboxedFilesystemLoader{ + LocalFilesystemLoader: fs, + }, nil +} + +// Move sandbox to a virtual fs + +/* +if len(set.SandboxDirectories) > 0 { + defer func() { + // Remove any ".." or other crap + resolvedPath = filepath.Clean(resolvedPath) + + // Make the path absolute + absPath, err := filepath.Abs(resolvedPath) + if err != nil { + panic(err) + } + resolvedPath = absPath + + // Check against the sandbox directories (once one pattern matches, we're done and can allow it) + for _, pattern := range set.SandboxDirectories { + matched, err := filepath.Match(pattern, resolvedPath) + if err != nil { + panic("Wrong sandbox directory match pattern (see http://golang.org/pkg/path/filepath/#Match).") + } + if matched { + // OK! + return + } + } + + // No pattern matched, we have to log+deny the request + set.logf("Access attempt outside of the sandbox directories (blocked): '%s'", resolvedPath) + resolvedPath = "" + }() +} +*/ diff --git a/vendor/github.com/flosch/pongo2/template_sets.go b/vendor/github.com/flosch/pongo2/template_sets.go new file mode 100644 index 0000000000..78b3c8d010 --- /dev/null +++ b/vendor/github.com/flosch/pongo2/template_sets.go @@ -0,0 +1,305 @@ +package pongo2 + +import ( + "fmt" + "io" + "io/ioutil" + "log" + "os" + "sync" + + "github.com/juju/errors" +) + +// TemplateLoader allows to implement a virtual file system. +type TemplateLoader interface { + // Abs calculates the path to a given template. Whenever a path must be resolved + // due to an import from another template, the base equals the parent template's path. + Abs(base, name string) string + + // Get returns an io.Reader where the template's content can be read from. + Get(path string) (io.Reader, error) +} + +// TemplateSet allows you to create your own group of templates with their own +// global context (which is shared among all members of the set) and their own +// configuration. +// It's useful for a separation of different kind of templates +// (e. g. web templates vs. mail templates). +type TemplateSet struct { + name string + loaders []TemplateLoader + + // Globals will be provided to all templates created within this template set + Globals Context + + // If debug is true (default false), ExecutionContext.Logf() will work and output + // to STDOUT. Furthermore, FromCache() won't cache the templates. + // Make sure to synchronize the access to it in case you're changing this + // variable during program execution (and template compilation/execution). + Debug bool + + // Options allow you to change the behavior of template-engine. + // You can change the options before calling the Execute method. + Options *Options + + // Sandbox features + // - Disallow access to specific tags and/or filters (using BanTag() and BanFilter()) + // + // For efficiency reasons you can ban tags/filters only *before* you have + // added your first template to the set (restrictions are statically checked). + // After you added one, it's not possible anymore (for your personal security). + firstTemplateCreated bool + bannedTags map[string]bool + bannedFilters map[string]bool + + // Template cache (for FromCache()) + templateCache map[string]*Template + templateCacheMutex sync.Mutex +} + +// NewSet can be used to create sets with different kind of templates +// (e. g. web from mail templates), with different globals or +// other configurations. +func NewSet(name string, loaders ...TemplateLoader) *TemplateSet { + if len(loaders) == 0 { + panic(fmt.Errorf("at least one template loader must be specified")) + } + + return &TemplateSet{ + name: name, + loaders: loaders, + Globals: make(Context), + bannedTags: make(map[string]bool), + bannedFilters: make(map[string]bool), + templateCache: make(map[string]*Template), + Options: newOptions(), + } +} + +func (set *TemplateSet) AddLoader(loaders ...TemplateLoader) { + set.loaders = append(set.loaders, loaders...) +} + +func (set *TemplateSet) resolveFilename(tpl *Template, path string) string { + return set.resolveFilenameForLoader(set.loaders[0], tpl, path) +} + +func (set *TemplateSet) resolveFilenameForLoader(loader TemplateLoader, tpl *Template, path string) string { + name := "" + if tpl != nil && tpl.isTplString { + return path + } + if tpl != nil { + name = tpl.name + } + + return loader.Abs(name, path) +} + +// BanTag bans a specific tag for this template set. See more in the documentation for TemplateSet. +func (set *TemplateSet) BanTag(name string) error { + _, has := tags[name] + if !has { + return errors.Errorf("tag '%s' not found", name) + } + if set.firstTemplateCreated { + return errors.New("you cannot ban any tags after you've added your first template to your template set") + } + _, has = set.bannedTags[name] + if has { + return errors.Errorf("tag '%s' is already banned", name) + } + set.bannedTags[name] = true + + return nil +} + +// BanFilter bans a specific filter for this template set. See more in the documentation for TemplateSet. +func (set *TemplateSet) BanFilter(name string) error { + _, has := filters[name] + if !has { + return errors.Errorf("filter '%s' not found", name) + } + if set.firstTemplateCreated { + return errors.New("you cannot ban any filters after you've added your first template to your template set") + } + _, has = set.bannedFilters[name] + if has { + return errors.Errorf("filter '%s' is already banned", name) + } + set.bannedFilters[name] = true + + return nil +} + +func (set *TemplateSet) resolveTemplate(tpl *Template, path string) (name string, loader TemplateLoader, fd io.Reader, err error) { + // iterate over loaders until we appear to have a valid template + for _, loader = range set.loaders { + name = set.resolveFilenameForLoader(loader, tpl, path) + fd, err = loader.Get(name) + if err == nil { + return + } + } + + return path, nil, nil, fmt.Errorf("unable to resolve template") +} + +// CleanCache cleans the template cache. If filenames is not empty, +// it will remove the template caches of those filenames. +// Or it will empty the whole template cache. It is thread-safe. +func (set *TemplateSet) CleanCache(filenames ...string) { + set.templateCacheMutex.Lock() + defer set.templateCacheMutex.Unlock() + + if len(filenames) == 0 { + set.templateCache = make(map[string]*Template, len(set.templateCache)) + } + + for _, filename := range filenames { + delete(set.templateCache, set.resolveFilename(nil, filename)) + } +} + +// FromCache is a convenient method to cache templates. It is thread-safe +// and will only compile the template associated with a filename once. +// If TemplateSet.Debug is true (for example during development phase), +// FromCache() will not cache the template and instead recompile it on any +// call (to make changes to a template live instantaneously). +func (set *TemplateSet) FromCache(filename string) (*Template, error) { + if set.Debug { + // Recompile on any request + return set.FromFile(filename) + } + // Cache the template + cleanedFilename := set.resolveFilename(nil, filename) + + set.templateCacheMutex.Lock() + defer set.templateCacheMutex.Unlock() + + tpl, has := set.templateCache[cleanedFilename] + + // Cache miss + if !has { + tpl, err := set.FromFile(cleanedFilename) + if err != nil { + return nil, err + } + set.templateCache[cleanedFilename] = tpl + return tpl, nil + } + + // Cache hit + return tpl, nil +} + +// FromString loads a template from string and returns a Template instance. +func (set *TemplateSet) FromString(tpl string) (*Template, error) { + set.firstTemplateCreated = true + + return newTemplateString(set, []byte(tpl)) +} + +// FromBytes loads a template from bytes and returns a Template instance. +func (set *TemplateSet) FromBytes(tpl []byte) (*Template, error) { + set.firstTemplateCreated = true + + return newTemplateString(set, tpl) +} + +// FromFile loads a template from a filename and returns a Template instance. +func (set *TemplateSet) FromFile(filename string) (*Template, error) { + set.firstTemplateCreated = true + + _, _, fd, err := set.resolveTemplate(nil, filename) + if err != nil { + return nil, &Error{ + Filename: filename, + Sender: "fromfile", + OrigError: err, + } + } + buf, err := ioutil.ReadAll(fd) + if err != nil { + return nil, &Error{ + Filename: filename, + Sender: "fromfile", + OrigError: err, + } + } + + return newTemplate(set, filename, false, buf) +} + +// RenderTemplateString is a shortcut and renders a template string directly. +func (set *TemplateSet) RenderTemplateString(s string, ctx Context) (string, error) { + set.firstTemplateCreated = true + + tpl := Must(set.FromString(s)) + result, err := tpl.Execute(ctx) + if err != nil { + return "", err + } + return result, nil +} + +// RenderTemplateBytes is a shortcut and renders template bytes directly. +func (set *TemplateSet) RenderTemplateBytes(b []byte, ctx Context) (string, error) { + set.firstTemplateCreated = true + + tpl := Must(set.FromBytes(b)) + result, err := tpl.Execute(ctx) + if err != nil { + return "", err + } + return result, nil +} + +// RenderTemplateFile is a shortcut and renders a template file directly. +func (set *TemplateSet) RenderTemplateFile(fn string, ctx Context) (string, error) { + set.firstTemplateCreated = true + + tpl := Must(set.FromFile(fn)) + result, err := tpl.Execute(ctx) + if err != nil { + return "", err + } + return result, nil +} + +func (set *TemplateSet) logf(format string, args ...interface{}) { + if set.Debug { + logger.Printf(fmt.Sprintf("[template set: %s] %s", set.name, format), args...) + } +} + +// Logging function (internally used) +func logf(format string, items ...interface{}) { + if debug { + logger.Printf(format, items...) + } +} + +var ( + debug bool // internal debugging + logger = log.New(os.Stdout, "[pongo2] ", log.LstdFlags|log.Lshortfile) + + // DefaultLoader allows the default un-sandboxed access to the local file + // system and is being used by the DefaultSet. + DefaultLoader = MustNewLocalFileSystemLoader("") + + // DefaultSet is a set created for you for convinience reasons. + DefaultSet = NewSet("default", DefaultLoader) + + // Methods on the default set + FromString = DefaultSet.FromString + FromBytes = DefaultSet.FromBytes + FromFile = DefaultSet.FromFile + FromCache = DefaultSet.FromCache + RenderTemplateString = DefaultSet.RenderTemplateString + RenderTemplateFile = DefaultSet.RenderTemplateFile + + // Globals for the default set + Globals = DefaultSet.Globals +) diff --git a/vendor/github.com/flosch/pongo2/value.go b/vendor/github.com/flosch/pongo2/value.go new file mode 100644 index 0000000000..df70bbc80c --- /dev/null +++ b/vendor/github.com/flosch/pongo2/value.go @@ -0,0 +1,520 @@ +package pongo2 + +import ( + "fmt" + "reflect" + "sort" + "strconv" + "strings" +) + +type Value struct { + val reflect.Value + safe bool // used to indicate whether a Value needs explicit escaping in the template +} + +// AsValue converts any given value to a pongo2.Value +// Usually being used within own functions passed to a template +// through a Context or within filter functions. +// +// Example: +// AsValue("my string") +func AsValue(i interface{}) *Value { + return &Value{ + val: reflect.ValueOf(i), + } +} + +// AsSafeValue works like AsValue, but does not apply the 'escape' filter. +func AsSafeValue(i interface{}) *Value { + return &Value{ + val: reflect.ValueOf(i), + safe: true, + } +} + +func (v *Value) getResolvedValue() reflect.Value { + if v.val.IsValid() && v.val.Kind() == reflect.Ptr { + return v.val.Elem() + } + return v.val +} + +// IsString checks whether the underlying value is a string +func (v *Value) IsString() bool { + return v.getResolvedValue().Kind() == reflect.String +} + +// IsBool checks whether the underlying value is a bool +func (v *Value) IsBool() bool { + return v.getResolvedValue().Kind() == reflect.Bool +} + +// IsFloat checks whether the underlying value is a float +func (v *Value) IsFloat() bool { + return v.getResolvedValue().Kind() == reflect.Float32 || + v.getResolvedValue().Kind() == reflect.Float64 +} + +// IsInteger checks whether the underlying value is an integer +func (v *Value) IsInteger() bool { + return v.getResolvedValue().Kind() == reflect.Int || + v.getResolvedValue().Kind() == reflect.Int8 || + v.getResolvedValue().Kind() == reflect.Int16 || + v.getResolvedValue().Kind() == reflect.Int32 || + v.getResolvedValue().Kind() == reflect.Int64 || + v.getResolvedValue().Kind() == reflect.Uint || + v.getResolvedValue().Kind() == reflect.Uint8 || + v.getResolvedValue().Kind() == reflect.Uint16 || + v.getResolvedValue().Kind() == reflect.Uint32 || + v.getResolvedValue().Kind() == reflect.Uint64 +} + +// IsNumber checks whether the underlying value is either an integer +// or a float. +func (v *Value) IsNumber() bool { + return v.IsInteger() || v.IsFloat() +} + +// IsNil checks whether the underlying value is NIL +func (v *Value) IsNil() bool { + //fmt.Printf("%+v\n", v.getResolvedValue().Type().String()) + return !v.getResolvedValue().IsValid() +} + +// String returns a string for the underlying value. If this value is not +// of type string, pongo2 tries to convert it. Currently the following +// types for underlying values are supported: +// +// 1. string +// 2. int/uint (any size) +// 3. float (any precision) +// 4. bool +// 5. time.Time +// 6. String() will be called on the underlying value if provided +// +// NIL values will lead to an empty string. Unsupported types are leading +// to their respective type name. +func (v *Value) String() string { + if v.IsNil() { + return "" + } + + switch v.getResolvedValue().Kind() { + case reflect.String: + return v.getResolvedValue().String() + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return strconv.FormatInt(v.getResolvedValue().Int(), 10) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return strconv.FormatUint(v.getResolvedValue().Uint(), 10) + case reflect.Float32, reflect.Float64: + return fmt.Sprintf("%f", v.getResolvedValue().Float()) + case reflect.Bool: + if v.Bool() { + return "True" + } + return "False" + case reflect.Struct: + if t, ok := v.Interface().(fmt.Stringer); ok { + return t.String() + } + } + + logf("Value.String() not implemented for type: %s\n", v.getResolvedValue().Kind().String()) + return v.getResolvedValue().String() +} + +// Integer returns the underlying value as an integer (converts the underlying +// value, if necessary). If it's not possible to convert the underlying value, +// it will return 0. +func (v *Value) Integer() int { + switch v.getResolvedValue().Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return int(v.getResolvedValue().Int()) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return int(v.getResolvedValue().Uint()) + case reflect.Float32, reflect.Float64: + return int(v.getResolvedValue().Float()) + case reflect.String: + // Try to convert from string to int (base 10) + f, err := strconv.ParseFloat(v.getResolvedValue().String(), 64) + if err != nil { + return 0 + } + return int(f) + default: + logf("Value.Integer() not available for type: %s\n", v.getResolvedValue().Kind().String()) + return 0 + } +} + +// Float returns the underlying value as a float (converts the underlying +// value, if necessary). If it's not possible to convert the underlying value, +// it will return 0.0. +func (v *Value) Float() float64 { + switch v.getResolvedValue().Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return float64(v.getResolvedValue().Int()) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return float64(v.getResolvedValue().Uint()) + case reflect.Float32, reflect.Float64: + return v.getResolvedValue().Float() + case reflect.String: + // Try to convert from string to float64 (base 10) + f, err := strconv.ParseFloat(v.getResolvedValue().String(), 64) + if err != nil { + return 0.0 + } + return f + default: + logf("Value.Float() not available for type: %s\n", v.getResolvedValue().Kind().String()) + return 0.0 + } +} + +// Bool returns the underlying value as bool. If the value is not bool, false +// will always be returned. If you're looking for true/false-evaluation of the +// underlying value, have a look on the IsTrue()-function. +func (v *Value) Bool() bool { + switch v.getResolvedValue().Kind() { + case reflect.Bool: + return v.getResolvedValue().Bool() + default: + logf("Value.Bool() not available for type: %s\n", v.getResolvedValue().Kind().String()) + return false + } +} + +// IsTrue tries to evaluate the underlying value the Pythonic-way: +// +// Returns TRUE in one the following cases: +// +// * int != 0 +// * uint != 0 +// * float != 0.0 +// * len(array/chan/map/slice/string) > 0 +// * bool == true +// * underlying value is a struct +// +// Otherwise returns always FALSE. +func (v *Value) IsTrue() bool { + switch v.getResolvedValue().Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return v.getResolvedValue().Int() != 0 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return v.getResolvedValue().Uint() != 0 + case reflect.Float32, reflect.Float64: + return v.getResolvedValue().Float() != 0 + case reflect.Array, reflect.Chan, reflect.Map, reflect.Slice, reflect.String: + return v.getResolvedValue().Len() > 0 + case reflect.Bool: + return v.getResolvedValue().Bool() + case reflect.Struct: + return true // struct instance is always true + default: + logf("Value.IsTrue() not available for type: %s\n", v.getResolvedValue().Kind().String()) + return false + } +} + +// Negate tries to negate the underlying value. It's mainly used for +// the NOT-operator and in conjunction with a call to +// return_value.IsTrue() afterwards. +// +// Example: +// AsValue(1).Negate().IsTrue() == false +func (v *Value) Negate() *Value { + switch v.getResolvedValue().Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + if v.Integer() != 0 { + return AsValue(0) + } + return AsValue(1) + case reflect.Float32, reflect.Float64: + if v.Float() != 0.0 { + return AsValue(float64(0.0)) + } + return AsValue(float64(1.1)) + case reflect.Array, reflect.Chan, reflect.Map, reflect.Slice, reflect.String: + return AsValue(v.getResolvedValue().Len() == 0) + case reflect.Bool: + return AsValue(!v.getResolvedValue().Bool()) + case reflect.Struct: + return AsValue(false) + default: + logf("Value.IsTrue() not available for type: %s\n", v.getResolvedValue().Kind().String()) + return AsValue(true) + } +} + +// Len returns the length for an array, chan, map, slice or string. +// Otherwise it will return 0. +func (v *Value) Len() int { + switch v.getResolvedValue().Kind() { + case reflect.Array, reflect.Chan, reflect.Map, reflect.Slice: + return v.getResolvedValue().Len() + case reflect.String: + runes := []rune(v.getResolvedValue().String()) + return len(runes) + default: + logf("Value.Len() not available for type: %s\n", v.getResolvedValue().Kind().String()) + return 0 + } +} + +// Slice slices an array, slice or string. Otherwise it will +// return an empty []int. +func (v *Value) Slice(i, j int) *Value { + switch v.getResolvedValue().Kind() { + case reflect.Array, reflect.Slice: + return AsValue(v.getResolvedValue().Slice(i, j).Interface()) + case reflect.String: + runes := []rune(v.getResolvedValue().String()) + return AsValue(string(runes[i:j])) + default: + logf("Value.Slice() not available for type: %s\n", v.getResolvedValue().Kind().String()) + return AsValue([]int{}) + } +} + +// Index gets the i-th item of an array, slice or string. Otherwise +// it will return NIL. +func (v *Value) Index(i int) *Value { + switch v.getResolvedValue().Kind() { + case reflect.Array, reflect.Slice: + if i >= v.Len() { + return AsValue(nil) + } + return AsValue(v.getResolvedValue().Index(i).Interface()) + case reflect.String: + //return AsValue(v.getResolvedValue().Slice(i, i+1).Interface()) + s := v.getResolvedValue().String() + runes := []rune(s) + if i < len(runes) { + return AsValue(string(runes[i])) + } + return AsValue("") + default: + logf("Value.Slice() not available for type: %s\n", v.getResolvedValue().Kind().String()) + return AsValue([]int{}) + } +} + +// Contains checks whether the underlying value (which must be of type struct, map, +// string, array or slice) contains of another Value (e. g. used to check +// whether a struct contains of a specific field or a map contains a specific key). +// +// Example: +// AsValue("Hello, World!").Contains(AsValue("World")) == true +func (v *Value) Contains(other *Value) bool { + switch v.getResolvedValue().Kind() { + case reflect.Struct: + fieldValue := v.getResolvedValue().FieldByName(other.String()) + return fieldValue.IsValid() + case reflect.Map: + var mapValue reflect.Value + switch other.Interface().(type) { + case int: + mapValue = v.getResolvedValue().MapIndex(other.getResolvedValue()) + case string: + mapValue = v.getResolvedValue().MapIndex(other.getResolvedValue()) + default: + logf("Value.Contains() does not support lookup type '%s'\n", other.getResolvedValue().Kind().String()) + return false + } + + return mapValue.IsValid() + case reflect.String: + return strings.Contains(v.getResolvedValue().String(), other.String()) + + case reflect.Slice, reflect.Array: + for i := 0; i < v.getResolvedValue().Len(); i++ { + item := v.getResolvedValue().Index(i) + if other.Interface() == item.Interface() { + return true + } + } + return false + + default: + logf("Value.Contains() not available for type: %s\n", v.getResolvedValue().Kind().String()) + return false + } +} + +// CanSlice checks whether the underlying value is of type array, slice or string. +// You normally would use CanSlice() before using the Slice() operation. +func (v *Value) CanSlice() bool { + switch v.getResolvedValue().Kind() { + case reflect.Array, reflect.Slice, reflect.String: + return true + } + return false +} + +// Iterate iterates over a map, array, slice or a string. It calls the +// function's first argument for every value with the following arguments: +// +// idx current 0-index +// count total amount of items +// key *Value for the key or item +// value *Value (only for maps, the respective value for a specific key) +// +// If the underlying value has no items or is not one of the types above, +// the empty function (function's second argument) will be called. +func (v *Value) Iterate(fn func(idx, count int, key, value *Value) bool, empty func()) { + v.IterateOrder(fn, empty, false, false) +} + +// IterateOrder behaves like Value.Iterate, but can iterate through an array/slice/string in reverse. Does +// not affect the iteration through a map because maps don't have any particular order. +// However, you can force an order using the `sorted` keyword (and even use `reversed sorted`). +func (v *Value) IterateOrder(fn func(idx, count int, key, value *Value) bool, empty func(), reverse bool, sorted bool) { + switch v.getResolvedValue().Kind() { + case reflect.Map: + keys := sortedKeys(v.getResolvedValue().MapKeys()) + if sorted { + if reverse { + sort.Sort(sort.Reverse(keys)) + } else { + sort.Sort(keys) + } + } + keyLen := len(keys) + for idx, key := range keys { + value := v.getResolvedValue().MapIndex(key) + if !fn(idx, keyLen, &Value{val: key}, &Value{val: value}) { + return + } + } + if keyLen == 0 { + empty() + } + return // done + case reflect.Array, reflect.Slice: + var items valuesList + + itemCount := v.getResolvedValue().Len() + for i := 0; i < itemCount; i++ { + items = append(items, &Value{val: v.getResolvedValue().Index(i)}) + } + + if sorted { + if reverse { + sort.Sort(sort.Reverse(items)) + } else { + sort.Sort(items) + } + } else { + if reverse { + for i := 0; i < itemCount/2; i++ { + items[i], items[itemCount-1-i] = items[itemCount-1-i], items[i] + } + } + } + + if len(items) > 0 { + for idx, item := range items { + if !fn(idx, itemCount, item, nil) { + return + } + } + } else { + empty() + } + return // done + case reflect.String: + if sorted { + // TODO(flosch): Handle sorted + panic("TODO: handle sort for type string") + } + + // TODO(flosch): Not utf8-compatible (utf8-decoding necessary) + charCount := v.getResolvedValue().Len() + if charCount > 0 { + if reverse { + for i := charCount - 1; i >= 0; i-- { + if !fn(i, charCount, &Value{val: v.getResolvedValue().Slice(i, i+1)}, nil) { + return + } + } + } else { + for i := 0; i < charCount; i++ { + if !fn(i, charCount, &Value{val: v.getResolvedValue().Slice(i, i+1)}, nil) { + return + } + } + } + } else { + empty() + } + return // done + default: + logf("Value.Iterate() not available for type: %s\n", v.getResolvedValue().Kind().String()) + } + empty() +} + +// Interface gives you access to the underlying value. +func (v *Value) Interface() interface{} { + if v.val.IsValid() { + return v.val.Interface() + } + return nil +} + +// EqualValueTo checks whether two values are containing the same value or object. +func (v *Value) EqualValueTo(other *Value) bool { + // comparison of uint with int fails using .Interface()-comparison (see issue #64) + if v.IsInteger() && other.IsInteger() { + return v.Integer() == other.Integer() + } + return v.Interface() == other.Interface() +} + +type sortedKeys []reflect.Value + +func (sk sortedKeys) Len() int { + return len(sk) +} + +func (sk sortedKeys) Less(i, j int) bool { + vi := &Value{val: sk[i]} + vj := &Value{val: sk[j]} + switch { + case vi.IsInteger() && vj.IsInteger(): + return vi.Integer() < vj.Integer() + case vi.IsFloat() && vj.IsFloat(): + return vi.Float() < vj.Float() + default: + return vi.String() < vj.String() + } +} + +func (sk sortedKeys) Swap(i, j int) { + sk[i], sk[j] = sk[j], sk[i] +} + +type valuesList []*Value + +func (vl valuesList) Len() int { + return len(vl) +} + +func (vl valuesList) Less(i, j int) bool { + vi := vl[i] + vj := vl[j] + switch { + case vi.IsInteger() && vj.IsInteger(): + return vi.Integer() < vj.Integer() + case vi.IsFloat() && vj.IsFloat(): + return vi.Float() < vj.Float() + default: + return vi.String() < vj.String() + } +} + +func (vl valuesList) Swap(i, j int) { + vl[i], vl[j] = vl[j], vl[i] +} diff --git a/vendor/github.com/flosch/pongo2/variable.go b/vendor/github.com/flosch/pongo2/variable.go new file mode 100644 index 0000000000..a506e376e4 --- /dev/null +++ b/vendor/github.com/flosch/pongo2/variable.go @@ -0,0 +1,695 @@ +package pongo2 + +import ( + "fmt" + "reflect" + "strconv" + "strings" + + "github.com/juju/errors" +) + +const ( + varTypeInt = iota + varTypeIdent +) + +var ( + typeOfValuePtr = reflect.TypeOf(new(Value)) + typeOfExecCtxPtr = reflect.TypeOf(new(ExecutionContext)) +) + +type variablePart struct { + typ int + s string + i int + + isFunctionCall bool + callingArgs []functionCallArgument // needed for a function call, represents all argument nodes (INode supports nested function calls) +} + +type functionCallArgument interface { + Evaluate(*ExecutionContext) (*Value, *Error) +} + +// TODO: Add location tokens +type stringResolver struct { + locationToken *Token + val string +} + +type intResolver struct { + locationToken *Token + val int +} + +type floatResolver struct { + locationToken *Token + val float64 +} + +type boolResolver struct { + locationToken *Token + val bool +} + +type variableResolver struct { + locationToken *Token + + parts []*variablePart +} + +type nodeFilteredVariable struct { + locationToken *Token + + resolver IEvaluator + filterChain []*filterCall +} + +type nodeVariable struct { + locationToken *Token + expr IEvaluator +} + +type executionCtxEval struct{} + +func (v *nodeFilteredVariable) Execute(ctx *ExecutionContext, writer TemplateWriter) *Error { + value, err := v.Evaluate(ctx) + if err != nil { + return err + } + writer.WriteString(value.String()) + return nil +} + +func (vr *variableResolver) Execute(ctx *ExecutionContext, writer TemplateWriter) *Error { + value, err := vr.Evaluate(ctx) + if err != nil { + return err + } + writer.WriteString(value.String()) + return nil +} + +func (s *stringResolver) Execute(ctx *ExecutionContext, writer TemplateWriter) *Error { + value, err := s.Evaluate(ctx) + if err != nil { + return err + } + writer.WriteString(value.String()) + return nil +} + +func (i *intResolver) Execute(ctx *ExecutionContext, writer TemplateWriter) *Error { + value, err := i.Evaluate(ctx) + if err != nil { + return err + } + writer.WriteString(value.String()) + return nil +} + +func (f *floatResolver) Execute(ctx *ExecutionContext, writer TemplateWriter) *Error { + value, err := f.Evaluate(ctx) + if err != nil { + return err + } + writer.WriteString(value.String()) + return nil +} + +func (b *boolResolver) Execute(ctx *ExecutionContext, writer TemplateWriter) *Error { + value, err := b.Evaluate(ctx) + if err != nil { + return err + } + writer.WriteString(value.String()) + return nil +} + +func (v *nodeFilteredVariable) GetPositionToken() *Token { + return v.locationToken +} + +func (vr *variableResolver) GetPositionToken() *Token { + return vr.locationToken +} + +func (s *stringResolver) GetPositionToken() *Token { + return s.locationToken +} + +func (i *intResolver) GetPositionToken() *Token { + return i.locationToken +} + +func (f *floatResolver) GetPositionToken() *Token { + return f.locationToken +} + +func (b *boolResolver) GetPositionToken() *Token { + return b.locationToken +} + +func (s *stringResolver) Evaluate(ctx *ExecutionContext) (*Value, *Error) { + return AsValue(s.val), nil +} + +func (i *intResolver) Evaluate(ctx *ExecutionContext) (*Value, *Error) { + return AsValue(i.val), nil +} + +func (f *floatResolver) Evaluate(ctx *ExecutionContext) (*Value, *Error) { + return AsValue(f.val), nil +} + +func (b *boolResolver) Evaluate(ctx *ExecutionContext) (*Value, *Error) { + return AsValue(b.val), nil +} + +func (s *stringResolver) FilterApplied(name string) bool { + return false +} + +func (i *intResolver) FilterApplied(name string) bool { + return false +} + +func (f *floatResolver) FilterApplied(name string) bool { + return false +} + +func (b *boolResolver) FilterApplied(name string) bool { + return false +} + +func (nv *nodeVariable) FilterApplied(name string) bool { + return nv.expr.FilterApplied(name) +} + +func (nv *nodeVariable) Execute(ctx *ExecutionContext, writer TemplateWriter) *Error { + value, err := nv.expr.Evaluate(ctx) + if err != nil { + return err + } + + if !nv.expr.FilterApplied("safe") && !value.safe && value.IsString() && ctx.Autoescape { + // apply escape filter + value, err = filters["escape"](value, nil) + if err != nil { + return err + } + } + + writer.WriteString(value.String()) + return nil +} + +func (executionCtxEval) Evaluate(ctx *ExecutionContext) (*Value, *Error) { + return AsValue(ctx), nil +} + +func (vr *variableResolver) FilterApplied(name string) bool { + return false +} + +func (vr *variableResolver) String() string { + parts := make([]string, 0, len(vr.parts)) + for _, p := range vr.parts { + switch p.typ { + case varTypeInt: + parts = append(parts, strconv.Itoa(p.i)) + case varTypeIdent: + parts = append(parts, p.s) + default: + panic("unimplemented") + } + } + return strings.Join(parts, ".") +} + +func (vr *variableResolver) resolve(ctx *ExecutionContext) (*Value, error) { + var current reflect.Value + var isSafe bool + + for idx, part := range vr.parts { + if idx == 0 { + // We're looking up the first part of the variable. + // First we're having a look in our private + // context (e. g. information provided by tags, like the forloop) + val, inPrivate := ctx.Private[vr.parts[0].s] + if !inPrivate { + // Nothing found? Then have a final lookup in the public context + val = ctx.Public[vr.parts[0].s] + } + current = reflect.ValueOf(val) // Get the initial value + } else { + // Next parts, resolve it from current + + // Before resolving the pointer, let's see if we have a method to call + // Problem with resolving the pointer is we're changing the receiver + isFunc := false + if part.typ == varTypeIdent { + funcValue := current.MethodByName(part.s) + if funcValue.IsValid() { + current = funcValue + isFunc = true + } + } + + if !isFunc { + // If current a pointer, resolve it + if current.Kind() == reflect.Ptr { + current = current.Elem() + if !current.IsValid() { + // Value is not valid (anymore) + return AsValue(nil), nil + } + } + + // Look up which part must be called now + switch part.typ { + case varTypeInt: + // Calling an index is only possible for: + // * slices/arrays/strings + switch current.Kind() { + case reflect.String, reflect.Array, reflect.Slice: + if part.i >= 0 && current.Len() > part.i { + current = current.Index(part.i) + } else { + // In Django, exceeding the length of a list is just empty. + return AsValue(nil), nil + } + default: + return nil, errors.Errorf("Can't access an index on type %s (variable %s)", + current.Kind().String(), vr.String()) + } + case varTypeIdent: + // debugging: + // fmt.Printf("now = %s (kind: %s)\n", part.s, current.Kind().String()) + + // Calling a field or key + switch current.Kind() { + case reflect.Struct: + current = current.FieldByName(part.s) + case reflect.Map: + current = current.MapIndex(reflect.ValueOf(part.s)) + default: + return nil, errors.Errorf("Can't access a field by name on type %s (variable %s)", + current.Kind().String(), vr.String()) + } + default: + panic("unimplemented") + } + } + } + + if !current.IsValid() { + // Value is not valid (anymore) + return AsValue(nil), nil + } + + // If current is a reflect.ValueOf(pongo2.Value), then unpack it + // Happens in function calls (as a return value) or by injecting + // into the execution context (e.g. in a for-loop) + if current.Type() == typeOfValuePtr { + tmpValue := current.Interface().(*Value) + current = tmpValue.val + isSafe = tmpValue.safe + } + + // Check whether this is an interface and resolve it where required + if current.Kind() == reflect.Interface { + current = reflect.ValueOf(current.Interface()) + } + + // Check if the part is a function call + if part.isFunctionCall || current.Kind() == reflect.Func { + // Check for callable + if current.Kind() != reflect.Func { + return nil, errors.Errorf("'%s' is not a function (it is %s)", vr.String(), current.Kind().String()) + } + + // Check for correct function syntax and types + // func(*Value, ...) *Value + t := current.Type() + currArgs := part.callingArgs + + // If an implicit ExecCtx is needed + if t.NumIn() > 0 && t.In(0) == typeOfExecCtxPtr { + currArgs = append([]functionCallArgument{executionCtxEval{}}, currArgs...) + } + + // Input arguments + if len(currArgs) != t.NumIn() && !(len(currArgs) >= t.NumIn()-1 && t.IsVariadic()) { + return nil, + errors.Errorf("Function input argument count (%d) of '%s' must be equal to the calling argument count (%d).", + t.NumIn(), vr.String(), len(currArgs)) + } + + // Output arguments + if t.NumOut() != 1 && t.NumOut() != 2 { + return nil, errors.Errorf("'%s' must have exactly 1 or 2 output arguments, the second argument must be of type error", vr.String()) + } + + // Evaluate all parameters + var parameters []reflect.Value + + numArgs := t.NumIn() + isVariadic := t.IsVariadic() + var fnArg reflect.Type + + for idx, arg := range currArgs { + pv, err := arg.Evaluate(ctx) + if err != nil { + return nil, err + } + + if isVariadic { + if idx >= t.NumIn()-1 { + fnArg = t.In(numArgs - 1).Elem() + } else { + fnArg = t.In(idx) + } + } else { + fnArg = t.In(idx) + } + + if fnArg != typeOfValuePtr { + // Function's argument is not a *pongo2.Value, then we have to check whether input argument is of the same type as the function's argument + if !isVariadic { + if fnArg != reflect.TypeOf(pv.Interface()) && fnArg.Kind() != reflect.Interface { + return nil, errors.Errorf("Function input argument %d of '%s' must be of type %s or *pongo2.Value (not %T).", + idx, vr.String(), fnArg.String(), pv.Interface()) + } + // Function's argument has another type, using the interface-value + parameters = append(parameters, reflect.ValueOf(pv.Interface())) + } else { + if fnArg != reflect.TypeOf(pv.Interface()) && fnArg.Kind() != reflect.Interface { + return nil, errors.Errorf("Function variadic input argument of '%s' must be of type %s or *pongo2.Value (not %T).", + vr.String(), fnArg.String(), pv.Interface()) + } + // Function's argument has another type, using the interface-value + parameters = append(parameters, reflect.ValueOf(pv.Interface())) + } + } else { + // Function's argument is a *pongo2.Value + parameters = append(parameters, reflect.ValueOf(pv)) + } + } + + // Check if any of the values are invalid + for _, p := range parameters { + if p.Kind() == reflect.Invalid { + return nil, errors.Errorf("Calling a function using an invalid parameter") + } + } + + // Call it and get first return parameter back + values := current.Call(parameters) + rv := values[0] + if t.NumOut() == 2 { + e := values[1].Interface() + if e != nil { + err, ok := e.(error) + if !ok { + return nil, errors.Errorf("The second return value is not an error") + } + if err != nil { + return nil, err + } + } + } + + if rv.Type() != typeOfValuePtr { + current = reflect.ValueOf(rv.Interface()) + } else { + // Return the function call value + current = rv.Interface().(*Value).val + isSafe = rv.Interface().(*Value).safe + } + } + + if !current.IsValid() { + // Value is not valid (e. g. NIL value) + return AsValue(nil), nil + } + } + + return &Value{val: current, safe: isSafe}, nil +} + +func (vr *variableResolver) Evaluate(ctx *ExecutionContext) (*Value, *Error) { + value, err := vr.resolve(ctx) + if err != nil { + return AsValue(nil), ctx.Error(err.Error(), vr.locationToken) + } + return value, nil +} + +func (v *nodeFilteredVariable) FilterApplied(name string) bool { + for _, filter := range v.filterChain { + if filter.name == name { + return true + } + } + return false +} + +func (v *nodeFilteredVariable) Evaluate(ctx *ExecutionContext) (*Value, *Error) { + value, err := v.resolver.Evaluate(ctx) + if err != nil { + return nil, err + } + + for _, filter := range v.filterChain { + value, err = filter.Execute(value, ctx) + if err != nil { + return nil, err + } + } + + return value, nil +} + +// IDENT | IDENT.(IDENT|NUMBER)... +func (p *Parser) parseVariableOrLiteral() (IEvaluator, *Error) { + t := p.Current() + + if t == nil { + return nil, p.Error("Unexpected EOF, expected a number, string, keyword or identifier.", p.lastToken) + } + + // Is first part a number or a string, there's nothing to resolve (because there's only to return the value then) + switch t.Typ { + case TokenNumber: + p.Consume() + + // One exception to the rule that we don't have float64 literals is at the beginning + // of an expression (or a variable name). Since we know we started with an integer + // which can't obviously be a variable name, we can check whether the first number + // is followed by dot (and then a number again). If so we're converting it to a float64. + + if p.Match(TokenSymbol, ".") != nil { + // float64 + t2 := p.MatchType(TokenNumber) + if t2 == nil { + return nil, p.Error("Expected a number after the '.'.", nil) + } + f, err := strconv.ParseFloat(fmt.Sprintf("%s.%s", t.Val, t2.Val), 64) + if err != nil { + return nil, p.Error(err.Error(), t) + } + fr := &floatResolver{ + locationToken: t, + val: f, + } + return fr, nil + } + i, err := strconv.Atoi(t.Val) + if err != nil { + return nil, p.Error(err.Error(), t) + } + nr := &intResolver{ + locationToken: t, + val: i, + } + return nr, nil + + case TokenString: + p.Consume() + sr := &stringResolver{ + locationToken: t, + val: t.Val, + } + return sr, nil + case TokenKeyword: + p.Consume() + switch t.Val { + case "true": + br := &boolResolver{ + locationToken: t, + val: true, + } + return br, nil + case "false": + br := &boolResolver{ + locationToken: t, + val: false, + } + return br, nil + default: + return nil, p.Error("This keyword is not allowed here.", nil) + } + } + + resolver := &variableResolver{ + locationToken: t, + } + + // First part of a variable MUST be an identifier + if t.Typ != TokenIdentifier { + return nil, p.Error("Expected either a number, string, keyword or identifier.", t) + } + + resolver.parts = append(resolver.parts, &variablePart{ + typ: varTypeIdent, + s: t.Val, + }) + + p.Consume() // we consumed the first identifier of the variable name + +variableLoop: + for p.Remaining() > 0 { + t = p.Current() + + if p.Match(TokenSymbol, ".") != nil { + // Next variable part (can be either NUMBER or IDENT) + t2 := p.Current() + if t2 != nil { + switch t2.Typ { + case TokenIdentifier: + resolver.parts = append(resolver.parts, &variablePart{ + typ: varTypeIdent, + s: t2.Val, + }) + p.Consume() // consume: IDENT + continue variableLoop + case TokenNumber: + i, err := strconv.Atoi(t2.Val) + if err != nil { + return nil, p.Error(err.Error(), t2) + } + resolver.parts = append(resolver.parts, &variablePart{ + typ: varTypeInt, + i: i, + }) + p.Consume() // consume: NUMBER + continue variableLoop + default: + return nil, p.Error("This token is not allowed within a variable name.", t2) + } + } else { + // EOF + return nil, p.Error("Unexpected EOF, expected either IDENTIFIER or NUMBER after DOT.", + p.lastToken) + } + } else if p.Match(TokenSymbol, "(") != nil { + // Function call + // FunctionName '(' Comma-separated list of expressions ')' + part := resolver.parts[len(resolver.parts)-1] + part.isFunctionCall = true + argumentLoop: + for { + if p.Remaining() == 0 { + return nil, p.Error("Unexpected EOF, expected function call argument list.", p.lastToken) + } + + if p.Peek(TokenSymbol, ")") == nil { + // No closing bracket, so we're parsing an expression + exprArg, err := p.ParseExpression() + if err != nil { + return nil, err + } + part.callingArgs = append(part.callingArgs, exprArg) + + if p.Match(TokenSymbol, ")") != nil { + // If there's a closing bracket after an expression, we will stop parsing the arguments + break argumentLoop + } else { + // If there's NO closing bracket, there MUST be an comma + if p.Match(TokenSymbol, ",") == nil { + return nil, p.Error("Missing comma or closing bracket after argument.", nil) + } + } + } else { + // We got a closing bracket, so stop parsing arguments + p.Consume() + break argumentLoop + } + + } + // We're done parsing the function call, next variable part + continue variableLoop + } + + // No dot or function call? Then we're done with the variable parsing + break + } + + return resolver, nil +} + +func (p *Parser) parseVariableOrLiteralWithFilter() (*nodeFilteredVariable, *Error) { + v := &nodeFilteredVariable{ + locationToken: p.Current(), + } + + // Parse the variable name + resolver, err := p.parseVariableOrLiteral() + if err != nil { + return nil, err + } + v.resolver = resolver + + // Parse all the filters +filterLoop: + for p.Match(TokenSymbol, "|") != nil { + // Parse one single filter + filter, err := p.parseFilter() + if err != nil { + return nil, err + } + + // Check sandbox filter restriction + if _, isBanned := p.template.set.bannedFilters[filter.name]; isBanned { + return nil, p.Error(fmt.Sprintf("Usage of filter '%s' is not allowed (sandbox restriction active).", filter.name), nil) + } + + v.filterChain = append(v.filterChain, filter) + + continue filterLoop + } + + return v, nil +} + +func (p *Parser) parseVariableElement() (INode, *Error) { + node := &nodeVariable{ + locationToken: p.Current(), + } + + p.Consume() // consume '{{' + + expr, err := p.ParseExpression() + if err != nil { + return nil, err + } + node.expr = expr + + if p.Match(TokenSymbol, "}}") == nil { + return nil, p.Error("'}}' expected", nil) + } + + return node, nil +} diff --git a/vendor/github.com/gorilla/websocket/.travis.yml b/vendor/github.com/gorilla/websocket/.travis.yml deleted file mode 100644 index a49db51c43..0000000000 --- a/vendor/github.com/gorilla/websocket/.travis.yml +++ /dev/null @@ -1,19 +0,0 @@ -language: go -sudo: false - -matrix: - include: - - go: 1.7.x - - go: 1.8.x - - go: 1.9.x - - go: 1.10.x - - go: 1.11.x - - go: tip - allow_failures: - - go: tip - -script: - - go get -t -v ./... - - diff -u <(echo -n) <(gofmt -d .) - - go vet $(go list ./... | grep -v /vendor/) - - go test -v -race ./... diff --git a/vendor/github.com/gorilla/websocket/README.md b/vendor/github.com/gorilla/websocket/README.md index 20e391f865..0827d059c1 100644 --- a/vendor/github.com/gorilla/websocket/README.md +++ b/vendor/github.com/gorilla/websocket/README.md @@ -1,11 +1,11 @@ # Gorilla WebSocket +[![GoDoc](https://godoc.org/github.com/gorilla/websocket?status.svg)](https://godoc.org/github.com/gorilla/websocket) +[![CircleCI](https://circleci.com/gh/gorilla/websocket.svg?style=svg)](https://circleci.com/gh/gorilla/websocket) + Gorilla WebSocket is a [Go](http://golang.org/) implementation of the [WebSocket](http://www.rfc-editor.org/rfc/rfc6455.txt) protocol. -[![Build Status](https://travis-ci.org/gorilla/websocket.svg?branch=master)](https://travis-ci.org/gorilla/websocket) -[![GoDoc](https://godoc.org/github.com/gorilla/websocket?status.svg)](https://godoc.org/github.com/gorilla/websocket) - ### Documentation * [API Reference](http://godoc.org/github.com/gorilla/websocket) @@ -27,7 +27,7 @@ package API is stable. ### Protocol Compliance The Gorilla WebSocket package passes the server tests in the [Autobahn Test -Suite](http://autobahn.ws/testsuite) using the application in the [examples/autobahn +Suite](https://github.com/crossbario/autobahn-testsuite) using the application in the [examples/autobahn subdirectory](https://github.com/gorilla/websocket/tree/master/examples/autobahn). ### Gorilla WebSocket compared with other packages @@ -40,7 +40,7 @@ subdirectory](https://github.com/gorilla/websocket/tree/master/examples/autobahn RFC 6455 Features -Passes Autobahn Test SuiteYesNo +Passes Autobahn Test SuiteYesNo Receive fragmented messageYesNo, see note 1 Send close messageYesNo Send pings and receive pongsYesNo diff --git a/vendor/github.com/gorilla/websocket/client.go b/vendor/github.com/gorilla/websocket/client.go index 2e32fd506e..962c06a391 100644 --- a/vendor/github.com/gorilla/websocket/client.go +++ b/vendor/github.com/gorilla/websocket/client.go @@ -70,7 +70,7 @@ type Dialer struct { // HandshakeTimeout specifies the duration for the handshake to complete. HandshakeTimeout time.Duration - // ReadBufferSize and WriteBufferSize specify I/O buffer sizes. If a buffer + // ReadBufferSize and WriteBufferSize specify I/O buffer sizes in bytes. If a buffer // size is zero, then a useful default size is used. The I/O buffer sizes // do not limit the size of the messages that can be sent or received. ReadBufferSize, WriteBufferSize int @@ -140,7 +140,7 @@ var nilDialer = *DefaultDialer // Use the response.Header to get the selected subprotocol // (Sec-WebSocket-Protocol) and cookies (Set-Cookie). // -// The context will be used in the request and in the Dialer +// The context will be used in the request and in the Dialer. // // If the WebSocket handshake fails, ErrBadHandshake is returned along with a // non-nil *http.Response so that callers can handle redirects, authentication, diff --git a/vendor/github.com/gorilla/websocket/conn.go b/vendor/github.com/gorilla/websocket/conn.go index d2a21c148b..6f17cd2998 100644 --- a/vendor/github.com/gorilla/websocket/conn.go +++ b/vendor/github.com/gorilla/websocket/conn.go @@ -260,10 +260,12 @@ type Conn struct { newCompressionWriter func(io.WriteCloser, int) io.WriteCloser // Read fields - reader io.ReadCloser // the current reader returned to the application - readErr error - br *bufio.Reader - readRemaining int64 // bytes remaining in current frame. + reader io.ReadCloser // the current reader returned to the application + readErr error + br *bufio.Reader + // bytes remaining in current frame. + // set setReadRemaining to safely update this value and prevent overflow + readRemaining int64 readFinal bool // true the current message has more frames. readLength int64 // Message size. readLimit int64 // Maximum message size. @@ -320,6 +322,17 @@ func newConn(conn net.Conn, isServer bool, readBufferSize, writeBufferSize int, return c } +// setReadRemaining tracks the number of bytes remaining on the connection. If n +// overflows, an ErrReadLimit is returned. +func (c *Conn) setReadRemaining(n int64) error { + if n < 0 { + return ErrReadLimit + } + + c.readRemaining = n + return nil +} + // Subprotocol returns the negotiated protocol for the connection. func (c *Conn) Subprotocol() string { return c.subprotocol @@ -451,7 +464,8 @@ func (c *Conn) WriteControl(messageType int, data []byte, deadline time.Time) er return err } -func (c *Conn) prepWrite(messageType int) error { +// beginMessage prepares a connection and message writer for a new message. +func (c *Conn) beginMessage(mw *messageWriter, messageType int) error { // Close previous writer if not already closed by the application. It's // probably better to return an error in this situation, but we cannot // change this without breaking existing applications. @@ -471,6 +485,10 @@ func (c *Conn) prepWrite(messageType int) error { return err } + mw.c = c + mw.frameType = messageType + mw.pos = maxFrameHeaderSize + if c.writeBuf == nil { wpd, ok := c.writePool.Get().(writePoolData) if ok { @@ -491,16 +509,11 @@ func (c *Conn) prepWrite(messageType int) error { // All message types (TextMessage, BinaryMessage, CloseMessage, PingMessage and // PongMessage) are supported. func (c *Conn) NextWriter(messageType int) (io.WriteCloser, error) { - if err := c.prepWrite(messageType); err != nil { + var mw messageWriter + if err := c.beginMessage(&mw, messageType); err != nil { return nil, err } - - mw := &messageWriter{ - c: c, - frameType: messageType, - pos: maxFrameHeaderSize, - } - c.writer = mw + c.writer = &mw if c.newCompressionWriter != nil && c.enableWriteCompression && isData(messageType) { w := c.newCompressionWriter(c.writer, c.compressionLevel) mw.compress = true @@ -517,10 +530,16 @@ type messageWriter struct { err error } -func (w *messageWriter) fatal(err error) error { +func (w *messageWriter) endMessage(err error) error { if w.err != nil { - w.err = err - w.c.writer = nil + return err + } + c := w.c + w.err = err + c.writer = nil + if c.writePool != nil { + c.writePool.Put(writePoolData{buf: c.writeBuf}) + c.writeBuf = nil } return err } @@ -534,7 +553,7 @@ func (w *messageWriter) flushFrame(final bool, extra []byte) error { // Check for invalid control frames. if isControl(w.frameType) && (!final || length > maxControlFramePayloadSize) { - return w.fatal(errInvalidControlFrame) + return w.endMessage(errInvalidControlFrame) } b0 := byte(w.frameType) @@ -579,7 +598,7 @@ func (w *messageWriter) flushFrame(final bool, extra []byte) error { copy(c.writeBuf[maxFrameHeaderSize-4:], key[:]) maskBytes(key, 0, c.writeBuf[maxFrameHeaderSize:w.pos]) if len(extra) > 0 { - return c.writeFatal(errors.New("websocket: internal error, extra used in client mode")) + return w.endMessage(c.writeFatal(errors.New("websocket: internal error, extra used in client mode"))) } } @@ -600,15 +619,11 @@ func (w *messageWriter) flushFrame(final bool, extra []byte) error { c.isWriting = false if err != nil { - return w.fatal(err) + return w.endMessage(err) } if final { - c.writer = nil - if c.writePool != nil { - c.writePool.Put(writePoolData{buf: c.writeBuf}) - c.writeBuf = nil - } + w.endMessage(errWriteClosed) return nil } @@ -706,11 +721,7 @@ func (w *messageWriter) Close() error { if w.err != nil { return w.err } - if err := w.flushFrame(true, nil); err != nil { - return err - } - w.err = errWriteClosed - return nil + return w.flushFrame(true, nil) } // WritePreparedMessage writes prepared message into connection. @@ -742,10 +753,10 @@ func (c *Conn) WriteMessage(messageType int, data []byte) error { if c.isServer && (c.newCompressionWriter == nil || !c.enableWriteCompression) { // Fast path with no allocations and single frame. - if err := c.prepWrite(messageType); err != nil { + var mw messageWriter + if err := c.beginMessage(&mw, messageType); err != nil { return err } - mw := messageWriter{c: c, frameType: messageType, pos: maxFrameHeaderSize} n := copy(c.writeBuf[mw.pos:], data) mw.pos += n data = data[n:] @@ -792,7 +803,7 @@ func (c *Conn) advanceFrame() (int, error) { final := p[0]&finalBit != 0 frameType := int(p[0] & 0xf) mask := p[1]&maskBit != 0 - c.readRemaining = int64(p[1] & 0x7f) + c.setReadRemaining(int64(p[1] & 0x7f)) c.readDecompress = false if c.newDecompressionReader != nil && (p[0]&rsv1Bit) != 0 { @@ -826,7 +837,17 @@ func (c *Conn) advanceFrame() (int, error) { return noFrame, c.handleProtocolError("unknown opcode " + strconv.Itoa(frameType)) } - // 3. Read and parse frame length. + // 3. Read and parse frame length as per + // https://tools.ietf.org/html/rfc6455#section-5.2 + // + // The length of the "Payload data", in bytes: if 0-125, that is the payload + // length. + // - If 126, the following 2 bytes interpreted as a 16-bit unsigned + // integer are the payload length. + // - If 127, the following 8 bytes interpreted as + // a 64-bit unsigned integer (the most significant bit MUST be 0) are the + // payload length. Multibyte length quantities are expressed in network byte + // order. switch c.readRemaining { case 126: @@ -834,13 +855,19 @@ func (c *Conn) advanceFrame() (int, error) { if err != nil { return noFrame, err } - c.readRemaining = int64(binary.BigEndian.Uint16(p)) + + if err := c.setReadRemaining(int64(binary.BigEndian.Uint16(p))); err != nil { + return noFrame, err + } case 127: p, err := c.read(8) if err != nil { return noFrame, err } - c.readRemaining = int64(binary.BigEndian.Uint64(p)) + + if err := c.setReadRemaining(int64(binary.BigEndian.Uint64(p))); err != nil { + return noFrame, err + } } // 4. Handle frame masking. @@ -863,6 +890,12 @@ func (c *Conn) advanceFrame() (int, error) { if frameType == continuationFrame || frameType == TextMessage || frameType == BinaryMessage { c.readLength += c.readRemaining + // Don't allow readLength to overflow in the presence of a large readRemaining + // counter. + if c.readLength < 0 { + return noFrame, ErrReadLimit + } + if c.readLimit > 0 && c.readLength > c.readLimit { c.WriteControl(CloseMessage, FormatCloseMessage(CloseMessageTooBig, ""), time.Now().Add(writeWait)) return noFrame, ErrReadLimit @@ -876,7 +909,7 @@ func (c *Conn) advanceFrame() (int, error) { var payload []byte if c.readRemaining > 0 { payload, err = c.read(int(c.readRemaining)) - c.readRemaining = 0 + c.setReadRemaining(0) if err != nil { return noFrame, err } @@ -949,6 +982,7 @@ func (c *Conn) NextReader() (messageType int, r io.Reader, err error) { c.readErr = hideTempErr(err) break } + if frameType == TextMessage || frameType == BinaryMessage { c.messageReader = &messageReader{c} c.reader = c.messageReader @@ -989,7 +1023,9 @@ func (r *messageReader) Read(b []byte) (int, error) { if c.isServer { c.readMaskPos = maskBytes(c.readMaskKey, c.readMaskPos, b[:n]) } - c.readRemaining -= int64(n) + rem := c.readRemaining + rem -= int64(n) + c.setReadRemaining(rem) if c.readRemaining > 0 && c.readErr == io.EOF { c.readErr = errUnexpectedEOF } @@ -1041,7 +1077,7 @@ func (c *Conn) SetReadDeadline(t time.Time) error { return c.conn.SetReadDeadline(t) } -// SetReadLimit sets the maximum size for a message read from the peer. If a +// SetReadLimit sets the maximum size in bytes for a message read from the peer. If a // message exceeds the limit, the connection sends a close message to the peer // and returns ErrReadLimit to the application. func (c *Conn) SetReadLimit(limit int64) { diff --git a/vendor/github.com/gorilla/websocket/doc.go b/vendor/github.com/gorilla/websocket/doc.go index dcce1a63c0..c6f4df8960 100644 --- a/vendor/github.com/gorilla/websocket/doc.go +++ b/vendor/github.com/gorilla/websocket/doc.go @@ -151,6 +151,53 @@ // checking. The application is responsible for checking the Origin header // before calling the Upgrade function. // +// Buffers +// +// Connections buffer network input and output to reduce the number +// of system calls when reading or writing messages. +// +// Write buffers are also used for constructing WebSocket frames. See RFC 6455, +// Section 5 for a discussion of message framing. A WebSocket frame header is +// written to the network each time a write buffer is flushed to the network. +// Decreasing the size of the write buffer can increase the amount of framing +// overhead on the connection. +// +// The buffer sizes in bytes are specified by the ReadBufferSize and +// WriteBufferSize fields in the Dialer and Upgrader. The Dialer uses a default +// size of 4096 when a buffer size field is set to zero. The Upgrader reuses +// buffers created by the HTTP server when a buffer size field is set to zero. +// The HTTP server buffers have a size of 4096 at the time of this writing. +// +// The buffer sizes do not limit the size of a message that can be read or +// written by a connection. +// +// Buffers are held for the lifetime of the connection by default. If the +// Dialer or Upgrader WriteBufferPool field is set, then a connection holds the +// write buffer only when writing a message. +// +// Applications should tune the buffer sizes to balance memory use and +// performance. Increasing the buffer size uses more memory, but can reduce the +// number of system calls to read or write the network. In the case of writing, +// increasing the buffer size can reduce the number of frame headers written to +// the network. +// +// Some guidelines for setting buffer parameters are: +// +// Limit the buffer sizes to the maximum expected message size. Buffers larger +// than the largest message do not provide any benefit. +// +// Depending on the distribution of message sizes, setting the buffer size to +// to a value less than the maximum expected message size can greatly reduce +// memory use with a small impact on performance. Here's an example: If 99% of +// the messages are smaller than 256 bytes and the maximum message size is 512 +// bytes, then a buffer size of 256 bytes will result in 1.01 more system calls +// than a buffer size of 512 bytes. The memory savings is 50%. +// +// A write buffer pool is useful when the application has a modest number +// writes over a large number of connections. when buffers are pooled, a larger +// buffer size has a reduced impact on total memory use and has the benefit of +// reducing system calls and frame overhead. +// // Compression EXPERIMENTAL // // Per message compression extensions (RFC 7692) are experimentally supported diff --git a/vendor/github.com/gorilla/websocket/go.mod b/vendor/github.com/gorilla/websocket/go.mod new file mode 100644 index 0000000000..1a7afd5028 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/go.mod @@ -0,0 +1,3 @@ +module github.com/gorilla/websocket + +go 1.12 diff --git a/vendor/github.com/gorilla/websocket/go.sum b/vendor/github.com/gorilla/websocket/go.sum new file mode 100644 index 0000000000..cf4fbbaa07 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/go.sum @@ -0,0 +1,2 @@ +github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= diff --git a/vendor/github.com/gorilla/websocket/join.go b/vendor/github.com/gorilla/websocket/join.go new file mode 100644 index 0000000000..c64f8c8290 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/join.go @@ -0,0 +1,42 @@ +// Copyright 2019 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package websocket + +import ( + "io" + "strings" +) + +// JoinMessages concatenates received messages to create a single io.Reader. +// The string term is appended to each message. The returned reader does not +// support concurrent calls to the Read method. +func JoinMessages(c *Conn, term string) io.Reader { + return &joinReader{c: c, term: term} +} + +type joinReader struct { + c *Conn + term string + r io.Reader +} + +func (r *joinReader) Read(p []byte) (int, error) { + if r.r == nil { + var err error + _, r.r, err = r.c.NextReader() + if err != nil { + return 0, err + } + if r.term != "" { + r.r = io.MultiReader(r.r, strings.NewReader(r.term)) + } + } + n, err := r.r.Read(p) + if err == io.EOF { + err = nil + r.r = nil + } + return n, err +} diff --git a/vendor/github.com/gorilla/websocket/proxy.go b/vendor/github.com/gorilla/websocket/proxy.go index bf2478e430..e87a8c9f0c 100644 --- a/vendor/github.com/gorilla/websocket/proxy.go +++ b/vendor/github.com/gorilla/websocket/proxy.go @@ -22,18 +22,18 @@ func (fn netDialerFunc) Dial(network, addr string) (net.Conn, error) { func init() { proxy_RegisterDialerType("http", func(proxyURL *url.URL, forwardDialer proxy_Dialer) (proxy_Dialer, error) { - return &httpProxyDialer{proxyURL: proxyURL, fowardDial: forwardDialer.Dial}, nil + return &httpProxyDialer{proxyURL: proxyURL, forwardDial: forwardDialer.Dial}, nil }) } type httpProxyDialer struct { - proxyURL *url.URL - fowardDial func(network, addr string) (net.Conn, error) + proxyURL *url.URL + forwardDial func(network, addr string) (net.Conn, error) } func (hpd *httpProxyDialer) Dial(network string, addr string) (net.Conn, error) { hostPort, _ := hostPortNoPort(hpd.proxyURL) - conn, err := hpd.fowardDial(network, hostPort) + conn, err := hpd.forwardDial(network, hostPort) if err != nil { return nil, err } diff --git a/vendor/github.com/gorilla/websocket/server.go b/vendor/github.com/gorilla/websocket/server.go index a761824b33..887d558918 100644 --- a/vendor/github.com/gorilla/websocket/server.go +++ b/vendor/github.com/gorilla/websocket/server.go @@ -27,7 +27,7 @@ type Upgrader struct { // HandshakeTimeout specifies the duration for the handshake to complete. HandshakeTimeout time.Duration - // ReadBufferSize and WriteBufferSize specify I/O buffer sizes. If a buffer + // ReadBufferSize and WriteBufferSize specify I/O buffer sizes in bytes. If a buffer // size is zero, then buffers allocated by the HTTP server are used. The // I/O buffer sizes do not limit the size of the messages that can be sent // or received. @@ -153,7 +153,7 @@ func (u *Upgrader) Upgrade(w http.ResponseWriter, r *http.Request, responseHeade challengeKey := r.Header.Get("Sec-Websocket-Key") if challengeKey == "" { - return u.returnError(w, r, http.StatusBadRequest, "websocket: not a websocket handshake: `Sec-WebSocket-Key' header is missing or blank") + return u.returnError(w, r, http.StatusBadRequest, "websocket: not a websocket handshake: 'Sec-WebSocket-Key' header is missing or blank") } subprotocol := u.selectSubprotocol(r, responseHeader) diff --git a/vendor/github.com/gorilla/websocket/util.go b/vendor/github.com/gorilla/websocket/util.go index 354001e1ed..7bf2f66c67 100644 --- a/vendor/github.com/gorilla/websocket/util.go +++ b/vendor/github.com/gorilla/websocket/util.go @@ -31,68 +31,113 @@ func generateChallengeKey() (string, error) { return base64.StdEncoding.EncodeToString(p), nil } -// Octet types from RFC 2616. -var octetTypes [256]byte - -const ( - isTokenOctet = 1 << iota - isSpaceOctet -) - -func init() { - // From RFC 2616 - // - // OCTET = - // CHAR = - // CTL = - // CR = - // LF = - // SP = - // HT = - // <"> = - // CRLF = CR LF - // LWS = [CRLF] 1*( SP | HT ) - // TEXT = - // separators = "(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\" | <"> - // | "/" | "[" | "]" | "?" | "=" | "{" | "}" | SP | HT - // token = 1* - // qdtext = > - - for c := 0; c < 256; c++ { - var t byte - isCtl := c <= 31 || c == 127 - isChar := 0 <= c && c <= 127 - isSeparator := strings.IndexRune(" \t\"(),/:;<=>?@[]\\{}", rune(c)) >= 0 - if strings.IndexRune(" \t\r\n", rune(c)) >= 0 { - t |= isSpaceOctet - } - if isChar && !isCtl && !isSeparator { - t |= isTokenOctet - } - octetTypes[c] = t - } +// Token octets per RFC 2616. +var isTokenOctet = [256]bool{ + '!': true, + '#': true, + '$': true, + '%': true, + '&': true, + '\'': true, + '*': true, + '+': true, + '-': true, + '.': true, + '0': true, + '1': true, + '2': true, + '3': true, + '4': true, + '5': true, + '6': true, + '7': true, + '8': true, + '9': true, + 'A': true, + 'B': true, + 'C': true, + 'D': true, + 'E': true, + 'F': true, + 'G': true, + 'H': true, + 'I': true, + 'J': true, + 'K': true, + 'L': true, + 'M': true, + 'N': true, + 'O': true, + 'P': true, + 'Q': true, + 'R': true, + 'S': true, + 'T': true, + 'U': true, + 'W': true, + 'V': true, + 'X': true, + 'Y': true, + 'Z': true, + '^': true, + '_': true, + '`': true, + 'a': true, + 'b': true, + 'c': true, + 'd': true, + 'e': true, + 'f': true, + 'g': true, + 'h': true, + 'i': true, + 'j': true, + 'k': true, + 'l': true, + 'm': true, + 'n': true, + 'o': true, + 'p': true, + 'q': true, + 'r': true, + 's': true, + 't': true, + 'u': true, + 'v': true, + 'w': true, + 'x': true, + 'y': true, + 'z': true, + '|': true, + '~': true, } +// skipSpace returns a slice of the string s with all leading RFC 2616 linear +// whitespace removed. func skipSpace(s string) (rest string) { i := 0 for ; i < len(s); i++ { - if octetTypes[s[i]]&isSpaceOctet == 0 { + if b := s[i]; b != ' ' && b != '\t' { break } } return s[i:] } +// nextToken returns the leading RFC 2616 token of s and the string following +// the token. func nextToken(s string) (token, rest string) { i := 0 for ; i < len(s); i++ { - if octetTypes[s[i]]&isTokenOctet == 0 { + if !isTokenOctet[s[i]] { break } } return s[:i], s[i:] } +// nextTokenOrQuoted returns the leading token or quoted string per RFC 2616 +// and the string following the token or quoted string. func nextTokenOrQuoted(s string) (value string, rest string) { if !strings.HasPrefix(s, "\"") { return nextToken(s) @@ -128,7 +173,8 @@ func nextTokenOrQuoted(s string) (value string, rest string) { return "", "" } -// equalASCIIFold returns true if s is equal to t with ASCII case folding. +// equalASCIIFold returns true if s is equal to t with ASCII case folding as +// defined in RFC 4790. func equalASCIIFold(s, t string) bool { for s != "" && t != "" { sr, size := utf8.DecodeRuneInString(s) diff --git a/vendor/github.com/lxc/lxd/AUTHORS b/vendor/github.com/lxc/lxd/AUTHORS new file mode 100644 index 0000000000..f7c0c6a2e3 --- /dev/null +++ b/vendor/github.com/lxc/lxd/AUTHORS @@ -0,0 +1,5 @@ +Unless mentioned otherwise in a specific file's header, all code in this +project is released under the Apache 2.0 license. + +The list of authors and contributors can be retrieved from the git +commit history and in some cases, the file headers. diff --git a/vendor/github.com/lxc/lxd/COPYING b/vendor/github.com/lxc/lxd/COPYING new file mode 100644 index 0000000000..d645695673 --- /dev/null +++ b/vendor/github.com/lxc/lxd/COPYING @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/lxc/lxd/shared/api/certificate.go b/vendor/github.com/lxc/lxd/shared/api/certificate.go new file mode 100644 index 0000000000..56664fa6cc --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/api/certificate.go @@ -0,0 +1,30 @@ +package api + +// CertificatesPost represents the fields of a new LXD certificate +type CertificatesPost struct { + CertificatePut `yaml:",inline"` + + Certificate string `json:"certificate" yaml:"certificate"` + Password string `json:"password" yaml:"password"` +} + +// CertificatePut represents the modifiable fields of a LXD certificate +// +// API extension: certificate_update +type CertificatePut struct { + Name string `json:"name" yaml:"name"` + Type string `json:"type" yaml:"type"` +} + +// Certificate represents a LXD certificate +type Certificate struct { + CertificatePut `yaml:",inline"` + + Certificate string `json:"certificate" yaml:"certificate"` + Fingerprint string `json:"fingerprint" yaml:"fingerprint"` +} + +// Writable converts a full Certificate struct into a CertificatePut struct (filters read-only fields) +func (cert *Certificate) Writable() CertificatePut { + return cert.CertificatePut +} diff --git a/vendor/github.com/lxc/lxd/shared/api/cluster.go b/vendor/github.com/lxc/lxd/shared/api/cluster.go new file mode 100644 index 0000000000..c773929c8d --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/api/cluster.go @@ -0,0 +1,63 @@ +package api + +// Cluster represents high-level information about a LXD cluster. +// +// API extension: clustering +type Cluster struct { + ServerName string `json:"server_name" yaml:"server_name"` + Enabled bool `json:"enabled" yaml:"enabled"` + + // API extension: clustering_join + MemberConfig []ClusterMemberConfigKey `json:"member_config" yaml:"member_config"` +} + +// ClusterMemberConfigKey represents a single config key that a new member of +// the cluster is required to provide when joining. +// +// The Value field is empty when getting clustering information with GET +// /1.0/cluster, and should be filled by the joining node when performing a PUT +// /1.0/cluster join request. +// +// API extension: clustering_join +type ClusterMemberConfigKey struct { + Entity string `json:"entity" yaml:"entity"` + Name string `json:"name" yaml:"name"` + Key string `json:"key" yaml:"key"` + Value string `json:"value" yaml:"value"` + Description string `json:"description" yaml:"description"` +} + +// ClusterPut represents the fields required to bootstrap or join a LXD +// cluster. +// +// API extension: clustering +type ClusterPut struct { + Cluster `yaml:",inline"` + ClusterAddress string `json:"cluster_address" yaml:"cluster_address"` + ClusterCertificate string `json:"cluster_certificate" yaml:"cluster_certificate"` + + // API extension: clustering_join + ServerAddress string `json:"server_address" yaml:"server_address"` + ClusterPassword string `json:"cluster_password" yaml:"cluster_password"` +} + +// ClusterMemberPost represents the fields required to rename a LXD node. +// +// API extension: clustering +type ClusterMemberPost struct { + ServerName string `json:"server_name" yaml:"server_name"` +} + +// ClusterMember represents the a LXD node in the cluster. +// +// API extension: clustering +type ClusterMember struct { + ServerName string `json:"server_name" yaml:"server_name"` + URL string `json:"url" yaml:"url"` + Database bool `json:"database" yaml:"database"` + Status string `json:"status" yaml:"status"` + Message string `json:"message" yaml:"message"` + + // API extension: clustering_roles + Roles []string `json:"roles" yaml:"roles"` +} diff --git a/vendor/github.com/lxc/lxd/shared/api/container.go b/vendor/github.com/lxc/lxd/shared/api/container.go new file mode 100644 index 0000000000..ed41a6e61e --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/api/container.go @@ -0,0 +1,141 @@ +package api + +import ( + "time" +) + +// ContainersPost represents the fields available for a new LXD container +type ContainersPost struct { + ContainerPut `yaml:",inline"` + + Name string `json:"name" yaml:"name"` + Source ContainerSource `json:"source" yaml:"source"` + + InstanceType string `json:"instance_type" yaml:"instance_type"` +} + +// ContainerPost represents the fields required to rename/move a LXD container +type ContainerPost struct { + // Used for renames + Name string `json:"name" yaml:"name"` + + // Used for migration + Migration bool `json:"migration" yaml:"migration"` + + // API extension: container_stateless_copy + Live bool `json:"live" yaml:"live"` + + // API extension: container_only_migration + ContainerOnly bool `json:"container_only" yaml:"container_only"` + + // API extension: container_push_target + Target *ContainerPostTarget `json:"target" yaml:"target"` +} + +// ContainerPostTarget represents the migration target host and operation +// +// API extension: container_push_target +type ContainerPostTarget struct { + Certificate string `json:"certificate" yaml:"certificate"` + Operation string `json:"operation,omitempty" yaml:"operation,omitempty"` + Websockets map[string]string `json:"secrets,omitempty" yaml:"secrets,omitempty"` +} + +// ContainerPut represents the modifiable fields of a LXD container +type ContainerPut struct { + Architecture string `json:"architecture" yaml:"architecture"` + Config map[string]string `json:"config" yaml:"config"` + Devices map[string]map[string]string `json:"devices" yaml:"devices"` + Ephemeral bool `json:"ephemeral" yaml:"ephemeral"` + Profiles []string `json:"profiles" yaml:"profiles"` + + // For snapshot restore + Restore string `json:"restore,omitempty" yaml:"restore,omitempty"` + Stateful bool `json:"stateful" yaml:"stateful"` + + // API extension: entity_description + Description string `json:"description" yaml:"description"` +} + +// Container represents a LXD container +type Container struct { + ContainerPut `yaml:",inline"` + + CreatedAt time.Time `json:"created_at" yaml:"created_at"` + ExpandedConfig map[string]string `json:"expanded_config" yaml:"expanded_config"` + ExpandedDevices map[string]map[string]string `json:"expanded_devices" yaml:"expanded_devices"` + Name string `json:"name" yaml:"name"` + Status string `json:"status" yaml:"status"` + StatusCode StatusCode `json:"status_code" yaml:"status_code"` + + // API extension: container_last_used_at + LastUsedAt time.Time `json:"last_used_at" yaml:"last_used_at"` + + // API extension: clustering + Location string `json:"location" yaml:"location"` +} + +// ContainerFull is a combination of Container, ContainerState and CotnainerSnapshot +// +// API extension: container_full +type ContainerFull struct { + Container `yaml:",inline"` + + Backups []ContainerBackup `json:"backups" yaml:"backups"` + State *ContainerState `json:"state" yaml:"state"` + Snapshots []ContainerSnapshot `json:"snapshots" yaml:"snapshots"` +} + +// Writable converts a full Container struct into a ContainerPut struct (filters read-only fields) +func (c *Container) Writable() ContainerPut { + return c.ContainerPut +} + +// IsActive checks whether the container state indicates the container is active +func (c Container) IsActive() bool { + switch c.StatusCode { + case Stopped: + return false + case Error: + return false + default: + return true + } +} + +// ContainerSource represents the creation source for a new container +type ContainerSource struct { + Type string `json:"type" yaml:"type"` + Certificate string `json:"certificate" yaml:"certificate"` + + // For "image" type + Alias string `json:"alias,omitempty" yaml:"alias,omitempty"` + Fingerprint string `json:"fingerprint,omitempty" yaml:"fingerprint,omitempty"` + Properties map[string]string `json:"properties,omitempty" yaml:"properties,omitempty"` + Server string `json:"server,omitempty" yaml:"server,omitempty"` + Secret string `json:"secret,omitempty" yaml:"secret,omitempty"` + Protocol string `json:"protocol,omitempty" yaml:"protocol,omitempty"` + + // For "migration" and "copy" types + BaseImage string `json:"base-image,omitempty" yaml:"base-image,omitempty"` + + // For "migration" type + Mode string `json:"mode,omitempty" yaml:"mode,omitempty"` + Operation string `json:"operation,omitempty" yaml:"operation,omitempty"` + Websockets map[string]string `json:"secrets,omitempty" yaml:"secrets,omitempty"` + + // For "copy" type + Source string `json:"source,omitempty" yaml:"source,omitempty"` + + // API extension: container_push + Live bool `json:"live,omitempty" yaml:"live,omitempty"` + + // API extension: container_only_migration + ContainerOnly bool `json:"container_only,omitempty" yaml:"container_only,omitempty"` + + // API extension: container_incremental_copy + Refresh bool `json:"refresh,omitempty" yaml:"refresh,omitempty"` + + // API extension: container_copy_project + Project string `json:"project,omitempty" yaml:"project,omitempty"` +} diff --git a/vendor/github.com/lxc/lxd/shared/api/container_backup.go b/vendor/github.com/lxc/lxd/shared/api/container_backup.go new file mode 100644 index 0000000000..8fe35e9964 --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/api/container_backup.go @@ -0,0 +1,29 @@ +package api + +import "time" + +// ContainerBackupsPost represents the fields available for a new LXD container backup +// API extension: container_backup +type ContainerBackupsPost struct { + Name string `json:"name" yaml:"name"` + ExpiresAt time.Time `json:"expires_at" yaml:"expires_at"` + ContainerOnly bool `json:"container_only" yaml:"container_only"` + OptimizedStorage bool `json:"optimized_storage" yaml:"optimized_storage"` +} + +// ContainerBackup represents a LXD container backup +// API extension: container_backup +type ContainerBackup struct { + Name string `json:"name" yaml:"name"` + CreatedAt time.Time `json:"created_at" yaml:"created_at"` + ExpiresAt time.Time `json:"expires_at" yaml:"expires_at"` + ContainerOnly bool `json:"container_only" yaml:"container_only"` + OptimizedStorage bool `json:"optimized_storage" yaml:"optimized_storage"` +} + +// ContainerBackupPost represents the fields available for the renaming of a +// container backup +// API extension: container_backup +type ContainerBackupPost struct { + Name string `json:"name" yaml:"name"` +} diff --git a/vendor/github.com/lxc/lxd/shared/api/container_console.go b/vendor/github.com/lxc/lxd/shared/api/container_console.go new file mode 100644 index 0000000000..56aff07aa4 --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/api/container_console.go @@ -0,0 +1,17 @@ +package api + +// ContainerConsoleControl represents a message on the container console "control" socket +// +// API extension: console +type ContainerConsoleControl struct { + Command string `json:"command" yaml:"command"` + Args map[string]string `json:"args" yaml:"args"` +} + +// ContainerConsolePost represents a LXD container console request +// +// API extension: console +type ContainerConsolePost struct { + Width int `json:"width" yaml:"width"` + Height int `json:"height" yaml:"height"` +} diff --git a/vendor/github.com/lxc/lxd/shared/api/container_exec.go b/vendor/github.com/lxc/lxd/shared/api/container_exec.go new file mode 100644 index 0000000000..7e724dc49f --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/api/container_exec.go @@ -0,0 +1,26 @@ +package api + +// ContainerExecControl represents a message on the container exec "control" socket +type ContainerExecControl struct { + Command string `json:"command" yaml:"command"` + Args map[string]string `json:"args" yaml:"args"` + Signal int `json:"signal" yaml:"signal"` +} + +// ContainerExecPost represents a LXD container exec request +type ContainerExecPost struct { + Command []string `json:"command" yaml:"command"` + WaitForWS bool `json:"wait-for-websocket" yaml:"wait-for-websocket"` + Interactive bool `json:"interactive" yaml:"interactive"` + Environment map[string]string `json:"environment" yaml:"environment"` + Width int `json:"width" yaml:"width"` + Height int `json:"height" yaml:"height"` + + // API extension: container_exec_recording + RecordOutput bool `json:"record-output" yaml:"record-output"` + + // API extension: container_user_group_cwd + User uint32 `json:"user" yaml:"user"` + Group uint32 `json:"group" yaml:"group"` + Cwd string `json:"cwd" yaml:"cwd"` +} diff --git a/vendor/github.com/lxc/lxd/shared/api/container_snapshot.go b/vendor/github.com/lxc/lxd/shared/api/container_snapshot.go new file mode 100644 index 0000000000..e68a0fb8cc --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/api/container_snapshot.go @@ -0,0 +1,53 @@ +package api + +import ( + "time" +) + +// ContainerSnapshotsPost represents the fields available for a new LXD container snapshot +type ContainerSnapshotsPost struct { + Name string `json:"name" yaml:"name"` + Stateful bool `json:"stateful" yaml:"stateful"` + + // API extension: snapshot_expiry_creation + ExpiresAt *time.Time `json:"expires_at" yaml:"expires_at"` +} + +// ContainerSnapshotPost represents the fields required to rename/move a LXD container snapshot +type ContainerSnapshotPost struct { + Name string `json:"name" yaml:"name"` + Migration bool `json:"migration" yaml:"migration"` + Target *ContainerPostTarget `json:"target" yaml:"target"` + + // API extension: container_snapshot_stateful_migration + Live bool `json:"live,omitempty" yaml:"live,omitempty"` +} + +// ContainerSnapshotPut represents the modifiable fields of a LXD container snapshot +// API extension: snapshot_expiry +type ContainerSnapshotPut struct { + Architecture string `json:"architecture" yaml:"architecture"` + Config map[string]string `json:"config" yaml:"config"` + Devices map[string]map[string]string `json:"devices" yaml:"devices"` + Ephemeral bool `json:"ephemeral" yaml:"ephemeral"` + Profiles []string `json:"profiles" yaml:"profiles"` + ExpiresAt time.Time `json:"expires_at" yaml:"expires_at"` +} + +// ContainerSnapshot represents a LXD conainer snapshot +type ContainerSnapshot struct { + ContainerSnapshotPut `yaml:",inline"` + + CreatedAt time.Time `json:"created_at" yaml:"created_at"` + ExpandedConfig map[string]string `json:"expanded_config" yaml:"expanded_config"` + ExpandedDevices map[string]map[string]string `json:"expanded_devices" yaml:"expanded_devices"` + LastUsedAt time.Time `json:"last_used_at" yaml:"last_used_at"` + Name string `json:"name" yaml:"name"` + Stateful bool `json:"stateful" yaml:"stateful"` +} + +// Writable converts a full ContainerSnapshot struct into a ContainerSnapshotPut struct +// (filters read-only fields) +func (c *ContainerSnapshot) Writable() ContainerSnapshotPut { + return c.ContainerSnapshotPut +} diff --git a/vendor/github.com/lxc/lxd/shared/api/container_state.go b/vendor/github.com/lxc/lxd/shared/api/container_state.go new file mode 100644 index 0000000000..f9e1cb9b53 --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/api/container_state.go @@ -0,0 +1,70 @@ +package api + +// ContainerStatePut represents the modifiable fields of a LXD container's state +type ContainerStatePut struct { + Action string `json:"action" yaml:"action"` + Timeout int `json:"timeout" yaml:"timeout"` + Force bool `json:"force" yaml:"force"` + Stateful bool `json:"stateful" yaml:"stateful"` +} + +// ContainerState represents a LXD container's state +type ContainerState struct { + Status string `json:"status" yaml:"status"` + StatusCode StatusCode `json:"status_code" yaml:"status_code"` + Disk map[string]ContainerStateDisk `json:"disk" yaml:"disk"` + Memory ContainerStateMemory `json:"memory" yaml:"memory"` + Network map[string]ContainerStateNetwork `json:"network" yaml:"network"` + Pid int64 `json:"pid" yaml:"pid"` + Processes int64 `json:"processes" yaml:"processes"` + + // API extension: container_cpu_time + CPU ContainerStateCPU `json:"cpu" yaml:"cpu"` +} + +// ContainerStateDisk represents the disk information section of a LXD container's state +type ContainerStateDisk struct { + Usage int64 `json:"usage" yaml:"usage"` +} + +// ContainerStateCPU represents the cpu information section of a LXD container's state +// +// API extension: container_cpu_time +type ContainerStateCPU struct { + Usage int64 `json:"usage" yaml:"usage"` +} + +// ContainerStateMemory represents the memory information section of a LXD container's state +type ContainerStateMemory struct { + Usage int64 `json:"usage" yaml:"usage"` + UsagePeak int64 `json:"usage_peak" yaml:"usage_peak"` + SwapUsage int64 `json:"swap_usage" yaml:"swap_usage"` + SwapUsagePeak int64 `json:"swap_usage_peak" yaml:"swap_usage_peak"` +} + +// ContainerStateNetwork represents the network information section of a LXD container's state +type ContainerStateNetwork struct { + Addresses []ContainerStateNetworkAddress `json:"addresses" yaml:"addresses"` + Counters ContainerStateNetworkCounters `json:"counters" yaml:"counters"` + Hwaddr string `json:"hwaddr" yaml:"hwaddr"` + HostName string `json:"host_name" yaml:"host_name"` + Mtu int `json:"mtu" yaml:"mtu"` + State string `json:"state" yaml:"state"` + Type string `json:"type" yaml:"type"` +} + +// ContainerStateNetworkAddress represents a network address as part of the network section of a LXD container's state +type ContainerStateNetworkAddress struct { + Family string `json:"family" yaml:"family"` + Address string `json:"address" yaml:"address"` + Netmask string `json:"netmask" yaml:"netmask"` + Scope string `json:"scope" yaml:"scope"` +} + +// ContainerStateNetworkCounters represents packet counters as part of the network section of a LXD container's state +type ContainerStateNetworkCounters struct { + BytesReceived int64 `json:"bytes_received" yaml:"bytes_received"` + BytesSent int64 `json:"bytes_sent" yaml:"bytes_sent"` + PacketsReceived int64 `json:"packets_received" yaml:"packets_received"` + PacketsSent int64 `json:"packets_sent" yaml:"packets_sent"` +} diff --git a/vendor/github.com/lxc/lxd/shared/api/doc.go b/vendor/github.com/lxc/lxd/shared/api/doc.go new file mode 100644 index 0000000000..f7400524dd --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/api/doc.go @@ -0,0 +1,13 @@ +// Package api contains Go structs for all LXD API objects +// +// Overview +// +// This package has Go structs for every API object, all the various +// structs are named after the object they represent and some variations of +// those structs exist for initial object creation, object update and +// object retrieval. +// +// A few convenience functions are also tied to those structs which let +// you convert between the various strucs for a given object and also query +// some of the more complex metadata that LXD can export. +package api diff --git a/vendor/github.com/lxc/lxd/shared/api/event.go b/vendor/github.com/lxc/lxd/shared/api/event.go new file mode 100644 index 0000000000..5b9e871857 --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/api/event.go @@ -0,0 +1,32 @@ +package api + +import ( + "encoding/json" + "time" +) + +// Event represents an event entry (over websocket) +type Event struct { + Type string `yaml:"type" json:"type"` + Timestamp time.Time `yaml:"timestamp" json:"timestamp"` + Metadata json.RawMessage `yaml:"metadata" json:"metadata"` + + // API extension: event_location + Location string `yaml:"location,omitempty" json:"location,omitempty"` +} + +// EventLogging represents a logging type event entry (admin only) +type EventLogging struct { + Message string `yaml:"message" json:"message"` + Level string `yaml:"level" json:"level"` + Context map[string]string `yaml:"context" json:"context"` +} + +// EventLifecycle represets a lifecycle type event entry +// +// API extension: event_lifecycle +type EventLifecycle struct { + Action string `yaml:"action" json:"action"` + Source string `yaml:"source" json:"source"` + Context map[string]interface{} `yaml:"context,omitempty" json:"context,omitempty"` +} diff --git a/vendor/github.com/lxc/lxd/shared/api/image.go b/vendor/github.com/lxc/lxd/shared/api/image.go new file mode 100644 index 0000000000..0eb4c392e6 --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/api/image.go @@ -0,0 +1,132 @@ +package api + +import ( + "time" +) + +// ImagesPost represents the fields available for a new LXD image +type ImagesPost struct { + ImagePut `yaml:",inline"` + + Filename string `json:"filename" yaml:"filename"` + Source *ImagesPostSource `json:"source" yaml:"source"` + + // API extension: image_compression_algorithm + CompressionAlgorithm string `json:"compression_algorithm" yaml:"compression_algorithm"` + + // API extension: image_create_aliases + Aliases []ImageAlias `json:"aliases" yaml:"aliases"` +} + +// ImagesPostSource represents the source of a new LXD image +type ImagesPostSource struct { + ImageSource `yaml:",inline"` + + Mode string `json:"mode" yaml:"mode"` + Type string `json:"type" yaml:"type"` + + // For protocol "direct" + URL string `json:"url" yaml:"url"` + + // For type "container" + Name string `json:"name" yaml:"name"` + + // For type "image" + Fingerprint string `json:"fingerprint" yaml:"fingerprint"` + Secret string `json:"secret" yaml:"secret"` +} + +// ImagePut represents the modifiable fields of a LXD image +type ImagePut struct { + AutoUpdate bool `json:"auto_update" yaml:"auto_update"` + Properties map[string]string `json:"properties" yaml:"properties"` + Public bool `json:"public" yaml:"public"` + + // API extension: images_expiry + ExpiresAt time.Time `json:"expires_at" yaml:"expires_at"` +} + +// Image represents a LXD image +type Image struct { + ImagePut `yaml:",inline"` + + Aliases []ImageAlias `json:"aliases" yaml:"aliases"` + Architecture string `json:"architecture" yaml:"architecture"` + Cached bool `json:"cached" yaml:"cached"` + Filename string `json:"filename" yaml:"filename"` + Fingerprint string `json:"fingerprint" yaml:"fingerprint"` + Size int64 `json:"size" yaml:"size"` + UpdateSource *ImageSource `json:"update_source,omitempty" yaml:"update_source,omitempty"` + + // API extension: image_types + Type string `json:"type" yaml:"type"` + + CreatedAt time.Time `json:"created_at" yaml:"created_at"` + LastUsedAt time.Time `json:"last_used_at" yaml:"last_used_at"` + UploadedAt time.Time `json:"uploaded_at" yaml:"uploaded_at"` +} + +// Writable converts a full Image struct into a ImagePut struct (filters read-only fields) +func (img *Image) Writable() ImagePut { + return img.ImagePut +} + +// ImageAlias represents an alias from the alias list of a LXD image +type ImageAlias struct { + Name string `json:"name" yaml:"name"` + Description string `json:"description" yaml:"description"` +} + +// ImageSource represents the source of a LXD image +type ImageSource struct { + Alias string `json:"alias" yaml:"alias"` + Certificate string `json:"certificate" yaml:"certificate"` + Protocol string `json:"protocol" yaml:"protocol"` + Server string `json:"server" yaml:"server"` + + // API extension: image_types + ImageType string `json:"image_type" yaml:"image_type"` +} + +// ImageAliasesPost represents a new LXD image alias +type ImageAliasesPost struct { + ImageAliasesEntry `yaml:",inline"` +} + +// ImageAliasesEntryPost represents the required fields to rename a LXD image alias +type ImageAliasesEntryPost struct { + Name string `json:"name" yaml:"name"` +} + +// ImageAliasesEntryPut represents the modifiable fields of a LXD image alias +type ImageAliasesEntryPut struct { + Description string `json:"description" yaml:"description"` + Target string `json:"target" yaml:"target"` +} + +// ImageAliasesEntry represents a LXD image alias +type ImageAliasesEntry struct { + ImageAliasesEntryPut `yaml:",inline"` + + Name string `json:"name" yaml:"name"` + + // API extension: image_types + Type string `json:"type" yaml:"type"` +} + +// ImageMetadata represents LXD image metadata +type ImageMetadata struct { + Architecture string `json:"architecture" yaml:"architecture"` + CreationDate int64 `json:"creation_date" yaml:"creation_date"` + ExpiryDate int64 `json:"expiry_date" yaml:"expiry_date"` + Properties map[string]string `json:"properties" yaml:"properties"` + Templates map[string]*ImageMetadataTemplate `json:"templates" yaml:"templates"` +} + +// ImageMetadataTemplate represents a template entry in image metadata +type ImageMetadataTemplate struct { + When []string `json:"when" yaml:"when"` + CreateOnly bool `json:"create_only" yaml:"create_only"` + Template string `json:"template" yaml:"template"` + Properties map[string]string `json:"properties" yaml:"properties"` +} diff --git a/vendor/github.com/lxc/lxd/shared/api/instance.go b/vendor/github.com/lxc/lxd/shared/api/instance.go new file mode 100644 index 0000000000..cec6b4a258 --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/api/instance.go @@ -0,0 +1,137 @@ +package api + +import ( + "time" +) + +// InstanceType represents the type if instance being returned or requested via the API. +type InstanceType string + +// InstanceTypeAny defines the instance type value for requesting any instance type. +const InstanceTypeAny = InstanceType("") + +// InstanceTypeContainer defines the instance type value for a container. +const InstanceTypeContainer = InstanceType("container") + +// InstanceTypeVM defines the instance type value for a virtual-machine. +const InstanceTypeVM = InstanceType("virtual-machine") + +// InstancesPost represents the fields available for a new LXD instance. +// +// API extension: instances +type InstancesPost struct { + InstancePut `yaml:",inline"` + + Name string `json:"name" yaml:"name"` + Source InstanceSource `json:"source" yaml:"source"` + InstanceType string `json:"instance_type" yaml:"instance_type"` + Type InstanceType `json:"type" yaml:"type"` +} + +// InstancePost represents the fields required to rename/move a LXD instance. +// +// API extension: instances +type InstancePost struct { + Name string `json:"name" yaml:"name"` + Migration bool `json:"migration" yaml:"migration"` + Live bool `json:"live" yaml:"live"` + InstanceOnly bool `json:"instance_only" yaml:"instance_only"` + ContainerOnly bool `json:"container_only" yaml:"container_only"` // Deprecated, use InstanceOnly. + Target *InstancePostTarget `json:"target" yaml:"target"` +} + +// InstancePostTarget represents the migration target host and operation. +// +// API extension: instances +type InstancePostTarget struct { + Certificate string `json:"certificate" yaml:"certificate"` + Operation string `json:"operation,omitempty" yaml:"operation,omitempty"` + Websockets map[string]string `json:"secrets,omitempty" yaml:"secrets,omitempty"` +} + +// InstancePut represents the modifiable fields of a LXD instance. +// +// API extension: instances +type InstancePut struct { + Architecture string `json:"architecture" yaml:"architecture"` + Config map[string]string `json:"config" yaml:"config"` + Devices map[string]map[string]string `json:"devices" yaml:"devices"` + Ephemeral bool `json:"ephemeral" yaml:"ephemeral"` + Profiles []string `json:"profiles" yaml:"profiles"` + Restore string `json:"restore,omitempty" yaml:"restore,omitempty"` + Stateful bool `json:"stateful" yaml:"stateful"` + Description string `json:"description" yaml:"description"` +} + +// Instance represents a LXD instance. +// +// API extension: instances +type Instance struct { + InstancePut `yaml:",inline"` + + CreatedAt time.Time `json:"created_at" yaml:"created_at"` + ExpandedConfig map[string]string `json:"expanded_config" yaml:"expanded_config"` + ExpandedDevices map[string]map[string]string `json:"expanded_devices" yaml:"expanded_devices"` + Name string `json:"name" yaml:"name"` + Status string `json:"status" yaml:"status"` + StatusCode StatusCode `json:"status_code" yaml:"status_code"` + LastUsedAt time.Time `json:"last_used_at" yaml:"last_used_at"` + Location string `json:"location" yaml:"location"` + Type string `json:"type" yaml:"type"` +} + +// InstanceFull is a combination of Instance, InstanceBackup, InstanceState and InstanceSnapshot. +// +// API extension: instances +type InstanceFull struct { + Instance `yaml:",inline"` + + Backups []InstanceBackup `json:"backups" yaml:"backups"` + State *InstanceState `json:"state" yaml:"state"` + Snapshots []InstanceSnapshot `json:"snapshots" yaml:"snapshots"` +} + +// Writable converts a full Instance struct into a InstancePut struct (filters read-only fields). +// +// API extension: instances +func (c *Instance) Writable() InstancePut { + return c.InstancePut +} + +// IsActive checks whether the instance state indicates the instance is active. +// +// API extension: instances +func (c Instance) IsActive() bool { + switch c.StatusCode { + case Stopped: + return false + case Error: + return false + default: + return true + } +} + +// InstanceSource represents the creation source for a new instance. +// +// API extension: instances +type InstanceSource struct { + Type string `json:"type" yaml:"type"` + Certificate string `json:"certificate" yaml:"certificate"` + Alias string `json:"alias,omitempty" yaml:"alias,omitempty"` + Fingerprint string `json:"fingerprint,omitempty" yaml:"fingerprint,omitempty"` + Properties map[string]string `json:"properties,omitempty" yaml:"properties,omitempty"` + Server string `json:"server,omitempty" yaml:"server,omitempty"` + Secret string `json:"secret,omitempty" yaml:"secret,omitempty"` + Protocol string `json:"protocol,omitempty" yaml:"protocol,omitempty"` + BaseImage string `json:"base-image,omitempty" yaml:"base-image,omitempty"` + Mode string `json:"mode,omitempty" yaml:"mode,omitempty"` + Operation string `json:"operation,omitempty" yaml:"operation,omitempty"` + Websockets map[string]string `json:"secrets,omitempty" yaml:"secrets,omitempty"` + Source string `json:"source,omitempty" yaml:"source,omitempty"` + Live bool `json:"live,omitempty" yaml:"live,omitempty"` + InstanceOnly bool `json:"instance_only,omitempty" yaml:"instance_only,omitempty"` + ContainerOnly bool `json:"container_only,omitempty" yaml:"container_only,omitempty"` // Deprecated, use InstanceOnly. + Refresh bool `json:"refresh,omitempty" yaml:"refresh,omitempty"` + Project string `json:"project,omitempty" yaml:"project,omitempty"` +} diff --git a/vendor/github.com/lxc/lxd/shared/api/instance_backup.go b/vendor/github.com/lxc/lxd/shared/api/instance_backup.go new file mode 100644 index 0000000000..093c0cd967 --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/api/instance_backup.go @@ -0,0 +1,36 @@ +package api + +import "time" + +// InstanceBackupsPost represents the fields available for a new LXD instance backup. +// +// API extension: instances +type InstanceBackupsPost struct { + Name string `json:"name" yaml:"name"` + ExpiresAt time.Time `json:"expires_at" yaml:"expires_at"` + InstanceOnly bool `json:"instance_only" yaml:"instance_only"` + ContainerOnly bool `json:"container_only" yaml:"container_only"` // Deprecated, use InstanceOnly. + OptimizedStorage bool `json:"optimized_storage" yaml:"optimized_storage"` + + // API extension: backup_compression_algorithm + CompressionAlgorithm string `json:"compression_algorithm" yaml:"compression_algorithm"` +} + +// InstanceBackup represents a LXD instance backup. +// +// API extension: instances +type InstanceBackup struct { + Name string `json:"name" yaml:"name"` + CreatedAt time.Time `json:"created_at" yaml:"created_at"` + ExpiresAt time.Time `json:"expires_at" yaml:"expires_at"` + InstanceOnly bool `json:"instance_only" yaml:"instance_only"` + ContainerOnly bool `json:"container_only" yaml:"container_only"` // Deprecated, use InstanceOnly. + OptimizedStorage bool `json:"optimized_storage" yaml:"optimized_storage"` +} + +// InstanceBackupPost represents the fields available for the renaming of a instance backup. +// +// API extension: instances +type InstanceBackupPost struct { + Name string `json:"name" yaml:"name"` +} diff --git a/vendor/github.com/lxc/lxd/shared/api/instance_console.go b/vendor/github.com/lxc/lxd/shared/api/instance_console.go new file mode 100644 index 0000000000..614beb1245 --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/api/instance_console.go @@ -0,0 +1,17 @@ +package api + +// InstanceConsoleControl represents a message on the instance console "control" socket. +// +// API extension: instances +type InstanceConsoleControl struct { + Command string `json:"command" yaml:"command"` + Args map[string]string `json:"args" yaml:"args"` +} + +// InstanceConsolePost represents a LXD instance console request. +// +// API extension: instances +type InstanceConsolePost struct { + Width int `json:"width" yaml:"width"` + Height int `json:"height" yaml:"height"` +} diff --git a/vendor/github.com/lxc/lxd/shared/api/instance_exec.go b/vendor/github.com/lxc/lxd/shared/api/instance_exec.go new file mode 100644 index 0000000000..4579b2c89d --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/api/instance_exec.go @@ -0,0 +1,26 @@ +package api + +// InstanceExecControl represents a message on the instance exec "control" socket. +// +// API extension: instances +type InstanceExecControl struct { + Command string `json:"command" yaml:"command"` + Args map[string]string `json:"args" yaml:"args"` + Signal int `json:"signal" yaml:"signal"` +} + +// InstanceExecPost represents a LXD instance exec request. +// +// API extension: instances +type InstanceExecPost struct { + Command []string `json:"command" yaml:"command"` + WaitForWS bool `json:"wait-for-websocket" yaml:"wait-for-websocket"` + Interactive bool `json:"interactive" yaml:"interactive"` + Environment map[string]string `json:"environment" yaml:"environment"` + Width int `json:"width" yaml:"width"` + Height int `json:"height" yaml:"height"` + RecordOutput bool `json:"record-output" yaml:"record-output"` + User uint32 `json:"user" yaml:"user"` + Group uint32 `json:"group" yaml:"group"` + Cwd string `json:"cwd" yaml:"cwd"` +} diff --git a/vendor/github.com/lxc/lxd/shared/api/instance_snapshot.go b/vendor/github.com/lxc/lxd/shared/api/instance_snapshot.go new file mode 100644 index 0000000000..bdd93544b2 --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/api/instance_snapshot.go @@ -0,0 +1,60 @@ +package api + +import ( + "time" +) + +// InstanceSnapshotsPost represents the fields available for a new LXD instance snapshot. +// +// API extension: instances +type InstanceSnapshotsPost struct { + Name string `json:"name" yaml:"name"` + Stateful bool `json:"stateful" yaml:"stateful"` + + // API extension: snapshot_expiry_creation + ExpiresAt *time.Time `json:"expires_at" yaml:"expires_at"` +} + +// InstanceSnapshotPost represents the fields required to rename/move a LXD instance snapshot. +// +// API extension: instances +type InstanceSnapshotPost struct { + Name string `json:"name" yaml:"name"` + Migration bool `json:"migration" yaml:"migration"` + Target *InstancePostTarget `json:"target" yaml:"target"` + Live bool `json:"live,omitempty" yaml:"live,omitempty"` +} + +// InstanceSnapshotPut represents the modifiable fields of a LXD instance snapshot. +// +// API extension: instances +type InstanceSnapshotPut struct { + Architecture string `json:"architecture" yaml:"architecture"` + Config map[string]string `json:"config" yaml:"config"` + Devices map[string]map[string]string `json:"devices" yaml:"devices"` + Ephemeral bool `json:"ephemeral" yaml:"ephemeral"` + Profiles []string `json:"profiles" yaml:"profiles"` + ExpiresAt time.Time `json:"expires_at" yaml:"expires_at"` +} + +// InstanceSnapshot represents a LXD instance snapshot. +// +// API extension: instances +type InstanceSnapshot struct { + InstanceSnapshotPut `yaml:",inline"` + + CreatedAt time.Time `json:"created_at" yaml:"created_at"` + ExpandedConfig map[string]string `json:"expanded_config" yaml:"expanded_config"` + ExpandedDevices map[string]map[string]string `json:"expanded_devices" yaml:"expanded_devices"` + LastUsedAt time.Time `json:"last_used_at" yaml:"last_used_at"` + Name string `json:"name" yaml:"name"` + Stateful bool `json:"stateful" yaml:"stateful"` +} + +// Writable converts a full InstanceSnapshot struct into a InstanceSnapshotPut struct +// (filters read-only fields). +// +// API extension: instances +func (c *InstanceSnapshot) Writable() InstanceSnapshotPut { + return c.InstanceSnapshotPut +} diff --git a/vendor/github.com/lxc/lxd/shared/api/instance_state.go b/vendor/github.com/lxc/lxd/shared/api/instance_state.go new file mode 100644 index 0000000000..cd7823cbac --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/api/instance_state.go @@ -0,0 +1,84 @@ +package api + +// InstanceStatePut represents the modifiable fields of a LXD instance's state. +// +// API extension: instances +type InstanceStatePut struct { + Action string `json:"action" yaml:"action"` + Timeout int `json:"timeout" yaml:"timeout"` + Force bool `json:"force" yaml:"force"` + Stateful bool `json:"stateful" yaml:"stateful"` +} + +// InstanceState represents a LXD instance's state. +// +// API extension: instances +type InstanceState struct { + Status string `json:"status" yaml:"status"` + StatusCode StatusCode `json:"status_code" yaml:"status_code"` + Disk map[string]InstanceStateDisk `json:"disk" yaml:"disk"` + Memory InstanceStateMemory `json:"memory" yaml:"memory"` + Network map[string]InstanceStateNetwork `json:"network" yaml:"network"` + Pid int64 `json:"pid" yaml:"pid"` + Processes int64 `json:"processes" yaml:"processes"` + CPU InstanceStateCPU `json:"cpu" yaml:"cpu"` +} + +// InstanceStateDisk represents the disk information section of a LXD instance's state. +// +// API extension: instances +type InstanceStateDisk struct { + Usage int64 `json:"usage" yaml:"usage"` +} + +// InstanceStateCPU represents the cpu information section of a LXD instance's state. +// +// API extension: instances +type InstanceStateCPU struct { + Usage int64 `json:"usage" yaml:"usage"` +} + +// InstanceStateMemory represents the memory information section of a LXD instance's state. +// +// API extension: instances +type InstanceStateMemory struct { + Usage int64 `json:"usage" yaml:"usage"` + UsagePeak int64 `json:"usage_peak" yaml:"usage_peak"` + SwapUsage int64 `json:"swap_usage" yaml:"swap_usage"` + SwapUsagePeak int64 `json:"swap_usage_peak" yaml:"swap_usage_peak"` +} + +// InstanceStateNetwork represents the network information section of a LXD instance's state. +// +// API extension: instances +type InstanceStateNetwork struct { + Addresses []InstanceStateNetworkAddress `json:"addresses" yaml:"addresses"` + Counters InstanceStateNetworkCounters `json:"counters" yaml:"counters"` + Hwaddr string `json:"hwaddr" yaml:"hwaddr"` + HostName string `json:"host_name" yaml:"host_name"` + Mtu int `json:"mtu" yaml:"mtu"` + State string `json:"state" yaml:"state"` + Type string `json:"type" yaml:"type"` +} + +// InstanceStateNetworkAddress represents a network address as part of the network section of a LXD +// instance's state. +// +// API extension: instances +type InstanceStateNetworkAddress struct { + Family string `json:"family" yaml:"family"` + Address string `json:"address" yaml:"address"` + Netmask string `json:"netmask" yaml:"netmask"` + Scope string `json:"scope" yaml:"scope"` +} + +// InstanceStateNetworkCounters represents packet counters as part of the network section of a LXD +// instance's state. +// +// API extension: instances +type InstanceStateNetworkCounters struct { + BytesReceived int64 `json:"bytes_received" yaml:"bytes_received"` + BytesSent int64 `json:"bytes_sent" yaml:"bytes_sent"` + PacketsReceived int64 `json:"packets_received" yaml:"packets_received"` + PacketsSent int64 `json:"packets_sent" yaml:"packets_sent"` +} diff --git a/vendor/github.com/lxc/lxd/shared/api/network.go b/vendor/github.com/lxc/lxd/shared/api/network.go new file mode 100644 index 0000000000..00478d26d7 --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/api/network.go @@ -0,0 +1,89 @@ +package api + +// NetworksPost represents the fields of a new LXD network +// +// API extension: network +type NetworksPost struct { + NetworkPut `yaml:",inline"` + + Managed bool `json:"managed" yaml:"managed"` + Name string `json:"name" yaml:"name"` + Type string `json:"type" yaml:"type"` +} + +// NetworkPost represents the fields required to rename a LXD network +// +// API extension: network +type NetworkPost struct { + Name string `json:"name" yaml:"name"` +} + +// NetworkPut represents the modifiable fields of a LXD network +// +// API extension: network +type NetworkPut struct { + Config map[string]string `json:"config" yaml:"config"` + + // API extension: entity_description + Description string `json:"description" yaml:"description"` +} + +// Network represents a LXD network +type Network struct { + NetworkPut `yaml:",inline"` + + Name string `json:"name" yaml:"name"` + Type string `json:"type" yaml:"type"` + UsedBy []string `json:"used_by" yaml:"used_by"` + + // API extension: network + Managed bool `json:"managed" yaml:"managed"` + + // API extension: clustering + Status string `json:"status" yaml:"status"` + Locations []string `json:"locations" yaml:"locations"` +} + +// Writable converts a full Network struct into a NetworkPut struct (filters read-only fields) +func (network *Network) Writable() NetworkPut { + return network.NetworkPut +} + +// NetworkLease represents a DHCP lease +// +// API extension: network_leases +type NetworkLease struct { + Hostname string `json:"hostname" yaml:"hostname"` + Hwaddr string `json:"hwaddr" yaml:"hwaddr"` + Address string `json:"address" yaml:"address"` + Type string `json:"type" yaml:"type"` + + // API extension: network_leases_location + Location string `json:"location" yaml:"location"` +} + +// NetworkState represents the network state +type NetworkState struct { + Addresses []NetworkStateAddress `json:"addresses" yaml:"addresses"` + Counters NetworkStateCounters `json:"counters" yaml:"counters"` + Hwaddr string `json:"hwaddr" yaml:"hwaddr"` + Mtu int `json:"mtu" yaml:"mtu"` + State string `json:"state" yaml:"state"` + Type string `json:"type" yaml:"type"` +} + +// NetworkStateAddress represents a network address +type NetworkStateAddress struct { + Family string `json:"family" yaml:"family"` + Address string `json:"address" yaml:"address"` + Netmask string `json:"netmask" yaml:"netmask"` + Scope string `json:"scope" yaml:"scope"` +} + +// NetworkStateCounters represents packet counters +type NetworkStateCounters struct { + BytesReceived int64 `json:"bytes_received" yaml:"bytes_received"` + BytesSent int64 `json:"bytes_sent" yaml:"bytes_sent"` + PacketsReceived int64 `json:"packets_received" yaml:"packets_received"` + PacketsSent int64 `json:"packets_sent" yaml:"packets_sent"` +} diff --git a/vendor/github.com/lxc/lxd/shared/api/operation.go b/vendor/github.com/lxc/lxd/shared/api/operation.go new file mode 100644 index 0000000000..98774b50d8 --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/api/operation.go @@ -0,0 +1,23 @@ +package api + +import ( + "time" +) + +// Operation represents a LXD background operation +type Operation struct { + ID string `json:"id" yaml:"id"` + Class string `json:"class" yaml:"class"` + Description string `json:"description" yaml:"description"` + CreatedAt time.Time `json:"created_at" yaml:"created_at"` + UpdatedAt time.Time `json:"updated_at" yaml:"updated_at"` + Status string `json:"status" yaml:"status"` + StatusCode StatusCode `json:"status_code" yaml:"status_code"` + Resources map[string][]string `json:"resources" yaml:"resources"` + Metadata map[string]interface{} `json:"metadata" yaml:"metadata"` + MayCancel bool `json:"may_cancel" yaml:"may_cancel"` + Err string `json:"err" yaml:"err"` + + // API extension: operation_location + Location string `json:"location" yaml:"location"` +} diff --git a/vendor/github.com/lxc/lxd/shared/api/profile.go b/vendor/github.com/lxc/lxd/shared/api/profile.go new file mode 100644 index 0000000000..3cc7a64280 --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/api/profile.go @@ -0,0 +1,35 @@ +package api + +// ProfilesPost represents the fields of a new LXD profile +type ProfilesPost struct { + ProfilePut `yaml:",inline"` + + Name string `json:"name" yaml:"name" db:"primary=yes"` +} + +// ProfilePost represents the fields required to rename a LXD profile +type ProfilePost struct { + Name string `json:"name" yaml:"name"` +} + +// ProfilePut represents the modifiable fields of a LXD profile +type ProfilePut struct { + Config map[string]string `json:"config" yaml:"config"` + Description string `json:"description" yaml:"description"` + Devices map[string]map[string]string `json:"devices" yaml:"devices"` +} + +// Profile represents a LXD profile +type Profile struct { + ProfilePut `yaml:",inline"` + + Name string `json:"name" yaml:"name" db:"primary=yes"` + + // API extension: profile_usedby + UsedBy []string `json:"used_by" yaml:"used_by"` +} + +// Writable converts a full Profile struct into a ProfilePut struct (filters read-only fields) +func (profile *Profile) Writable() ProfilePut { + return profile.ProfilePut +} diff --git a/vendor/github.com/lxc/lxd/shared/api/project.go b/vendor/github.com/lxc/lxd/shared/api/project.go new file mode 100644 index 0000000000..376e7dc139 --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/api/project.go @@ -0,0 +1,42 @@ +package api + +// ProjectsPost represents the fields of a new LXD project +// +// API extension: projects +type ProjectsPost struct { + ProjectPut `yaml:",inline"` + + Name string `json:"name" yaml:"name"` +} + +// ProjectPost represents the fields required to rename a LXD project +// +// API extension: projects +type ProjectPost struct { + Name string `json:"name" yaml:"name"` +} + +// ProjectPut represents the modifiable fields of a LXD project +// +// API extension: projects +type ProjectPut struct { + Description string `json:"description" yaml:"description"` + Config map[string]string `json:"config" yaml:"config"` +} + +// Project represents a LXD project +// +// API extension: projects +type Project struct { + ProjectPut `yaml:",inline"` + + Name string `json:"name" yaml:"name"` + UsedBy []string `json:"used_by" yaml:"used_by"` +} + +// Writable converts a full Project struct into a ProjectPut struct (filters read-only fields) +// +// API extension: projects +func (project *Project) Writable() ProjectPut { + return project.ProjectPut +} diff --git a/vendor/github.com/lxc/lxd/shared/api/resource.go b/vendor/github.com/lxc/lxd/shared/api/resource.go new file mode 100644 index 0000000000..f2a9619d15 --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/api/resource.go @@ -0,0 +1,293 @@ +package api + +// Resources represents the system resources avaible for LXD +// API extension: resources +type Resources struct { + CPU ResourcesCPU `json:"cpu" yaml:"cpu"` + Memory ResourcesMemory `json:"memory" yaml:"memory"` + + // API extension: resources_gpu + GPU ResourcesGPU `json:"gpu" yaml:"gpu"` + + // API extension: resources_v2 + Network ResourcesNetwork `json:"network" yaml:"network"` + Storage ResourcesStorage `json:"storage" yaml:"storage"` +} + +// ResourcesCPU represents the cpu resources available on the system +// API extension: resources +type ResourcesCPU struct { + // API extension: resources_v2 + Architecture string `json:"architecture" yaml:"architecture"` + + Sockets []ResourcesCPUSocket `json:"sockets" yaml:"sockets"` + Total uint64 `json:"total" yaml:"total"` +} + +// ResourcesCPUSocket represents a CPU socket on the system +// API extension: resources_v2 +type ResourcesCPUSocket struct { + Name string `json:"name,omitempty" yaml:"name,omitempty"` + Vendor string `json:"vendor,omitempty" yaml:"vendor,omitempty"` + + Socket uint64 `json:"socket" yaml:"socket"` + Cache []ResourcesCPUCache `json:"cache,omitempty" yaml:"cache,omitempty"` + Cores []ResourcesCPUCore `json:"cores" yaml:"cores"` + + Frequency uint64 `json:"frequency,omitempty" yaml:"frequency,omitempty"` + FrequencyMinimum uint64 `json:"frequency_minimum,omitempty" yaml:"frequency_minimum,omitempty"` + FrequencyTurbo uint64 `json:"frequency_turbo,omitempty" yaml:"frequency_turbo,omitempty"` +} + +// ResourcesCPUCache represents a CPU cache +// API extension: resources_v2 +type ResourcesCPUCache struct { + Level uint64 `json:"level" yaml:"level"` + Type string `json:"type" yaml:"type"` + Size uint64 `json:"size" yaml:"size"` +} + +// ResourcesCPUCore represents a CPU core on the system +// API extension: resources_v2 +type ResourcesCPUCore struct { + Core uint64 `json:"core" yaml:"core"` + NUMANode uint64 `json:"numa_node" yaml:"numa_node"` + + Threads []ResourcesCPUThread `json:"threads" yaml:"threads"` + + Frequency uint64 `json:"frequency,omitempty" yaml:"frequency,omitempty"` +} + +// ResourcesCPUThread represents a CPU thread on the system +// API extension: resources_v2 +type ResourcesCPUThread struct { + ID int64 `json:"id" yaml:"id"` + Thread uint64 `json:"thread" yaml:"thread"` + Online bool `json:"online" yaml:"online"` +} + +// ResourcesGPU represents the GPU resources available on the system +// API extension: resources_gpu +type ResourcesGPU struct { + Cards []ResourcesGPUCard `json:"cards" yaml:"cards"` + Total uint64 `json:"total" yaml:"total"` +} + +// ResourcesGPUCard represents a GPU card on the system +// API extension: resources_v2 +type ResourcesGPUCard struct { + Driver string `json:"driver,omitempty" yaml:"driver,omitempty"` + DriverVersion string `json:"driver_version,omitempty" yaml:"driver_version,omitempty"` + + DRM *ResourcesGPUCardDRM `json:"drm,omitempty" yaml:"drm,omitempty"` + SRIOV *ResourcesGPUCardSRIOV `json:"sriov,omitempty" yaml:"sriov,omitempty"` + Nvidia *ResourcesGPUCardNvidia `json:"nvidia,omitempty" yaml:"nvidia,omitempty"` + + NUMANode uint64 `json:"numa_node" yaml:"numa_node"` + PCIAddress string `json:"pci_address,omitempty" yaml:"pci_address,omitempty"` + + Vendor string `json:"vendor,omitempty" yaml:"vendor,omitempty"` + VendorID string `json:"vendor_id,omitempty" yaml:"vendor_id,omitempty"` + Product string `json:"product,omitempty" yaml:"product,omitempty"` + ProductID string `json:"product_id,omitempty" yaml:"product_id,omitempty"` +} + +// ResourcesGPUCardDRM represents the Linux DRM configuration of the GPU +// API extension: resources_v2 +type ResourcesGPUCardDRM struct { + ID uint64 `json:"id" yaml:"id"` + + CardName string `json:"card_name" yaml:"card_name"` + CardDevice string `json:"card_device" yaml:"card_device"` + + ControlName string `json:"control_name,omitempty" yaml:"control_name,omitempty"` + ControlDevice string `json:"control_device,omitempty" yaml:"control_device,omitempty"` + + RenderName string `json:"render_name,omitempty" yaml:"render_name,omitempty"` + RenderDevice string `json:"render_device,omitempty" yaml:"render_device,omitempty"` +} + +// ResourcesGPUCardSRIOV represents the SRIOV configuration of the GPU +// API extension: resources_v2 +type ResourcesGPUCardSRIOV struct { + CurrentVFs uint64 `json:"current_vfs" yaml:"current_vfs"` + MaximumVFs uint64 `json:"maximum_vfs" yaml:"maximum_vfs"` + + VFs []ResourcesGPUCard `json:"vfs" yaml:"vfs"` +} + +// ResourcesGPUCardNvidia represents additional information for NVIDIA GPUs +// API extension: resources_gpu +type ResourcesGPUCardNvidia struct { + CUDAVersion string `json:"cuda_version,omitempty" yaml:"cuda_version,omitempty"` + NVRMVersion string `json:"nvrm_version,omitempty" yaml:"nvrm_version,omitempty"` + + Brand string `json:"brand" yaml:"brand"` + Model string `json:"model" yaml:"model"` + UUID string `json:"uuid,omitempty" yaml:"uuid,omitempty"` + Architecture string `json:"architecture,omitempty" yaml:"architecture,omitempty"` + + // API extension: resources_v2 + CardName string `json:"card_name" yaml:"card_name"` + CardDevice string `json:"card_device" yaml:"card_device"` +} + +// ResourcesNetwork represents the network cards available on the system +// API extension: resources_v2 +type ResourcesNetwork struct { + Cards []ResourcesNetworkCard `json:"cards" yaml:"cards"` + Total uint64 `json:"total" yaml:"total"` +} + +// ResourcesNetworkCard represents a network card on the system +// API extension: resources_v2 +type ResourcesNetworkCard struct { + Driver string `json:"driver,omitempty" yaml:"driver,omitempty"` + DriverVersion string `json:"driver_version,omitempty" yaml:"driver_version,omitempty"` + + Ports []ResourcesNetworkCardPort `json:"ports,omitempty" yaml:"ports,omitempty"` + SRIOV *ResourcesNetworkCardSRIOV `json:"sriov,omitempty" yaml:"sriov,omitempty"` + + NUMANode uint64 `json:"numa_node" yaml:"numa_node"` + PCIAddress string `json:"pci_address,omitempty" yaml:"pci_address,omitempty"` + + Vendor string `json:"vendor,omitempty" yaml:"vendor,omitempty"` + VendorID string `json:"vendor_id,omitempty" yaml:"vendor_id,omitempty"` + Product string `json:"product,omitempty" yaml:"product,omitempty"` + ProductID string `json:"product_id,omitempty" yaml:"product_id,omitempty"` + + // API extension: resources_network_firmware + FirmwareVersion string `json:"firmware_version,omitempty" yaml:"firmware_version,omitempty"` +} + +// ResourcesNetworkCardPort represents a network port on the system +// API extension: resources_v2 +type ResourcesNetworkCardPort struct { + ID string `json:"id" yaml:"id"` + Address string `json:"address,omitempty" yaml:"address,omitempty"` + Port uint64 `json:"port" yaml:"port"` + Protocol string `json:"protocol" yaml:"protocol"` + + SupportedModes []string `json:"supported_modes,omitempty" yaml:"supported_modes,omitempty"` + SupportedPorts []string `json:"supported_ports,omitempty" yaml:"supported_ports,omitempty"` + + PortType string `json:"port_type,omitempty" yaml:"port_type,omitempty"` + TransceiverType string `json:"transceiver_type,omitempty" yaml:"transceiver_type,omitempty"` + + AutoNegotiation bool `json:"auto_negotiation" yaml:"auto_negotiation"` + LinkDetected bool `json:"link_detected" yaml:"link_detected"` + LinkSpeed uint64 `json:"link_speed,omitempty" yaml:"link_speed,omitempty"` + LinkDuplex string `json:"link_duplex,omitempty" yaml:"link_duplex,omitempty"` + + // API extension: resources_infiniband + Infiniband *ResourcesNetworkCardPortInfiniband `json:"infiniband,omitempty" yaml:"infiniband,omitempty"` +} + +// ResourcesNetworkCardPortInfiniband represents the Linux Infiniband configuration for the port +// API extension: resources_infiniband +type ResourcesNetworkCardPortInfiniband struct { + IsSMName string `json:"issm_name,omitempty" yaml:"issm_name,omitempty"` + IsSMDevice string `json:"issm_device,omitempty" yaml:"issm_device,omitempty"` + + MADName string `json:"mad_name,omitempty" yaml:"mad_name,omitempty"` + MADDevice string `json:"mad_device,omitempty" yaml:"mad_device,omitempty"` + + VerbName string `json:"verb_name,omitempty" yaml:"verb_name,omitempty"` + VerbDevice string `json:"verb_device,omitempty" yaml:"verb_device,omitempty"` +} + +// ResourcesNetworkCardSRIOV represents the SRIOV configuration of the network card +// API extension: resources_v2 +type ResourcesNetworkCardSRIOV struct { + CurrentVFs uint64 `json:"current_vfs" yaml:"current_vfs"` + MaximumVFs uint64 `json:"maximum_vfs" yaml:"maximum_vfs"` + + VFs []ResourcesNetworkCard `json:"vfs" yaml:"vfs"` +} + +// ResourcesStorage represents the local storage +// API extension: resources_v2 +type ResourcesStorage struct { + Disks []ResourcesStorageDisk `json:"disks" yaml:"disks"` + Total uint64 `json:"total" yaml:"total"` +} + +// ResourcesStorageDisk represents a disk +// API extension: resources_v2 +type ResourcesStorageDisk struct { + ID string `json:"id" yaml:"id"` + Device string `json:"device" yaml:"device"` + Model string `json:"model,omitempty" yaml:"model,omitempty"` + Type string `json:"type,omitempty" yaml:"type,omitempty"` + ReadOnly bool `json:"read_only" yaml:"read_only"` + Size uint64 `json:"size" yaml:"size"` + + Removable bool `json:"removable" yaml:"removable"` + WWN string `json:"wwn,omitempty" yaml:"wwn,omitempty"` + NUMANode uint64 `json:"numa_node" yaml:"numa_node"` + + // API extension: resources_disk_sata + DevicePath string `json:"device_path" yaml:"device_path"` + BlockSize uint64 `json:"block_size" yaml:"block_size"` + FirmwareVersion string `json:"firmware_version,omitempty" yaml:"firmware_version,omitempty"` + RPM uint64 `json:"rpm" yaml:"rpm"` + Serial string `json:"serial,omitempty" yaml:"serial,omitempty"` + + Partitions []ResourcesStorageDiskPartition `json:"partitions" yaml:"partitions"` +} + +// ResourcesStorageDiskPartition represents a partition on a disk +// API extension: resources_v2 +type ResourcesStorageDiskPartition struct { + ID string `json:"id" yaml:"id"` + Device string `json:"device" yaml:"device"` + ReadOnly bool `json:"read_only" yaml:"read_only"` + Size uint64 `json:"size" yaml:"size"` + + Partition uint64 `json:"partition" yaml:"partition"` +} + +// ResourcesMemory represents the memory resources available on the system +// API extension: resources +type ResourcesMemory struct { + // API extension: resources_v2 + Nodes []ResourcesMemoryNode `json:"nodes,omitempty" yaml:"nodes,omitempty"` + HugepagesTotal uint64 `json:"hugepages_total" yaml:"hugepages_total"` + HugepagesUsed uint64 `json:"hugepages_used" yaml:"hugepages_used"` + HugepagesSize uint64 `json:"hugepages_size" yaml:"hugepages_size"` + + Used uint64 `json:"used" yaml:"used"` + Total uint64 `json:"total" yaml:"total"` +} + +// ResourcesMemoryNode represents the node-specific memory resources available on the system +// API extension: resources_v2 +type ResourcesMemoryNode struct { + NUMANode uint64 `json:"numa_node" yaml:"numa_node"` + HugepagesUsed uint64 `json:"hugepages_used" yaml:"hugepages_used"` + HugepagesTotal uint64 `json:"hugepages_total" yaml:"hugepages_total"` + + Used uint64 `json:"used" yaml:"used"` + Total uint64 `json:"total" yaml:"total"` +} + +// ResourcesStoragePool represents the resources available to a given storage pool +// API extension: resources +type ResourcesStoragePool struct { + Space ResourcesStoragePoolSpace `json:"space,omitempty" yaml:"space,omitempty"` + Inodes ResourcesStoragePoolInodes `json:"inodes,omitempty" yaml:"inodes,omitempty"` +} + +// ResourcesStoragePoolSpace represents the space available to a given storage pool +// API extension: resources +type ResourcesStoragePoolSpace struct { + Used uint64 `json:"used,omitempty" yaml:"used,omitempty"` + Total uint64 `json:"total" yaml:"total"` +} + +// ResourcesStoragePoolInodes represents the inodes available to a given storage pool +// API extension: resources +type ResourcesStoragePoolInodes struct { + Used uint64 `json:"used" yaml:"used"` + Total uint64 `json:"total" yaml:"total"` +} diff --git a/vendor/github.com/lxc/lxd/shared/api/response.go b/vendor/github.com/lxc/lxd/shared/api/response.go new file mode 100644 index 0000000000..4f4e044977 --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/api/response.go @@ -0,0 +1,90 @@ +package api + +import ( + "encoding/json" +) + +// ResponseRaw represents a LXD operation in its original form +type ResponseRaw struct { + Type ResponseType `json:"type" yaml:"type"` + + // Valid only for Sync responses + Status string `json:"status" yaml:"status"` + StatusCode int `json:"status_code" yaml:"status_code"` + + // Valid only for Async responses + Operation string `json:"operation" yaml:"operation"` + + // Valid only for Error responses + Code int `json:"error_code" yaml:"error_code"` + Error string `json:"error" yaml:"error"` + + Metadata interface{} `json:"metadata" yaml:"metadata"` +} + +// Response represents a LXD operation +type Response struct { + Type ResponseType `json:"type" yaml:"type"` + + // Valid only for Sync responses + Status string `json:"status" yaml:"status"` + StatusCode int `json:"status_code" yaml:"status_code"` + + // Valid only for Async responses + Operation string `json:"operation" yaml:"operation"` + + // Valid only for Error responses + Code int `json:"error_code" yaml:"error_code"` + Error string `json:"error" yaml:"error"` + + // Valid for Sync and Error responses + Metadata json.RawMessage `json:"metadata" yaml:"metadata"` +} + +// MetadataAsMap parses the Response metadata into a map +func (r *Response) MetadataAsMap() (map[string]interface{}, error) { + ret := map[string]interface{}{} + err := r.MetadataAsStruct(&ret) + if err != nil { + return nil, err + } + + return ret, nil +} + +// MetadataAsOperation turns the Response metadata into an Operation +func (r *Response) MetadataAsOperation() (*Operation, error) { + op := Operation{} + err := r.MetadataAsStruct(&op) + if err != nil { + return nil, err + } + + return &op, nil +} + +// MetadataAsStringSlice parses the Response metadata into a slice of string +func (r *Response) MetadataAsStringSlice() ([]string, error) { + sl := []string{} + err := r.MetadataAsStruct(&sl) + if err != nil { + return nil, err + } + + return sl, nil +} + +// MetadataAsStruct parses the Response metadata into a provided struct +func (r *Response) MetadataAsStruct(target interface{}) error { + return json.Unmarshal(r.Metadata, &target) +} + +// ResponseType represents a valid LXD response type +type ResponseType string + +// LXD response types +const ( + SyncResponse ResponseType = "sync" + AsyncResponse ResponseType = "async" + ErrorResponse ResponseType = "error" +) diff --git a/vendor/github.com/lxc/lxd/shared/api/server.go b/vendor/github.com/lxc/lxd/shared/api/server.go new file mode 100644 index 0000000000..c40b44e410 --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/api/server.go @@ -0,0 +1,65 @@ +package api + +// ServerEnvironment represents the read-only environment fields of a LXD server +type ServerEnvironment struct { + Addresses []string `json:"addresses" yaml:"addresses"` + Architectures []string `json:"architectures" yaml:"architectures"` + Certificate string `json:"certificate" yaml:"certificate"` + CertificateFingerprint string `json:"certificate_fingerprint" yaml:"certificate_fingerprint"` + Driver string `json:"driver" yaml:"driver"` + DriverVersion string `json:"driver_version" yaml:"driver_version"` + Kernel string `json:"kernel" yaml:"kernel"` + KernelArchitecture string `json:"kernel_architecture" yaml:"kernel_architecture"` + + // API extension: kernel_features + KernelFeatures map[string]string `json:"kernel_features" yaml:"kernel_features"` + + KernelVersion string `json:"kernel_version" yaml:"kernel_version"` + + // API extension: lxc_features + LXCFeatures map[string]string `json:"lxc_features" yaml:"lxc_features"` + + // API extension: projects + Project string `json:"project" yaml:"project"` + + Server string `json:"server" yaml:"server"` + + // API extension: clustering + ServerClustered bool `json:"server_clustered" yaml:"server_clustered"` + ServerName string `json:"server_name" yaml:"server_name"` + + ServerPid int `json:"server_pid" yaml:"server_pid"` + ServerVersion string `json:"server_version" yaml:"server_version"` + Storage string `json:"storage" yaml:"storage"` + StorageVersion string `json:"storage_version" yaml:"storage_version"` +} + +// ServerPut represents the modifiable fields of a LXD server configuration +type ServerPut struct { + Config map[string]interface{} `json:"config" yaml:"config"` +} + +// ServerUntrusted represents a LXD server for an untrusted client +type ServerUntrusted struct { + APIExtensions []string `json:"api_extensions" yaml:"api_extensions"` + APIStatus string `json:"api_status" yaml:"api_status"` + APIVersion string `json:"api_version" yaml:"api_version"` + Auth string `json:"auth" yaml:"auth"` + Public bool `json:"public" yaml:"public"` + + // API extension: macaroon_authentication + AuthMethods []string `json:"auth_methods" yaml:"auth_methods"` +} + +// Server represents a LXD server +type Server struct { + ServerPut `yaml:",inline"` + ServerUntrusted `yaml:",inline"` + + Environment ServerEnvironment `json:"environment" yaml:"environment"` +} + +// Writable converts a full Server struct into a ServerPut struct (filters read-only fields) +func (srv *Server) Writable() ServerPut { + return srv.ServerPut +} diff --git a/vendor/github.com/lxc/lxd/shared/api/status_code.go b/vendor/github.com/lxc/lxd/shared/api/status_code.go new file mode 100644 index 0000000000..bf2986607b --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/api/status_code.go @@ -0,0 +1,53 @@ +package api + +// StatusCode represents a valid LXD operation and container status +type StatusCode int + +// LXD status codes +const ( + OperationCreated StatusCode = 100 + Started StatusCode = 101 + Stopped StatusCode = 102 + Running StatusCode = 103 + Cancelling StatusCode = 104 + Pending StatusCode = 105 + Starting StatusCode = 106 + Stopping StatusCode = 107 + Aborting StatusCode = 108 + Freezing StatusCode = 109 + Frozen StatusCode = 110 + Thawed StatusCode = 111 + Error StatusCode = 112 + + Success StatusCode = 200 + + Failure StatusCode = 400 + Cancelled StatusCode = 401 +) + +// String returns a suitable string representation for the status code +func (o StatusCode) String() string { + return map[StatusCode]string{ + OperationCreated: "Operation created", + Started: "Started", + Stopped: "Stopped", + Running: "Running", + Cancelling: "Cancelling", + Pending: "Pending", + Success: "Success", + Failure: "Failure", + Cancelled: "Cancelled", + Starting: "Starting", + Stopping: "Stopping", + Aborting: "Aborting", + Freezing: "Freezing", + Frozen: "Frozen", + Thawed: "Thawed", + Error: "Error", + }[o] +} + +// IsFinal will return true if the status code indicates an end state +func (o StatusCode) IsFinal() bool { + return int(o) >= 200 +} diff --git a/vendor/github.com/lxc/lxd/shared/api/storage_pool.go b/vendor/github.com/lxc/lxd/shared/api/storage_pool.go new file mode 100644 index 0000000000..16cf4fad30 --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/api/storage_pool.go @@ -0,0 +1,42 @@ +package api + +// StoragePoolsPost represents the fields of a new LXD storage pool +// +// API extension: storage +type StoragePoolsPost struct { + StoragePoolPut `yaml:",inline"` + + Name string `json:"name" yaml:"name"` + Driver string `json:"driver" yaml:"driver"` +} + +// StoragePool represents the fields of a LXD storage pool. +// +// API extension: storage +type StoragePool struct { + StoragePoolPut `yaml:",inline"` + + Name string `json:"name" yaml:"name"` + Driver string `json:"driver" yaml:"driver"` + UsedBy []string `json:"used_by" yaml:"used_by"` + + // API extension: clustering + Status string `json:"status" yaml:"status"` + Locations []string `json:"locations" yaml:"locations"` +} + +// StoragePoolPut represents the modifiable fields of a LXD storage pool. +// +// API extension: storage +type StoragePoolPut struct { + Config map[string]string `json:"config" yaml:"config"` + + // API extension: entity_description + Description string `json:"description" yaml:"description"` +} + +// Writable converts a full StoragePool struct into a StoragePoolPut struct +// (filters read-only fields). +func (storagePool *StoragePool) Writable() StoragePoolPut { + return storagePool.StoragePoolPut +} diff --git a/vendor/github.com/lxc/lxd/shared/api/storage_pool_volume.go b/vendor/github.com/lxc/lxd/shared/api/storage_pool_volume.go new file mode 100644 index 0000000000..db916ff0fa --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/api/storage_pool_volume.go @@ -0,0 +1,92 @@ +package api + +// StorageVolumesPost represents the fields of a new LXD storage pool volume +// +// API extension: storage +type StorageVolumesPost struct { + StorageVolumePut `yaml:",inline"` + + Name string `json:"name" yaml:"name"` + Type string `json:"type" yaml:"type"` + + // API extension: storage_api_local_volume_handling + Source StorageVolumeSource `json:"source" yaml:"source"` +} + +// StorageVolumePost represents the fields required to rename a LXD storage pool volume +// +// API extension: storage_api_volume_rename +type StorageVolumePost struct { + Name string `json:"name" yaml:"name"` + + // API extension: storage_api_local_volume_handling + Pool string `json:"pool,omitempty" yaml:"pool,omitempty"` + + // API extension: storage_api_remote_volume_handling + Migration bool `json:"migration" yaml:"migration"` + + // API extension: storage_api_remote_volume_handling + Target *StorageVolumePostTarget `json:"target" yaml:"target"` + + // API extension: storage_api_remote_volume_snapshots + VolumeOnly bool `json:"volume_only" yaml:"volume_only"` +} + +// StorageVolumePostTarget represents the migration target host and operation +// +// API extension: storage_api_remote_volume_handling +type StorageVolumePostTarget struct { + Certificate string `json:"certificate" yaml:"certificate"` + Operation string `json:"operation,omitempty" yaml:"operation,omitempty"` + Websockets map[string]string `json:"secrets,omitempty" yaml:"secrets,omitempty"` +} + +// StorageVolume represents the fields of a LXD storage volume. +// +// API extension: storage +type StorageVolume struct { + StorageVolumePut `yaml:",inline"` + Name string `json:"name" yaml:"name"` + Type string `json:"type" yaml:"type"` + UsedBy []string `json:"used_by" yaml:"used_by"` + + // API extension: clustering + Location string `json:"location" yaml:"location"` +} + +// StorageVolumePut represents the modifiable fields of a LXD storage volume. +// +// API extension: storage +type StorageVolumePut struct { + Config map[string]string `json:"config" yaml:"config"` + + // API extension: entity_description + Description string `json:"description" yaml:"description"` + + // API extension: storage_api_volume_snapshots + Restore string `json:"restore,omitempty" yaml:"restore,omitempty"` +} + +// StorageVolumeSource represents the creation source for a new storage volume. +// +// API extension: storage_api_local_volume_handling +type StorageVolumeSource struct { + Name string `json:"name" yaml:"name"` + Type string `json:"type" yaml:"type"` + Pool string `json:"pool" yaml:"pool"` + + // API extension: storage_api_remote_volume_handling + Certificate string `json:"certificate" yaml:"certificate"` + Mode string `json:"mode,omitempty" yaml:"mode,omitempty"` + Operation string `json:"operation,omitempty" yaml:"operation,omitempty"` + Websockets map[string]string `json:"secrets,omitempty" yaml:"secrets,omitempty"` + + // API extension: storage_api_volume_snapshots + VolumeOnly bool `json:"volume_only" yaml:"volume_only"` +} + +// Writable converts a full StorageVolume struct into a StorageVolumePut struct +// (filters read-only fields). +func (storageVolume *StorageVolume) Writable() StorageVolumePut { + return storageVolume.StorageVolumePut +} diff --git a/vendor/github.com/lxc/lxd/shared/api/storage_pool_volume_snapshot.go b/vendor/github.com/lxc/lxd/shared/api/storage_pool_volume_snapshot.go new file mode 100644 index 0000000000..4ba21da05d --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/api/storage_pool_volume_snapshot.go @@ -0,0 +1,31 @@ +package api + +// StorageVolumeSnapshotsPost represents the fields available for a new LXD storage volume snapshot +// +// API extension: storage_api_volume_snapshots +type StorageVolumeSnapshotsPost struct { + Name string `json:"name" yaml:"name"` +} + +// StorageVolumeSnapshotPost represents the fields required to rename/move a LXD storage volume snapshot +// +// API extension: storage_api_volume_snapshots +type StorageVolumeSnapshotPost struct { + Name string `json:"name" yaml:"name"` +} + +// StorageVolumeSnapshot represents a LXD storage volume snapshot +// +// API extension: storage_api_volume_snapshots +type StorageVolumeSnapshot struct { + Name string `json:"name" yaml:"name"` + Config map[string]string `json:"config" yaml:"config"` + Description string `json:"description" yaml:"description"` +} + +// StorageVolumeSnapshotPut represents the modifiable fields of a LXD storage volume +// +// API extension: storage_api_volume_snapshots +type StorageVolumeSnapshotPut struct { + Description string `json:"description" yaml:"description"` +} diff --git a/vendor/github.com/lxc/lxd/shared/archive_linux.go b/vendor/github.com/lxc/lxd/shared/archive_linux.go new file mode 100644 index 0000000000..a77a09b70a --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/archive_linux.go @@ -0,0 +1,147 @@ +package shared + +import ( + "bytes" + "fmt" + "io" + "os" + "strings" + + "golang.org/x/sys/unix" + + "github.com/lxc/lxd/shared/ioprogress" + "github.com/lxc/lxd/shared/logger" +) + +func DetectCompression(fname string) ([]string, string, []string, error) { + f, err := os.Open(fname) + if err != nil { + return nil, "", nil, err + } + defer f.Close() + + return DetectCompressionFile(f) +} + +func DetectCompressionFile(f io.ReadSeeker) ([]string, string, []string, error) { + // read header parts to detect compression method + // bz2 - 2 bytes, 'BZ' signature/magic number + // gz - 2 bytes, 0x1f 0x8b + // lzma - 6 bytes, { [0x000, 0xE0], '7', 'z', 'X', 'Z', 0x00 } - + // xy - 6 bytes, header format { 0xFD, '7', 'z', 'X', 'Z', 0x00 } + // tar - 263 bytes, trying to get ustar from 257 - 262 + header := make([]byte, 263) + _, err := f.Read(header) + if err != nil { + return nil, "", nil, err + } + + switch { + case bytes.Equal(header[0:2], []byte{'B', 'Z'}): + return []string{"-jxf"}, ".tar.bz2", []string{"bzip2", "-d"}, nil + case bytes.Equal(header[0:2], []byte{0x1f, 0x8b}): + return []string{"-zxf"}, ".tar.gz", []string{"gzip", "-d"}, nil + case (bytes.Equal(header[1:5], []byte{'7', 'z', 'X', 'Z'}) && header[0] == 0xFD): + return []string{"-Jxf"}, ".tar.xz", []string{"xz", "-d"}, nil + case (bytes.Equal(header[1:5], []byte{'7', 'z', 'X', 'Z'}) && header[0] != 0xFD): + return []string{"--lzma", "-xf"}, ".tar.lzma", []string{"lzma", "-d"}, nil + case bytes.Equal(header[0:3], []byte{0x5d, 0x00, 0x00}): + return []string{"--lzma", "-xf"}, ".tar.lzma", []string{"lzma", "-d"}, nil + case bytes.Equal(header[257:262], []byte{'u', 's', 't', 'a', 'r'}): + return []string{"-xf"}, ".tar", []string{}, nil + case bytes.Equal(header[0:4], []byte{'h', 's', 'q', 's'}): + return []string{"-xf"}, ".squashfs", + []string{"sqfs2tar", "--no-skip"}, nil + default: + return nil, "", nil, fmt.Errorf("Unsupported compression") + } +} + +func Unpack(file string, path string, blockBackend bool, runningInUserns bool, tracker *ioprogress.ProgressTracker) error { + extractArgs, extension, _, err := DetectCompression(file) + if err != nil { + return err + } + + command := "" + args := []string{} + var reader io.Reader + if strings.HasPrefix(extension, ".tar") { + command = "tar" + if runningInUserns { + args = append(args, "--wildcards") + args = append(args, "--exclude=dev/*") + args = append(args, "--exclude=./dev/*") + args = append(args, "--exclude=rootfs/dev/*") + args = append(args, "--exclude=rootfs/./dev/*") + } + args = append(args, "-C", path, "--numeric-owner", "--xattrs-include=*") + args = append(args, extractArgs...) + args = append(args, "-") + + f, err := os.Open(file) + if err != nil { + return err + } + defer f.Close() + + reader = f + + // Attach the ProgressTracker if supplied. + if tracker != nil { + fsinfo, err := f.Stat() + if err != nil { + return err + } + + tracker.Length = fsinfo.Size() + reader = &ioprogress.ProgressReader{ + ReadCloser: f, + Tracker: tracker, + } + } + } else if strings.HasPrefix(extension, ".squashfs") { + // unsquashfs does not support reading from stdin, + // so ProgressTracker is not possible. + command = "unsquashfs" + args = append(args, "-f", "-d", path, "-n") + + // Limit unsquashfs chunk size to 10% of memory and up to 256MB (default) + // When running on a low memory system, also disable multi-processing + mem, err := DeviceTotalMemory() + mem = mem / 1024 / 1024 / 10 + if err == nil && mem < 256 { + args = append(args, "-da", fmt.Sprintf("%d", mem), "-fr", fmt.Sprintf("%d", mem), "-p", "1") + } + + args = append(args, file) + } else { + return fmt.Errorf("Unsupported image format: %s", extension) + } + + err = RunCommandWithFds(reader, nil, command, args...) + if err != nil { + // Check if we ran out of space + fs := unix.Statfs_t{} + + err1 := unix.Statfs(path, &fs) + if err1 != nil { + return err1 + } + + // Check if we're running out of space + if int64(fs.Bfree) < int64(2*fs.Bsize) { + if blockBackend { + return fmt.Errorf("Unable to unpack image, run out of disk space (consider increasing your pool's volume.size)") + } else { + return fmt.Errorf("Unable to unpack image, run out of disk space") + } + } + + logger.Debugf("Unpacking failed") + logger.Debugf(err.Error()) + return fmt.Errorf("Unpack failed, %s.", err) + } + + return nil +} diff --git a/vendor/github.com/lxc/lxd/shared/cancel/canceler.go b/vendor/github.com/lxc/lxd/shared/cancel/canceler.go new file mode 100644 index 0000000000..b3356cf37e --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/cancel/canceler.go @@ -0,0 +1,73 @@ +package cancel + +import ( + "fmt" + "net/http" + "sync" +) + +// Canceler tracks a cancelable operation +type Canceler struct { + reqChCancel map[*http.Request]chan struct{} + lock sync.Mutex +} + +// NewCanceler returns a new Canceler struct +func NewCanceler() *Canceler { + c := Canceler{} + + c.lock.Lock() + c.reqChCancel = make(map[*http.Request]chan struct{}) + c.lock.Unlock() + + return &c +} + +// Cancelable indicates whether there are operations that support cancelation +func (c *Canceler) Cancelable() bool { + c.lock.Lock() + length := len(c.reqChCancel) + c.lock.Unlock() + + return length > 0 +} + +// Cancel will attempt to cancel all ongoing operations +func (c *Canceler) Cancel() error { + if !c.Cancelable() { + return fmt.Errorf("This operation can't be canceled at this time") + } + + c.lock.Lock() + for req, ch := range c.reqChCancel { + close(ch) + delete(c.reqChCancel, req) + } + c.lock.Unlock() + + return nil +} + +// CancelableDownload performs an http request and allows for it to be canceled at any time +func CancelableDownload(c *Canceler, client *http.Client, req *http.Request) (*http.Response, chan bool, error) { + chDone := make(chan bool) + chCancel := make(chan struct{}) + if c != nil { + c.lock.Lock() + c.reqChCancel[req] = chCancel + c.lock.Unlock() + } + req.Cancel = chCancel + + go func() { + <-chDone + if c != nil { + c.lock.Lock() + delete(c.reqChCancel, req) + c.lock.Unlock() + } + }() + + resp, err := client.Do(req) + return resp, chDone, err +} diff --git a/vendor/github.com/lxc/lxd/shared/cert.go b/vendor/github.com/lxc/lxd/shared/cert.go new file mode 100644 index 0000000000..b38fa93a67 --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/cert.go @@ -0,0 +1,531 @@ +// http://golang.org/src/pkg/crypto/tls/generate_cert.go +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package shared + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "io/ioutil" + "math/big" + "net" + "net/http" + "os" + "os/user" + "path" + "path/filepath" + "time" +) + +// KeyPairAndCA returns a CertInfo object with a reference to the key pair and +// (optionally) CA certificate located in the given directory and having the +// given name prefix +// +// The naming conversion for the various files is: +// +// .crt -> public key +// .key -> private key +// .ca -> CA certificate +// +// If no public/private key files are found, a new key pair will be generated +// and saved on disk. +// +// If a CA certificate is found, it will be returned as well as second return +// value (otherwise it will be nil). +func KeyPairAndCA(dir, prefix string, kind CertKind) (*CertInfo, error) { + certFilename := filepath.Join(dir, prefix+".crt") + keyFilename := filepath.Join(dir, prefix+".key") + + // Ensure that the certificate exists, or create a new one if it does + // not. + err := FindOrGenCert(certFilename, keyFilename, kind == CertClient) + if err != nil { + return nil, err + } + + // Load the certificate. + keypair, err := tls.LoadX509KeyPair(certFilename, keyFilename) + if err != nil { + return nil, err + } + + // If available, load the CA data as well. + caFilename := filepath.Join(dir, prefix+".ca") + var ca *x509.Certificate + if PathExists(caFilename) { + ca, err = ReadCert(caFilename) + if err != nil { + return nil, err + } + } + + info := &CertInfo{ + keypair: keypair, + ca: ca, + } + return info, nil +} + +// CertInfo captures TLS certificate information about a certain public/private +// keypair and an optional CA certificate. +// +// Given LXD's support for PKI setups, these two bits of information are +// normally used and passed around together, so this structure helps with that +// (see doc/security.md for more details). +type CertInfo struct { + keypair tls.Certificate + ca *x509.Certificate +} + +// KeyPair returns the public/private key pair. +func (c *CertInfo) KeyPair() tls.Certificate { + return c.keypair +} + +// CA returns the CA certificate. +func (c *CertInfo) CA() *x509.Certificate { + return c.ca +} + +// PublicKey is a convenience to encode the underlying public key to ASCII. +func (c *CertInfo) PublicKey() []byte { + data := c.KeyPair().Certificate[0] + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: data}) +} + +// PrivateKey is a convenience to encode the underlying private key. +func (c *CertInfo) PrivateKey() []byte { + ecKey, ok := c.KeyPair().PrivateKey.(*ecdsa.PrivateKey) + if ok { + data, err := x509.MarshalECPrivateKey(ecKey) + if err != nil { + return nil + } + + return pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: data}) + } + + rsaKey, ok := c.KeyPair().PrivateKey.(*rsa.PrivateKey) + if ok { + data := x509.MarshalPKCS1PrivateKey(rsaKey) + return pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: data}) + } + + return nil +} + +// Fingerprint returns the fingerprint of the public key. +func (c *CertInfo) Fingerprint() string { + fingerprint, err := CertFingerprintStr(string(c.PublicKey())) + // Parsing should never fail, since we generated the cert ourselves, + // but let's check the error for good measure. + if err != nil { + panic("invalid public key material") + } + return fingerprint +} + +// CertKind defines the kind of certificate to generate from scratch in +// KeyPairAndCA when it's not there. +// +// The two possible kinds are client and server, and they differ in the +// ext-key-usage bitmaps. See GenerateMemCert for more details. +type CertKind int + +// Possible kinds of certificates. +const ( + CertClient CertKind = iota + CertServer +) + +// TestingKeyPair returns CertInfo object initialized with a test keypair. It's +// meant to be used only by tests. +func TestingKeyPair() *CertInfo { + keypair, err := tls.X509KeyPair(testCertPEMBlock, testKeyPEMBlock) + if err != nil { + panic(fmt.Sprintf("invalid X509 keypair material: %v", err)) + } + cert := &CertInfo{ + keypair: keypair, + } + return cert +} + +// TestingAltKeyPair returns CertInfo object initialized with a test keypair +// which differs from the one returned by TestCertInfo. It's meant to be used +// only by tests. +func TestingAltKeyPair() *CertInfo { + keypair, err := tls.X509KeyPair(testAltCertPEMBlock, testAltKeyPEMBlock) + if err != nil { + panic(fmt.Sprintf("invalid X509 keypair material: %v", err)) + } + cert := &CertInfo{ + keypair: keypair, + } + return cert +} + +/* + * Generate a list of names for which the certificate will be valid. + * This will include the hostname and ip address + */ +func mynames() ([]string, error) { + h, err := os.Hostname() + if err != nil { + return nil, err + } + + ret := []string{h} + + ifs, err := net.Interfaces() + if err != nil { + return nil, err + } + + for _, iface := range ifs { + if IsLoopback(&iface) { + continue + } + + addrs, err := iface.Addrs() + if err != nil { + return nil, err + } + + for _, addr := range addrs { + ret = append(ret, addr.String()) + } + } + + return ret, nil +} + +func FindOrGenCert(certf string, keyf string, certtype bool) error { + if PathExists(certf) && PathExists(keyf) { + return nil + } + + /* If neither stat succeeded, then this is our first run and we + * need to generate cert and privkey */ + err := GenCert(certf, keyf, certtype) + if err != nil { + return err + } + + return nil +} + +// GenCert will create and populate a certificate file and a key file +func GenCert(certf string, keyf string, certtype bool) error { + /* Create the basenames if needed */ + dir := path.Dir(certf) + err := os.MkdirAll(dir, 0750) + if err != nil { + return err + } + dir = path.Dir(keyf) + err = os.MkdirAll(dir, 0750) + if err != nil { + return err + } + + certBytes, keyBytes, err := GenerateMemCert(certtype) + if err != nil { + return err + } + + certOut, err := os.Create(certf) + if err != nil { + return fmt.Errorf("Failed to open %s for writing: %v", certf, err) + } + certOut.Write(certBytes) + certOut.Close() + + keyOut, err := os.OpenFile(keyf, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) + if err != nil { + return fmt.Errorf("Failed to open %s for writing: %v", keyf, err) + } + keyOut.Write(keyBytes) + keyOut.Close() + return nil +} + +// GenerateMemCert creates client or server certificate and key pair, +// returning them as byte arrays in memory. +func GenerateMemCert(client bool) ([]byte, []byte, error) { + privk, err := ecdsa.GenerateKey(elliptic.P384(), rand.Reader) + if err != nil { + return nil, nil, fmt.Errorf("Failed to generate key: %v", err) + } + + hosts, err := mynames() + if err != nil { + return nil, nil, fmt.Errorf("Failed to get my hostname: %v", err) + } + + validFrom := time.Now() + validTo := validFrom.Add(10 * 365 * 24 * time.Hour) + + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) + serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) + if err != nil { + return nil, nil, fmt.Errorf("Failed to generate serial number: %v", err) + } + + userEntry, err := user.Current() + var username string + if err == nil { + username = userEntry.Username + if username == "" { + username = "UNKNOWN" + } + } else { + username = "UNKNOWN" + } + + hostname, err := os.Hostname() + if err != nil { + hostname = "UNKNOWN" + } + + template := x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{ + Organization: []string{"linuxcontainers.org"}, + CommonName: fmt.Sprintf("%s@%s", username, hostname), + }, + NotBefore: validFrom, + NotAfter: validTo, + + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + BasicConstraintsValid: true, + } + + if client { + template.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth} + } else { + template.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth} + } + + for _, h := range hosts { + if ip, _, err := net.ParseCIDR(h); err == nil { + if !ip.IsLinkLocalUnicast() && !ip.IsLinkLocalMulticast() { + template.IPAddresses = append(template.IPAddresses, ip) + } + } else { + template.DNSNames = append(template.DNSNames, h) + } + } + + derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &privk.PublicKey, privk) + if err != nil { + return nil, nil, fmt.Errorf("Failed to create certificate: %v", err) + } + + data, err := x509.MarshalECPrivateKey(privk) + if err != nil { + return nil, nil, err + } + + cert := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) + key := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: data}) + + return cert, key, nil +} + +func ReadCert(fpath string) (*x509.Certificate, error) { + cf, err := ioutil.ReadFile(fpath) + if err != nil { + return nil, err + } + + certBlock, _ := pem.Decode(cf) + if certBlock == nil { + return nil, fmt.Errorf("Invalid certificate file") + } + + return x509.ParseCertificate(certBlock.Bytes) +} + +func CertFingerprint(cert *x509.Certificate) string { + return fmt.Sprintf("%x", sha256.Sum256(cert.Raw)) +} + +func CertFingerprintStr(c string) (string, error) { + pemCertificate, _ := pem.Decode([]byte(c)) + if pemCertificate == nil { + return "", fmt.Errorf("invalid certificate") + } + + cert, err := x509.ParseCertificate(pemCertificate.Bytes) + if err != nil { + return "", err + } + + return CertFingerprint(cert), nil +} + +func GetRemoteCertificate(address string) (*x509.Certificate, error) { + // Setup a permissive TLS config + tlsConfig, err := GetTLSConfig("", "", "", nil) + if err != nil { + return nil, err + } + + tlsConfig.InsecureSkipVerify = true + + // Support disabling of strict ciphers + if IsTrue(os.Getenv("LXD_INSECURE_TLS")) { + tlsConfig.CipherSuites = nil + } + + tr := &http.Transport{ + TLSClientConfig: tlsConfig, + Dial: RFC3493Dialer, + Proxy: ProxyFromEnvironment, + } + + // Connect + client := &http.Client{Transport: tr} + resp, err := client.Get(address) + if err != nil { + return nil, err + } + + // Retrieve the certificate + if resp.TLS == nil || len(resp.TLS.PeerCertificates) == 0 { + return nil, fmt.Errorf("Unable to read remote TLS certificate") + } + + return resp.TLS.PeerCertificates[0], nil +} + +var testCertPEMBlock = []byte(`-----BEGIN CERTIFICATE----- +MIIFzjCCA7igAwIBAgIRAKnCQRdpkZ86oXYOd9hGrPgwCwYJKoZIhvcNAQELMB4x +HDAaBgNVBAoTE2xpbnV4Y29udGFpbmVycy5vcmcwHhcNMTUwNzE1MDQ1NjQ0WhcN +MjUwNzEyMDQ1NjQ0WjAeMRwwGgYDVQQKExNsaW51eGNvbnRhaW5lcnMub3JnMIIC +IjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAyViJkCzoxa1NYilXqGJog6xz +lSm4xt8KIzayc0JdB9VxEdIVdJqUzBAUtyCS4KZ9MbPmMEOX9NbBASL0tRK58/7K +Scq99Kj4XbVMLU1P/y5aW0ymnF0OpKbG6unmgAI2k/duRlbYHvGRdhlswpKl0Yst +l8i2kXOK0Rxcz90FewcEXGSnIYW21sz8YpBLfIZqOx6XEV36mOdi3MLrhUSAhXDw +Pay33Y7NonCQUBtiO7BT938cqI14FJrWdKon1UnODtzONcVBLTWtoe7D41+mx7EE +Taq5OPxBSe0DD6KQcPOZ7ZSJEhIqVKMvzLyiOJpyShmhm4OuGNoAG6jAuSij/9Kc +aLU4IitcrvFOuAo8M9OpiY9ZCR7Gb/qaPAXPAxE7Ci3f9DDNKXtPXDjhj3YG01+h +fNXMW3kCkMImn0A/+mZUMdCL87GWN2AN3Do5qaIc5XVEt1gp+LVqJeMoZ/lAeZWT +IbzcnkneOzE25m+bjw3r3WlR26amhyrWNwjGzRkgfEpw336kniX/GmwaCNgdNk+g +5aIbVxIHO0DbgkDBtdljR3VOic4djW/LtUIYIQ2egnPPyRR3fcFI+x5EQdVQYUXf +jpGIwovUDyG0Lkam2tpdeEXvLMZr8+Lhzu+H6vUFSj3cz6gcw/Xepw40FOkYdAI9 +LYB6nwpZLTVaOqZCJ2ECAwEAAaOCAQkwggEFMA4GA1UdDwEB/wQEAwIAoDATBgNV +HSUEDDAKBggrBgEFBQcDATAMBgNVHRMBAf8EAjAAMIHPBgNVHREEgccwgcSCCVVi +dW50dVByb4IRMTAuMTY3LjE2MC4xODMvMjSCHzIwMDE6MTVjMDo2NzM1OmVlMDA6 +OmU6ZTMxMy8xMjiCKWZkNTc6Yzg3ZDpmMWVlOmVlMDA6MjFkOjdkZmY6ZmUwOToz +NzUzLzY0gikyMDAxOjE1YzA6NjczNTplZTAwOjIxZDo3ZGZmOmZlMDk6Mzc1My82 +NIIbZmU4MDo6MjFkOjdkZmY6ZmUwOTozNzUzLzY0ghAxOTIuMTY4LjEyMi4xLzI0 +MAsGCSqGSIb3DQEBCwOCAgEAmcJUSBH7cLw3auEEV1KewtdqY1ARVB/pafAtbe9F +7ZKBbxUcS7cP3P1hRs5FH1bH44bIJKHxckctNUPqvC+MpXSryKinQ5KvGPNjGdlW +6EPlQr23btizC6hRdQ6RjEkCnQxhyTLmQ9n78nt47hjA96rFAhCUyfPdv9dI4Zux +bBTJekhCx5taamQKoxr7tql4Y2TchVlwASZvOfar8I0GxBRFT8w9IjckOSLoT9/s +OhlvXpeoxxFT7OHwqXEXdRUvw/8MGBo6JDnw+J/NGDBw3Z0goebG4FMT//xGSHia +czl3A0M0flk4/45L7N6vctwSqi+NxVaJRKeiYPZyzOO9K/d+No+WVBPwKmyP8icQ +b7FGTelPJOUolC6kmoyM+vyaNUoU4nz6lgOSHAtuqGNDWZWuX/gqzZw77hzDIgkN +qisOHZWPVlG/iUh1JBkbglBaPeaa3zf0XwSdgwwf4v8Z+YtEiRqkuFgQY70eQKI/ +CIkj1p0iW5IBEsEAGUGklz4ZwqJwH3lQIqDBzIgHe3EP4cXaYsx6oYhPSDdHLPv4 +HMZhl05DP75CEkEWRD0AIaL7SHdyuYUmCZ2zdrMI7TEDrAqcUuPbYpHcdJ2wnYmi +2G8XHJibfu4PCpIm1J8kPL8rqpdgW3moKR8Mp0HJQOH4tSBr1Ep7xNLP1wg6PIe+ +p7U= +-----END CERTIFICATE----- +`) + +var testKeyPEMBlock = []byte(`-----BEGIN RSA PRIVATE KEY----- +MIIJKAIBAAKCAgEAyViJkCzoxa1NYilXqGJog6xzlSm4xt8KIzayc0JdB9VxEdIV +dJqUzBAUtyCS4KZ9MbPmMEOX9NbBASL0tRK58/7KScq99Kj4XbVMLU1P/y5aW0ym +nF0OpKbG6unmgAI2k/duRlbYHvGRdhlswpKl0Ystl8i2kXOK0Rxcz90FewcEXGSn +IYW21sz8YpBLfIZqOx6XEV36mOdi3MLrhUSAhXDwPay33Y7NonCQUBtiO7BT938c +qI14FJrWdKon1UnODtzONcVBLTWtoe7D41+mx7EETaq5OPxBSe0DD6KQcPOZ7ZSJ +EhIqVKMvzLyiOJpyShmhm4OuGNoAG6jAuSij/9KcaLU4IitcrvFOuAo8M9OpiY9Z +CR7Gb/qaPAXPAxE7Ci3f9DDNKXtPXDjhj3YG01+hfNXMW3kCkMImn0A/+mZUMdCL +87GWN2AN3Do5qaIc5XVEt1gp+LVqJeMoZ/lAeZWTIbzcnkneOzE25m+bjw3r3WlR +26amhyrWNwjGzRkgfEpw336kniX/GmwaCNgdNk+g5aIbVxIHO0DbgkDBtdljR3VO +ic4djW/LtUIYIQ2egnPPyRR3fcFI+x5EQdVQYUXfjpGIwovUDyG0Lkam2tpdeEXv +LMZr8+Lhzu+H6vUFSj3cz6gcw/Xepw40FOkYdAI9LYB6nwpZLTVaOqZCJ2ECAwEA +AQKCAgBCe8GwoaOa4kaTCyOurg/kqqTftA8XW751MjJqbJdbZtcXE0+SWRiY6RZu +AYt+MntUVhrEBQ3AAsloHqq+v5g3QQJ6qz9d8g1Qo/SrYMPxdtTPINhC+VdEdu1n +1CQQUKrE4QbAoxxp20o0vOB0vweR0WsUm2ntTUGhGsRqvoh4vzBpcbLeFtDwzG7p +/MtwKtIZA1jOm0GMC5tRWet67cuiRFCPjOCJgAXWhWShjuk43FhdeNN1tIDaDOaT +Tzwn6V7o+W/9wUxsKTVUKwrzoTno5kKNgrn2XxUP2/sOxpb7NPS2xj0cgnMHz3qR +GBhYqGbkoOID/88U1acDew1oFktQL24yd8/cvooh7KLN3k5oSKjpKmGAKaMMwsSv +ccRSM9EkTtgTANLpSFiVF738drZw7UXUsvVTCF8WHhMtGD50XOahR02D1kZnpqpe +SdxJ9qFNEeozk6w56cTerJNz4od18/gQtNADcPI6WE+8NBrqYjN/X4CBNS76IEtp +5ddGbi6+4HgO5B0pU87f2bZH4BwR8XJ07wdMRyXXhmnKcnirkyqUtgHmLF3LZnGX ++Fph5KmhBGs/ZovBvnBI2nREsMfNvzffK7x3hyFXv6J+XxILk4i3LkgKLJFC+RY0 +sjWNQB5tHuA1dbq3AtsbfJcTK764kSaUsq0JoqPQgiSuiNoCIQKCAQEA1Fk4SR5I +H1QHlXeQ/k1sg6B5H0uosPAnAQxjuI8SvYkty+b4diP+CJIS4IphgLIItROORUFE +bOi6pj2D2oK04J55fhlJaE8LQs7i90nFXT4B09Ut4oBYGCz5aE/wAUxUanaq1dxj +K17y+ejlqh7yKTwupHOvIm4ddDwU1U5H9J/Cyywvp5fznVIGMJynVk7zriXYM6aC +tioNCbOTHwQxjYEaG3AwymXaI6sNwdNiAzgq6M7v43GF3IOj8SYK2VhVdLqLJPnL +6G5OqMRxxQtxOcSctFOuicu+Jq/KVWJGDaERQZJloHcBJCtO34ONswGJqC/PGoU+ +Ny/BOaZdLQDIpwKCAQEA8rxOKaLuOWEi4MDJuAgQYqpO9JxY0h3yN1YrspBuGezR +4Lzdh0vUh9Jr4npV723gGwA7r8AcqIPZvSk8MmcYVuwoxz9VWYeNP8P6cRc3bDO8 +shnSvFxV32gKTEH8fOH3/BlJOnbn62tebSFHnGxyh2WPsRbzAMOKj9Q3Yq6ad3DD +6rJhtopIedC3AWc3aVeO2FHPC+Lza0PhUVsHf5X7Bg+zQlHaaEXB0lysruXkDlU9 +WdW+Ajvo0enhOROgEa7QBC74NsKZF4KJGMGTaglydRtVYbqfx4QbfgDU5h2zaUnB +lRINZvKNYGRXDN944ymynE9bo4xfOERbWc68GFaItwKCAQBCY+qvIaKW+OSuHIXe +nEJTHPcBi9wgBdWMBF2hNEo9rAf/eiUweqxP7autPFajsAX85zJSAMft7Q1+MDlr +NfZrS+DcRfenfx8cMibP/eaQ8nQL0NjZuhrQ5C7OKD/3h+/UoWlkF9WBl9wLun8j +oy0/KyvCCtE0yIy47Jfu4NyqZNC4SQZVNbLa+uwogrHm0CRrzDU+YM75OUh+QgC7 +b8o2XajV70ux3ApJoI9ajEZWj1cLFrf1umaJvTaijKxTq8R8DF64nsjb0LETHugb +HSq3TvtXfdpSBrtayRdPfrw8QqFsiOLxOoPG1SuBwlWpI8/wH5J2zjXXdzzIU3VK +PrZ9AoIBAQDazTjbuT1pxZCN7donJEW42nHPdvttc4b5sJg1HpHQlrNdFIHPyl/q +iperD8FU0MM5M42Zz99FW4yzQW88s8ex2rCrYgCKcnC1cO/YbygLRduq4zIdjlHt +zrexo6132K0TtqtWowZNJHx6fIwziWH3gGn1JI2pO5o0KgQ+1MryLVi8v0zrIV1R +SP0dq6+8Kivd/GhY+5uWLhr1nct1i3k6Ln7Uojnw0ihzegxCn4FiFh32U4AyPVSR +m3PkYjdgmSZzDu+5VNJw6b6w7RT3eUqOGzRsorASRZgOjatbPpyRpOV1fU9NZAhi +QjBhrzMl+VlCIxqkowzWCHAb1QmiGqajAoIBAGYKD5h7jTgPFKFlMViTg8LoMcQl +9vbpmWkB+WdY5xXOwO0hO99rFDmLx6elsmYjdpq8zJkOFTnSB2o3IpenxZltNMsI ++aDlZWxDxokTxr6gbQPPrjePT1oON0/6sLEYkDOln8H1P9jmLPqTrET0DxCMgE5D +NE9TAEuUKVhRTWy6FSdP58hUimyVnlbnvbGOh2tviNO+TK/H7k0WjRg57Sz9XTHO +q36ob5TEsQngkTATEoksE9xhXFxtmTm/nu/26wN2Py49LSwu2aAYTfX/KhQKklNX +P/tP5//z+hGeba8/xv8YhEr7vhbnlBdwp0wHJj5g7nHAbYfo9ELbXSON8wc= +-----END RSA PRIVATE KEY----- +`) + +var testAltCertPEMBlock = []byte(`-----BEGIN CERTIFICATE----- +MIICEzCCAXygAwIBAgIQMIMChMLGrR+QvmQvpwAU6zANBgkqhkiG9w0BAQsFADAS +MRAwDgYDVQQKEwdBY21lIENvMCAXDTcwMDEwMTAwMDAwMFoYDzIwODQwMTI5MTYw +MDAwWjASMRAwDgYDVQQKEwdBY21lIENvMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCB +iQKBgQDuLnQAI3mDgey3VBzWnB2L39JUU4txjeVE6myuDqkM/uGlfjb9SjY1bIw4 +iA5sBBZzHi3z0h1YV8QPuxEbi4nW91IJm2gsvvZhIrCHS3l6afab4pZBl2+XsDul +rKBxKKtD1rGxlG4LjncdabFn9gvLZad2bSysqz/qTAUStTvqJQIDAQABo2gwZjAO +BgNVHQ8BAf8EBAMCAqQwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDwYDVR0TAQH/BAUw +AwEB/zAuBgNVHREEJzAlggtleGFtcGxlLmNvbYcEfwAAAYcQAAAAAAAAAAAAAAAA +AAAAATANBgkqhkiG9w0BAQsFAAOBgQCEcetwO59EWk7WiJsG4x8SY+UIAA+flUI9 +tyC4lNhbcF2Idq9greZwbYCqTTTr2XiRNSMLCOjKyI7ukPoPjo16ocHj+P3vZGfs +h1fIw3cSS2OolhloGw/XM6RWPWtPAlGykKLciQrBru5NAPvCMsb/I1DAceTiotQM +fblo6RBxUQ== +-----END CERTIFICATE-----`) + +var testAltKeyPEMBlock = []byte(`-----BEGIN RSA PRIVATE KEY----- +MIICXgIBAAKBgQDuLnQAI3mDgey3VBzWnB2L39JUU4txjeVE6myuDqkM/uGlfjb9 +SjY1bIw4iA5sBBZzHi3z0h1YV8QPuxEbi4nW91IJm2gsvvZhIrCHS3l6afab4pZB +l2+XsDulrKBxKKtD1rGxlG4LjncdabFn9gvLZad2bSysqz/qTAUStTvqJQIDAQAB +AoGAGRzwwir7XvBOAy5tM/uV6e+Zf6anZzus1s1Y1ClbjbE6HXbnWWF/wbZGOpet +3Zm4vD6MXc7jpTLryzTQIvVdfQbRc6+MUVeLKwZatTXtdZrhu+Jk7hx0nTPy8Jcb +uJqFk541aEw+mMogY/xEcfbWd6IOkp+4xqjlFLBEDytgbIECQQDvH/E6nk+hgN4H +qzzVtxxr397vWrjrIgPbJpQvBsafG7b0dA4AFjwVbFLmQcj2PprIMmPcQrooz8vp +jy4SHEg1AkEA/v13/5M47K9vCxmb8QeD/asydfsgS5TeuNi8DoUBEmiSJwma7FXY +fFUtxuvL7XvjwjN5B30pNEbc6Iuyt7y4MQJBAIt21su4b3sjXNueLKH85Q+phy2U +fQtuUE9txblTu14q3N7gHRZB4ZMhFYyDy8CKrN2cPg/Fvyt0Xlp/DoCzjA0CQQDU +y2ptGsuSmgUtWj3NM9xuwYPm+Z/F84K6+ARYiZ6PYj013sovGKUFfYAqVXVlxtIX +qyUBnu3X9ps8ZfjLZO7BAkEAlT4R5Yl6cGhaJQYZHOde3JEMhNRcVFMO8dJDaFeo +f9Oeos0UUothgiDktdQHxdNEwLjQf7lJJBzV+5OtwswCWA== +-----END RSA PRIVATE KEY-----`) diff --git a/vendor/github.com/lxc/lxd/shared/cgo.go b/vendor/github.com/lxc/lxd/shared/cgo.go new file mode 100644 index 0000000000..898f41e94f --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/cgo.go @@ -0,0 +1,12 @@ +// +build linux,cgo + +package shared + +// #cgo CFLAGS: -std=gnu11 -Wvla -Werror -fvisibility=hidden -Winit-self +// #cgo CFLAGS: -Wformat=2 -Wshadow -Wendif-labels -fasynchronous-unwind-tables +// #cgo CFLAGS: -pipe --param=ssp-buffer-size=4 -g -Wunused +// #cgo CFLAGS: -Werror=implicit-function-declaration +// #cgo CFLAGS: -Werror=return-type -Wendif-labels -Werror=overflow +// #cgo CFLAGS: -Wnested-externs -fexceptions +// #cgo LDFLAGS: -lutil -lpthread +import "C" diff --git a/vendor/github.com/lxc/lxd/shared/container.go b/vendor/github.com/lxc/lxd/shared/container.go new file mode 100644 index 0000000000..869c6c1fc5 --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/container.go @@ -0,0 +1,425 @@ +package shared + +import ( + "fmt" + "regexp" + "strconv" + "strings" + "time" + + "github.com/pkg/errors" + "gopkg.in/robfig/cron.v2" + + "github.com/lxc/lxd/shared/units" +) + +type ContainerAction string + +const ( + Stop ContainerAction = "stop" + Start ContainerAction = "start" + Restart ContainerAction = "restart" + Freeze ContainerAction = "freeze" + Unfreeze ContainerAction = "unfreeze" +) + +func IsInt64(value string) error { + if value == "" { + return nil + } + + _, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return fmt.Errorf("Invalid value for an integer: %s", value) + } + + return nil +} + +func IsUint8(value string) error { + if value == "" { + return nil + } + + _, err := strconv.ParseUint(value, 10, 8) + if err != nil { + return fmt.Errorf("Invalid value for an integer: %s. Must be between 0 and 255", value) + } + + return nil +} + +func IsUint32(value string) error { + if value == "" { + return nil + } + + _, err := strconv.ParseUint(value, 10, 32) + if err != nil { + return fmt.Errorf("Invalid value for uint32: %s: %v", value, err) + } + + return nil +} + +func IsPriority(value string) error { + if value == "" { + return nil + } + + valueInt, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return fmt.Errorf("Invalid value for an integer: %s", value) + } + + if valueInt < 0 || valueInt > 10 { + return fmt.Errorf("Invalid value for a limit '%s'. Must be between 0 and 10", value) + } + + return nil +} + +func IsBool(value string) error { + if value == "" { + return nil + } + + if !StringInSlice(strings.ToLower(value), []string{"true", "false", "yes", "no", "1", "0", "on", "off"}) { + return fmt.Errorf("Invalid value for a boolean: %s", value) + } + + return nil +} + +func IsOneOf(value string, valid []string) error { + if value == "" { + return nil + } + + if !StringInSlice(value, valid) { + return fmt.Errorf("Invalid value: %s (not one of %s)", value, valid) + } + + return nil +} + +func IsAny(value string) error { + return nil +} + +func IsNotEmpty(value string) error { + if value == "" { + return fmt.Errorf("Required value") + } + + return nil +} + +// IsDeviceID validates string is four lowercase hex characters suitable as Vendor or Device ID. +func IsDeviceID(value string) error { + if value == "" { + return nil + } + + regexHexLc, err := regexp.Compile("^[0-9a-f]+$") + if err != nil { + return err + } + + if len(value) != 4 || !regexHexLc.MatchString(value) { + return fmt.Errorf("Invalid value, must be four lower case hex characters") + } + + return nil +} + +// IsRootDiskDevice returns true if the given device representation is configured as root disk for +// a container. It typically get passed a specific entry of api.Container.Devices. +func IsRootDiskDevice(device map[string]string) bool { + // Root disk devices also need a non-empty "pool" property, but we can't check that here + // because this function is used with clients talking to older servers where there was no + // concept of a storage pool, and also it is used for migrating from old to new servers. + // The validation of the non-empty "pool" property is done inside the disk device itself. + if device["type"] == "disk" && device["path"] == "/" && device["source"] == "" { + return true + } + + return false +} + +// GetRootDiskDevice returns the container device that is configured as root disk +func GetRootDiskDevice(devices map[string]map[string]string) (string, map[string]string, error) { + var devName string + var dev map[string]string + + for n, d := range devices { + if IsRootDiskDevice(d) { + if devName != "" { + return "", nil, fmt.Errorf("More than one root device found") + } + + devName = n + dev = d + } + } + + if devName != "" { + return devName, dev, nil + } + + return "", nil, fmt.Errorf("No root device could be found") +} + +// KnownContainerConfigKeys maps all fully defined, well-known config keys +// to an appropriate checker function, which validates whether or not a +// given value is syntactically legal. +var KnownContainerConfigKeys = map[string]func(value string) error{ + "boot.autostart": IsBool, + "boot.autostart.delay": IsInt64, + "boot.autostart.priority": IsInt64, + "boot.stop.priority": IsInt64, + "boot.host_shutdown_timeout": IsInt64, + + "limits.cpu": func(value string) error { + if value == "" { + return nil + } + + // Validate the character set + match, _ := regexp.MatchString("^[-,0-9]*$", value) + if !match { + return fmt.Errorf("Invalid CPU limit syntax") + } + + // Validate first character + if strings.HasPrefix(value, "-") || strings.HasPrefix(value, ",") { + return fmt.Errorf("CPU limit can't start with a separator") + } + + // Validate last character + if strings.HasSuffix(value, "-") || strings.HasSuffix(value, ",") { + return fmt.Errorf("CPU limit can't end with a separator") + } + + return nil + }, + "limits.cpu.allowance": func(value string) error { + if value == "" { + return nil + } + + if strings.HasSuffix(value, "%") { + // Percentage based allocation + _, err := strconv.Atoi(strings.TrimSuffix(value, "%")) + if err != nil { + return err + } + + return nil + } + + // Time based allocation + fields := strings.SplitN(value, "/", 2) + if len(fields) != 2 { + return fmt.Errorf("Invalid allowance: %s", value) + } + + _, err := strconv.Atoi(strings.TrimSuffix(fields[0], "ms")) + if err != nil { + return err + } + + _, err = strconv.Atoi(strings.TrimSuffix(fields[1], "ms")) + if err != nil { + return err + } + + return nil + }, + "limits.cpu.priority": IsPriority, + + "limits.disk.priority": IsPriority, + + "limits.memory": func(value string) error { + if value == "" { + return nil + } + + if strings.HasSuffix(value, "%") { + _, err := strconv.ParseInt(strings.TrimSuffix(value, "%"), 10, 64) + if err != nil { + return err + } + + return nil + } + + _, err := units.ParseByteSizeString(value) + if err != nil { + return err + } + + return nil + }, + "limits.memory.enforce": func(value string) error { + return IsOneOf(value, []string{"soft", "hard"}) + }, + "limits.memory.swap": IsBool, + "limits.memory.swap.priority": IsPriority, + + "limits.network.priority": IsPriority, + + "limits.processes": IsInt64, + + "linux.kernel_modules": IsAny, + + "migration.incremental.memory": IsBool, + "migration.incremental.memory.iterations": IsUint32, + "migration.incremental.memory.goal": IsUint32, + + "nvidia.runtime": IsBool, + "nvidia.driver.capabilities": IsAny, + "nvidia.require.cuda": IsAny, + "nvidia.require.driver": IsAny, + + "security.nesting": IsBool, + "security.privileged": IsBool, + "security.devlxd": IsBool, + "security.devlxd.images": IsBool, + + "security.protection.delete": IsBool, + "security.protection.shift": IsBool, + + "security.idmap.base": IsUint32, + "security.idmap.isolated": IsBool, + "security.idmap.size": IsUint32, + + "security.syscalls.blacklist_default": IsBool, + "security.syscalls.blacklist_compat": IsBool, + "security.syscalls.blacklist": IsAny, + "security.syscalls.intercept.mknod": IsBool, + "security.syscalls.intercept.mount": IsBool, + "security.syscalls.intercept.mount.allowed": IsAny, + "security.syscalls.intercept.mount.shift": IsBool, + "security.syscalls.intercept.setxattr": IsBool, + "security.syscalls.whitelist": IsAny, + + "snapshots.schedule": func(value string) error { + if value == "" { + return nil + } + + if len(strings.Split(value, " ")) != 5 { + return fmt.Errorf("Schedule must be of the form: ") + } + + _, err := cron.Parse(fmt.Sprintf("* %s", value)) + if err != nil { + return errors.Wrap(err, "Error parsing schedule") + } + + return nil + }, + "snapshots.schedule.stopped": IsBool, + "snapshots.pattern": IsAny, + "snapshots.expiry": func(value string) error { + // Validate expression + _, err := GetSnapshotExpiry(time.Time{}, value) + return err + }, + + // Caller is responsible for full validation of any raw.* value + "raw.apparmor": IsAny, + "raw.lxc": IsAny, + "raw.seccomp": IsAny, + "raw.idmap": IsAny, + + "volatile.apply_template": IsAny, + "volatile.base_image": IsAny, + "volatile.last_state.idmap": IsAny, + "volatile.last_state.power": IsAny, + "volatile.idmap.base": IsAny, + "volatile.idmap.current": IsAny, + "volatile.idmap.next": IsAny, + "volatile.apply_quota": IsAny, +} + +// ConfigKeyChecker returns a function that will check whether or not +// a provide value is valid for the associate config key. Returns an +// error if the key is not known. The checker function only performs +// syntactic checking of the value, semantic and usage checking must +// be done by the caller. User defined keys are always considered to +// be valid, e.g. user.* and environment.* keys. +func ConfigKeyChecker(key string) (func(value string) error, error) { + if f, ok := KnownContainerConfigKeys[key]; ok { + return f, nil + } + + if strings.HasPrefix(key, "volatile.") { + if strings.HasSuffix(key, ".hwaddr") { + return IsAny, nil + } + + if strings.HasSuffix(key, ".name") { + return IsAny, nil + } + + if strings.HasSuffix(key, ".host_name") { + return IsAny, nil + } + + if strings.HasSuffix(key, ".mtu") { + return IsAny, nil + } + + if strings.HasSuffix(key, ".created") { + return IsAny, nil + } + + if strings.HasSuffix(key, ".id") { + return IsAny, nil + } + + if strings.HasSuffix(key, ".vlan") { + return IsAny, nil + } + + if strings.HasSuffix(key, ".spoofcheck") { + return IsAny, nil + } + + if strings.HasSuffix(key, ".apply_quota") { + return IsAny, nil + } + } + + if strings.HasPrefix(key, "environment.") { + return IsAny, nil + } + + if strings.HasPrefix(key, "user.") { + return IsAny, nil + } + + if strings.HasPrefix(key, "image.") { + return IsAny, nil + } + + if strings.HasPrefix(key, "limits.kernel.") && + (len(key) > len("limits.kernel.")) { + return IsAny, nil + } + + return nil, fmt.Errorf("Unknown configuration key: %s", key) +} + +// ContainerGetParentAndSnapshotName returns the parent container name, snapshot +// name, and whether it actually was a snapshot name. +func ContainerGetParentAndSnapshotName(name string) (string, string, bool) { + fields := strings.SplitN(name, SnapshotDelimiter, 2) + if len(fields) == 1 { + return name, "", false + } + + return fields[0], fields[1], true +} diff --git a/vendor/github.com/lxc/lxd/shared/eagain/file_unix.go b/vendor/github.com/lxc/lxd/shared/eagain/file_unix.go new file mode 100644 index 0000000000..bd671df906 --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/eagain/file_unix.go @@ -0,0 +1,53 @@ +package eagain + +import ( + "io" + + "golang.org/x/sys/unix" + + "github.com/lxc/lxd/shared" +) + +// Reader represents an io.Reader that handles EAGAIN +type Reader struct { + Reader io.Reader +} + +// Read behaves like io.Reader.Read but will retry on EAGAIN +func (er Reader) Read(p []byte) (int, error) { +again: + n, err := er.Reader.Read(p) + if err == nil { + return n, nil + } + + // keep retrying on EAGAIN + errno, ok := shared.GetErrno(err) + if ok && (errno == unix.EAGAIN || errno == unix.EINTR) { + goto again + } + + return n, err +} + +// Writer represents an io.Writer that handles EAGAIN +type Writer struct { + Writer io.Writer +} + +// Write behaves like io.Writer.Write but will retry on EAGAIN +func (ew Writer) Write(p []byte) (int, error) { +again: + n, err := ew.Writer.Write(p) + if err == nil { + return n, nil + } + + // keep retrying on EAGAIN + errno, ok := shared.GetErrno(err) + if ok && (errno == unix.EAGAIN || errno == unix.EINTR) { + goto again + } + + return n, err +} diff --git a/vendor/github.com/lxc/lxd/shared/ioprogress/data.go b/vendor/github.com/lxc/lxd/shared/ioprogress/data.go new file mode 100644 index 0000000000..59a7905889 --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/ioprogress/data.go @@ -0,0 +1,16 @@ +package ioprogress + +// The ProgressData struct represents new progress information on an operation +type ProgressData struct { + // Preferred string repreentation of progress (always set) + Text string + + // Progress in percent + Percentage int + + // Number of bytes transferred (for files) + TransferredBytes int64 + + // Total number of bytes (for files) + TotalBytes int64 +} diff --git a/vendor/github.com/lxc/lxd/shared/ioprogress/reader.go b/vendor/github.com/lxc/lxd/shared/ioprogress/reader.go new file mode 100644 index 0000000000..299cb6b29f --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/ioprogress/reader.go @@ -0,0 +1,25 @@ +package ioprogress + +import ( + "io" +) + +// ProgressReader is a wrapper around ReadCloser which allows for progress tracking +type ProgressReader struct { + io.ReadCloser + Tracker *ProgressTracker +} + +// Read in ProgressReader is the same as io.Read +func (pt *ProgressReader) Read(p []byte) (int, error) { + // Do normal reader tasks + n, err := pt.ReadCloser.Read(p) + + // Do the actual progress tracking + if pt.Tracker != nil { + pt.Tracker.total += int64(n) + pt.Tracker.update(n) + } + + return n, err +} diff --git a/vendor/github.com/lxc/lxd/shared/ioprogress/tracker.go b/vendor/github.com/lxc/lxd/shared/ioprogress/tracker.go new file mode 100644 index 0000000000..494f4b66c4 --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/ioprogress/tracker.go @@ -0,0 +1,77 @@ +package ioprogress + +import ( + "time" +) + +// ProgressTracker provides the stream information needed for tracking +type ProgressTracker struct { + Length int64 + Handler func(int64, int64) + + percentage float64 + total int64 + start *time.Time + last *time.Time +} + +func (pt *ProgressTracker) update(n int) { + // Skip the rest if no handler attached + if pt.Handler == nil { + return + } + + // Initialize start time if needed + if pt.start == nil { + cur := time.Now() + pt.start = &cur + pt.last = pt.start + } + + // Skip if no data to count + if n <= 0 { + return + } + + // Update interval handling + var percentage float64 + if pt.Length > 0 { + // If running in relative mode, check that we increased by at least 1% + percentage = float64(pt.total) / float64(pt.Length) * float64(100) + if percentage-pt.percentage < 0.9 { + return + } + } else { + // If running in absolute mode, check that at least a second elapsed + interval := time.Since(*pt.last).Seconds() + if interval < 1 { + return + } + } + + // Determine speed + speedInt := int64(0) + duration := time.Since(*pt.start).Seconds() + if duration > 0 { + speed := float64(pt.total) / duration + speedInt = int64(speed) + } + + // Determine progress + var progressInt int64 + if pt.Length > 0 { + pt.percentage = percentage + progressInt = int64(1 - (int(percentage) % 1) + int(percentage)) + if progressInt > 100 { + progressInt = 100 + } + } else { + progressInt = pt.total + + // Update timestamp + cur := time.Now() + pt.last = &cur + } + + pt.Handler(progressInt, speedInt) +} diff --git a/vendor/github.com/lxc/lxd/shared/ioprogress/writer.go b/vendor/github.com/lxc/lxd/shared/ioprogress/writer.go new file mode 100644 index 0000000000..f45b45e8b9 --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/ioprogress/writer.go @@ -0,0 +1,25 @@ +package ioprogress + +import ( + "io" +) + +// ProgressWriter is a wrapper around WriteCloser which allows for progress tracking +type ProgressWriter struct { + io.WriteCloser + Tracker *ProgressTracker +} + +// Write in ProgressWriter is the same as io.Write +func (pt *ProgressWriter) Write(p []byte) (int, error) { + // Do normal writer tasks + n, err := pt.WriteCloser.Write(p) + + // Do the actual progress tracking + if pt.Tracker != nil { + pt.Tracker.total += int64(n) + pt.Tracker.update(n) + } + + return n, err +} diff --git a/vendor/github.com/lxc/lxd/shared/json.go b/vendor/github.com/lxc/lxd/shared/json.go new file mode 100644 index 0000000000..09f1066653 --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/json.go @@ -0,0 +1,63 @@ +package shared + +import ( + "bytes" + "encoding/json" + "fmt" + + "github.com/lxc/lxd/shared/logger" +) + +type Jmap map[string]interface{} + +func (m Jmap) GetString(key string) (string, error) { + if val, ok := m[key]; !ok { + return "", fmt.Errorf("Response was missing `%s`", key) + } else if val, ok := val.(string); !ok { + return "", fmt.Errorf("`%s` was not a string", key) + } else { + return val, nil + } +} + +func (m Jmap) GetMap(key string) (Jmap, error) { + if val, ok := m[key]; !ok { + return nil, fmt.Errorf("Response was missing `%s`", key) + } else if val, ok := val.(map[string]interface{}); !ok { + return nil, fmt.Errorf("`%s` was not a map, got %T", key, m[key]) + } else { + return val, nil + } +} + +func (m Jmap) GetInt(key string) (int, error) { + if val, ok := m[key]; !ok { + return -1, fmt.Errorf("Response was missing `%s`", key) + } else if val, ok := val.(float64); !ok { + return -1, fmt.Errorf("`%s` was not an int", key) + } else { + return int(val), nil + } +} + +func (m Jmap) GetBool(key string) (bool, error) { + if val, ok := m[key]; !ok { + return false, fmt.Errorf("Response was missing `%s`", key) + } else if val, ok := val.(bool); !ok { + return false, fmt.Errorf("`%s` was not an int", key) + } else { + return val, nil + } +} + +func DebugJson(r *bytes.Buffer) { + pretty := &bytes.Buffer{} + if err := json.Indent(pretty, r.Bytes(), "\t", "\t"); err != nil { + logger.Debugf("error indenting json: %s", err) + return + } + + // Print the JSON without the last "\n" + str := pretty.String() + logger.Debugf("\n\t%s", str[0:len(str)-1]) +} diff --git a/vendor/github.com/lxc/lxd/shared/logger/format.go b/vendor/github.com/lxc/lxd/shared/logger/format.go new file mode 100644 index 0000000000..dcd11d3d19 --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/logger/format.go @@ -0,0 +1,25 @@ +package logger + +import ( + "encoding/json" + "fmt" + "runtime" +) + +// Pretty will attempt to convert any Go structure into a string suitable for logging +func Pretty(input interface{}) string { + pretty, err := json.MarshalIndent(input, "\t", "\t") + if err != nil { + return fmt.Sprintf("%v", input) + } + + return fmt.Sprintf("\n\t%s", pretty) +} + +// GetStack will convert the Go stack into a string suitable for logging +func GetStack() string { + buf := make([]byte, 1<<16) + n := runtime.Stack(buf, true) + + return fmt.Sprintf("\n\t%s", buf[:n]) +} diff --git a/vendor/github.com/lxc/lxd/shared/logger/log.go b/vendor/github.com/lxc/lxd/shared/logger/log.go new file mode 100644 index 0000000000..a031d8c14c --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/logger/log.go @@ -0,0 +1,101 @@ +// +build !logdebug + +package logger + +import ( + "fmt" +) + +// Logger is the main logging interface +type Logger interface { + Debug(msg string, ctx ...interface{}) + Info(msg string, ctx ...interface{}) + Warn(msg string, ctx ...interface{}) + Error(msg string, ctx ...interface{}) + Crit(msg string, ctx ...interface{}) +} + +// Log contains the logger used by all the logging functions +var Log Logger + +type nullLogger struct{} + +func (nl nullLogger) Debug(msg string, ctx ...interface{}) {} +func (nl nullLogger) Info(msg string, ctx ...interface{}) {} +func (nl nullLogger) Warn(msg string, ctx ...interface{}) {} +func (nl nullLogger) Error(msg string, ctx ...interface{}) {} +func (nl nullLogger) Crit(msg string, ctx ...interface{}) {} + +func init() { + Log = nullLogger{} +} + +// Debug logs a message (with optional context) at the DEBUG log level +func Debug(msg string, ctx ...interface{}) { + if Log != nil { + Log.Debug(msg, ctx...) + } +} + +// Info logs a message (with optional context) at the INFO log level +func Info(msg string, ctx ...interface{}) { + if Log != nil { + Log.Info(msg, ctx...) + } +} + +// Warn logs a message (with optional context) at the WARNING log level +func Warn(msg string, ctx ...interface{}) { + if Log != nil { + Log.Warn(msg, ctx...) + } +} + +// Error logs a message (with optional context) at the ERROR log level +func Error(msg string, ctx ...interface{}) { + if Log != nil { + Log.Error(msg, ctx...) + } +} + +// Crit logs a message (with optional context) at the CRITICAL log level +func Crit(msg string, ctx ...interface{}) { + if Log != nil { + Log.Crit(msg, ctx...) + } +} + +// Infof logs at the INFO log level using a standard printf format string +func Infof(format string, args ...interface{}) { + if Log != nil { + Log.Info(fmt.Sprintf(format, args...)) + } +} + +// Debugf logs at the DEBUG log level using a standard printf format string +func Debugf(format string, args ...interface{}) { + if Log != nil { + Log.Debug(fmt.Sprintf(format, args...)) + } +} + +// Warnf logs at the WARNING log level using a standard printf format string +func Warnf(format string, args ...interface{}) { + if Log != nil { + Log.Warn(fmt.Sprintf(format, args...)) + } +} + +// Errorf logs at the ERROR log level using a standard printf format string +func Errorf(format string, args ...interface{}) { + if Log != nil { + Log.Error(fmt.Sprintf(format, args...)) + } +} + +// Critf logs at the CRITICAL log level using a standard printf format string +func Critf(format string, args ...interface{}) { + if Log != nil { + Log.Crit(fmt.Sprintf(format, args...)) + } +} diff --git a/vendor/github.com/lxc/lxd/shared/logger/log_debug.go b/vendor/github.com/lxc/lxd/shared/logger/log_debug.go new file mode 100644 index 0000000000..49185537b3 --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/logger/log_debug.go @@ -0,0 +1,124 @@ +// +build logdebug + +package logger + +import ( + "fmt" + "runtime" +) + +type Logger interface { + Debug(msg string, ctx ...interface{}) + Info(msg string, ctx ...interface{}) + Warn(msg string, ctx ...interface{}) + Error(msg string, ctx ...interface{}) + Crit(msg string, ctx ...interface{}) +} + +var Log Logger + +type nullLogger struct{} + +func (nl nullLogger) Debug(msg string, ctx ...interface{}) {} +func (nl nullLogger) Info(msg string, ctx ...interface{}) {} +func (nl nullLogger) Warn(msg string, ctx ...interface{}) {} +func (nl nullLogger) Error(msg string, ctx ...interface{}) {} +func (nl nullLogger) Crit(msg string, ctx ...interface{}) {} + +func init() { + Log = nullLogger{} +} + +// General wrappers around Logger interface functions. +func Debug(msg string, ctx ...interface{}) { + if Log != nil { + pc, fn, line, _ := runtime.Caller(1) + msg := fmt.Sprintf("%s: %d: %s: %s", fn, line, runtime.FuncForPC(pc).Name(), msg) + Log.Debug(msg, ctx...) + } +} + +func Info(msg string, ctx ...interface{}) { + if Log != nil { + pc, fn, line, _ := runtime.Caller(1) + msg := fmt.Sprintf("%s: %d: %s: %s", fn, line, runtime.FuncForPC(pc).Name(), msg) + Log.Info(msg, ctx...) + } +} + +func Warn(msg string, ctx ...interface{}) { + if Log != nil { + pc, fn, line, _ := runtime.Caller(1) + msg := fmt.Sprintf("%s: %d: %s: %s", fn, line, runtime.FuncForPC(pc).Name(), msg) + Log.Warn(msg, ctx...) + } +} + +func Error(msg string, ctx ...interface{}) { + if Log != nil { + pc, fn, line, _ := runtime.Caller(1) + msg := fmt.Sprintf("%s: %d: %s: %s", fn, line, runtime.FuncForPC(pc).Name(), msg) + Log.Error(msg, ctx...) + } +} + +func Crit(msg string, ctx ...interface{}) { + if Log != nil { + pc, fn, line, _ := runtime.Caller(1) + msg := fmt.Sprintf("%s: %d: %s: %s", fn, line, runtime.FuncForPC(pc).Name(), msg) + Log.Crit(msg, ctx...) + } +} + +// Wrappers around Logger interface functions that send a string to the Logger +// by running it through fmt.Sprintf(). +func Infof(format string, args ...interface{}) { + if Log != nil { + msg := fmt.Sprintf(format, args...) + pc, fn, line, _ := runtime.Caller(1) + msg = fmt.Sprintf("%s: %d: %s: %s", fn, line, runtime.FuncForPC(pc).Name(), msg) + Log.Info(msg) + } +} + +func Debugf(format string, args ...interface{}) { + if Log != nil { + msg := fmt.Sprintf(format, args...) + pc, fn, line, _ := runtime.Caller(1) + msg = fmt.Sprintf("%s: %d: %s: %s", fn, line, runtime.FuncForPC(pc).Name(), msg) + Log.Debug(msg) + } +} + +func Warnf(format string, args ...interface{}) { + if Log != nil { + msg := fmt.Sprintf(format, args...) + pc, fn, line, _ := runtime.Caller(1) + msg = fmt.Sprintf("%s: %d: %s: %s", fn, line, runtime.FuncForPC(pc).Name(), msg) + Log.Warn(msg) + } +} + +func Errorf(format string, args ...interface{}) { + if Log != nil { + msg := fmt.Sprintf(format, args...) + pc, fn, line, _ := runtime.Caller(1) + msg = fmt.Sprintf("%s: %d: %s: %s", fn, line, runtime.FuncForPC(pc).Name(), msg) + Log.Error(msg) + } +} + +func Critf(format string, args ...interface{}) { + if Log != nil { + msg := fmt.Sprintf(format, args...) + pc, fn, line, _ := runtime.Caller(1) + msg = fmt.Sprintf("%s: %d: %s: %s", fn, line, runtime.FuncForPC(pc).Name(), msg) + Log.Crit(msg) + } +} + +func PrintStack() { + buf := make([]byte, 1<<16) + runtime.Stack(buf, true) + Errorf("%s", buf) +} diff --git a/vendor/github.com/lxc/lxd/shared/network.go b/vendor/github.com/lxc/lxd/shared/network.go new file mode 100644 index 0000000000..beb927e66c --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/network.go @@ -0,0 +1,568 @@ +package shared + +import ( + "crypto/tls" + "crypto/x509" + "encoding/pem" + "fmt" + "io" + "io/ioutil" + "net" + "net/http" + "strconv" + "strings" + "sync" + "time" + + "github.com/gorilla/websocket" + + "github.com/lxc/lxd/shared/api" + "github.com/lxc/lxd/shared/logger" +) + +func RFC3493Dialer(network, address string) (net.Conn, error) { + host, port, err := net.SplitHostPort(address) + if err != nil { + return nil, err + } + + addrs, err := net.LookupHost(host) + if err != nil { + return nil, err + } + for _, a := range addrs { + c, err := net.DialTimeout(network, net.JoinHostPort(a, port), 10*time.Second) + if err != nil { + continue + } + if tc, ok := c.(*net.TCPConn); ok { + tc.SetKeepAlive(true) + tc.SetKeepAlivePeriod(3 * time.Second) + } + return c, err + } + return nil, fmt.Errorf("Unable to connect to: " + address) +} + +// InitTLSConfig returns a tls.Config populated with default encryption +// parameters. This is used as baseline config for both client and server +// certificates used by LXD. +func InitTLSConfig() *tls.Config { + return &tls.Config{ + MinVersion: tls.VersionTLS12, + CipherSuites: []uint16{ + tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, + tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, + tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, + }, + PreferServerCipherSuites: true, + } +} + +func finalizeTLSConfig(tlsConfig *tls.Config, tlsRemoteCert *x509.Certificate) { + // Setup RootCA + if tlsConfig.RootCAs == nil { + tlsConfig.RootCAs, _ = systemCertPool() + } + + // Trusted certificates + if tlsRemoteCert != nil { + if tlsConfig.RootCAs == nil { + tlsConfig.RootCAs = x509.NewCertPool() + } + + // Make it a valid RootCA + tlsRemoteCert.IsCA = true + tlsRemoteCert.KeyUsage = x509.KeyUsageCertSign + + // Setup the pool + tlsConfig.RootCAs.AddCert(tlsRemoteCert) + + // Set the ServerName + if tlsRemoteCert.DNSNames != nil { + tlsConfig.ServerName = tlsRemoteCert.DNSNames[0] + } + } + + tlsConfig.BuildNameToCertificate() +} + +func GetTLSConfig(tlsClientCertFile string, tlsClientKeyFile string, tlsClientCAFile string, tlsRemoteCert *x509.Certificate) (*tls.Config, error) { + tlsConfig := InitTLSConfig() + + // Client authentication + if tlsClientCertFile != "" && tlsClientKeyFile != "" { + cert, err := tls.LoadX509KeyPair(tlsClientCertFile, tlsClientKeyFile) + if err != nil { + return nil, err + } + + tlsConfig.Certificates = []tls.Certificate{cert} + } + + if tlsClientCAFile != "" { + caCertificates, err := ioutil.ReadFile(tlsClientCAFile) + if err != nil { + return nil, err + } + + caPool := x509.NewCertPool() + caPool.AppendCertsFromPEM(caCertificates) + + tlsConfig.RootCAs = caPool + } + + finalizeTLSConfig(tlsConfig, tlsRemoteCert) + return tlsConfig, nil +} + +func GetTLSConfigMem(tlsClientCert string, tlsClientKey string, tlsClientCA string, tlsRemoteCertPEM string, insecureSkipVerify bool) (*tls.Config, error) { + tlsConfig := InitTLSConfig() + tlsConfig.InsecureSkipVerify = insecureSkipVerify + // Client authentication + if tlsClientCert != "" && tlsClientKey != "" { + cert, err := tls.X509KeyPair([]byte(tlsClientCert), []byte(tlsClientKey)) + if err != nil { + return nil, err + } + + tlsConfig.Certificates = []tls.Certificate{cert} + } + + var tlsRemoteCert *x509.Certificate + if tlsRemoteCertPEM != "" { + // Ignore any content outside of the PEM bytes we care about + certBlock, _ := pem.Decode([]byte(tlsRemoteCertPEM)) + if certBlock == nil { + return nil, fmt.Errorf("Invalid remote certificate") + } + + var err error + tlsRemoteCert, err = x509.ParseCertificate(certBlock.Bytes) + if err != nil { + return nil, err + } + } + + if tlsClientCA != "" { + caPool := x509.NewCertPool() + caPool.AppendCertsFromPEM([]byte(tlsClientCA)) + + tlsConfig.RootCAs = caPool + } + + finalizeTLSConfig(tlsConfig, tlsRemoteCert) + + return tlsConfig, nil +} + +func IsLoopback(iface *net.Interface) bool { + return int(iface.Flags&net.FlagLoopback) > 0 +} + +func WebsocketSendStream(conn *websocket.Conn, r io.Reader, bufferSize int) chan bool { + ch := make(chan bool) + + if r == nil { + close(ch) + return ch + } + + go func(conn *websocket.Conn, r io.Reader) { + in := ReaderToChannel(r, bufferSize) + for { + buf, ok := <-in + if !ok { + break + } + + w, err := conn.NextWriter(websocket.BinaryMessage) + if err != nil { + logger.Debugf("Got error getting next writer %s", err) + break + } + + _, err = w.Write(buf) + w.Close() + if err != nil { + logger.Debugf("Got err writing %s", err) + break + } + } + conn.WriteMessage(websocket.TextMessage, []byte{}) + ch <- true + }(conn, r) + + return ch +} + +func WebsocketRecvStream(w io.Writer, conn *websocket.Conn) chan bool { + ch := make(chan bool) + + go func(w io.Writer, conn *websocket.Conn) { + for { + mt, r, err := conn.NextReader() + if mt == websocket.CloseMessage { + logger.Debugf("Got close message for reader") + break + } + + if mt == websocket.TextMessage { + logger.Debugf("got message barrier") + break + } + + if err != nil { + logger.Debugf("Got error getting next reader %s, %s", err, w) + break + } + + buf, err := ioutil.ReadAll(r) + if err != nil { + logger.Debugf("Got error writing to writer %s", err) + break + } + + if w == nil { + continue + } + + i, err := w.Write(buf) + if i != len(buf) { + logger.Debugf("Didn't write all of buf") + break + } + if err != nil { + logger.Debugf("Error writing buf %s", err) + break + } + } + ch <- true + }(w, conn) + + return ch +} + +func WebsocketProxy(source *websocket.Conn, target *websocket.Conn) chan bool { + forward := func(in *websocket.Conn, out *websocket.Conn, ch chan bool) { + for { + mt, r, err := in.NextReader() + if err != nil { + break + } + + w, err := out.NextWriter(mt) + if err != nil { + break + } + + _, err = io.Copy(w, r) + w.Close() + if err != nil { + break + } + } + + ch <- true + } + + chSend := make(chan bool) + go forward(source, target, chSend) + + chRecv := make(chan bool) + go forward(target, source, chRecv) + + ch := make(chan bool) + go func() { + select { + case <-chSend: + case <-chRecv: + } + + source.Close() + target.Close() + + ch <- true + }() + + return ch +} + +func defaultReader(conn *websocket.Conn, r io.ReadCloser, readDone chan<- bool) { + /* For now, we don't need to adjust buffer sizes in + * WebsocketMirror, since it's used for interactive things like + * exec. + */ + in := ReaderToChannel(r, -1) + for { + buf, ok := <-in + if !ok { + r.Close() + logger.Debugf("sending write barrier") + conn.WriteMessage(websocket.TextMessage, []byte{}) + readDone <- true + return + } + w, err := conn.NextWriter(websocket.BinaryMessage) + if err != nil { + logger.Debugf("Got error getting next writer %s", err) + break + } + + _, err = w.Write(buf) + w.Close() + if err != nil { + logger.Debugf("Got err writing %s", err) + break + } + } + closeMsg := websocket.FormatCloseMessage(websocket.CloseNormalClosure, "") + conn.WriteMessage(websocket.CloseMessage, closeMsg) + readDone <- true + r.Close() +} + +func DefaultWriter(conn *websocket.Conn, w io.WriteCloser, writeDone chan<- bool) { + for { + mt, r, err := conn.NextReader() + if err != nil { + logger.Debugf("Got error getting next reader %s", err) + break + } + + if mt == websocket.CloseMessage { + logger.Debugf("Got close message for reader") + break + } + + if mt == websocket.TextMessage { + logger.Debugf("Got message barrier, resetting stream") + break + } + + buf, err := ioutil.ReadAll(r) + if err != nil { + logger.Debugf("Got error writing to writer %s", err) + break + } + i, err := w.Write(buf) + if i != len(buf) { + logger.Debugf("Didn't write all of buf") + break + } + if err != nil { + logger.Debugf("Error writing buf %s", err) + break + } + } + writeDone <- true + w.Close() +} + +// WebsocketIO is a wrapper implementing ReadWriteCloser on top of websocket +type WebsocketIO struct { + Conn *websocket.Conn + reader io.Reader + mu sync.Mutex +} + +func (w *WebsocketIO) Read(p []byte) (n int, err error) { + for { + // First read from this message + if w.reader == nil { + var mt int + + mt, w.reader, err = w.Conn.NextReader() + if err != nil { + return -1, err + } + + if mt == websocket.CloseMessage { + return 0, io.EOF + } + + if mt == websocket.TextMessage { + return 0, io.EOF + } + } + + // Perform the read itself + n, err := w.reader.Read(p) + if err == io.EOF { + // At the end of the message, reset reader + w.reader = nil + return n, nil + } + + if err != nil { + return -1, err + } + + return n, nil + } +} + +func (w *WebsocketIO) Write(p []byte) (n int, err error) { + w.mu.Lock() + defer w.mu.Unlock() + wr, err := w.Conn.NextWriter(websocket.BinaryMessage) + if err != nil { + return -1, err + } + defer wr.Close() + + n, err = wr.Write(p) + if err != nil { + return -1, err + } + + return n, nil +} + +// Close sends a control message indicating the stream is finished, but it does not actually close +// the socket. +func (w *WebsocketIO) Close() error { + w.mu.Lock() + defer w.mu.Unlock() + // Target expects to get a control message indicating stream is finished. + return w.Conn.WriteMessage(websocket.TextMessage, []byte{}) +} + +// WebsocketMirror allows mirroring a reader to a websocket and taking the +// result and writing it to a writer. This function allows for multiple +// mirrorings and correctly negotiates stream endings. However, it means any +// websocket.Conns passed to it are live when it returns, and must be closed +// explicitly. +type WebSocketMirrorReader func(conn *websocket.Conn, r io.ReadCloser, readDone chan<- bool) +type WebSocketMirrorWriter func(conn *websocket.Conn, w io.WriteCloser, writeDone chan<- bool) + +func WebsocketMirror(conn *websocket.Conn, w io.WriteCloser, r io.ReadCloser, Reader WebSocketMirrorReader, Writer WebSocketMirrorWriter) (chan bool, chan bool) { + readDone := make(chan bool, 1) + writeDone := make(chan bool, 1) + + ReadFunc := Reader + if ReadFunc == nil { + ReadFunc = defaultReader + } + + WriteFunc := Writer + if WriteFunc == nil { + WriteFunc = DefaultWriter + } + + go ReadFunc(conn, r, readDone) + go WriteFunc(conn, w, writeDone) + + return readDone, writeDone +} + +func WebsocketConsoleMirror(conn *websocket.Conn, w io.WriteCloser, r io.ReadCloser) (chan bool, chan bool) { + readDone := make(chan bool, 1) + writeDone := make(chan bool, 1) + + go DefaultWriter(conn, w, writeDone) + + go func(conn *websocket.Conn, r io.ReadCloser) { + in := ReaderToChannel(r, -1) + for { + buf, ok := <-in + if !ok { + r.Close() + logger.Debugf("sending write barrier") + conn.WriteMessage(websocket.TextMessage, []byte{}) + readDone <- true + return + } + w, err := conn.NextWriter(websocket.BinaryMessage) + if err != nil { + logger.Debugf("Got error getting next writer %s", err) + break + } + + _, err = w.Write(buf) + w.Close() + if err != nil { + logger.Debugf("Got err writing %s", err) + break + } + } + closeMsg := websocket.FormatCloseMessage(websocket.CloseNormalClosure, "") + conn.WriteMessage(websocket.CloseMessage, closeMsg) + readDone <- true + r.Close() + }(conn, r) + + return readDone, writeDone +} + +var WebsocketUpgrader = websocket.Upgrader{ + CheckOrigin: func(r *http.Request) bool { return true }, +} + +// AllocatePort asks the kernel for a free open port that is ready to use +func AllocatePort() (int, error) { + addr, err := net.ResolveTCPAddr("tcp", "localhost:0") + if err != nil { + return -1, err + } + + l, err := net.ListenTCP("tcp", addr) + if err != nil { + return -1, err + } + defer l.Close() + return l.Addr().(*net.TCPAddr).Port, nil +} + +func NetworkGetCounters(ifName string) api.NetworkStateCounters { + counters := api.NetworkStateCounters{} + // Get counters + content, err := ioutil.ReadFile("/proc/net/dev") + if err == nil { + for _, line := range strings.Split(string(content), "\n") { + fields := strings.Fields(line) + + if len(fields) != 17 { + continue + } + + intName := strings.TrimSuffix(fields[0], ":") + if intName != ifName { + continue + } + + rxBytes, err := strconv.ParseInt(fields[1], 10, 64) + if err != nil { + continue + } + + rxPackets, err := strconv.ParseInt(fields[2], 10, 64) + if err != nil { + continue + } + + txBytes, err := strconv.ParseInt(fields[9], 10, 64) + if err != nil { + continue + } + + txPackets, err := strconv.ParseInt(fields[10], 10, 64) + if err != nil { + continue + } + + counters.BytesSent = txBytes + counters.BytesReceived = rxBytes + counters.PacketsSent = txPackets + counters.PacketsReceived = rxPackets + break + } + } + + return counters +} diff --git a/vendor/github.com/lxc/lxd/shared/network_unix.go b/vendor/github.com/lxc/lxd/shared/network_unix.go new file mode 100644 index 0000000000..1e5cdc7cca --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/network_unix.go @@ -0,0 +1,26 @@ +// +build !windows + +package shared + +import ( + "crypto/x509" + "io/ioutil" +) + +func systemCertPool() (*x509.CertPool, error) { + // Get the system pool + pool, err := x509.SystemCertPool() + if err != nil { + return nil, err + } + + // Attempt to load the system's pool too (for snaps) + if PathExists("/var/lib/snapd/hostfs/etc/ssl/certs/ca-certificates.crt") { + snapCerts, err := ioutil.ReadFile("/var/lib/snapd/hostfs/etc/ssl/certs/ca-certificates.crt") + if err == nil { + pool.AppendCertsFromPEM(snapCerts) + } + } + + return pool, nil +} diff --git a/vendor/github.com/lxc/lxd/shared/network_windows.go b/vendor/github.com/lxc/lxd/shared/network_windows.go new file mode 100644 index 0000000000..e07aa27824 --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/network_windows.go @@ -0,0 +1,60 @@ +// +build windows + +package shared + +import ( + "crypto/x509" + "fmt" + "sync" + "unsafe" + + "golang.org/x/sys/windows" +) + +var once sync.Once +var systemRoots *x509.CertPool + +func systemCertPool() (*x509.CertPool, error) { + once.Do(initSystemRoots) + if systemRoots == nil { + return nil, fmt.Errorf("Bad system root pool") + } + return systemRoots, nil +} + +func initSystemRoots() { + const CRYPT_E_NOT_FOUND = 0x80092004 + + store, err := windows.CertOpenSystemStore(0, windows.StringToUTF16Ptr("ROOT")) + if err != nil { + systemRoots = nil + return + } + defer windows.CertCloseStore(store, 0) + + roots := x509.NewCertPool() + var cert *windows.CertContext + for { + cert, err = windows.CertEnumCertificatesInStore(store, cert) + if err != nil { + if errno, ok := err.(windows.Errno); ok { + if errno == CRYPT_E_NOT_FOUND { + break + } + } + systemRoots = nil + return + } + if cert == nil { + break + } + // Copy the buf, since ParseCertificate does not create its own copy. + buf := (*[1 << 20]byte)(unsafe.Pointer(cert.EncodedCert))[:] + buf2 := make([]byte, cert.Length) + copy(buf2, buf) + if c, err := x509.ParseCertificate(buf2); err == nil { + roots.AddCert(c) + } + } + systemRoots = roots +} diff --git a/vendor/github.com/lxc/lxd/shared/proxy.go b/vendor/github.com/lxc/lxd/shared/proxy.go new file mode 100644 index 0000000000..56bb19a9e8 --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/proxy.go @@ -0,0 +1,162 @@ +package shared + +import ( + "fmt" + "net" + "net/http" + "net/url" + "os" + "strings" + "sync" +) + +var ( + httpProxyEnv = &envOnce{ + names: []string{"HTTP_PROXY", "http_proxy"}, + } + httpsProxyEnv = &envOnce{ + names: []string{"HTTPS_PROXY", "https_proxy"}, + } + noProxyEnv = &envOnce{ + names: []string{"NO_PROXY", "no_proxy"}, + } +) + +type envOnce struct { + names []string + once sync.Once + val string +} + +func (e *envOnce) Get() string { + e.once.Do(e.init) + return e.val +} + +func (e *envOnce) init() { + for _, n := range e.names { + e.val = os.Getenv(n) + if e.val != "" { + return + } + } +} + +// This is basically the same as golang's ProxyFromEnvironment, except it +// doesn't fall back to http_proxy when https_proxy isn't around, which is +// incorrect behavior. It still respects HTTP_PROXY, HTTPS_PROXY, and NO_PROXY. +func ProxyFromEnvironment(req *http.Request) (*url.URL, error) { + return ProxyFromConfig("", "", "")(req) +} + +func ProxyFromConfig(httpsProxy string, httpProxy string, noProxy string) func(req *http.Request) (*url.URL, error) { + return func(req *http.Request) (*url.URL, error) { + var proxy, port string + var err error + + switch req.URL.Scheme { + case "https": + proxy = httpsProxy + if proxy == "" { + proxy = httpsProxyEnv.Get() + } + port = ":443" + case "http": + proxy = httpProxy + if proxy == "" { + proxy = httpProxyEnv.Get() + } + port = ":80" + default: + return nil, fmt.Errorf("unknown scheme %s", req.URL.Scheme) + } + + if proxy == "" { + return nil, nil + } + + addr := req.URL.Host + if !hasPort(addr) { + addr = addr + port + } + + use, err := useProxy(addr, noProxy) + if err != nil { + return nil, err + } + if !use { + return nil, nil + } + + proxyURL, err := url.Parse(proxy) + if err != nil || !strings.HasPrefix(proxyURL.Scheme, "http") { + // proxy was bogus. Try prepending "http://" to it and + // see if that parses correctly. If not, we fall + // through and complain about the original one. + if proxyURL, err := url.Parse("http://" + proxy); err == nil { + return proxyURL, nil + } + } + if err != nil { + return nil, fmt.Errorf("invalid proxy address %q: %v", proxy, err) + } + return proxyURL, nil + } +} + +func hasPort(s string) bool { + return strings.LastIndex(s, ":") > strings.LastIndex(s, "]") +} + +func useProxy(addr string, noProxy string) (bool, error) { + if noProxy == "" { + noProxy = noProxyEnv.Get() + } + + if len(addr) == 0 { + return true, nil + } + host, _, err := net.SplitHostPort(addr) + if err != nil { + return false, nil + } + if host == "localhost" { + return false, nil + } + if ip := net.ParseIP(host); ip != nil { + if ip.IsLoopback() { + return false, nil + } + } + + if noProxy == "*" { + return false, nil + } + + addr = strings.ToLower(strings.TrimSpace(addr)) + if hasPort(addr) { + addr = addr[:strings.LastIndex(addr, ":")] + } + + for _, p := range strings.Split(noProxy, ",") { + p = strings.ToLower(strings.TrimSpace(p)) + if len(p) == 0 { + continue + } + if hasPort(p) { + p = p[:strings.LastIndex(p, ":")] + } + if addr == p { + return false, nil + } + if p[0] == '.' && (strings.HasSuffix(addr, p) || addr == p[1:]) { + // noProxy ".foo.com" matches "bar.foo.com" or "foo.com" + return false, nil + } + if p[0] != '.' && strings.HasSuffix(addr, p) && addr[len(addr)-len(p)-1] == '.' { + // noProxy "foo.com" matches "bar.foo.com" + return false, nil + } + } + return true, nil +} diff --git a/vendor/github.com/lxc/lxd/shared/units/units.go b/vendor/github.com/lxc/lxd/shared/units/units.go new file mode 100644 index 0000000000..23514c9c30 --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/units/units.go @@ -0,0 +1,162 @@ +package units + +import ( + "fmt" + "strconv" +) + +// ParseByteSizeString parses a human representation of an amount of +// data into a number of bytes +func ParseByteSizeString(input string) (int64, error) { + // Empty input + if input == "" { + return 0, nil + } + + // Find where the suffix begins + suffixLen := 0 + for i, chr := range []byte(input) { + _, err := strconv.Atoi(string([]byte{chr})) + if err != nil { + suffixLen = len(input) - i + break + } + } + + if suffixLen == len(input) { + return -1, fmt.Errorf("Invalid value: %s", input) + } + + // Extract the suffix + suffix := input[len(input)-suffixLen:] + + // Extract the value + value := input[0 : len(input)-suffixLen] + valueInt, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return -1, fmt.Errorf("Invalid integer: %s", input) + } + + // Figure out the multiplicator + multiplicator := int64(0) + switch suffix { + case "", "B", " bytes": + multiplicator = 1 + case "kB": + multiplicator = 1000 + case "MB": + multiplicator = 1000 * 1000 + case "GB": + multiplicator = 1000 * 1000 * 1000 + case "TB": + multiplicator = 1000 * 1000 * 1000 * 1000 + case "PB": + multiplicator = 1000 * 1000 * 1000 * 1000 * 1000 + case "EB": + multiplicator = 1000 * 1000 * 1000 * 1000 * 1000 * 1000 + case "KiB": + multiplicator = 1024 + case "MiB": + multiplicator = 1024 * 1024 + case "GiB": + multiplicator = 1024 * 1024 * 1024 + case "TiB": + multiplicator = 1024 * 1024 * 1024 * 1024 + case "PiB": + multiplicator = 1024 * 1024 * 1024 * 1024 * 1024 + case "EiB": + multiplicator = 1024 * 1024 * 1024 * 1024 * 1024 * 1024 + default: + return -1, fmt.Errorf("Invalid value: %s", input) + } + + return valueInt * multiplicator, nil +} + +// ParseBitSizeString parses a human representation of an amount of +// data into a number of bits +func ParseBitSizeString(input string) (int64, error) { + // Empty input + if input == "" { + return 0, nil + } + + // Find where the suffix begins + suffixLen := 0 + for i, chr := range []byte(input) { + _, err := strconv.Atoi(string([]byte{chr})) + if err != nil { + suffixLen = len(input) - i + break + } + } + + if suffixLen == len(input) { + return -1, fmt.Errorf("Invalid value: %s", input) + } + + // Extract the suffix + suffix := input[len(input)-suffixLen:] + + // Extract the value + value := input[0 : len(input)-suffixLen] + valueInt, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return -1, fmt.Errorf("Invalid integer: %s", input) + } + + // Figure out the multiplicator + multiplicator := int64(0) + switch suffix { + case "", "bit": + multiplicator = 1 + case "kbit": + multiplicator = 1000 + case "Mbit": + multiplicator = 1000 * 1000 + case "Gbit": + multiplicator = 1000 * 1000 * 1000 + case "Tbit": + multiplicator = 1000 * 1000 * 1000 * 1000 + case "Pbit": + multiplicator = 1000 * 1000 * 1000 * 1000 * 1000 + case "Ebit": + multiplicator = 1000 * 1000 * 1000 * 1000 * 1000 * 1000 + case "Kibit": + multiplicator = 1024 + case "Mibit": + multiplicator = 1024 * 1024 + case "Gibit": + multiplicator = 1024 * 1024 * 1024 + case "Tibit": + multiplicator = 1024 * 1024 * 1024 * 1024 + case "Pibit": + multiplicator = 1024 * 1024 * 1024 * 1024 * 1024 + case "Eibit": + multiplicator = 1024 * 1024 * 1024 * 1024 * 1024 * 1024 + + default: + return -1, fmt.Errorf("Unsupported suffix: %s", suffix) + } + + return valueInt * multiplicator, nil +} + +// GetByteSizeString takes a number of bytes and precision and returns a +// human representation of the amount of data +func GetByteSizeString(input int64, precision uint) string { + if input < 1000 { + return fmt.Sprintf("%dB", input) + } + + value := float64(input) + + for _, unit := range []string{"kB", "MB", "GB", "TB", "PB", "EB"} { + value = value / 1000 + if value < 1000 { + return fmt.Sprintf("%.*f%s", precision, value, unit) + } + } + + return fmt.Sprintf("%.*fEB", precision, value) +} diff --git a/vendor/github.com/lxc/lxd/shared/util.go b/vendor/github.com/lxc/lxd/shared/util.go new file mode 100644 index 0000000000..e307f985fe --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/util.go @@ -0,0 +1,1115 @@ +package shared + +import ( + "bufio" + "bytes" + "crypto/rand" + "encoding/gob" + "encoding/hex" + "encoding/json" + "fmt" + "hash" + "io" + "io/ioutil" + "net/http" + "net/url" + "os" + "os/exec" + "path" + "path/filepath" + "reflect" + "regexp" + "runtime" + "strconv" + "strings" + "time" + + "github.com/flosch/pongo2" + "github.com/pkg/errors" + + "github.com/lxc/lxd/shared/cancel" + "github.com/lxc/lxd/shared/ioprogress" + "github.com/lxc/lxd/shared/units" +) + +const SnapshotDelimiter = "/" +const DefaultPort = "8443" + +// URLEncode encodes a path and query parameters to a URL. +func URLEncode(path string, query map[string]string) (string, error) { + u, err := url.Parse(path) + if err != nil { + return "", err + } + + params := url.Values{} + for key, value := range query { + params.Add(key, value) + } + u.RawQuery = params.Encode() + return u.String(), nil +} + +// AddSlash adds a slash to the end of paths if they don't already have one. +// This can be useful for rsyncing things, since rsync has behavior present on +// the presence or absence of a trailing slash. +func AddSlash(path string) string { + if path[len(path)-1] != '/' { + return path + "/" + } + + return path +} + +func PathExists(name string) bool { + _, err := os.Lstat(name) + if err != nil && os.IsNotExist(err) { + return false + } + return true +} + +// PathIsEmpty checks if the given path is empty. +func PathIsEmpty(path string) (bool, error) { + f, err := os.Open(path) + if err != nil { + return false, err + } + defer f.Close() + + // read in ONLY one file + _, err = f.Readdir(1) + + // and if the file is EOF... well, the dir is empty. + if err == io.EOF { + return true, nil + } + return false, err +} + +// IsDir returns true if the given path is a directory. +func IsDir(name string) bool { + stat, err := os.Stat(name) + if err != nil { + return false + } + return stat.IsDir() +} + +// IsUnixSocket returns true if the given path is either a Unix socket +// or a symbolic link pointing at a Unix socket. +func IsUnixSocket(path string) bool { + stat, err := os.Stat(path) + if err != nil { + return false + } + return (stat.Mode() & os.ModeSocket) == os.ModeSocket +} + +// HostPath returns the host path for the provided path +// On a normal system, this does nothing +// When inside of a snap environment, returns the real path +func HostPath(path string) string { + // Ignore empty paths + if len(path) == 0 { + return path + } + + // Don't prefix stdin/stdout + if path == "-" { + return path + } + + // Check if we're running in a snap package + _, inSnap := os.LookupEnv("SNAP") + snapName := os.Getenv("SNAP_NAME") + if !inSnap || snapName != "lxd" { + return path + } + + // Handle relative paths + if path[0] != os.PathSeparator { + // Use the cwd of the parent as snap-confine alters our own cwd on launch + ppid := os.Getppid() + if ppid < 1 { + return path + } + + pwd, err := os.Readlink(fmt.Sprintf("/proc/%d/cwd", ppid)) + if err != nil { + return path + } + + path = filepath.Clean(strings.Join([]string{pwd, path}, string(os.PathSeparator))) + } + + // Check if the path is already snap-aware + for _, prefix := range []string{"/dev", "/snap", "/var/snap", "/var/lib/snapd"} { + if path == prefix || strings.HasPrefix(path, fmt.Sprintf("%s/", prefix)) { + return path + } + } + + return fmt.Sprintf("/var/lib/snapd/hostfs%s", path) +} + +// VarPath returns the provided path elements joined by a slash and +// appended to the end of $LXD_DIR, which defaults to /var/lib/lxd. +func VarPath(path ...string) string { + varDir := os.Getenv("LXD_DIR") + if varDir == "" { + varDir = "/var/lib/lxd" + } + + items := []string{varDir} + items = append(items, path...) + return filepath.Join(items...) +} + +// CachePath returns the directory that LXD should its cache under. If LXD_DIR is +// set, this path is $LXD_DIR/cache, otherwise it is /var/cache/lxd. +func CachePath(path ...string) string { + varDir := os.Getenv("LXD_DIR") + logDir := "/var/cache/lxd" + if varDir != "" { + logDir = filepath.Join(varDir, "cache") + } + items := []string{logDir} + items = append(items, path...) + return filepath.Join(items...) +} + +// LogPath returns the directory that LXD should put logs under. If LXD_DIR is +// set, this path is $LXD_DIR/logs, otherwise it is /var/log/lxd. +func LogPath(path ...string) string { + varDir := os.Getenv("LXD_DIR") + logDir := "/var/log/lxd" + if varDir != "" { + logDir = filepath.Join(varDir, "logs") + } + items := []string{logDir} + items = append(items, path...) + return filepath.Join(items...) +} + +func ParseLXDFileHeaders(headers http.Header) (uid int64, gid int64, mode int, type_ string, write string) { + uid, err := strconv.ParseInt(headers.Get("X-LXD-uid"), 10, 64) + if err != nil { + uid = -1 + } + + gid, err = strconv.ParseInt(headers.Get("X-LXD-gid"), 10, 64) + if err != nil { + gid = -1 + } + + mode, err = strconv.Atoi(headers.Get("X-LXD-mode")) + if err != nil { + mode = -1 + } else { + rawMode, err := strconv.ParseInt(headers.Get("X-LXD-mode"), 0, 0) + if err == nil { + mode = int(os.FileMode(rawMode) & os.ModePerm) + } + } + + type_ = headers.Get("X-LXD-type") + /* backwards compat: before "type" was introduced, we could only + * manipulate files + */ + if type_ == "" { + type_ = "file" + } + + write = headers.Get("X-LXD-write") + /* backwards compat: before "write" was introduced, we could only + * overwrite files + */ + if write == "" { + write = "overwrite" + } + + return uid, gid, mode, type_, write +} + +func ReadToJSON(r io.Reader, req interface{}) error { + buf, err := ioutil.ReadAll(r) + if err != nil { + return err + } + + return json.Unmarshal(buf, req) +} + +func ReaderToChannel(r io.Reader, bufferSize int) <-chan []byte { + if bufferSize <= 128*1024 { + bufferSize = 128 * 1024 + } + + ch := make(chan ([]byte)) + + go func() { + readSize := 128 * 1024 + offset := 0 + buf := make([]byte, bufferSize) + + for { + read := buf[offset : offset+readSize] + nr, err := r.Read(read) + offset += nr + if offset > 0 && (offset+readSize >= bufferSize || err != nil) { + ch <- buf[0:offset] + offset = 0 + buf = make([]byte, bufferSize) + } + + if err != nil { + close(ch) + break + } + } + }() + + return ch +} + +// Returns a random base64 encoded string from crypto/rand. +func RandomCryptoString() (string, error) { + buf := make([]byte, 32) + n, err := rand.Read(buf) + if err != nil { + return "", err + } + + if n != len(buf) { + return "", fmt.Errorf("not enough random bytes read") + } + + return hex.EncodeToString(buf), nil +} + +func SplitExt(fpath string) (string, string) { + b := path.Base(fpath) + ext := path.Ext(fpath) + return b[:len(b)-len(ext)], ext +} + +func AtoiEmptyDefault(s string, def int) (int, error) { + if s == "" { + return def, nil + } + + return strconv.Atoi(s) +} + +func ReadStdin() ([]byte, error) { + buf := bufio.NewReader(os.Stdin) + line, _, err := buf.ReadLine() + if err != nil { + return nil, err + } + return line, nil +} + +func WriteAll(w io.Writer, data []byte) error { + buf := bytes.NewBuffer(data) + + toWrite := int64(buf.Len()) + for { + n, err := io.Copy(w, buf) + if err != nil { + return err + } + + toWrite -= n + if toWrite <= 0 { + return nil + } + } +} + +// FileMove tries to move a file by using os.Rename, +// if that fails it tries to copy the file and remove the source. +func FileMove(oldPath string, newPath string) error { + err := os.Rename(oldPath, newPath) + if err == nil { + return nil + } + + err = FileCopy(oldPath, newPath) + if err != nil { + return err + } + + os.Remove(oldPath) + + return nil +} + +// FileCopy copies a file, overwriting the target if it exists. +func FileCopy(source string, dest string) error { + fi, err := os.Lstat(source) + if err != nil { + return err + } + + _, uid, gid := GetOwnerMode(fi) + + if fi.Mode()&os.ModeSymlink != 0 { + target, err := os.Readlink(source) + if err != nil { + return err + } + + if PathExists(dest) { + err = os.Remove(dest) + if err != nil { + return err + } + } + + err = os.Symlink(target, dest) + if err != nil { + return err + } + + if runtime.GOOS != "windows" { + return os.Lchown(dest, uid, gid) + } + + return nil + } + + s, err := os.Open(source) + if err != nil { + return err + } + defer s.Close() + + d, err := os.Create(dest) + if err != nil { + if os.IsExist(err) { + d, err = os.OpenFile(dest, os.O_WRONLY, fi.Mode()) + if err != nil { + return err + } + } else { + return err + } + } + defer d.Close() + + _, err = io.Copy(d, s) + if err != nil { + return err + } + + /* chown not supported on windows */ + if runtime.GOOS != "windows" { + return d.Chown(uid, gid) + } + + return nil +} + +// DirCopy copies a directory recursively, overwriting the target if it exists. +func DirCopy(source string, dest string) error { + // Get info about source. + info, err := os.Stat(source) + if err != nil { + return errors.Wrapf(err, "failed to get source directory info") + } + + if !info.IsDir() { + return fmt.Errorf("source is not a directory") + } + + // Remove dest if it already exists. + if PathExists(dest) { + err := os.RemoveAll(dest) + if err != nil { + return errors.Wrapf(err, "failed to remove destination directory %s", dest) + } + } + + // Create dest. + err = os.MkdirAll(dest, info.Mode()) + if err != nil { + return errors.Wrapf(err, "failed to create destination directory %s", dest) + } + + // Copy all files. + entries, err := ioutil.ReadDir(source) + if err != nil { + return errors.Wrapf(err, "failed to read source directory %s", source) + } + + for _, entry := range entries { + + sourcePath := filepath.Join(source, entry.Name()) + destPath := filepath.Join(dest, entry.Name()) + + if entry.IsDir() { + err := DirCopy(sourcePath, destPath) + if err != nil { + return errors.Wrapf(err, "failed to copy sub-directory from %s to %s", sourcePath, destPath) + } + } else { + err := FileCopy(sourcePath, destPath) + if err != nil { + return errors.Wrapf(err, "failed to copy file from %s to %s", sourcePath, destPath) + } + } + + } + + return nil +} + +type BytesReadCloser struct { + Buf *bytes.Buffer +} + +func (r BytesReadCloser) Read(b []byte) (n int, err error) { + return r.Buf.Read(b) +} + +func (r BytesReadCloser) Close() error { + /* no-op since we're in memory */ + return nil +} + +func IsSnapshot(name string) bool { + return strings.Contains(name, SnapshotDelimiter) +} + +func MkdirAllOwner(path string, perm os.FileMode, uid int, gid int) error { + // This function is a slightly modified version of MkdirAll from the Go standard library. + // https://golang.org/src/os/path.go?s=488:535#L9 + + // Fast path: if we can tell whether path is a directory or file, stop with success or error. + dir, err := os.Stat(path) + if err == nil { + if dir.IsDir() { + return nil + } + return fmt.Errorf("path exists but isn't a directory") + } + + // Slow path: make sure parent exists and then call Mkdir for path. + i := len(path) + for i > 0 && os.IsPathSeparator(path[i-1]) { // Skip trailing path separator. + i-- + } + + j := i + for j > 0 && !os.IsPathSeparator(path[j-1]) { // Scan backward over element. + j-- + } + + if j > 1 { + // Create parent + err = MkdirAllOwner(path[0:j-1], perm, uid, gid) + if err != nil { + return err + } + } + + // Parent now exists; invoke Mkdir and use its result. + err = os.Mkdir(path, perm) + + err_chown := os.Chown(path, uid, gid) + if err_chown != nil { + return err_chown + } + + if err != nil { + // Handle arguments like "foo/." by + // double-checking that directory doesn't exist. + dir, err1 := os.Lstat(path) + if err1 == nil && dir.IsDir() { + return nil + } + return err + } + return nil +} + +func StringInSlice(key string, list []string) bool { + for _, entry := range list { + if entry == key { + return true + } + } + return false +} + +func IntInSlice(key int, list []int) bool { + for _, entry := range list { + if entry == key { + return true + } + } + return false +} + +func Int64InSlice(key int64, list []int64) bool { + for _, entry := range list { + if entry == key { + return true + } + } + return false +} + +func IsTrue(value string) bool { + if StringInSlice(strings.ToLower(value), []string{"true", "1", "yes", "on"}) { + return true + } + + return false +} + +// StringMapHasStringKey returns true if any of the supplied keys are present in the map. +func StringMapHasStringKey(m map[string]string, keys ...string) bool { + for _, k := range keys { + if _, ok := m[k]; ok { + return true + } + } + + return false +} + +func IsUnixDev(path string) bool { + stat, err := os.Stat(path) + if err != nil { + return false + + } + + if (stat.Mode() & os.ModeDevice) == 0 { + return false + } + + return true +} + +func IsBlockdev(fm os.FileMode) bool { + return ((fm&os.ModeDevice != 0) && (fm&os.ModeCharDevice == 0)) +} + +func IsBlockdevPath(pathName string) bool { + sb, err := os.Stat(pathName) + if err != nil { + return false + } + + fm := sb.Mode() + return ((fm&os.ModeDevice != 0) && (fm&os.ModeCharDevice == 0)) +} + +// DeepCopy copies src to dest by using encoding/gob so its not that fast. +func DeepCopy(src, dest interface{}) error { + buff := new(bytes.Buffer) + enc := gob.NewEncoder(buff) + dec := gob.NewDecoder(buff) + if err := enc.Encode(src); err != nil { + return err + } + + if err := dec.Decode(dest); err != nil { + return err + } + + return nil +} + +func RunningInUserNS() bool { + file, err := os.Open("/proc/self/uid_map") + if err != nil { + return false + } + defer file.Close() + + buf := bufio.NewReader(file) + l, _, err := buf.ReadLine() + if err != nil { + return false + } + + line := string(l) + var a, b, c int64 + fmt.Sscanf(line, "%d %d %d", &a, &b, &c) + if a == 0 && b == 0 && c == 4294967295 { + return false + } + return true +} + +func ValidHostname(name string) bool { + // Validate length + if len(name) < 1 || len(name) > 63 { + return false + } + + // Validate first character + if strings.HasPrefix(name, "-") { + return false + } + + if _, err := strconv.Atoi(string(name[0])); err == nil { + return false + } + + // Validate last character + if strings.HasSuffix(name, "-") { + return false + } + + // Validate the character set + match, _ := regexp.MatchString("^[-a-zA-Z0-9]*$", name) + if !match { + return false + } + + return true +} + +// Spawn the editor with a temporary YAML file for editing configs +func TextEditor(inPath string, inContent []byte) ([]byte, error) { + var f *os.File + var err error + var path string + + // Detect the text editor to use + editor := os.Getenv("VISUAL") + if editor == "" { + editor = os.Getenv("EDITOR") + if editor == "" { + for _, p := range []string{"editor", "vi", "emacs", "nano"} { + _, err := exec.LookPath(p) + if err == nil { + editor = p + break + } + } + if editor == "" { + return []byte{}, fmt.Errorf("No text editor found, please set the EDITOR environment variable") + } + } + } + + if inPath == "" { + // If provided input, create a new file + f, err = ioutil.TempFile("", "lxd_editor_") + if err != nil { + return []byte{}, err + } + + err = os.Chmod(f.Name(), 0600) + if err != nil { + f.Close() + os.Remove(f.Name()) + return []byte{}, err + } + + f.Write(inContent) + f.Close() + + path = fmt.Sprintf("%s.yaml", f.Name()) + os.Rename(f.Name(), path) + defer os.Remove(path) + } else { + path = inPath + } + + cmdParts := strings.Fields(editor) + cmd := exec.Command(cmdParts[0], append(cmdParts[1:], path)...) + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err = cmd.Run() + if err != nil { + return []byte{}, err + } + + content, err := ioutil.ReadFile(path) + if err != nil { + return []byte{}, err + } + + return content, nil +} + +func ParseMetadata(metadata interface{}) (map[string]interface{}, error) { + newMetadata := make(map[string]interface{}) + s := reflect.ValueOf(metadata) + if !s.IsValid() { + return nil, nil + } + + if s.Kind() == reflect.Map { + for _, k := range s.MapKeys() { + if k.Kind() != reflect.String { + return nil, fmt.Errorf("Invalid metadata provided (key isn't a string)") + } + newMetadata[k.String()] = s.MapIndex(k).Interface() + } + } else if s.Kind() == reflect.Ptr && !s.Elem().IsValid() { + return nil, nil + } else { + return nil, fmt.Errorf("Invalid metadata provided (type isn't a map)") + } + + return newMetadata, nil +} + +// RemoveDuplicatesFromString removes all duplicates of the string 'sep' +// from the specified string 's'. Leading and trailing occurrences of sep +// are NOT removed (duplicate leading/trailing are). Performs poorly if +// there are multiple consecutive redundant separators. +func RemoveDuplicatesFromString(s string, sep string) string { + dup := sep + sep + for s = strings.Replace(s, dup, sep, -1); strings.Contains(s, dup); s = strings.Replace(s, dup, sep, -1) { + + } + return s +} + +type RunError struct { + msg string + Err error + Stdout string + Stderr string +} + +func (e RunError) Error() string { + return e.msg +} + +// RunCommandSplit runs a command with a supplied environment and optional arguments and returns the +// resulting stdout and stderr output as separate variables. If the supplied environment is nil then +// the default environment is used. If the command fails to start or returns a non-zero exit code +// then an error is returned containing the output of stderr too. +func RunCommandSplit(env []string, name string, arg ...string) (string, string, error) { + cmd := exec.Command(name, arg...) + + if env != nil { + cmd.Env = env + } + + var stdout bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + err := cmd.Run() + if err != nil { + err := RunError{ + msg: fmt.Sprintf("Failed to run: %s %s: %s", name, strings.Join(arg, " "), strings.TrimSpace(string(stderr.Bytes()))), + Stdout: string(stdout.Bytes()), + Stderr: string(stderr.Bytes()), + Err: err, + } + return string(stdout.Bytes()), string(stderr.Bytes()), err + } + + return string(stdout.Bytes()), string(stderr.Bytes()), nil +} + +// RunCommand runs a command with optional arguments and returns stdout. If the command fails to +// start or returns a non-zero exit code then an error is returned containing the output of stderr. +func RunCommand(name string, arg ...string) (string, error) { + stdout, _, err := RunCommandSplit(nil, name, arg...) + return stdout, err +} + +// RunCommandCLocale runs a command with a LANG=C.UTF-8 environment set with optional arguments and +// returns stdout. If the command fails to start or returns a non-zero exit code then an error is +// returned containing the output of stderr. +func RunCommandCLocale(name string, arg ...string) (string, error) { + stdout, _, err := RunCommandSplit(append(os.Environ(), "LANG=C.UTF-8"), name, arg...) + return stdout, err +} + +func RunCommandWithFds(stdin io.Reader, stdout io.Writer, name string, arg ...string) error { + cmd := exec.Command(name, arg...) + + if stdin != nil { + cmd.Stdin = stdin + } + + if stdout != nil { + cmd.Stdout = stdout + } + + var buffer bytes.Buffer + cmd.Stderr = &buffer + + err := cmd.Run() + if err != nil { + err := RunError{ + msg: fmt.Sprintf("Failed to run: %s %s: %s", name, strings.Join(arg, " "), + strings.TrimSpace(buffer.String())), + Err: err, + } + + return err + } + + return nil +} + +func TryRunCommand(name string, arg ...string) (string, error) { + var err error + var output string + + for i := 0; i < 20; i++ { + output, err = RunCommand(name, arg...) + if err == nil { + break + } + + time.Sleep(500 * time.Millisecond) + } + + return output, err +} + +func TimeIsSet(ts time.Time) bool { + if ts.Unix() <= 0 { + return false + } + + if ts.UTC().Unix() <= 0 { + return false + } + + return true +} + +// WriteTempFile creates a temp file with the specified content +func WriteTempFile(dir string, prefix string, content string) (string, error) { + f, err := ioutil.TempFile(dir, prefix) + if err != nil { + return "", err + } + defer f.Close() + + _, err = f.WriteString(content) + return f.Name(), err +} + +// EscapePathFstab escapes a path fstab-style. +// This ensures that getmntent_r() and friends can correctly parse stuff like +// /some/wacky path with spaces /some/wacky target with spaces +func EscapePathFstab(path string) string { + r := strings.NewReplacer( + " ", "\\040", + "\t", "\\011", + "\n", "\\012", + "\\", "\\\\") + return r.Replace(path) +} + +func SetProgressMetadata(metadata map[string]interface{}, stage, displayPrefix string, percent, processed, speed int64) { + progress := make(map[string]string) + // stage, percent, speed sent for API callers. + progress["stage"] = stage + if processed > 0 { + progress["processed"] = strconv.FormatInt(processed, 10) + } + + if percent > 0 { + progress["percent"] = strconv.FormatInt(percent, 10) + } + + progress["speed"] = strconv.FormatInt(speed, 10) + metadata["progress"] = progress + + // _progress with formatted text sent for lxc cli. + if percent > 0 { + metadata[stage+"_progress"] = fmt.Sprintf("%s: %d%% (%s/s)", displayPrefix, percent, units.GetByteSizeString(speed, 2)) + } else if processed > 0 { + metadata[stage+"_progress"] = fmt.Sprintf("%s: %s (%s/s)", displayPrefix, units.GetByteSizeString(processed, 2), units.GetByteSizeString(speed, 2)) + } else { + metadata[stage+"_progress"] = fmt.Sprintf("%s: %s/s", displayPrefix, units.GetByteSizeString(speed, 2)) + } +} + +func DownloadFileHash(httpClient *http.Client, useragent string, progress func(progress ioprogress.ProgressData), canceler *cancel.Canceler, filename string, url string, hash string, hashFunc hash.Hash, target io.WriteSeeker) (int64, error) { + // Always seek to the beginning + target.Seek(0, 0) + + // Prepare the download request + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return -1, err + } + + if useragent != "" { + req.Header.Set("User-Agent", useragent) + } + + // Perform the request + r, doneCh, err := cancel.CancelableDownload(canceler, httpClient, req) + if err != nil { + return -1, err + } + defer r.Body.Close() + defer close(doneCh) + + if r.StatusCode != http.StatusOK { + return -1, fmt.Errorf("Unable to fetch %s: %s", url, r.Status) + } + + // Handle the data + body := r.Body + if progress != nil { + body = &ioprogress.ProgressReader{ + ReadCloser: r.Body, + Tracker: &ioprogress.ProgressTracker{ + Length: r.ContentLength, + Handler: func(percent int64, speed int64) { + if filename != "" { + progress(ioprogress.ProgressData{Text: fmt.Sprintf("%s: %d%% (%s/s)", filename, percent, units.GetByteSizeString(speed, 2))}) + } else { + progress(ioprogress.ProgressData{Text: fmt.Sprintf("%d%% (%s/s)", percent, units.GetByteSizeString(speed, 2))}) + } + }, + }, + } + } + + var size int64 + + if hashFunc != nil { + size, err = io.Copy(io.MultiWriter(target, hashFunc), body) + if err != nil { + return -1, err + } + + result := fmt.Sprintf("%x", hashFunc.Sum(nil)) + if result != hash { + return -1, fmt.Errorf("Hash mismatch for %s: %s != %s", url, result, hash) + } + } else { + size, err = io.Copy(target, body) + if err != nil { + return -1, err + } + } + + return size, nil +} + +func ParseNumberFromFile(file string) (int64, error) { + f, err := os.Open(file) + if err != nil { + return int64(0), err + } + defer f.Close() + + buf := make([]byte, 4096) + n, err := f.Read(buf) + if err != nil { + return int64(0), err + } + + str := strings.TrimSpace(string(buf[0:n])) + nr, err := strconv.Atoi(str) + if err != nil { + return int64(0), err + } + + return int64(nr), nil +} + +type ReadSeeker struct { + io.Reader + io.Seeker +} + +func NewReadSeeker(reader io.Reader, seeker io.Seeker) *ReadSeeker { + return &ReadSeeker{Reader: reader, Seeker: seeker} +} + +func (r *ReadSeeker) Read(p []byte) (n int, err error) { + return r.Reader.Read(p) +} + +func (r *ReadSeeker) Seek(offset int64, whence int) (int64, error) { + return r.Seeker.Seek(offset, whence) +} + +// RenderTemplate renders a pongo2 template. +func RenderTemplate(template string, ctx pongo2.Context) (string, error) { + // Load template from string + tpl, err := pongo2.FromString("{% autoescape off %}" + template + "{% endautoescape %}") + if err != nil { + return "", err + } + + // Get rendered template + ret, err := tpl.Execute(ctx) + if err != nil { + return ret, err + } + + // Looks like we're nesting templates so run pongo again + if strings.Contains(ret, "{{") || strings.Contains(ret, "{%") { + return RenderTemplate(ret, ctx) + } + + return ret, err +} + +func GetSnapshotExpiry(refDate time.Time, s string) (time.Time, error) { + expr := strings.TrimSpace(s) + + if expr == "" { + return time.Time{}, nil + } + + re := regexp.MustCompile(`^(\d+)(M|H|d|w|m|y)$`) + expiry := map[string]int{ + "M": 0, + "H": 0, + "d": 0, + "w": 0, + "m": 0, + "y": 0, + } + + values := strings.Split(expr, " ") + + if len(values) == 0 { + return time.Time{}, nil + } + + for _, value := range values { + fields := re.FindStringSubmatch(value) + if fields == nil { + return time.Time{}, fmt.Errorf("Invalid expiry expression") + } + + if expiry[fields[2]] > 0 { + // We don't allow fields to be set multiple times + return time.Time{}, fmt.Errorf("Invalid expiry expression") + } + + val, err := strconv.Atoi(fields[1]) + if err != nil { + return time.Time{}, err + } + + expiry[fields[2]] = val + + } + + t := refDate.AddDate(expiry["y"], expiry["m"], expiry["d"]+expiry["w"]*7).Add( + time.Hour*time.Duration(expiry["H"]) + time.Minute*time.Duration(expiry["M"])) + + return t, nil +} diff --git a/vendor/github.com/lxc/lxd/shared/util_linux.go b/vendor/github.com/lxc/lxd/shared/util_linux.go new file mode 100644 index 0000000000..f2b8827416 --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/util_linux.go @@ -0,0 +1,378 @@ +// +build linux + +package shared + +import ( + "bufio" + "fmt" + "os" + "path/filepath" + "reflect" + "strings" + "syscall" + "unsafe" + + "golang.org/x/sys/unix" + + "github.com/lxc/lxd/shared/units" +) + +// --- pure Go functions --- + +func GetFileStat(p string) (uid int, gid int, major uint32, minor uint32, inode uint64, nlink int, err error) { + var stat unix.Stat_t + err = unix.Lstat(p, &stat) + if err != nil { + return + } + uid = int(stat.Uid) + gid = int(stat.Gid) + inode = uint64(stat.Ino) + nlink = int(stat.Nlink) + if stat.Mode&unix.S_IFBLK != 0 || stat.Mode&unix.S_IFCHR != 0 { + major = unix.Major(stat.Rdev) + minor = unix.Minor(stat.Rdev) + } + + return +} + +// GetPathMode returns a os.FileMode for the provided path +func GetPathMode(path string) (os.FileMode, error) { + fi, err := os.Stat(path) + if err != nil { + return os.FileMode(0000), err + } + + mode, _, _ := GetOwnerMode(fi) + return mode, nil +} + +func parseMountinfo(name string) int { + // In case someone uses symlinks we need to look for the actual + // mountpoint. + actualPath, err := filepath.EvalSymlinks(name) + if err != nil { + return -1 + } + + f, err := os.Open("/proc/self/mountinfo") + if err != nil { + return -1 + } + defer f.Close() + + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := scanner.Text() + tokens := strings.Fields(line) + if len(tokens) < 5 { + return -1 + } + cleanPath := filepath.Clean(tokens[4]) + if cleanPath == actualPath { + return 1 + } + } + + return 0 +} + +func IsMountPoint(name string) bool { + ret := parseMountinfo(name) + if ret == 1 { + return true + } + + stat, err := os.Stat(name) + if err != nil { + return false + } + + rootStat, err := os.Lstat(name + "/..") + if err != nil { + return false + } + // If the directory has the same device as parent, then it's not a mountpoint. + return stat.Sys().(*syscall.Stat_t).Dev != rootStat.Sys().(*syscall.Stat_t).Dev +} + +func SetSize(fd int, width int, height int) (err error) { + var dimensions [4]uint16 + dimensions[0] = uint16(height) + dimensions[1] = uint16(width) + + if _, _, err := unix.Syscall6(unix.SYS_IOCTL, uintptr(fd), uintptr(unix.TIOCSWINSZ), uintptr(unsafe.Pointer(&dimensions)), 0, 0, 0); err != 0 { + return err + } + return nil +} + +// This uses ssize_t llistxattr(const char *path, char *list, size_t size); to +// handle symbolic links (should it in the future be possible to set extended +// attributed on symlinks): If path is a symbolic link the extended attributes +// associated with the link itself are retrieved. +func llistxattr(path string, list []byte) (sz int, err error) { + var _p0 *byte + _p0, err = unix.BytePtrFromString(path) + if err != nil { + return + } + var _p1 unsafe.Pointer + if len(list) > 0 { + _p1 = unsafe.Pointer(&list[0]) + } else { + _p1 = unsafe.Pointer(nil) + } + r0, _, e1 := unix.Syscall(unix.SYS_LLISTXATTR, uintptr(unsafe.Pointer(_p0)), uintptr(_p1), uintptr(len(list))) + sz = int(r0) + if e1 != 0 { + err = e1 + } + return +} + +// GetAllXattr retrieves all extended attributes associated with a file, +// directory or symbolic link. +func GetAllXattr(path string) (xattrs map[string]string, err error) { + e1 := fmt.Errorf("Extended attributes changed during retrieval") + + // Call llistxattr() twice: First, to determine the size of the buffer + // we need to allocate to store the extended attributes, second, to + // actually store the extended attributes in the buffer. Also, check if + // the size/number of extended attributes hasn't changed between the two + // calls. + pre, err := llistxattr(path, nil) + if err != nil || pre < 0 { + return nil, err + } + if pre == 0 { + return nil, nil + } + + dest := make([]byte, pre) + + post, err := llistxattr(path, dest) + if err != nil || post < 0 { + return nil, err + } + if post != pre { + return nil, e1 + } + + split := strings.Split(string(dest), "\x00") + if split == nil { + return nil, fmt.Errorf("No valid extended attribute key found") + } + // *listxattr functions return a list of names as an unordered array + // of null-terminated character strings (attribute names are separated + // by null bytes ('\0')), like this: user.name1\0system.name1\0user.name2\0 + // Since we split at the '\0'-byte the last element of the slice will be + // the empty string. We remove it: + if split[len(split)-1] == "" { + split = split[:len(split)-1] + } + + xattrs = make(map[string]string, len(split)) + + for _, x := range split { + xattr := string(x) + // Call Getxattr() twice: First, to determine the size of the + // buffer we need to allocate to store the extended attributes, + // second, to actually store the extended attributes in the + // buffer. Also, check if the size of the extended attribute + // hasn't changed between the two calls. + pre, err = unix.Getxattr(path, xattr, nil) + if err != nil || pre < 0 { + return nil, err + } + + dest = make([]byte, pre) + post := 0 + if pre > 0 { + post, err = unix.Getxattr(path, xattr, dest) + if err != nil || post < 0 { + return nil, err + } + } + + if post != pre { + return nil, e1 + } + + xattrs[xattr] = string(dest) + } + + return xattrs, nil +} + +var ObjectFound = fmt.Errorf("Found requested object") + +func LookupUUIDByBlockDevPath(diskDevice string) (string, error) { + uuid := "" + readUUID := func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if (info.Mode() & os.ModeSymlink) == os.ModeSymlink { + link, err := os.Readlink(path) + if err != nil { + return err + } + + // filepath.Join() will call Clean() on the result and + // thus resolve those ugly "../../" parts that make it + // hard to compare the strings. + absPath := filepath.Join("/dev/disk/by-uuid", link) + if absPath == diskDevice { + uuid = path + // Will allows us to avoid needlessly travers + // the whole directory. + return ObjectFound + } + } + return nil + } + + err := filepath.Walk("/dev/disk/by-uuid", readUUID) + if err != nil && err != ObjectFound { + return "", fmt.Errorf("Failed to detect UUID: %s", err) + } + + if uuid == "" { + return "", fmt.Errorf("Failed to detect UUID") + } + + lastSlash := strings.LastIndex(uuid, "/") + return uuid[lastSlash+1:], nil +} + +// Detect whether err is an errno. +func GetErrno(err error) (errno error, iserrno bool) { + sysErr, ok := err.(*os.SyscallError) + if ok { + return sysErr.Err, true + } + + pathErr, ok := err.(*os.PathError) + if ok { + return pathErr.Err, true + } + + tmpErrno, ok := err.(unix.Errno) + if ok { + return tmpErrno, true + } + + return nil, false +} + +// Utsname returns the same info as unix.Utsname, as strings +type Utsname struct { + Sysname string + Nodename string + Release string + Version string + Machine string + Domainname string +} + +// Uname returns Utsname as strings +func Uname() (*Utsname, error) { + /* + * Based on: https://groups.google.com/forum/#!topic/golang-nuts/Jel8Bb-YwX8 + * there is really no better way to do this, which is + * unfortunate. Also, we ditch the more accepted CharsToString + * version in that thread, since it doesn't seem as portable, + * viz. github issue #206. + */ + + uname := unix.Utsname{} + err := unix.Uname(&uname) + if err != nil { + return nil, err + } + + return &Utsname{ + Sysname: intArrayToString(uname.Sysname), + Nodename: intArrayToString(uname.Nodename), + Release: intArrayToString(uname.Release), + Version: intArrayToString(uname.Version), + Machine: intArrayToString(uname.Machine), + Domainname: intArrayToString(uname.Domainname), + }, nil +} + +func intArrayToString(arr interface{}) string { + slice := reflect.ValueOf(arr) + s := "" + for i := 0; i < slice.Len(); i++ { + val := slice.Index(i) + valInt := int64(-1) + + switch val.Kind() { + case reflect.Int: + case reflect.Int8: + valInt = int64(val.Int()) + case reflect.Uint: + case reflect.Uint8: + valInt = int64(val.Uint()) + default: + continue + } + + if valInt == 0 { + break + } + + s += string(byte(valInt)) + } + + return s +} + +func Statvfs(path string) (*unix.Statfs_t, error) { + var st unix.Statfs_t + + err := unix.Statfs(path, &st) + if err != nil { + return nil, err + } + + return &st, nil +} + +func DeviceTotalMemory() (int64, error) { + // Open /proc/meminfo + f, err := os.Open("/proc/meminfo") + if err != nil { + return -1, err + } + defer f.Close() + + // Read it line by line + scan := bufio.NewScanner(f) + for scan.Scan() { + line := scan.Text() + + // We only care about MemTotal + if !strings.HasPrefix(line, "MemTotal:") { + continue + } + + // Extract the before last (value) and last (unit) fields + fields := strings.Split(line, " ") + value := fields[len(fields)-2] + fields[len(fields)-1] + + // Feed the result to units.ParseByteSizeString to get an int value + valueBytes, err := units.ParseByteSizeString(value) + if err != nil { + return -1, err + } + + return valueBytes, nil + } + + return -1, fmt.Errorf("Couldn't find MemTotal") +} diff --git a/vendor/github.com/lxc/lxd/shared/util_linux_cgo.go b/vendor/github.com/lxc/lxd/shared/util_linux_cgo.go new file mode 100644 index 0000000000..bcf7e4f1cb --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/util_linux_cgo.go @@ -0,0 +1,461 @@ +// +build linux +// +build cgo + +package shared + +import ( + "errors" + "fmt" + "io" + "os" + "sync" + "sync/atomic" + "unsafe" + + "golang.org/x/sys/unix" + + "github.com/lxc/lxd/shared/logger" +) + +/* +#ifndef _GNU_SOURCE +#define _GNU_SOURCE 1 +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define ABSTRACT_UNIX_SOCK_LEN sizeof(((struct sockaddr_un *)0)->sun_path) + +// This is an adaption from https://codereview.appspot.com/4589049, to be +// included in the stdlib with the stdlib's license. + +void configure_pty(int fd) { + struct termios term_settings; + struct winsize win; + + if (tcgetattr(fd, &term_settings) < 0) { + fprintf(stderr, "Failed to get settings: %s\n", strerror(errno)); + return; + } + + term_settings.c_iflag |= IMAXBEL; + term_settings.c_iflag |= IUTF8; + term_settings.c_iflag |= BRKINT; + term_settings.c_iflag |= IXANY; + + term_settings.c_cflag |= HUPCL; + + if (tcsetattr(fd, TCSANOW, &term_settings) < 0) { + fprintf(stderr, "Failed to set settings: %s\n", strerror(errno)); + return; + } + + if (ioctl(fd, TIOCGWINSZ, &win) < 0) { + fprintf(stderr, "Failed to get the terminal size: %s\n", strerror(errno)); + return; + } + + win.ws_col = 80; + win.ws_row = 25; + + if (ioctl(fd, TIOCSWINSZ, &win) < 0) { + fprintf(stderr, "Failed to set the terminal size: %s\n", strerror(errno)); + return; + } + + if (fcntl(fd, F_SETFD, FD_CLOEXEC) < 0) { + fprintf(stderr, "Failed to set FD_CLOEXEC: %s\n", strerror(errno)); + return; + } + + return; +} + +void create_pty(int *master, int *slave, uid_t uid, gid_t gid) { + if (openpty(master, slave, NULL, NULL, NULL) < 0) { + fprintf(stderr, "Failed to openpty: %s\n", strerror(errno)); + return; + } + + configure_pty(*master); + configure_pty(*slave); + + if (fchown(*slave, uid, gid) < 0) { + fprintf(stderr, "Warning: error chowning pty to container root\n"); + fprintf(stderr, "Continuing...\n"); + } +} + +void create_pipe(int *master, int *slave) { + int pipefd[2]; + + if (pipe2(pipefd, O_CLOEXEC) < 0) { + fprintf(stderr, "Failed to create a pipe: %s\n", strerror(errno)); + return; + } + + *master = pipefd[0]; + *slave = pipefd[1]; +} + +int get_poll_revents(int lfd, int timeout, int flags, int *revents, int *saved_errno) +{ + int ret; + struct pollfd pfd = {lfd, flags, 0}; + +again: + ret = poll(&pfd, 1, timeout); + if (ret < 0) { + if (errno == EINTR) + goto again; + + *saved_errno = errno; + fprintf(stderr, "Failed to poll() on file descriptor.\n"); + return -1; + } + + *revents = pfd.revents; + + return ret; +} +*/ +import "C" + +const ABSTRACT_UNIX_SOCK_LEN int = C.ABSTRACT_UNIX_SOCK_LEN + +const POLLIN int = C.POLLIN +const POLLPRI int = C.POLLPRI +const POLLNVAL int = C.POLLNVAL +const POLLERR int = C.POLLERR +const POLLHUP int = C.POLLHUP +const POLLRDHUP int = C.POLLRDHUP + +func GetPollRevents(fd int, timeout int, flags int) (int, int, error) { + var err error + revents := C.int(0) + saved_errno := C.int(0) + + ret := C.get_poll_revents(C.int(fd), C.int(timeout), C.int(flags), &revents, &saved_errno) + if int(ret) < 0 { + err = unix.Errno(saved_errno) + } + + return int(ret), int(revents), err +} + +func OpenPty(uid, gid int64) (master *os.File, slave *os.File, err error) { + fd_master := C.int(-1) + fd_slave := C.int(-1) + rootUid := C.uid_t(uid) + rootGid := C.gid_t(gid) + + C.create_pty(&fd_master, &fd_slave, rootUid, rootGid) + + if fd_master == -1 || fd_slave == -1 { + return nil, nil, errors.New("Failed to create a new pts pair") + } + + master = os.NewFile(uintptr(fd_master), "master") + slave = os.NewFile(uintptr(fd_slave), "slave") + + return master, slave, nil +} + +func Pipe() (master *os.File, slave *os.File, err error) { + fd_master := C.int(-1) + fd_slave := C.int(-1) + + C.create_pipe(&fd_master, &fd_slave) + + if fd_master == -1 || fd_slave == -1 { + return nil, nil, errors.New("Failed to create a new pipe") + } + + master = os.NewFile(uintptr(fd_master), "master") + slave = os.NewFile(uintptr(fd_slave), "slave") + + return master, slave, nil +} + +// UserId is an adaption from https://codereview.appspot.com/4589049. +func UserId(name string) (int, error) { + var pw C.struct_passwd + var result *C.struct_passwd + + bufSize := C.sysconf(C._SC_GETPW_R_SIZE_MAX) + if bufSize < 0 { + bufSize = 4096 + } + + buf := C.malloc(C.size_t(bufSize)) + if buf == nil { + return -1, fmt.Errorf("allocation failed") + } + defer C.free(buf) + + cname := C.CString(name) + defer C.free(unsafe.Pointer(cname)) + +again: + rv, errno := C.getpwnam_r(cname, + &pw, + (*C.char)(buf), + C.size_t(bufSize), + &result) + if rv < 0 { + // OOM killer will take care of us if we end up doing this too + // often. + if errno == unix.ERANGE { + bufSize *= 2 + tmp := C.realloc(buf, C.size_t(bufSize)) + if tmp == nil { + return -1, fmt.Errorf("allocation failed") + } + buf = tmp + goto again + } + return -1, fmt.Errorf("failed user lookup: %s", unix.Errno(rv)) + } + + if result == nil { + return -1, fmt.Errorf("unknown user %s", name) + } + + return int(C.int(result.pw_uid)), nil +} + +// GroupId is an adaption from https://codereview.appspot.com/4589049. +func GroupId(name string) (int, error) { + var grp C.struct_group + var result *C.struct_group + + bufSize := C.sysconf(C._SC_GETGR_R_SIZE_MAX) + if bufSize < 0 { + bufSize = 4096 + } + + buf := C.malloc(C.size_t(bufSize)) + if buf == nil { + return -1, fmt.Errorf("allocation failed") + } + + cname := C.CString(name) + defer C.free(unsafe.Pointer(cname)) + +again: + rv, errno := C.getgrnam_r(cname, + &grp, + (*C.char)(buf), + C.size_t(bufSize), + &result) + if rv != 0 { + // OOM killer will take care of us if we end up doing this too + // often. + if errno == unix.ERANGE { + bufSize *= 2 + tmp := C.realloc(buf, C.size_t(bufSize)) + if tmp == nil { + return -1, fmt.Errorf("allocation failed") + } + buf = tmp + goto again + } + + C.free(buf) + return -1, fmt.Errorf("failed group lookup: %s", unix.Errno(rv)) + } + C.free(buf) + + if result == nil { + return -1, fmt.Errorf("unknown group %s", name) + } + + return int(C.int(result.gr_gid)), nil +} + +// Extensively commented directly in the code. Please leave the comments! +// Looking at this in a couple of months noone will know why and how this works +// anymore. +func ExecReaderToChannel(r io.Reader, bufferSize int, exited <-chan bool, fd int) <-chan []byte { + if bufferSize <= (128 * 1024) { + bufferSize = (128 * 1024) + } + + ch := make(chan ([]byte)) + + // Takes care that the closeChannel() function is exactly executed once. + // This allows us to avoid using a mutex. + var once sync.Once + closeChannel := func() { + close(ch) + } + + // [1]: This function has just one job: Dealing with the case where we + // are running an interactive shell session where we put a process in + // the background that does hold stdin/stdout open, but does not + // generate any output at all. This case cannot be dealt with in the + // following function call. Here's why: Assume the above case, now the + // attached child (the shell in this example) exits. This will not + // generate any poll() event: We won't get POLLHUP because the + // background process is holding stdin/stdout open and noone is writing + // to it. So we effectively block on GetPollRevents() in the function + // below. Hence, we use another go routine here who's only job is to + // handle that case: When we detect that the child has exited we check + // whether a POLLIN or POLLHUP event has been generated. If not, we know + // that there's nothing buffered on stdout and exit. + var attachedChildIsDead int32 = 0 + go func() { + <-exited + + atomic.StoreInt32(&attachedChildIsDead, 1) + + ret, revents, err := GetPollRevents(fd, 0, (POLLIN | POLLPRI | POLLERR | POLLHUP | POLLRDHUP | POLLNVAL)) + if ret < 0 { + logger.Errorf("Failed to poll(POLLIN | POLLPRI | POLLHUP | POLLRDHUP) on file descriptor: %s.", err) + } else if ret > 0 { + if (revents & POLLERR) > 0 { + logger.Warnf("Detected poll(POLLERR) event.") + } else if (revents & POLLNVAL) > 0 { + logger.Warnf("Detected poll(POLLNVAL) event.") + } + } else if ret == 0 { + logger.Debugf("No data in stdout: exiting.") + once.Do(closeChannel) + return + } + }() + + go func() { + readSize := (128 * 1024) + offset := 0 + buf := make([]byte, bufferSize) + avoidAtomicLoad := false + + defer once.Do(closeChannel) + for { + nr := 0 + var err error + + ret, revents, err := GetPollRevents(fd, -1, (POLLIN | POLLPRI | POLLERR | POLLHUP | POLLRDHUP | POLLNVAL)) + if ret < 0 { + // This condition is only reached in cases where we are massively f*cked since we even handle + // EINTR in the underlying C wrapper around poll(). So let's exit here. + logger.Errorf("Failed to poll(POLLIN | POLLPRI | POLLERR | POLLHUP | POLLRDHUP) on file descriptor: %s. Exiting.", err) + return + } + + // [2]: If the process exits before all its data has been read by us and no other process holds stdin or + // stdout open, then we will observe a (POLLHUP | POLLRDHUP | POLLIN) event. This means, we need to + // keep on reading from the pty file descriptor until we get a simple POLLHUP back. + both := ((revents & (POLLIN | POLLPRI)) > 0) && ((revents & (POLLHUP | POLLRDHUP)) > 0) + if both { + logger.Debugf("Detected poll(POLLIN | POLLPRI | POLLHUP | POLLRDHUP) event.") + read := buf[offset : offset+readSize] + nr, err = r.Read(read) + } + + if (revents & POLLERR) > 0 { + logger.Warnf("Detected poll(POLLERR) event: exiting.") + return + } else if (revents & POLLNVAL) > 0 { + logger.Warnf("Detected poll(POLLNVAL) event: exiting.") + return + } + + if ((revents & (POLLIN | POLLPRI)) > 0) && !both { + // This might appear unintuitive at first but is actually a nice trick: Assume we are running + // a shell session in a container and put a process in the background that is writing to + // stdout. Now assume the attached process (aka the shell in this example) exits because we + // used Ctrl+D to send EOF or something. If no other process would be holding stdout open we + // would expect to observe either a (POLLHUP | POLLRDHUP | POLLIN | POLLPRI) event if there + // is still data buffered from the previous process or a simple (POLLHUP | POLLRDHUP) if + // no data is buffered. The fact that we only observe a (POLLIN | POLLPRI) event means that + // another process is holding stdout open and is writing to it. + // One counter argument that can be leveraged is (brauner looks at tycho :)) + // "Hey, you need to write at least one additional tty buffer to make sure that + // everything that the attached child has written is actually shown." + // The answer to that is: + // "This case can only happen if the process has exited and has left data in stdout which + // would generate a (POLLIN | POLLPRI | POLLHUP | POLLRDHUP) event and this case is already + // handled and triggers another codepath. (See [2].)" + if avoidAtomicLoad || atomic.LoadInt32(&attachedChildIsDead) == 1 { + avoidAtomicLoad = true + // Handle race between atomic.StorInt32() in the go routine + // explained in [1] and atomic.LoadInt32() in the go routine + // here: + // We need to check for (POLLHUP | POLLRDHUP) here again since we might + // still be handling a pure POLLIN event from a write prior to the childs + // exit. But the child might have exited right before and performed + // atomic.StoreInt32() to update attachedChildIsDead before we + // performed our atomic.LoadInt32(). This means we accidentally hit this + // codepath and are misinformed about the available poll() events. So we + // need to perform a non-blocking poll() again to exclude that case: + // + // - If we detect no (POLLHUP | POLLRDHUP) event we know the child + // has already exited but someone else is holding stdin/stdout open and + // writing to it. + // Note that his case should only ever be triggered in situations like + // running a shell and doing stuff like: + // > ./lxc exec xen1 -- bash + // root@xen1:~# yes & + // . + // . + // . + // now send Ctrl+D or type "exit". By the time the Ctrl+D/exit event is + // triggered, we will have read all of the childs data it has written to + // stdout and so we can assume that anything that comes now belongs to + // the process that is holding stdin/stdout open. + // + // - If we detect a (POLLHUP | POLLRDHUP) event we know that we've + // hit this codepath on accident caused by the race between + // atomic.StoreInt32() in the go routine explained in [1] and + // atomic.LoadInt32() in this go routine. So the next call to + // GetPollRevents() will either return + // (POLLIN | POLLPRI | POLLERR | POLLHUP | POLLRDHUP) + // or (POLLHUP | POLLRDHUP). Both will trigger another codepath (See [2].) + // that takes care that all data of the child that is buffered in + // stdout is written out. + ret, revents, err := GetPollRevents(fd, 0, (POLLIN | POLLPRI | POLLERR | POLLHUP | POLLRDHUP | POLLNVAL)) + if ret < 0 { + logger.Errorf("Failed to poll(POLLIN | POLLPRI | POLLERR | POLLHUP | POLLRDHUP) on file descriptor: %s. Exiting.", err) + return + } else if (revents & (POLLHUP | POLLRDHUP | POLLERR | POLLNVAL)) == 0 { + logger.Debugf("Exiting but background processes are still running.") + return + } + } + read := buf[offset : offset+readSize] + nr, err = r.Read(read) + } + + // The attached process has exited and we have read all data that may have + // been buffered. + if ((revents & (POLLHUP | POLLRDHUP)) > 0) && !both { + logger.Debugf("Detected poll(POLLHUP) event: exiting.") + return + } + + offset += nr + if offset > 0 && (offset+readSize >= bufferSize || err != nil) { + ch <- buf[0:offset] + offset = 0 + buf = make([]byte, bufferSize) + } + } + }() + + return ch +} diff --git a/vendor/github.com/lxc/lxd/shared/util_linux_notcgo.go b/vendor/github.com/lxc/lxd/shared/util_linux_notcgo.go new file mode 100644 index 0000000000..9f82988c8d --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/util_linux_notcgo.go @@ -0,0 +1,5 @@ +// +build linux,!cgo + +package shared + +const ABSTRACT_UNIX_SOCK_LEN int = 107 diff --git a/vendor/github.com/lxc/lxd/shared/util_unix.go b/vendor/github.com/lxc/lxd/shared/util_unix.go new file mode 100644 index 0000000000..f87a109333 --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/util_unix.go @@ -0,0 +1,15 @@ +// +build !windows + +package shared + +import ( + "os" + "syscall" +) + +func GetOwnerMode(fInfo os.FileInfo) (os.FileMode, int, int) { + mode := fInfo.Mode() + uid := int(fInfo.Sys().(*syscall.Stat_t).Uid) + gid := int(fInfo.Sys().(*syscall.Stat_t).Gid) + return mode, uid, gid +} diff --git a/vendor/github.com/lxc/lxd/shared/util_windows.go b/vendor/github.com/lxc/lxd/shared/util_windows.go new file mode 100644 index 0000000000..7a480f5bc2 --- /dev/null +++ b/vendor/github.com/lxc/lxd/shared/util_windows.go @@ -0,0 +1,11 @@ +// +build windows + +package shared + +import ( + "os" +) + +func GetOwnerMode(fInfo os.FileInfo) (os.FileMode, int, int) { + return fInfo.Mode(), -1, -1 +} diff --git a/vendor/github.com/rancher/dynamiclistener/listener.go b/vendor/github.com/rancher/dynamiclistener/listener.go index 7ba1a6904a..eaec9b57a6 100644 --- a/vendor/github.com/rancher/dynamiclistener/listener.go +++ b/vendor/github.com/rancher/dynamiclistener/listener.go @@ -68,7 +68,7 @@ func (l *listener) Accept() (net.Conn, error) { return conn, err } - addr := conn.RemoteAddr() + addr := conn.LocalAddr() if addr == nil { return conn, nil } @@ -79,7 +79,11 @@ func (l *listener) Accept() (net.Conn, error) { return conn, nil } - return conn, l.updateCert(host) + if err := l.updateCert(host); err != nil { + logrus.Infof("failed to create TLS cert for: %s", host) + } + + return conn, nil } func (l *listener) getCertificate(hello *tls.ClientHelloInfo) (*tls.Certificate, error) { diff --git a/vendor/github.com/rancher/dynamiclistener/storage/kubernetes/controller.go b/vendor/github.com/rancher/dynamiclistener/storage/kubernetes/controller.go index 64ed53643c..d6c163c4f1 100644 --- a/vendor/github.com/rancher/dynamiclistener/storage/kubernetes/controller.go +++ b/vendor/github.com/rancher/dynamiclistener/storage/kubernetes/controller.go @@ -73,6 +73,11 @@ func (s *storage) init(secrets v1controller.SecretController) { return secret, nil }) s.secrets = secrets + + secret, err := s.storage.Get() + if err == nil && secret != nil { + s.saveInK8s(secret) + } } func (s *storage) Get() (*v1.Secret, error) { @@ -115,10 +120,10 @@ func (s *storage) saveInK8s(secret *v1.Secret) (*v1.Secret, error) { targetSecret.Data = secret.Data if targetSecret.UID == "" { - logrus.Infof("Creating new TLS secret for %v", targetSecret.Annotations) + logrus.Infof("Creating new TLS secret for %v (count: %d)", targetSecret.Name, len(targetSecret.Data)-1) return s.secrets.Create(targetSecret) } else { - logrus.Infof("Updating TLS secret for %v", targetSecret.Annotations) + logrus.Infof("Updating TLS secret for %v (count: %d)", targetSecret.Name, len(targetSecret.Data)-1) return s.secrets.Update(targetSecret) } } @@ -127,7 +132,16 @@ func (s *storage) Update(secret *v1.Secret) (err error) { s.Lock() defer s.Unlock() - secret, err = s.saveInK8s(secret) + for i := 0; i < 3; i++ { + secret, err = s.saveInK8s(secret) + if errors.IsConflict(err) { + continue + } else if err != nil { + return err + } + break + } + if err != nil { return err } diff --git a/vendor/github.com/rancher/kine/pkg/drivers/dqlite/dqlite.go b/vendor/github.com/rancher/kine/pkg/drivers/dqlite/dqlite.go new file mode 100644 index 0000000000..5e19d6f0d5 --- /dev/null +++ b/vendor/github.com/rancher/kine/pkg/drivers/dqlite/dqlite.go @@ -0,0 +1,228 @@ +// +build dqlite + +package dqlite + +import ( + "context" + "database/sql" + "fmt" + "net/url" + "os" + "strconv" + "strings" + + "github.com/canonical/go-dqlite/client" + "github.com/canonical/go-dqlite/driver" + "github.com/pkg/errors" + "github.com/rancher/kine/pkg/drivers/sqlite" + "github.com/rancher/kine/pkg/server" + "github.com/sirupsen/logrus" +) + +var ( + Dialer = client.DefaultDialFunc + Logger = client.DefaultLogFunc +) + +type opts struct { + peers []client.NodeInfo + peerFile string + dsn string +} + +func AddPeers(ctx context.Context, nodeStore client.NodeStore, additionalPeers ...client.NodeInfo) error { + existing, err := nodeStore.Get(ctx) + if err != nil { + return err + } + + var peers []client.NodeInfo + +outer: + for _, peer := range additionalPeers { + for _, check := range existing { + if check.Address == peer.Address { + continue outer + } + } + peers = append(peers, peer) + } + + if len(peers) > 0 { + err = nodeStore.Set(ctx, append(existing, peers...)) + if err != nil { + return err + } + } + + return nil +} + +func New(ctx context.Context, datasourceName string) (server.Backend, error) { + opts, err := parseOpts(datasourceName) + if err != nil { + return nil, err + } + + var nodeStore client.NodeStore + if opts.peerFile != "" { + nodeStore, err = client.DefaultNodeStore(opts.peerFile) + if err != nil { + return nil, err + } + } else { + nodeStore = client.NewInmemNodeStore() + } + + if err := AddPeers(ctx, nodeStore, opts.peers...); err != nil { + return nil, err + } + + d, err := driver.New(nodeStore, + driver.WithLogFunc(Logger), + driver.WithContext(ctx), + driver.WithDialFunc(Dialer)) + if err != nil { + return nil, err + } + + sql.Register("dqlite", d) + backend, generic, err := sqlite.NewVariant("dqlite", opts.dsn) + if err != nil { + return nil, err + } + + if err := migrate(ctx, generic.DB); err != nil { + return nil, errors.Wrap(err, "failed to migrate DB from sqlite") + } + + generic.LockWrites = true + generic.Retry = func(err error) bool { + if err, ok := err.(driver.Error); ok { + return err.Code == driver.ErrBusy + } + return false + } + + return backend, nil +} + +func migrate(ctx context.Context, newDB *sql.DB) (exitErr error) { + row := newDB.QueryRowContext(ctx, "SELECT COUNT(*) FROM kine") + var count int64 + if err := row.Scan(&count); err != nil { + return err + } + if count > 0 { + return nil + } + + if _, err := os.Stat("./db/state.db"); err != nil { + return nil + } + + oldDB, err := sql.Open("sqlite3", "./db/state.db") + if err != nil { + return nil + } + defer oldDB.Close() + + oldData, err := oldDB.QueryContext(ctx, "SELECT id, name, created, deleted, create_revision, prev_revision, lease, value, old_value FROM kine") + if err != nil { + logrus.Errorf("failed to find old data to migrate: %v", err) + return nil + } + defer oldData.Close() + + tx, err := newDB.BeginTx(ctx, nil) + if err != nil { + return err + } + defer func() { + if exitErr == nil { + exitErr = tx.Commit() + } else { + tx.Rollback() + } + }() + + for oldData.Next() { + row := []interface{}{ + new(int), + new(string), + new(int), + new(int), + new(int), + new(int), + new(int), + new([]byte), + new([]byte), + } + if err := oldData.Scan(row...); err != nil { + return err + } + + if _, err := newDB.ExecContext(ctx, "INSERT INTO kine(id, name, created, deleted, create_revision, prev_revision, lease, value, old_value) values(?, ?, ?, ?, ?, ?, ?, ?, ?)", + row...); err != nil { + return err + } + } + + if err := oldData.Err(); err != nil { + return err + } + + return nil +} + +func parseOpts(dsn string) (opts, error) { + result := opts{ + dsn: dsn, + } + + parts := strings.SplitN(dsn, "?", 2) + if len(parts) == 1 { + return result, nil + } + + values, err := url.ParseQuery(parts[1]) + if err != nil { + return result, err + } + + for k, vs := range values { + if len(vs) == 0 { + continue + } + + switch k { + case "peer": + for _, v := range vs { + parts := strings.SplitN(v, ":", 3) + if len(parts) != 3 { + return result, fmt.Errorf("must be ID:IP:PORT format got: %s", v) + } + id, err := strconv.ParseUint(parts[0], 10, 64) + if err != nil { + return result, errors.Wrapf(err, "failed to parse %s", parts[0]) + } + result.peers = append(result.peers, client.NodeInfo{ + ID: id, + Address: parts[1] + ":" + parts[2], + }) + } + delete(values, k) + case "peer-file": + result.peerFile = vs[0] + delete(values, k) + } + } + + if len(values) == 0 { + result.dsn = parts[0] + } else { + result.dsn = fmt.Sprintf("%s?%s", parts[0], values.Encode()) + } + + return result, nil +} diff --git a/vendor/github.com/rancher/kine/pkg/drivers/dqlite/no_dqlite.go b/vendor/github.com/rancher/kine/pkg/drivers/dqlite/no_dqlite.go new file mode 100644 index 0000000000..85f4636687 --- /dev/null +++ b/vendor/github.com/rancher/kine/pkg/drivers/dqlite/no_dqlite.go @@ -0,0 +1,14 @@ +// +build !dqlite + +package dqlite + +import ( + "context" + "fmt" + + "github.com/rancher/kine/pkg/server" +) + +func New(ctx context.Context, datasourceName string) (server.Backend, error) { + return nil, fmt.Errorf("dqlite is not support, compile with \"-tags dqlite\"") +} diff --git a/vendor/github.com/rancher/kine/pkg/drivers/generic/generic.go b/vendor/github.com/rancher/kine/pkg/drivers/generic/generic.go index eeabc83667..66a8bb5831 100644 --- a/vendor/github.com/rancher/kine/pkg/drivers/generic/generic.go +++ b/vendor/github.com/rancher/kine/pkg/drivers/generic/generic.go @@ -7,6 +7,11 @@ import ( "regexp" "strconv" "strings" + "sync" + "time" + + "github.com/Rican7/retry/backoff" + "github.com/Rican7/retry/strategy" "github.com/sirupsen/logrus" ) @@ -57,7 +62,12 @@ func (s Stripped) String() string { return regexp.MustCompile("[\t ]+").ReplaceAllString(str, " ") } +type ErrRetry func(error) bool + type Generic struct { + sync.Mutex + + LockWrites bool LastInsertID bool DB *sql.DB GetCurrentSQL string @@ -72,6 +82,7 @@ type Generic struct { InsertSQL string FillSQL string InsertLastInsertIDSQL string + Retry ErrRetry } func q(sql, param string, numbered bool) string { @@ -179,9 +190,27 @@ func (d *Generic) queryRow(ctx context.Context, sql string, args ...interface{}) return d.DB.QueryRowContext(ctx, sql, args...) } -func (d *Generic) execute(ctx context.Context, sql string, args ...interface{}) (sql.Result, error) { - logrus.Tracef("EXEC %v : %s", args, Stripped(sql)) - return d.DB.ExecContext(ctx, sql, args...) +func (d *Generic) execute(ctx context.Context, sql string, args ...interface{}) (result sql.Result, err error) { + if d.LockWrites { + d.Lock() + defer d.Unlock() + } + + wait := strategy.Backoff(backoff.Linear(100 + time.Millisecond)) + for i := uint(0); i < 20; i++ { + if i > 2 { + logrus.Infof("EXEC (%d) %v : %s", i, args, Stripped(sql)) + } else { + logrus.Tracef("EXEC (%d) %v : %s", i, args, Stripped(sql)) + } + result, err = d.DB.ExecContext(ctx, sql, args...) + if err != nil && d.Retry != nil && d.Retry(err) { + wait(i) + continue + } + return result, err + } + return } func (d *Generic) GetCompactRevision(ctx context.Context) (int64, error) { diff --git a/vendor/github.com/rancher/kine/pkg/drivers/sqlite/sqlite.go b/vendor/github.com/rancher/kine/pkg/drivers/sqlite/sqlite.go index e951ce485a..78754c6f1b 100644 --- a/vendor/github.com/rancher/kine/pkg/drivers/sqlite/sqlite.go +++ b/vendor/github.com/rancher/kine/pkg/drivers/sqlite/sqlite.go @@ -34,25 +34,30 @@ var ( ) func New(dataSourceName string) (server.Backend, error) { + backend, _, err := NewVariant("sqlite3", dataSourceName) + return backend, err +} + +func NewVariant(driverName, dataSourceName string) (server.Backend, *generic.Generic, error) { if dataSourceName == "" { if err := os.MkdirAll("./db", 0700); err != nil { - return nil, err + return nil, nil, err } dataSourceName = "./db/state.db?_journal=WAL&cache=shared" } - dialect, err := generic.Open("sqlite3", dataSourceName, "?", false) + dialect, err := generic.Open(driverName, dataSourceName, "?", false) if err != nil { - return nil, err + return nil, nil, err } dialect.LastInsertID = true if err := setup(dialect.DB); err != nil { - return nil, err + return nil, nil, err } dialect.Migrate(context.Background()) - return logstructured.New(sqllog.New(dialect)), nil + return logstructured.New(sqllog.New(dialect)), dialect, nil } func setup(db *sql.DB) error { diff --git a/vendor/github.com/rancher/kine/pkg/endpoint/endpoint.go b/vendor/github.com/rancher/kine/pkg/endpoint/endpoint.go index bdb31dc1b8..c1bd303887 100644 --- a/vendor/github.com/rancher/kine/pkg/endpoint/endpoint.go +++ b/vendor/github.com/rancher/kine/pkg/endpoint/endpoint.go @@ -7,6 +7,7 @@ import ( "os" "strings" + "github.com/rancher/kine/pkg/drivers/dqlite" "github.com/rancher/kine/pkg/drivers/mysql" "github.com/rancher/kine/pkg/drivers/pgsql" "github.com/rancher/kine/pkg/drivers/sqlite" @@ -19,6 +20,7 @@ import ( const ( KineSocket = "unix://kine.sock" SQLiteBackend = "sqlite" + DQLiteBackend = "dqlite" ETCDBackend = "etcd3" MySQLBackend = "mysql" PostgresBackend = "postgres" @@ -48,7 +50,7 @@ func Listen(ctx context.Context, config Config) (ETCDConfig, error) { }, nil } - leaderelect, backend, err := getKineStorageBackend(driver, dsn, config) + leaderelect, backend, err := getKineStorageBackend(ctx, driver, dsn, config) if err != nil { return ETCDConfig{}, err } @@ -112,7 +114,7 @@ func grpcServer(config Config) *grpc.Server { return grpc.NewServer() } -func getKineStorageBackend(driver, dsn string, cfg Config) (bool, server.Backend, error) { +func getKineStorageBackend(ctx context.Context, driver, dsn string, cfg Config) (bool, server.Backend, error) { var ( backend server.Backend leaderElect = true @@ -122,6 +124,8 @@ func getKineStorageBackend(driver, dsn string, cfg Config) (bool, server.Backend case SQLiteBackend: leaderElect = false backend, err = sqlite.New(dsn) + case DQLiteBackend: + backend, err = dqlite.New(ctx, dsn) case PostgresBackend: backend, err = pgsql.New(dsn, cfg.Config) case MySQLBackend: diff --git a/vendor/github.com/rancher/kine/pkg/logstructured/sqllog/sql.go b/vendor/github.com/rancher/kine/pkg/logstructured/sqllog/sql.go index c777331a5f..9a82853d25 100644 --- a/vendor/github.com/rancher/kine/pkg/logstructured/sqllog/sql.go +++ b/vendor/github.com/rancher/kine/pkg/logstructured/sqllog/sql.go @@ -41,10 +41,9 @@ type Dialect interface { IsFill(key string) bool } -func (s *SQLLog) Start(ctx context.Context) error { +func (s *SQLLog) Start(ctx context.Context) (err error) { s.ctx = ctx - go s.compact() - return nil + return } func (s *SQLLog) compact() { @@ -266,14 +265,22 @@ func filter(events interface{}, checkPrefix bool, prefix string) ([]*server.Even } func (s *SQLLog) startWatch() (chan interface{}, error) { + pollStart, err := s.d.GetCompactRevision(s.ctx) + if err != nil { + return nil, err + } + c := make(chan interface{}) - go s.poll(c) + // start compaction and polling at the same time to watch starts + // at the oldest revision, but compaction doesn't create gaps + go s.compact() + go s.poll(c, pollStart) return c, nil } -func (s *SQLLog) poll(result chan interface{}) { +func (s *SQLLog) poll(result chan interface{}, pollStart int64) { var ( - last int64 + last = pollStart skip int64 skipTime time.Time ) @@ -293,15 +300,6 @@ func (s *SQLLog) poll(result chan interface{}) { case <-wait.C: } - if last == 0 { - if currentRev, err := s.CurrentRevision(s.ctx); err != nil { - logrus.Errorf("failed to find current revision: %v", err) - continue - } else { - last = currentRev - } - } - rows, err := s.d.After(s.ctx, "%", last) if err != nil { logrus.Errorf("fail to list latest changes: %v", err) diff --git a/vendor/go.uber.org/atomic/.travis.yml b/vendor/go.uber.org/atomic/.travis.yml index 762d22c972..0f3769e5fa 100644 --- a/vendor/go.uber.org/atomic/.travis.yml +++ b/vendor/go.uber.org/atomic/.travis.yml @@ -3,11 +3,13 @@ language: go go_import_path: go.uber.org/atomic go: - - 1.7.x - - 1.8.x - - 1.9.x - - 1.10.x - - 1.x # latest release + - 1.11.x + - 1.12.x + +matrix: + include: + - go: 1.12.x + env: NO_TEST=yes LINT=yes cache: directories: @@ -17,9 +19,9 @@ install: - make install_ci script: - - make test_ci - - scripts/test-ubergo.sh - - make lint + - test -n "$NO_TEST" || make test_ci + - test -n "$NO_TEST" || scripts/test-ubergo.sh + - test -z "$LINT" || make install_lint lint after_success: - bash <(curl -s https://codecov.io/bash) diff --git a/vendor/go.uber.org/atomic/Makefile b/vendor/go.uber.org/atomic/Makefile index dfc63d9db4..1ef263075d 100644 --- a/vendor/go.uber.org/atomic/Makefile +++ b/vendor/go.uber.org/atomic/Makefile @@ -1,24 +1,13 @@ -PACKAGES := $(shell glide nv) # Many Go tools take file globs or directories as arguments instead of packages. PACKAGE_FILES ?= *.go - -# The linting tools evolve with each Go version, so run them only on the latest -# stable release. -GO_VERSION := $(shell go version | cut -d " " -f 3) -GO_MINOR_VERSION := $(word 2,$(subst ., ,$(GO_VERSION))) -LINTABLE_MINOR_VERSIONS := 7 8 -ifneq ($(filter $(LINTABLE_MINOR_VERSIONS),$(GO_MINOR_VERSION)),) -SHOULD_LINT := true -endif - - +# For pre go1.6 export GO15VENDOREXPERIMENT=1 .PHONY: build build: - go build -i $(PACKAGES) + go build -i ./... .PHONY: install @@ -29,7 +18,7 @@ install: .PHONY: test test: - go test -cover -race $(PACKAGES) + go test -cover -race ./... .PHONY: install_ci @@ -37,26 +26,24 @@ install_ci: install go get github.com/wadey/gocovmerge go get github.com/mattn/goveralls go get golang.org/x/tools/cmd/cover -ifdef SHOULD_LINT - go get github.com/golang/lint/golint -endif + +.PHONY: install_lint +install_lint: + go get golang.org/x/lint/golint + .PHONY: lint lint: -ifdef SHOULD_LINT @rm -rf lint.log @echo "Checking formatting..." @gofmt -d -s $(PACKAGE_FILES) 2>&1 | tee lint.log @echo "Checking vet..." - @$(foreach dir,$(PACKAGE_FILES),go tool vet $(dir) 2>&1 | tee -a lint.log;) + @go vet ./... 2>&1 | tee -a lint.log;) @echo "Checking lint..." - @$(foreach dir,$(PKGS),golint $(dir) 2>&1 | tee -a lint.log;) + @golint $$(go list ./...) 2>&1 | tee -a lint.log @echo "Checking for unresolved FIXMEs..." @git grep -i fixme | grep -v -e vendor -e Makefile | tee -a lint.log @[ ! -s lint.log ] -else - @echo "Skipping linters on" $(GO_VERSION) -endif .PHONY: test_ci diff --git a/vendor/go.uber.org/atomic/README.md b/vendor/go.uber.org/atomic/README.md index a871d2b5f5..62eb8e5760 100644 --- a/vendor/go.uber.org/atomic/README.md +++ b/vendor/go.uber.org/atomic/README.md @@ -28,8 +28,8 @@ Released under the [MIT License](LICENSE.txt). [doc-img]: https://godoc.org/github.com/uber-go/atomic?status.svg [doc]: https://godoc.org/go.uber.org/atomic -[ci-img]: https://travis-ci.org/uber-go/atomic.svg?branch=master -[ci]: https://travis-ci.org/uber-go/atomic +[ci-img]: https://travis-ci.com/uber-go/atomic.svg?branch=master +[ci]: https://travis-ci.com/uber-go/atomic [cov-img]: https://codecov.io/gh/uber-go/atomic/branch/master/graph/badge.svg [cov]: https://codecov.io/gh/uber-go/atomic [reportcard-img]: https://goreportcard.com/badge/go.uber.org/atomic diff --git a/vendor/go.uber.org/multierr/.travis.yml b/vendor/go.uber.org/multierr/.travis.yml index fc3936befd..5ffa8fed48 100644 --- a/vendor/go.uber.org/multierr/.travis.yml +++ b/vendor/go.uber.org/multierr/.travis.yml @@ -9,7 +9,7 @@ env: go: - 1.7 - 1.8 - - 1.9 + - tip cache: directories: diff --git a/vendor/go.uber.org/multierr/error.go b/vendor/go.uber.org/multierr/error.go index 150fd95d91..de6ce4736c 100644 --- a/vendor/go.uber.org/multierr/error.go +++ b/vendor/go.uber.org/multierr/error.go @@ -33,7 +33,7 @@ // If only two errors are being combined, the Append function may be used // instead. // -// err = multierr.Append(reader.Close(), writer.Close()) +// err = multierr.Combine(reader.Close(), writer.Close()) // // This makes it possible to record resource cleanup failures from deferred // blocks with the help of named return values. diff --git a/vendor/go.uber.org/zap/.travis.yml b/vendor/go.uber.org/zap/.travis.yml index a3321fa2dc..ada5ebdcc9 100644 --- a/vendor/go.uber.org/zap/.travis.yml +++ b/vendor/go.uber.org/zap/.travis.yml @@ -1,8 +1,8 @@ language: go sudo: false go: - - 1.9.x - - 1.10.x + - 1.11.x + - 1.12.x go_import_path: go.uber.org/zap env: global: diff --git a/vendor/go.uber.org/zap/CHANGELOG.md b/vendor/go.uber.org/zap/CHANGELOG.md index 17d5b49f33..28d10677eb 100644 --- a/vendor/go.uber.org/zap/CHANGELOG.md +++ b/vendor/go.uber.org/zap/CHANGELOG.md @@ -1,5 +1,22 @@ # Changelog +## 1.10.0 (29 Apr 2019) + +Bugfixes: +* [#657][]: Fix `MapObjectEncoder.AppendByteString` not adding value as a + string. +* [#706][]: Fix incorrect call depth to determine caller in Go 1.12. + +Enhancements: +* [#610][]: Add `zaptest.WrapOptions` to wrap `zap.Option` for creating test + loggers. +* [#675][]: Don't panic when encoding a String field. +* [#704][]: Disable HTML escaping for JSON objects encoded using the + reflect-based encoder. + +Thanks to @iaroslav-ciupin, @lelenanam, @joa, @NWilson for their contributions +to this release. + ## v1.9.1 (06 Aug 2018) Bugfixes: @@ -303,3 +320,8 @@ upgrade to the upcoming stable release. [#572]: https://github.com/uber-go/zap/pull/572 [#606]: https://github.com/uber-go/zap/pull/606 [#614]: https://github.com/uber-go/zap/pull/614 +[#657]: https://github.com/uber-go/zap/pull/657 +[#706]: https://github.com/uber-go/zap/pull/706 +[#610]: https://github.com/uber-go/zap/pull/610 +[#675]: https://github.com/uber-go/zap/pull/675 +[#704]: https://github.com/uber-go/zap/pull/704 diff --git a/vendor/go.uber.org/zap/Makefile b/vendor/go.uber.org/zap/Makefile index ef7893b3b0..073e9aa910 100644 --- a/vendor/go.uber.org/zap/Makefile +++ b/vendor/go.uber.org/zap/Makefile @@ -9,7 +9,7 @@ PKG_FILES ?= *.go zapcore benchmarks buffer zapgrpc zaptest zaptest/observer int # stable release. GO_VERSION := $(shell go version | cut -d " " -f 3) GO_MINOR_VERSION := $(word 2,$(subst ., ,$(GO_VERSION))) -LINTABLE_MINOR_VERSIONS := 10 +LINTABLE_MINOR_VERSIONS := 12 ifneq ($(filter $(LINTABLE_MINOR_VERSIONS),$(GO_MINOR_VERSION)),) SHOULD_LINT := true endif @@ -45,7 +45,7 @@ ifdef SHOULD_LINT @echo "Installing test dependencies for vet..." @go test -i $(PKGS) @echo "Checking vet..." - @$(foreach dir,$(PKG_FILES),go tool vet $(VET_RULES) $(dir) 2>&1 | tee -a lint.log;) + @go vet $(VET_RULES) $(PKGS) 2>&1 | tee -a lint.log @echo "Checking lint..." @$(foreach dir,$(PKGS),golint $(dir) 2>&1 | tee -a lint.log;) @echo "Checking for unresolved FIXMEs..." diff --git a/vendor/go.uber.org/zap/global.go b/vendor/go.uber.org/zap/global.go index d02232e39f..c1ac0507cd 100644 --- a/vendor/go.uber.org/zap/global.go +++ b/vendor/go.uber.org/zap/global.go @@ -31,7 +31,6 @@ import ( ) const ( - _stdLogDefaultDepth = 2 _loggerWriterDepth = 2 _programmerErrorTemplate = "You've found a bug in zap! Please file a bug at " + "https://github.com/uber-go/zap/issues/new and reference this error: %v" diff --git a/vendor/go.uber.org/zap/global_go112.go b/vendor/go.uber.org/zap/global_go112.go new file mode 100644 index 0000000000..6b5dbda807 --- /dev/null +++ b/vendor/go.uber.org/zap/global_go112.go @@ -0,0 +1,26 @@ +// Copyright (c) 2019 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +// See #682 for more information. +// +build go1.12 + +package zap + +const _stdLogDefaultDepth = 1 diff --git a/vendor/go.uber.org/zap/global_prego112.go b/vendor/go.uber.org/zap/global_prego112.go new file mode 100644 index 0000000000..d3ab9af933 --- /dev/null +++ b/vendor/go.uber.org/zap/global_prego112.go @@ -0,0 +1,26 @@ +// Copyright (c) 2019 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +// See #682 for more information. +// +build !go1.12 + +package zap + +const _stdLogDefaultDepth = 2 diff --git a/vendor/go.uber.org/zap/zapcore/field.go b/vendor/go.uber.org/zap/zapcore/field.go index 6a5e33e2f7..ae772e4a17 100644 --- a/vendor/go.uber.org/zap/zapcore/field.go +++ b/vendor/go.uber.org/zap/zapcore/field.go @@ -160,7 +160,7 @@ func (f Field) AddTo(enc ObjectEncoder) { case NamespaceType: enc.OpenNamespace(f.Key) case StringerType: - enc.AddString(f.Key, f.Interface.(fmt.Stringer).String()) + err = encodeStringer(f.Key, f.Interface, enc) case ErrorType: encodeError(f.Key, f.Interface.(error), enc) case SkipType: @@ -199,3 +199,14 @@ func addFields(enc ObjectEncoder, fields []Field) { fields[i].AddTo(enc) } } + +func encodeStringer(key string, stringer interface{}, enc ObjectEncoder) (err error) { + defer func() { + if v := recover(); v != nil { + err = fmt.Errorf("PANIC=%v", v) + } + }() + + enc.AddString(key, stringer.(fmt.Stringer).String()) + return +} diff --git a/vendor/go.uber.org/zap/zapcore/json_encoder.go b/vendor/go.uber.org/zap/zapcore/json_encoder.go index 2dc67d81e7..9aec4eada3 100644 --- a/vendor/go.uber.org/zap/zapcore/json_encoder.go +++ b/vendor/go.uber.org/zap/zapcore/json_encoder.go @@ -137,6 +137,9 @@ func (enc *jsonEncoder) resetReflectBuf() { if enc.reflectBuf == nil { enc.reflectBuf = bufferpool.Get() enc.reflectEnc = json.NewEncoder(enc.reflectBuf) + + // For consistency with our custom JSON encoder. + enc.reflectEnc.SetEscapeHTML(false) } else { enc.reflectBuf.Reset() } diff --git a/vendor/go.uber.org/zap/zapcore/memory_encoder.go b/vendor/go.uber.org/zap/zapcore/memory_encoder.go index 6ef85b09c7..dfead0829d 100644 --- a/vendor/go.uber.org/zap/zapcore/memory_encoder.go +++ b/vendor/go.uber.org/zap/zapcore/memory_encoder.go @@ -158,7 +158,7 @@ func (s *sliceArrayEncoder) AppendReflected(v interface{}) error { } func (s *sliceArrayEncoder) AppendBool(v bool) { s.elems = append(s.elems, v) } -func (s *sliceArrayEncoder) AppendByteString(v []byte) { s.elems = append(s.elems, v) } +func (s *sliceArrayEncoder) AppendByteString(v []byte) { s.elems = append(s.elems, string(v)) } func (s *sliceArrayEncoder) AppendComplex128(v complex128) { s.elems = append(s.elems, v) } func (s *sliceArrayEncoder) AppendComplex64(v complex64) { s.elems = append(s.elems, v) } func (s *sliceArrayEncoder) AppendDuration(v time.Duration) { s.elems = append(s.elems, v) } diff --git a/vendor/google.golang.org/genproto/googleapis/api/annotations/annotations.pb.go b/vendor/google.golang.org/genproto/googleapis/api/annotations/annotations.pb.go index 9521b50e9e..bf2f703fff 100644 --- a/vendor/google.golang.org/genproto/googleapis/api/annotations/annotations.pb.go +++ b/vendor/google.golang.org/genproto/googleapis/api/annotations/annotations.pb.go @@ -1,12 +1,15 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // source: google/api/annotations.proto -package annotations // import "google.golang.org/genproto/googleapis/api/annotations" +package annotations -import proto "github.com/golang/protobuf/proto" -import fmt "fmt" -import math "math" -import descriptor "github.com/golang/protobuf/protoc-gen-go/descriptor" +import ( + fmt "fmt" + math "math" + + proto "github.com/golang/protobuf/proto" + descriptor "github.com/golang/protobuf/protoc-gen-go/descriptor" +) // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal @@ -17,7 +20,7 @@ var _ = math.Inf // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package var E_Http = &proto.ExtensionDesc{ ExtendedType: (*descriptor.MethodOptions)(nil), @@ -32,11 +35,9 @@ func init() { proto.RegisterExtension(E_Http) } -func init() { - proto.RegisterFile("google/api/annotations.proto", fileDescriptor_annotations_55609bb51d80951d) -} +func init() { proto.RegisterFile("google/api/annotations.proto", fileDescriptor_c591c5aa9fb79aab) } -var fileDescriptor_annotations_55609bb51d80951d = []byte{ +var fileDescriptor_c591c5aa9fb79aab = []byte{ // 208 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x49, 0xcf, 0xcf, 0x4f, 0xcf, 0x49, 0xd5, 0x4f, 0x2c, 0xc8, 0xd4, 0x4f, 0xcc, 0xcb, 0xcb, 0x2f, 0x49, 0x2c, 0xc9, 0xcc, diff --git a/vendor/google.golang.org/genproto/googleapis/api/annotations/client.pb.go b/vendor/google.golang.org/genproto/googleapis/api/annotations/client.pb.go index d64b32280f..867fc0c3fa 100644 --- a/vendor/google.golang.org/genproto/googleapis/api/annotations/client.pb.go +++ b/vendor/google.golang.org/genproto/googleapis/api/annotations/client.pb.go @@ -1,12 +1,15 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // source: google/api/client.proto -package annotations // import "google.golang.org/genproto/googleapis/api/annotations" +package annotations -import proto "github.com/golang/protobuf/proto" -import fmt "fmt" -import math "math" -import descriptor "github.com/golang/protobuf/protoc-gen-go/descriptor" +import ( + fmt "fmt" + math "math" + + proto "github.com/golang/protobuf/proto" + descriptor "github.com/golang/protobuf/protoc-gen-go/descriptor" +) // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal @@ -17,14 +20,14 @@ var _ = math.Inf // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package var E_MethodSignature = &proto.ExtensionDesc{ ExtendedType: (*descriptor.MethodOptions)(nil), ExtensionType: ([]string)(nil), Field: 1051, Name: "google.api.method_signature", - Tag: "bytes,1051,rep,name=method_signature,json=methodSignature", + Tag: "bytes,1051,rep,name=method_signature", Filename: "google/api/client.proto", } @@ -33,7 +36,7 @@ var E_DefaultHost = &proto.ExtensionDesc{ ExtensionType: (*string)(nil), Field: 1049, Name: "google.api.default_host", - Tag: "bytes,1049,opt,name=default_host,json=defaultHost", + Tag: "bytes,1049,opt,name=default_host", Filename: "google/api/client.proto", } @@ -42,7 +45,7 @@ var E_OauthScopes = &proto.ExtensionDesc{ ExtensionType: (*string)(nil), Field: 1050, Name: "google.api.oauth_scopes", - Tag: "bytes,1050,opt,name=oauth_scopes,json=oauthScopes", + Tag: "bytes,1050,opt,name=oauth_scopes", Filename: "google/api/client.proto", } @@ -52,9 +55,9 @@ func init() { proto.RegisterExtension(E_OauthScopes) } -func init() { proto.RegisterFile("google/api/client.proto", fileDescriptor_client_1608614df476619f) } +func init() { proto.RegisterFile("google/api/client.proto", fileDescriptor_78f2c6f7c3a942c1) } -var fileDescriptor_client_1608614df476619f = []byte{ +var fileDescriptor_78f2c6f7c3a942c1 = []byte{ // 262 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x90, 0x3f, 0x4f, 0xc3, 0x30, 0x10, 0xc5, 0x55, 0x40, 0xa8, 0x75, 0x11, 0xa0, 0x2c, 0x20, 0x06, 0xc8, 0xd8, 0xc9, 0x1e, 0xd8, diff --git a/vendor/google.golang.org/genproto/googleapis/api/annotations/field_behavior.pb.go b/vendor/google.golang.org/genproto/googleapis/api/annotations/field_behavior.pb.go index 9a9ab1242f..31f87dd00d 100644 --- a/vendor/google.golang.org/genproto/googleapis/api/annotations/field_behavior.pb.go +++ b/vendor/google.golang.org/genproto/googleapis/api/annotations/field_behavior.pb.go @@ -1,12 +1,15 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // source: google/api/field_behavior.proto -package annotations // import "google.golang.org/genproto/googleapis/api/annotations" +package annotations -import proto "github.com/golang/protobuf/proto" -import fmt "fmt" -import math "math" -import descriptor "github.com/golang/protobuf/protoc-gen-go/descriptor" +import ( + fmt "fmt" + math "math" + + proto "github.com/golang/protobuf/proto" + descriptor "github.com/golang/protobuf/protoc-gen-go/descriptor" +) // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal @@ -17,7 +20,7 @@ var _ = math.Inf // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package // An indicator of the behavior of a given field (for example, that a field // is required in requests, or given as output but ignored as input). @@ -61,6 +64,7 @@ var FieldBehavior_name = map[int32]string{ 4: "INPUT_ONLY", 5: "IMMUTABLE", } + var FieldBehavior_value = map[string]int32{ "FIELD_BEHAVIOR_UNSPECIFIED": 0, "OPTIONAL": 1, @@ -73,8 +77,9 @@ var FieldBehavior_value = map[string]int32{ func (x FieldBehavior) String() string { return proto.EnumName(FieldBehavior_name, int32(x)) } + func (FieldBehavior) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_field_behavior_ddf5c982f789c6a3, []int{0} + return fileDescriptor_4648f18fd5079967, []int{0} } var E_FieldBehavior = &proto.ExtensionDesc{ @@ -82,7 +87,7 @@ var E_FieldBehavior = &proto.ExtensionDesc{ ExtensionType: ([]FieldBehavior)(nil), Field: 1052, Name: "google.api.field_behavior", - Tag: "varint,1052,rep,name=field_behavior,json=fieldBehavior,enum=google.api.FieldBehavior", + Tag: "varint,1052,rep,name=field_behavior,enum=google.api.FieldBehavior", Filename: "google/api/field_behavior.proto", } @@ -91,11 +96,9 @@ func init() { proto.RegisterExtension(E_FieldBehavior) } -func init() { - proto.RegisterFile("google/api/field_behavior.proto", fileDescriptor_field_behavior_ddf5c982f789c6a3) -} +func init() { proto.RegisterFile("google/api/field_behavior.proto", fileDescriptor_4648f18fd5079967) } -var fileDescriptor_field_behavior_ddf5c982f789c6a3 = []byte{ +var fileDescriptor_4648f18fd5079967 = []byte{ // 303 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x54, 0x90, 0x4f, 0x4f, 0xb3, 0x30, 0x1c, 0xc7, 0x9f, 0xfd, 0x79, 0xcc, 0xac, 0x0e, 0x49, 0x4f, 0xba, 0x44, 0xdd, 0xd1, 0x78, 0x28, diff --git a/vendor/google.golang.org/genproto/googleapis/api/annotations/http.pb.go b/vendor/google.golang.org/genproto/googleapis/api/annotations/http.pb.go index ca20ad3d61..a63870374d 100644 --- a/vendor/google.golang.org/genproto/googleapis/api/annotations/http.pb.go +++ b/vendor/google.golang.org/genproto/googleapis/api/annotations/http.pb.go @@ -1,11 +1,14 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // source: google/api/http.proto -package annotations // import "google.golang.org/genproto/googleapis/api/annotations" +package annotations -import proto "github.com/golang/protobuf/proto" -import fmt "fmt" -import math "math" +import ( + fmt "fmt" + math "math" + + proto "github.com/golang/protobuf/proto" +) // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal @@ -16,7 +19,7 @@ var _ = math.Inf // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package // Defines the HTTP configuration for an API service. It contains a list of // [HttpRule][google.api.HttpRule], each specifying the mapping of an RPC method @@ -42,16 +45,17 @@ func (m *Http) Reset() { *m = Http{} } func (m *Http) String() string { return proto.CompactTextString(m) } func (*Http) ProtoMessage() {} func (*Http) Descriptor() ([]byte, []int) { - return fileDescriptor_http_5af6bbacbb935ee3, []int{0} + return fileDescriptor_ff9994be407cdcc9, []int{0} } + func (m *Http) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Http.Unmarshal(m, b) } func (m *Http) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Http.Marshal(b, m, deterministic) } -func (dst *Http) XXX_Merge(src proto.Message) { - xxx_messageInfo_Http.Merge(dst, src) +func (m *Http) XXX_Merge(src proto.Message) { + xxx_messageInfo_Http.Merge(m, src) } func (m *Http) XXX_Size() int { return xxx_messageInfo_Http.Size(m) @@ -389,16 +393,17 @@ func (m *HttpRule) Reset() { *m = HttpRule{} } func (m *HttpRule) String() string { return proto.CompactTextString(m) } func (*HttpRule) ProtoMessage() {} func (*HttpRule) Descriptor() ([]byte, []int) { - return fileDescriptor_http_5af6bbacbb935ee3, []int{1} + return fileDescriptor_ff9994be407cdcc9, []int{1} } + func (m *HttpRule) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_HttpRule.Unmarshal(m, b) } func (m *HttpRule) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_HttpRule.Marshal(b, m, deterministic) } -func (dst *HttpRule) XXX_Merge(src proto.Message) { - xxx_messageInfo_HttpRule.Merge(dst, src) +func (m *HttpRule) XXX_Merge(src proto.Message) { + xxx_messageInfo_HttpRule.Merge(m, src) } func (m *HttpRule) XXX_Size() int { return xxx_messageInfo_HttpRule.Size(m) @@ -526,9 +531,9 @@ func (m *HttpRule) GetAdditionalBindings() []*HttpRule { return nil } -// XXX_OneofFuncs is for the internal use of the proto package. -func (*HttpRule) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { - return _HttpRule_OneofMarshaler, _HttpRule_OneofUnmarshaler, _HttpRule_OneofSizer, []interface{}{ +// XXX_OneofWrappers is for the internal use of the proto package. +func (*HttpRule) XXX_OneofWrappers() []interface{} { + return []interface{}{ (*HttpRule_Get)(nil), (*HttpRule_Put)(nil), (*HttpRule_Post)(nil), @@ -538,124 +543,6 @@ func (*HttpRule) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) erro } } -func _HttpRule_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { - m := msg.(*HttpRule) - // pattern - switch x := m.Pattern.(type) { - case *HttpRule_Get: - b.EncodeVarint(2<<3 | proto.WireBytes) - b.EncodeStringBytes(x.Get) - case *HttpRule_Put: - b.EncodeVarint(3<<3 | proto.WireBytes) - b.EncodeStringBytes(x.Put) - case *HttpRule_Post: - b.EncodeVarint(4<<3 | proto.WireBytes) - b.EncodeStringBytes(x.Post) - case *HttpRule_Delete: - b.EncodeVarint(5<<3 | proto.WireBytes) - b.EncodeStringBytes(x.Delete) - case *HttpRule_Patch: - b.EncodeVarint(6<<3 | proto.WireBytes) - b.EncodeStringBytes(x.Patch) - case *HttpRule_Custom: - b.EncodeVarint(8<<3 | proto.WireBytes) - if err := b.EncodeMessage(x.Custom); err != nil { - return err - } - case nil: - default: - return fmt.Errorf("HttpRule.Pattern has unexpected type %T", x) - } - return nil -} - -func _HttpRule_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { - m := msg.(*HttpRule) - switch tag { - case 2: // pattern.get - if wire != proto.WireBytes { - return true, proto.ErrInternalBadWireType - } - x, err := b.DecodeStringBytes() - m.Pattern = &HttpRule_Get{x} - return true, err - case 3: // pattern.put - if wire != proto.WireBytes { - return true, proto.ErrInternalBadWireType - } - x, err := b.DecodeStringBytes() - m.Pattern = &HttpRule_Put{x} - return true, err - case 4: // pattern.post - if wire != proto.WireBytes { - return true, proto.ErrInternalBadWireType - } - x, err := b.DecodeStringBytes() - m.Pattern = &HttpRule_Post{x} - return true, err - case 5: // pattern.delete - if wire != proto.WireBytes { - return true, proto.ErrInternalBadWireType - } - x, err := b.DecodeStringBytes() - m.Pattern = &HttpRule_Delete{x} - return true, err - case 6: // pattern.patch - if wire != proto.WireBytes { - return true, proto.ErrInternalBadWireType - } - x, err := b.DecodeStringBytes() - m.Pattern = &HttpRule_Patch{x} - return true, err - case 8: // pattern.custom - if wire != proto.WireBytes { - return true, proto.ErrInternalBadWireType - } - msg := new(CustomHttpPattern) - err := b.DecodeMessage(msg) - m.Pattern = &HttpRule_Custom{msg} - return true, err - default: - return false, nil - } -} - -func _HttpRule_OneofSizer(msg proto.Message) (n int) { - m := msg.(*HttpRule) - // pattern - switch x := m.Pattern.(type) { - case *HttpRule_Get: - n += 1 // tag and wire - n += proto.SizeVarint(uint64(len(x.Get))) - n += len(x.Get) - case *HttpRule_Put: - n += 1 // tag and wire - n += proto.SizeVarint(uint64(len(x.Put))) - n += len(x.Put) - case *HttpRule_Post: - n += 1 // tag and wire - n += proto.SizeVarint(uint64(len(x.Post))) - n += len(x.Post) - case *HttpRule_Delete: - n += 1 // tag and wire - n += proto.SizeVarint(uint64(len(x.Delete))) - n += len(x.Delete) - case *HttpRule_Patch: - n += 1 // tag and wire - n += proto.SizeVarint(uint64(len(x.Patch))) - n += len(x.Patch) - case *HttpRule_Custom: - s := proto.Size(x.Custom) - n += 1 // tag and wire - n += proto.SizeVarint(uint64(s)) - n += s - case nil: - default: - panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) - } - return n -} - // A custom pattern is used for defining custom HTTP verb. type CustomHttpPattern struct { // The name of this custom HTTP verb. @@ -671,16 +558,17 @@ func (m *CustomHttpPattern) Reset() { *m = CustomHttpPattern{} } func (m *CustomHttpPattern) String() string { return proto.CompactTextString(m) } func (*CustomHttpPattern) ProtoMessage() {} func (*CustomHttpPattern) Descriptor() ([]byte, []int) { - return fileDescriptor_http_5af6bbacbb935ee3, []int{2} + return fileDescriptor_ff9994be407cdcc9, []int{2} } + func (m *CustomHttpPattern) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CustomHttpPattern.Unmarshal(m, b) } func (m *CustomHttpPattern) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CustomHttpPattern.Marshal(b, m, deterministic) } -func (dst *CustomHttpPattern) XXX_Merge(src proto.Message) { - xxx_messageInfo_CustomHttpPattern.Merge(dst, src) +func (m *CustomHttpPattern) XXX_Merge(src proto.Message) { + xxx_messageInfo_CustomHttpPattern.Merge(m, src) } func (m *CustomHttpPattern) XXX_Size() int { return xxx_messageInfo_CustomHttpPattern.Size(m) @@ -711,9 +599,9 @@ func init() { proto.RegisterType((*CustomHttpPattern)(nil), "google.api.CustomHttpPattern") } -func init() { proto.RegisterFile("google/api/http.proto", fileDescriptor_http_5af6bbacbb935ee3) } +func init() { proto.RegisterFile("google/api/http.proto", fileDescriptor_ff9994be407cdcc9) } -var fileDescriptor_http_5af6bbacbb935ee3 = []byte{ +var fileDescriptor_ff9994be407cdcc9 = []byte{ // 419 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x92, 0xc1, 0x8e, 0xd3, 0x30, 0x10, 0x86, 0x49, 0x9b, 0x76, 0xdb, 0xe9, 0x82, 0x84, 0x59, 0x90, 0x85, 0x40, 0x54, 0xe5, 0x52, diff --git a/vendor/google.golang.org/genproto/googleapis/api/annotations/resource.pb.go b/vendor/google.golang.org/genproto/googleapis/api/annotations/resource.pb.go index 036ae3e16b..af057b90be 100644 --- a/vendor/google.golang.org/genproto/googleapis/api/annotations/resource.pb.go +++ b/vendor/google.golang.org/genproto/googleapis/api/annotations/resource.pb.go @@ -1,12 +1,15 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // source: google/api/resource.proto -package annotations // import "google.golang.org/genproto/googleapis/api/annotations" +package annotations -import proto "github.com/golang/protobuf/proto" -import fmt "fmt" -import math "math" -import descriptor "github.com/golang/protobuf/protoc-gen-go/descriptor" +import ( + fmt "fmt" + math "math" + + proto "github.com/golang/protobuf/proto" + descriptor "github.com/golang/protobuf/protoc-gen-go/descriptor" +) // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal @@ -17,7 +20,7 @@ var _ = math.Inf // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package // A description of the historical or future-looking state of the // resource pattern. @@ -40,6 +43,7 @@ var ResourceDescriptor_History_name = map[int32]string{ 1: "ORIGINALLY_SINGLE_PATTERN", 2: "FUTURE_MULTI_PATTERN", } + var ResourceDescriptor_History_value = map[string]int32{ "HISTORY_UNSPECIFIED": 0, "ORIGINALLY_SINGLE_PATTERN": 1, @@ -49,8 +53,9 @@ var ResourceDescriptor_History_value = map[string]int32{ func (x ResourceDescriptor_History) String() string { return proto.EnumName(ResourceDescriptor_History_name, int32(x)) } + func (ResourceDescriptor_History) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_resource_1953877c7bf00bf4, []int{0, 0} + return fileDescriptor_465e9122405d1bb5, []int{0, 0} } // A simple descriptor of a resource type. @@ -86,18 +91,18 @@ func (ResourceDescriptor_History) EnumDescriptor() ([]byte, []int) { // }; // } type ResourceDescriptor struct { - // The full name of the resource type. It must be in the format of - // {service_name}/{resource_type_kind}. The resource type names are - // singular and do not contain version numbers. + // The resource type. It must be in the format of + // {service_name}/{resource_type_kind}. The `resource_type_kind` must be + // singular and must not include version numbers. // - // For example: `storage.googleapis.com/Bucket` + // Example: `storage.googleapis.com/Bucket` // // The value of the resource_type_kind must follow the regular expression - // /[A-Z][a-zA-Z0-9]+/. It must start with upper case character and - // recommended to use PascalCase (UpperCamelCase). The maximum number of - // characters allowed for the resource_type_kind is 100. + // /[A-Za-z][a-zA-Z0-9]+/. It should start with an upper case character and + // should use PascalCase (UpperCamelCase). The maximum number of + // characters allowed for the `resource_type_kind` is 100. Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` - // Required. The valid pattern or patterns for this resource's names. + // Optional. The valid resource name pattern(s) for this resource type. // // Examples: // - "projects/{project}/topics/{topic}" @@ -119,7 +124,8 @@ type ResourceDescriptor struct { // message InspectTemplate { // option (google.api.resource) = { // type: "dlp.googleapis.com/InspectTemplate" - // pattern: "organizations/{organization}/inspectTemplates/{inspect_template}" + // pattern: + // "organizations/{organization}/inspectTemplates/{inspect_template}" // pattern: "projects/{project}/inspectTemplates/{inspect_template}" // history: ORIGINALLY_SINGLE_PATTERN // }; @@ -134,16 +140,17 @@ func (m *ResourceDescriptor) Reset() { *m = ResourceDescriptor{} } func (m *ResourceDescriptor) String() string { return proto.CompactTextString(m) } func (*ResourceDescriptor) ProtoMessage() {} func (*ResourceDescriptor) Descriptor() ([]byte, []int) { - return fileDescriptor_resource_1953877c7bf00bf4, []int{0} + return fileDescriptor_465e9122405d1bb5, []int{0} } + func (m *ResourceDescriptor) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ResourceDescriptor.Unmarshal(m, b) } func (m *ResourceDescriptor) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_ResourceDescriptor.Marshal(b, m, deterministic) } -func (dst *ResourceDescriptor) XXX_Merge(src proto.Message) { - xxx_messageInfo_ResourceDescriptor.Merge(dst, src) +func (m *ResourceDescriptor) XXX_Merge(src proto.Message) { + xxx_messageInfo_ResourceDescriptor.Merge(m, src) } func (m *ResourceDescriptor) XXX_Size() int { return xxx_messageInfo_ResourceDescriptor.Size(m) @@ -182,11 +189,9 @@ func (m *ResourceDescriptor) GetHistory() ResourceDescriptor_History { return ResourceDescriptor_HISTORY_UNSPECIFIED } -// An annotation designating that this field is a reference to a resource -// defined by another message. +// Defines a proto annotation that describes a field that refers to a resource. type ResourceReference struct { - // The unified resource type name of the type that this field references. - // Marks this as a field referring to a resource in another message. + // The resource type that the annotated field references. // // Example: // @@ -196,11 +201,9 @@ type ResourceReference struct { // }]; // } Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` - // The fully-qualified message name of a child of the type that this field - // references. - // - // This is useful for `parent` fields where a resource has more than one - // possible type of parent. + // The resource type of a child collection that the annotated field + // references. This is useful for `parent` fields where a resource has more + // than one possible type of parent. // // Example: // @@ -209,14 +212,6 @@ type ResourceReference struct { // child_type: "logging.googleapis.com/LogEntry" // }; // } - // - // If the referenced message is in the same proto package, the service name - // may be omitted: - // - // message ListLogEntriesRequest { - // string parent = 1 - // [(google.api.resource_reference).child_type = "LogEntry"]; - // } ChildType string `protobuf:"bytes,2,opt,name=child_type,json=childType,proto3" json:"child_type,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` @@ -227,16 +222,17 @@ func (m *ResourceReference) Reset() { *m = ResourceReference{} } func (m *ResourceReference) String() string { return proto.CompactTextString(m) } func (*ResourceReference) ProtoMessage() {} func (*ResourceReference) Descriptor() ([]byte, []int) { - return fileDescriptor_resource_1953877c7bf00bf4, []int{1} + return fileDescriptor_465e9122405d1bb5, []int{1} } + func (m *ResourceReference) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ResourceReference.Unmarshal(m, b) } func (m *ResourceReference) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_ResourceReference.Marshal(b, m, deterministic) } -func (dst *ResourceReference) XXX_Merge(src proto.Message) { - xxx_messageInfo_ResourceReference.Merge(dst, src) +func (m *ResourceReference) XXX_Merge(src proto.Message) { + xxx_messageInfo_ResourceReference.Merge(m, src) } func (m *ResourceReference) XXX_Size() int { return xxx_messageInfo_ResourceReference.Size(m) @@ -266,7 +262,7 @@ var E_ResourceReference = &proto.ExtensionDesc{ ExtensionType: (*ResourceReference)(nil), Field: 1055, Name: "google.api.resource_reference", - Tag: "bytes,1055,opt,name=resource_reference,json=resourceReference", + Tag: "bytes,1055,opt,name=resource_reference", Filename: "google/api/resource.proto", } @@ -280,16 +276,16 @@ var E_Resource = &proto.ExtensionDesc{ } func init() { + proto.RegisterEnum("google.api.ResourceDescriptor_History", ResourceDescriptor_History_name, ResourceDescriptor_History_value) proto.RegisterType((*ResourceDescriptor)(nil), "google.api.ResourceDescriptor") proto.RegisterType((*ResourceReference)(nil), "google.api.ResourceReference") - proto.RegisterEnum("google.api.ResourceDescriptor_History", ResourceDescriptor_History_name, ResourceDescriptor_History_value) proto.RegisterExtension(E_ResourceReference) proto.RegisterExtension(E_Resource) } -func init() { proto.RegisterFile("google/api/resource.proto", fileDescriptor_resource_1953877c7bf00bf4) } +func init() { proto.RegisterFile("google/api/resource.proto", fileDescriptor_465e9122405d1bb5) } -var fileDescriptor_resource_1953877c7bf00bf4 = []byte{ +var fileDescriptor_465e9122405d1bb5 = []byte{ // 430 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x52, 0x41, 0x6f, 0xd3, 0x30, 0x18, 0x25, 0x59, 0x45, 0xd7, 0x0f, 0x31, 0x6d, 0x06, 0x89, 0x0c, 0x29, 0x10, 0xf5, 0x80, 0x7a, diff --git a/vendor/google.golang.org/genproto/googleapis/rpc/status/status.pb.go b/vendor/google.golang.org/genproto/googleapis/rpc/status/status.pb.go index 57ae35f6b5..0b9907f89b 100644 --- a/vendor/google.golang.org/genproto/googleapis/rpc/status/status.pb.go +++ b/vendor/google.golang.org/genproto/googleapis/rpc/status/status.pb.go @@ -1,12 +1,15 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // source: google/rpc/status.proto -package status // import "google.golang.org/genproto/googleapis/rpc/status" +package status -import proto "github.com/golang/protobuf/proto" -import fmt "fmt" -import math "math" -import any "github.com/golang/protobuf/ptypes/any" +import ( + fmt "fmt" + math "math" + + proto "github.com/golang/protobuf/proto" + any "github.com/golang/protobuf/ptypes/any" +) // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal @@ -17,7 +20,7 @@ var _ = math.Inf // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package // The `Status` type defines a logical error model that is suitable for // different programming environments, including REST APIs and RPC APIs. It is @@ -93,16 +96,17 @@ func (m *Status) Reset() { *m = Status{} } func (m *Status) String() string { return proto.CompactTextString(m) } func (*Status) ProtoMessage() {} func (*Status) Descriptor() ([]byte, []int) { - return fileDescriptor_status_ced6ddf76350620b, []int{0} + return fileDescriptor_24d244abaf643bfe, []int{0} } + func (m *Status) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Status.Unmarshal(m, b) } func (m *Status) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Status.Marshal(b, m, deterministic) } -func (dst *Status) XXX_Merge(src proto.Message) { - xxx_messageInfo_Status.Merge(dst, src) +func (m *Status) XXX_Merge(src proto.Message) { + xxx_messageInfo_Status.Merge(m, src) } func (m *Status) XXX_Size() int { return xxx_messageInfo_Status.Size(m) @@ -138,9 +142,9 @@ func init() { proto.RegisterType((*Status)(nil), "google.rpc.Status") } -func init() { proto.RegisterFile("google/rpc/status.proto", fileDescriptor_status_ced6ddf76350620b) } +func init() { proto.RegisterFile("google/rpc/status.proto", fileDescriptor_24d244abaf643bfe) } -var fileDescriptor_status_ced6ddf76350620b = []byte{ +var fileDescriptor_24d244abaf643bfe = []byte{ // 209 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x12, 0x4f, 0xcf, 0xcf, 0x4f, 0xcf, 0x49, 0xd5, 0x2f, 0x2a, 0x48, 0xd6, 0x2f, 0x2e, 0x49, 0x2c, 0x29, 0x2d, 0xd6, 0x2b, 0x28, diff --git a/vendor/gopkg.in/robfig/cron.v2/.gitignore b/vendor/gopkg.in/robfig/cron.v2/.gitignore new file mode 100644 index 0000000000..00268614f0 --- /dev/null +++ b/vendor/gopkg.in/robfig/cron.v2/.gitignore @@ -0,0 +1,22 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe diff --git a/vendor/gopkg.in/robfig/cron.v2/.travis.yml b/vendor/gopkg.in/robfig/cron.v2/.travis.yml new file mode 100644 index 0000000000..4f2ee4d973 --- /dev/null +++ b/vendor/gopkg.in/robfig/cron.v2/.travis.yml @@ -0,0 +1 @@ +language: go diff --git a/vendor/gopkg.in/robfig/cron.v2/LICENSE b/vendor/gopkg.in/robfig/cron.v2/LICENSE new file mode 100644 index 0000000000..3a0f627ffe --- /dev/null +++ b/vendor/gopkg.in/robfig/cron.v2/LICENSE @@ -0,0 +1,21 @@ +Copyright (C) 2012 Rob Figueiredo +All Rights Reserved. + +MIT LICENSE + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/gopkg.in/robfig/cron.v2/README.md b/vendor/gopkg.in/robfig/cron.v2/README.md new file mode 100644 index 0000000000..a9db98c35f --- /dev/null +++ b/vendor/gopkg.in/robfig/cron.v2/README.md @@ -0,0 +1 @@ +[![GoDoc](http://godoc.org/github.com/robfig/cron?status.png)](http://godoc.org/github.com/robfig/cron) diff --git a/vendor/gopkg.in/robfig/cron.v2/constantdelay.go b/vendor/gopkg.in/robfig/cron.v2/constantdelay.go new file mode 100644 index 0000000000..cd6e7b1be9 --- /dev/null +++ b/vendor/gopkg.in/robfig/cron.v2/constantdelay.go @@ -0,0 +1,27 @@ +package cron + +import "time" + +// ConstantDelaySchedule represents a simple recurring duty cycle, e.g. "Every 5 minutes". +// It does not support jobs more frequent than once a second. +type ConstantDelaySchedule struct { + Delay time.Duration +} + +// Every returns a crontab Schedule that activates once every duration. +// Delays of less than a second are not supported (will round up to 1 second). +// Any fields less than a Second are truncated. +func Every(duration time.Duration) ConstantDelaySchedule { + if duration < time.Second { + duration = time.Second + } + return ConstantDelaySchedule{ + Delay: duration - time.Duration(duration.Nanoseconds())%time.Second, + } +} + +// Next returns the next time this should be run. +// This rounds so that the next activation time will be on the second. +func (schedule ConstantDelaySchedule) Next(t time.Time) time.Time { + return t.Add(schedule.Delay - time.Duration(t.Nanosecond())*time.Nanosecond) +} diff --git a/vendor/gopkg.in/robfig/cron.v2/cron.go b/vendor/gopkg.in/robfig/cron.v2/cron.go new file mode 100644 index 0000000000..62d2d839e0 --- /dev/null +++ b/vendor/gopkg.in/robfig/cron.v2/cron.go @@ -0,0 +1,236 @@ +// Package cron implements a cron spec parser and runner. +package cron // import "gopkg.in/robfig/cron.v2" + +import ( + "sort" + "time" +) + +// Cron keeps track of any number of entries, invoking the associated func as +// specified by the schedule. It may be started, stopped, and the entries may +// be inspected while running. +type Cron struct { + entries []*Entry + stop chan struct{} + add chan *Entry + remove chan EntryID + snapshot chan []Entry + running bool + nextID EntryID +} + +// Job is an interface for submitted cron jobs. +type Job interface { + Run() +} + +// Schedule describes a job's duty cycle. +type Schedule interface { + // Next returns the next activation time, later than the given time. + // Next is invoked initially, and then each time the job is run. + Next(time.Time) time.Time +} + +// EntryID identifies an entry within a Cron instance +type EntryID int + +// Entry consists of a schedule and the func to execute on that schedule. +type Entry struct { + // ID is the cron-assigned ID of this entry, which may be used to look up a + // snapshot or remove it. + ID EntryID + + // Schedule on which this job should be run. + Schedule Schedule + + // Next time the job will run, or the zero time if Cron has not been + // started or this entry's schedule is unsatisfiable + Next time.Time + + // Prev is the last time this job was run, or the zero time if never. + Prev time.Time + + // Job is the thing to run when the Schedule is activated. + Job Job +} + +// Valid returns true if this is not the zero entry. +func (e Entry) Valid() bool { return e.ID != 0 } + +// byTime is a wrapper for sorting the entry array by time +// (with zero time at the end). +type byTime []*Entry + +func (s byTime) Len() int { return len(s) } +func (s byTime) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s byTime) Less(i, j int) bool { + // Two zero times should return false. + // Otherwise, zero is "greater" than any other time. + // (To sort it at the end of the list.) + if s[i].Next.IsZero() { + return false + } + if s[j].Next.IsZero() { + return true + } + return s[i].Next.Before(s[j].Next) +} + +// New returns a new Cron job runner. +func New() *Cron { + return &Cron{ + entries: nil, + add: make(chan *Entry), + stop: make(chan struct{}), + snapshot: make(chan []Entry), + remove: make(chan EntryID), + running: false, + } +} + +// FuncJob is a wrapper that turns a func() into a cron.Job +type FuncJob func() + +func (f FuncJob) Run() { f() } + +// AddFunc adds a func to the Cron to be run on the given schedule. +func (c *Cron) AddFunc(spec string, cmd func()) (EntryID, error) { + return c.AddJob(spec, FuncJob(cmd)) +} + +// AddJob adds a Job to the Cron to be run on the given schedule. +func (c *Cron) AddJob(spec string, cmd Job) (EntryID, error) { + schedule, err := Parse(spec) + if err != nil { + return 0, err + } + return c.Schedule(schedule, cmd), nil +} + +// Schedule adds a Job to the Cron to be run on the given schedule. +func (c *Cron) Schedule(schedule Schedule, cmd Job) EntryID { + c.nextID++ + entry := &Entry{ + ID: c.nextID, + Schedule: schedule, + Job: cmd, + } + if !c.running { + c.entries = append(c.entries, entry) + } else { + c.add <- entry + } + return entry.ID +} + +// Entries returns a snapshot of the cron entries. +func (c *Cron) Entries() []Entry { + if c.running { + c.snapshot <- nil + return <-c.snapshot + } + return c.entrySnapshot() +} + +// Entry returns a snapshot of the given entry, or nil if it couldn't be found. +func (c *Cron) Entry(id EntryID) Entry { + for _, entry := range c.Entries() { + if id == entry.ID { + return entry + } + } + return Entry{} +} + +// Remove an entry from being run in the future. +func (c *Cron) Remove(id EntryID) { + if c.running { + c.remove <- id + } else { + c.removeEntry(id) + } +} + +// Start the cron scheduler in its own go-routine. +func (c *Cron) Start() { + c.running = true + go c.run() +} + +// run the scheduler.. this is private just due to the need to synchronize +// access to the 'running' state variable. +func (c *Cron) run() { + // Figure out the next activation times for each entry. + now := time.Now().Local() + for _, entry := range c.entries { + entry.Next = entry.Schedule.Next(now) + } + + for { + // Determine the next entry to run. + sort.Sort(byTime(c.entries)) + + var effective time.Time + if len(c.entries) == 0 || c.entries[0].Next.IsZero() { + // If there are no entries yet, just sleep - it still handles new entries + // and stop requests. + effective = now.AddDate(10, 0, 0) + } else { + effective = c.entries[0].Next + } + + select { + case now = <-time.After(effective.Sub(now)): + // Run every entry whose next time was this effective time. + for _, e := range c.entries { + if e.Next != effective { + break + } + go e.Job.Run() + e.Prev = e.Next + e.Next = e.Schedule.Next(effective) + } + continue + + case newEntry := <-c.add: + c.entries = append(c.entries, newEntry) + newEntry.Next = newEntry.Schedule.Next(now) + + case <-c.snapshot: + c.snapshot <- c.entrySnapshot() + + case id := <-c.remove: + c.removeEntry(id) + + case <-c.stop: + return + } + + now = time.Now().Local() + } +} + +// Stop the cron scheduler. +func (c *Cron) Stop() { + c.stop <- struct{}{} + c.running = false +} + +// entrySnapshot returns a copy of the current cron entry list. +func (c *Cron) entrySnapshot() []Entry { + var entries = make([]Entry, len(c.entries)) + for i, e := range c.entries { + entries[i] = *e + } + return entries +} + +func (c *Cron) removeEntry(id EntryID) { + var entries []*Entry + for _, e := range c.entries { + if e.ID != id { + entries = append(entries, e) + } + } + c.entries = entries +} diff --git a/vendor/gopkg.in/robfig/cron.v2/doc.go b/vendor/gopkg.in/robfig/cron.v2/doc.go new file mode 100644 index 0000000000..31cd74a62e --- /dev/null +++ b/vendor/gopkg.in/robfig/cron.v2/doc.go @@ -0,0 +1,132 @@ +/* +Package cron implements a cron spec parser and job runner. + +Usage + +Callers may register Funcs to be invoked on a given schedule. Cron will run +them in their own goroutines. + + c := cron.New() + c.AddFunc("0 30 * * * *", func() { fmt.Println("Every hour on the half hour") }) + c.AddFunc("TZ=Asia/Tokyo 30 04 * * * *", func() { fmt.Println("Runs at 04:30 Tokyo time every day") }) + c.AddFunc("@hourly", func() { fmt.Println("Every hour") }) + c.AddFunc("@every 1h30m", func() { fmt.Println("Every hour thirty") }) + c.Start() + .. + // Funcs are invoked in their own goroutine, asynchronously. + ... + // Funcs may also be added to a running Cron + c.AddFunc("@daily", func() { fmt.Println("Every day") }) + .. + // Inspect the cron job entries' next and previous run times. + inspect(c.Entries()) + .. + c.Stop() // Stop the scheduler (does not stop any jobs already running). + +CRON Expression Format + +A cron expression represents a set of times, using 6 space-separated fields. + + Field name | Mandatory? | Allowed values | Allowed special characters + ---------- | ---------- | -------------- | -------------------------- + Seconds | No | 0-59 | * / , - + Minutes | Yes | 0-59 | * / , - + Hours | Yes | 0-23 | * / , - + Day of month | Yes | 1-31 | * / , - ? + Month | Yes | 1-12 or JAN-DEC | * / , - + Day of week | Yes | 0-6 or SUN-SAT | * / , - ? + +Note: Month and Day-of-week field values are case insensitive. "SUN", "Sun", +and "sun" are equally accepted. + +Special Characters + +Asterisk ( * ) + +The asterisk indicates that the cron expression will match for all values of the +field; e.g., using an asterisk in the 5th field (month) would indicate every +month. + +Slash ( / ) + +Slashes are used to describe increments of ranges. For example 3-59/15 in the +1st field (minutes) would indicate the 3rd minute of the hour and every 15 +minutes thereafter. The form "*\/..." is equivalent to the form "first-last/...", +that is, an increment over the largest possible range of the field. The form +"N/..." is accepted as meaning "N-MAX/...", that is, starting at N, use the +increment until the end of that specific range. It does not wrap around. + +Comma ( , ) + +Commas are used to separate items of a list. For example, using "MON,WED,FRI" in +the 5th field (day of week) would mean Mondays, Wednesdays and Fridays. + +Hyphen ( - ) + +Hyphens are used to define ranges. For example, 9-17 would indicate every +hour between 9am and 5pm inclusive. + +Question mark ( ? ) + +Question mark may be used instead of '*' for leaving either day-of-month or +day-of-week blank. + +Predefined schedules + +You may use one of several pre-defined schedules in place of a cron expression. + + Entry | Description | Equivalent To + ----- | ----------- | ------------- + @yearly (or @annually) | Run once a year, midnight, Jan. 1st | 0 0 0 1 1 * + @monthly | Run once a month, midnight, first of month | 0 0 0 1 * * + @weekly | Run once a week, midnight on Sunday | 0 0 0 * * 0 + @daily (or @midnight) | Run once a day, midnight | 0 0 0 * * * + @hourly | Run once an hour, beginning of hour | 0 0 * * * * + +Intervals + +You may also schedule a job to execute at fixed intervals. This is supported by +formatting the cron spec like this: + + @every + +where "duration" is a string accepted by time.ParseDuration +(http://golang.org/pkg/time/#ParseDuration). + +For example, "@every 1h30m10s" would indicate a schedule that activates every +1 hour, 30 minutes, 10 seconds. + +Note: The interval does not take the job runtime into account. For example, +if a job takes 3 minutes to run, and it is scheduled to run every 5 minutes, +it will have only 2 minutes of idle time between each run. + +Time zones + +By default, all interpretation and scheduling is done in the machine's local +time zone (as provided by the Go time package http://www.golang.org/pkg/time). +The time zone may be overridden by providing an additional space-separated field +at the beginning of the cron spec, of the form "TZ=Asia/Tokyo" + +Be aware that jobs scheduled during daylight-savings leap-ahead transitions will +not be run! + +Thread safety + +Since the Cron service runs concurrently with the calling code, some amount of +care must be taken to ensure proper synchronization. + +All cron methods are designed to be correctly synchronized as long as the caller +ensures that invocations have a clear happens-before ordering between them. + +Implementation + +Cron entries are stored in an array, sorted by their next activation time. Cron +sleeps until the next job is due to be run. + +Upon waking: + - it runs each entry that is active on that second + - it calculates the next run times for the jobs that were run + - it re-sorts the array of entries by next activation time. + - it goes to sleep until the soonest job. +*/ +package cron diff --git a/vendor/gopkg.in/robfig/cron.v2/parser.go b/vendor/gopkg.in/robfig/cron.v2/parser.go new file mode 100644 index 0000000000..a9e6f947ac --- /dev/null +++ b/vendor/gopkg.in/robfig/cron.v2/parser.go @@ -0,0 +1,246 @@ +package cron + +import ( + "fmt" + "log" + "math" + "strconv" + "strings" + "time" +) + +// Parse returns a new crontab schedule representing the given spec. +// It returns a descriptive error if the spec is not valid. +// +// It accepts +// - Full crontab specs, e.g. "* * * * * ?" +// - Descriptors, e.g. "@midnight", "@every 1h30m" +func Parse(spec string) (_ Schedule, err error) { + // Convert panics into errors + defer func() { + if recovered := recover(); recovered != nil { + err = fmt.Errorf("%v", recovered) + } + }() + + // Extract timezone if present + var loc = time.Local + if strings.HasPrefix(spec, "TZ=") { + i := strings.Index(spec, " ") + if loc, err = time.LoadLocation(spec[3:i]); err != nil { + log.Panicf("Provided bad location %s: %v", spec[3:i], err) + } + spec = strings.TrimSpace(spec[i:]) + } + + // Handle named schedules (descriptors) + if strings.HasPrefix(spec, "@") { + return parseDescriptor(spec, loc), nil + } + + // Split on whitespace. We require 5 or 6 fields. + // (second, optional) (minute) (hour) (day of month) (month) (day of week) + fields := strings.Fields(spec) + if len(fields) != 5 && len(fields) != 6 { + log.Panicf("Expected 5 or 6 fields, found %d: %s", len(fields), spec) + } + + // Add 0 for second field if necessary. + if len(fields) == 5 { + fields = append([]string{"0"}, fields...) + } + + schedule := &SpecSchedule{ + Second: getField(fields[0], seconds), + Minute: getField(fields[1], minutes), + Hour: getField(fields[2], hours), + Dom: getField(fields[3], dom), + Month: getField(fields[4], months), + Dow: getField(fields[5], dow), + Location: loc, + } + + return schedule, nil +} + +// getField returns an Int with the bits set representing all of the times that +// the field represents. A "field" is a comma-separated list of "ranges". +func getField(field string, r bounds) uint64 { + // list = range {"," range} + var bits uint64 + ranges := strings.FieldsFunc(field, func(r rune) bool { return r == ',' }) + for _, expr := range ranges { + bits |= getRange(expr, r) + } + return bits +} + +// getRange returns the bits indicated by the given expression: +// number | number "-" number [ "/" number ] +func getRange(expr string, r bounds) uint64 { + var ( + start, end, step uint + rangeAndStep = strings.Split(expr, "/") + lowAndHigh = strings.Split(rangeAndStep[0], "-") + singleDigit = len(lowAndHigh) == 1 + extraStar uint64 + ) + if lowAndHigh[0] == "*" || lowAndHigh[0] == "?" { + start = r.min + end = r.max + extraStar = starBit + } else { + start = parseIntOrName(lowAndHigh[0], r.names) + switch len(lowAndHigh) { + case 1: + end = start + case 2: + end = parseIntOrName(lowAndHigh[1], r.names) + default: + log.Panicf("Too many hyphens: %s", expr) + } + } + + switch len(rangeAndStep) { + case 1: + step = 1 + case 2: + step = mustParseInt(rangeAndStep[1]) + + // Special handling: "N/step" means "N-max/step". + if singleDigit { + end = r.max + } + default: + log.Panicf("Too many slashes: %s", expr) + } + + if start < r.min { + log.Panicf("Beginning of range (%d) below minimum (%d): %s", start, r.min, expr) + } + if end > r.max { + log.Panicf("End of range (%d) above maximum (%d): %s", end, r.max, expr) + } + if start > end { + log.Panicf("Beginning of range (%d) beyond end of range (%d): %s", start, end, expr) + } + + return getBits(start, end, step) | extraStar +} + +// parseIntOrName returns the (possibly-named) integer contained in expr. +func parseIntOrName(expr string, names map[string]uint) uint { + if names != nil { + if namedInt, ok := names[strings.ToLower(expr)]; ok { + return namedInt + } + } + return mustParseInt(expr) +} + +// mustParseInt parses the given expression as an int or panics. +func mustParseInt(expr string) uint { + num, err := strconv.Atoi(expr) + if err != nil { + log.Panicf("Failed to parse int from %s: %s", expr, err) + } + if num < 0 { + log.Panicf("Negative number (%d) not allowed: %s", num, expr) + } + + return uint(num) +} + +// getBits sets all bits in the range [min, max], modulo the given step size. +func getBits(min, max, step uint) uint64 { + var bits uint64 + + // If step is 1, use shifts. + if step == 1 { + return ^(math.MaxUint64 << (max + 1)) & (math.MaxUint64 << min) + } + + // Else, use a simple loop. + for i := min; i <= max; i += step { + bits |= 1 << i + } + return bits +} + +// all returns all bits within the given bounds. (plus the star bit) +func all(r bounds) uint64 { + return getBits(r.min, r.max, 1) | starBit +} + +// parseDescriptor returns a pre-defined schedule for the expression, or panics +// if none matches. +func parseDescriptor(spec string, loc *time.Location) Schedule { + switch spec { + case "@yearly", "@annually": + return &SpecSchedule{ + Second: 1 << seconds.min, + Minute: 1 << minutes.min, + Hour: 1 << hours.min, + Dom: 1 << dom.min, + Month: 1 << months.min, + Dow: all(dow), + Location: loc, + } + + case "@monthly": + return &SpecSchedule{ + Second: 1 << seconds.min, + Minute: 1 << minutes.min, + Hour: 1 << hours.min, + Dom: 1 << dom.min, + Month: all(months), + Dow: all(dow), + Location: loc, + } + + case "@weekly": + return &SpecSchedule{ + Second: 1 << seconds.min, + Minute: 1 << minutes.min, + Hour: 1 << hours.min, + Dom: all(dom), + Month: all(months), + Dow: 1 << dow.min, + Location: loc, + } + + case "@daily", "@midnight": + return &SpecSchedule{ + Second: 1 << seconds.min, + Minute: 1 << minutes.min, + Hour: 1 << hours.min, + Dom: all(dom), + Month: all(months), + Dow: all(dow), + Location: loc, + } + + case "@hourly": + return &SpecSchedule{ + Second: 1 << seconds.min, + Minute: 1 << minutes.min, + Hour: all(hours), + Dom: all(dom), + Month: all(months), + Dow: all(dow), + Location: loc, + } + } + + const every = "@every " + if strings.HasPrefix(spec, every) { + duration, err := time.ParseDuration(spec[len(every):]) + if err != nil { + log.Panicf("Failed to parse duration %s: %s", spec, err) + } + return Every(duration) + } + + log.Panicf("Unrecognized descriptor: %s", spec) + return nil +} diff --git a/vendor/gopkg.in/robfig/cron.v2/spec.go b/vendor/gopkg.in/robfig/cron.v2/spec.go new file mode 100644 index 0000000000..3dfd3e088a --- /dev/null +++ b/vendor/gopkg.in/robfig/cron.v2/spec.go @@ -0,0 +1,165 @@ +package cron + +import "time" + +// SpecSchedule specifies a duty cycle (to the second granularity), based on a +// traditional crontab specification. It is computed initially and stored as bit sets. +type SpecSchedule struct { + Second, Minute, Hour, Dom, Month, Dow uint64 + Location *time.Location +} + +// bounds provides a range of acceptable values (plus a map of name to value). +type bounds struct { + min, max uint + names map[string]uint +} + +// The bounds for each field. +var ( + seconds = bounds{0, 59, nil} + minutes = bounds{0, 59, nil} + hours = bounds{0, 23, nil} + dom = bounds{1, 31, nil} + months = bounds{1, 12, map[string]uint{ + "jan": 1, + "feb": 2, + "mar": 3, + "apr": 4, + "may": 5, + "jun": 6, + "jul": 7, + "aug": 8, + "sep": 9, + "oct": 10, + "nov": 11, + "dec": 12, + }} + dow = bounds{0, 6, map[string]uint{ + "sun": 0, + "mon": 1, + "tue": 2, + "wed": 3, + "thu": 4, + "fri": 5, + "sat": 6, + }} +) + +const ( + // Set the top bit if a star was included in the expression. + starBit = 1 << 63 +) + +// Next returns the next time this schedule is activated, greater than the given +// time. If no time can be found to satisfy the schedule, return the zero time. +func (s *SpecSchedule) Next(t time.Time) time.Time { + // General approach: + // For Month, Day, Hour, Minute, Second: + // Check if the time value matches. If yes, continue to the next field. + // If the field doesn't match the schedule, then increment the field until it matches. + // While incrementing the field, a wrap-around brings it back to the beginning + // of the field list (since it is necessary to re-verify previous field + // values) + + // Convert the given time into the schedule's timezone. + // Save the original timezone so we can convert back after we find a time. + origLocation := t.Location() + t = t.In(s.Location) + + // Start at the earliest possible time (the upcoming second). + t = t.Add(1*time.Second - time.Duration(t.Nanosecond())*time.Nanosecond) + + // This flag indicates whether a field has been incremented. + added := false + + // If no time is found within five years, return zero. + yearLimit := t.Year() + 5 + +WRAP: + if t.Year() > yearLimit { + return time.Time{} + } + + // Find the first applicable month. + // If it's this month, then do nothing. + for 1< 0 + dowMatch bool = 1< 0 + ) + + if s.Dom&starBit > 0 || s.Dow&starBit > 0 { + return domMatch && dowMatch + } + return domMatch || dowMatch +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 6c36df822b..3049b1f5d9 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -70,6 +70,11 @@ github.com/NYTimes/gziphandler github.com/PuerkitoBio/purell # github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 github.com/PuerkitoBio/urlesc +# github.com/Rican7/retry v0.1.0 +github.com/Rican7/retry/backoff +github.com/Rican7/retry/strategy +github.com/Rican7/retry/jitter +github.com/Rican7/retry # github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e github.com/armon/circbuf # github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a @@ -124,6 +129,13 @@ github.com/blang/semver github.com/bronze1man/goStrongswanVici # github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23 github.com/buger/jsonparser +# github.com/canonical/go-dqlite v1.1.0 +github.com/canonical/go-dqlite/client +github.com/canonical/go-dqlite +github.com/canonical/go-dqlite/internal/logging +github.com/canonical/go-dqlite/internal/protocol +github.com/canonical/go-dqlite/internal/bindings +github.com/canonical/go-dqlite/driver # github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 github.com/chai2010/gettext-go/gettext github.com/chai2010/gettext-go/gettext/mo @@ -359,13 +371,13 @@ github.com/coreos/flannel/subnet github.com/coreos/go-iptables/iptables # github.com/coreos/go-oidc v2.1.0+incompatible github.com/coreos/go-oidc -# github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e => github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7 +# github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f => github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7 github.com/coreos/go-systemd/activation github.com/coreos/go-systemd/daemon github.com/coreos/go-systemd/dbus github.com/coreos/go-systemd/util github.com/coreos/go-systemd/journal -# github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea +# github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f github.com/coreos/pkg/dlopen github.com/coreos/pkg/capnslog # github.com/cyphar/filepath-securejoin v0.2.2 @@ -440,6 +452,8 @@ github.com/evanphx/json-patch github.com/exponent-io/jsonpath # github.com/fatih/camelcase v1.0.0 github.com/fatih/camelcase +# github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4 +github.com/flosch/pongo2 # github.com/fsnotify/fsnotify v1.4.7 github.com/fsnotify/fsnotify # github.com/ghodss/yaml v1.0.0 @@ -495,7 +509,7 @@ github.com/golang/protobuf/ptypes/duration github.com/golang/protobuf/ptypes/timestamp github.com/golang/protobuf/protoc-gen-go/descriptor github.com/golang/protobuf/ptypes/wrappers -# github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c +# github.com/google/btree v1.0.0 github.com/google/btree # github.com/google/cadvisor v0.34.0 github.com/google/cadvisor/container/common @@ -593,7 +607,7 @@ github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/lbaas_v2/l github.com/gophercloud/gophercloud/openstack/identity/v2/tenants # github.com/gorilla/mux v1.7.3 github.com/gorilla/mux -# github.com/gorilla/websocket v1.4.0 +# github.com/gorilla/websocket v1.4.1 github.com/gorilla/websocket # github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7 github.com/gregjones/httpcache @@ -630,6 +644,14 @@ github.com/lib/pq/scram github.com/liggitt/tabwriter # github.com/lithammer/dedent v1.1.0 github.com/lithammer/dedent +# github.com/lxc/lxd v0.0.0-20191108214106-60ea15630455 +github.com/lxc/lxd/shared/eagain +github.com/lxc/lxd/shared +github.com/lxc/lxd/shared/api +github.com/lxc/lxd/shared/cancel +github.com/lxc/lxd/shared/ioprogress +github.com/lxc/lxd/shared/logger +github.com/lxc/lxd/shared/units # github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63 github.com/mailru/easyjson/jlexer github.com/mailru/easyjson/jwriter @@ -712,7 +734,7 @@ github.com/pquerna/cachecontrol/cacheobject github.com/prometheus/client_golang/prometheus github.com/prometheus/client_golang/prometheus/internal github.com/prometheus/client_golang/prometheus/promhttp -# github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 => github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 +# github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 => github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 github.com/prometheus/client_model/go # github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 => github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 github.com/prometheus/common/expfmt @@ -726,7 +748,7 @@ github.com/prometheus/procfs/internal/util # github.com/rakelkar/gonetsh v0.0.0-20190719023240-501daadcadf8 github.com/rakelkar/gonetsh/netroute github.com/rakelkar/gonetsh/netsh -# github.com/rancher/dynamiclistener v0.1.1-0.20191108205817-245f86cc340a +# github.com/rancher/dynamiclistener v0.1.1-0.20191110035254-aaa5bc0d2a07 github.com/rancher/dynamiclistener github.com/rancher/dynamiclistener/factory github.com/rancher/dynamiclistener/storage/file @@ -747,8 +769,9 @@ github.com/rancher/helm-controller/pkg/generated/informers/externalversions/helm github.com/rancher/helm-controller/pkg/generated/listers/helm.cattle.io/v1 github.com/rancher/helm-controller/pkg/generated/informers/externalversions/internalinterfaces github.com/rancher/helm-controller/pkg/apis/helm.cattle.io -# github.com/rancher/kine v0.1.2-0.20191107225357-527576e3452f +# github.com/rancher/kine v0.2.0 => ../kine github.com/rancher/kine/pkg/endpoint +github.com/rancher/kine/pkg/drivers/dqlite github.com/rancher/kine/pkg/drivers/mysql github.com/rancher/kine/pkg/drivers/pgsql github.com/rancher/kine/pkg/drivers/sqlite @@ -885,11 +908,11 @@ go.opencensus.io/stats/internal go.opencensus.io/internal/tagencoding go.opencensus.io/metric/metricproducer go.opencensus.io/resource -# go.uber.org/atomic v0.0.0-20181018215023-8dc6146f7569 +# go.uber.org/atomic v1.4.0 go.uber.org/atomic -# go.uber.org/multierr v0.0.0-20180122172545-ddea229ff1df +# go.uber.org/multierr v1.1.0 go.uber.org/multierr -# go.uber.org/zap v0.0.0-20180814183419-67bc79d13d15 +# go.uber.org/zap v1.10.0 go.uber.org/zap go.uber.org/zap/zapcore go.uber.org/zap/internal/bufferpool @@ -1033,7 +1056,7 @@ google.golang.org/appengine/internal/base google.golang.org/appengine/internal/datastore google.golang.org/appengine/internal/log google.golang.org/appengine/internal/remote_api -# google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873 +# google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 google.golang.org/genproto/googleapis/rpc/status google.golang.org/genproto/googleapis/api/annotations # google.golang.org/grpc v1.23.0 @@ -1081,6 +1104,8 @@ gopkg.in/gcfg.v1/types gopkg.in/inf.v0 # gopkg.in/natefinch/lumberjack.v2 v2.0.0 gopkg.in/natefinch/lumberjack.v2 +# gopkg.in/robfig/cron.v2 v2.0.0-20150107220207-be2e0b0deed5 +gopkg.in/robfig/cron.v2 # gopkg.in/square/go-jose.v2 v2.2.2 gopkg.in/square/go-jose.v2 gopkg.in/square/go-jose.v2/jwt @@ -1214,11 +1239,11 @@ k8s.io/apimachinery/pkg/runtime/serializer/streaming k8s.io/apimachinery/pkg/conversion/queryparams k8s.io/apimachinery/pkg/api/validation/path k8s.io/apimachinery/pkg/apis/meta/internalversion -k8s.io/apimachinery/pkg/version k8s.io/apimachinery/pkg/util/uuid k8s.io/apimachinery/pkg/util/rand k8s.io/apimachinery/pkg/util/jsonmergepatch k8s.io/apimachinery/pkg/util/validation/field +k8s.io/apimachinery/pkg/version k8s.io/apimachinery/pkg/runtime/serializer/json k8s.io/apimachinery/pkg/runtime/serializer/protobuf k8s.io/apimachinery/pkg/runtime/serializer/recognizer @@ -1260,14 +1285,14 @@ k8s.io/apiserver/pkg/authorization/authorizerfactory k8s.io/apiserver/pkg/authentication/user k8s.io/apiserver/pkg/apis/audit k8s.io/apiserver/pkg/util/term +k8s.io/apiserver/pkg/endpoints/filters +k8s.io/apiserver/pkg/server/filters k8s.io/apiserver/pkg/admission k8s.io/apiserver/pkg/endpoints/openapi k8s.io/apiserver/pkg/features -k8s.io/apiserver/pkg/server/filters k8s.io/apiserver/pkg/server/options k8s.io/apiserver/pkg/storage/etcd3/preflight k8s.io/apiserver/pkg/util/webhook -k8s.io/apiserver/pkg/endpoints/filters k8s.io/apiserver/pkg/admission/plugin/namespace/lifecycle k8s.io/apiserver/pkg/admission/plugin/webhook/mutating k8s.io/apiserver/pkg/admission/plugin/webhook/validating @@ -1746,6 +1771,7 @@ k8s.io/kubernetes/pkg/client/metrics/prometheus k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes k8s.io/kubernetes/pkg/version/prometheus k8s.io/kubernetes/cmd/cloud-controller-manager/app +k8s.io/kubernetes/cmd/controller-manager/app k8s.io/kubernetes/cmd/kube-apiserver/app k8s.io/kubernetes/cmd/kube-controller-manager/app k8s.io/kubernetes/cmd/kube-scheduler/app @@ -1887,7 +1913,6 @@ k8s.io/kubernetes/pkg/volume/util/subpath k8s.io/kubernetes/pkg/volume/vsphere_volume k8s.io/kubernetes/cmd/cloud-controller-manager/app/config k8s.io/kubernetes/cmd/cloud-controller-manager/app/options -k8s.io/kubernetes/cmd/controller-manager/app k8s.io/kubernetes/pkg/controller/cloud k8s.io/kubernetes/pkg/controller/route k8s.io/kubernetes/pkg/controller/service From 29b270dce6d792cad7189f1e376917942b93a797 Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Sat, 9 Nov 2019 06:07:12 +0000 Subject: [PATCH 03/12] Wait for apiserver to be health, not just running --- pkg/daemons/control/server.go | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/pkg/daemons/control/server.go b/pkg/daemons/control/server.go index 0b52c1e578..9990ddd3f9 100644 --- a/pkg/daemons/control/server.go +++ b/pkg/daemons/control/server.go @@ -31,9 +31,10 @@ import ( "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apiserver/pkg/authentication/authenticator" - "k8s.io/client-go/discovery" + "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" ccmapp "k8s.io/kubernetes/cmd/cloud-controller-manager/app" + app2 "k8s.io/kubernetes/cmd/controller-manager/app" "k8s.io/kubernetes/cmd/kube-apiserver/app" cmapp "k8s.io/kubernetes/cmd/kube-controller-manager/app" sapp "k8s.io/kubernetes/cmd/kube-scheduler/app" @@ -858,25 +859,24 @@ func waitForAPIServer(ctx context.Context, runtime *config.ControlRuntime) error return err } - discoveryclient, err := discovery.NewDiscoveryClientForConfig(restConfig) + k8sClient, err := kubernetes.NewForConfig(restConfig) if err != nil { return err } - for i := 0; i < 60; i++ { - info, err := discoveryclient.ServerVersion() - if err == nil { - logrus.Infof("apiserver %s is up and running", info) - return nil - } - logrus.Infof("waiting for apiserver to become available") - select { - case <-ctx.Done(): - return ctx.Err() - case <-time.After(time.Second): - continue - } + select { + case <-ctx.Done(): + return ctx.Err() + case err := <-promise(func() error { return app2.WaitForAPIServer(k8sClient, 5*time.Minute) }): + return err } - - return fmt.Errorf("timeout waiting for apiserver") +} + +func promise(f func() error) <-chan error { + c := make(chan error, 1) + go func() { + c <- f() + close(c) + }() + return c } From 561bc960498484011a6e1c29d5e657fa1f22c1ad Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Sat, 9 Nov 2019 16:06:03 +0000 Subject: [PATCH 04/12] Change sonobuoy logging --- scripts/sonobuoy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/sonobuoy b/scripts/sonobuoy index cc0275a061..25f6c98080 100755 --- a/scripts/sonobuoy +++ b/scripts/sonobuoy @@ -212,7 +212,7 @@ echo "Started ${K3S_AGENT}" # --- timeout --foreground 1m bash -c 'wait-for-nodes 2' -timeout --foreground 1m bash -c 'wait-for-services coredns local-path-provisioner metrics-server' +timeout --foreground 3m bash -c 'wait-for-services coredns local-path-provisioner metrics-server' if [ "$ARCH" = 'arm' ]; then echo "Aborting sonobuoy tests, images not available for $ARCH" From 24fa3785a4a9df02d924f1e25e1fdf463507b230 Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Mon, 11 Nov 2019 22:16:18 +0000 Subject: [PATCH 05/12] Include nsswitch so golang will read /etc/hosts file in docker container --- package/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/package/Dockerfile b/package/Dockerfile index cc364b8cb0..b849cf759c 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -9,6 +9,8 @@ RUN cd image/bin && \ FROM scratch COPY --from=base /image / +RUN mkdir -p /etc && \ + echo 'hosts: files dns' > /etc/nsswitch.conf RUN chmod 1777 /tmp VOLUME /var/lib/kubelet VOLUME /var/lib/rancher/k3s From 3f5fb70116580f359c1fb6499cd615d14f6cccce Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Mon, 11 Nov 2019 22:16:46 +0000 Subject: [PATCH 06/12] Move server arguments to experimental for dqlite related --- pkg/cli/cmds/server.go | 64 +++++++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/pkg/cli/cmds/server.go b/pkg/cli/cmds/server.go index 9368b84c03..fa63fff7a0 100644 --- a/pkg/cli/cmds/server.go +++ b/pkg/cli/cmds/server.go @@ -129,38 +129,6 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command { Destination: &ServerConfig.TokenFile, EnvVar: "K3S_TOKEN_FILE", }, - cli.StringFlag{ - Name: "agent-token", - Usage: "(cluster) Shared secret used to join agents to the cluster, but not agents", - Destination: &ServerConfig.AgentToken, - EnvVar: "K3S_AGENT_TOKEN", - }, - cli.StringFlag{ - Name: "agent-token-file", - Usage: "(cluster) File containing the agent secret", - Destination: &ServerConfig.AgentTokenFile, - EnvVar: "K3S_AGENT_TOKEN_FILE", - }, - cli.StringFlag{ - Name: "server,s", - Usage: "(cluster) Server to connect to, used to join a cluster", - EnvVar: "K3S_URL", - Destination: &ServerConfig.ServerURL, - }, - cli.BoolFlag{ - Name: "cluster-init", - Hidden: hideDqlite, - Usage: "(cluster) Initialize new cluster master", - EnvVar: "K3S_CLUSTER_INIT", - Destination: &ServerConfig.ClusterInit, - }, - cli.BoolFlag{ - Name: "cluster-reset", - Hidden: hideDqlite, - Usage: "(cluster) Forget all peers and become a single cluster new cluster master", - EnvVar: "K3S_CLUSTER_RESET", - Destination: &ServerConfig.ClusterReset, - }, cli.StringFlag{ Name: "write-kubeconfig,o", Usage: "(client) Write kubeconfig for admin client to this file", @@ -260,6 +228,38 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command { Usage: "(experimental) Run rootless", Destination: &ServerConfig.Rootless, }, + cli.StringFlag{ + Name: "agent-token", + Usage: "(experimental/cluster) Shared secret used to join agents to the cluster, but not agents", + Destination: &ServerConfig.AgentToken, + EnvVar: "K3S_AGENT_TOKEN", + }, + cli.StringFlag{ + Name: "agent-token-file", + Usage: "(experimental/cluster) File containing the agent secret", + Destination: &ServerConfig.AgentTokenFile, + EnvVar: "K3S_AGENT_TOKEN_FILE", + }, + cli.StringFlag{ + Name: "server,s", + Usage: "(experimental/cluster) Server to connect to, used to join a cluster", + EnvVar: "K3S_URL", + Destination: &ServerConfig.ServerURL, + }, + cli.BoolFlag{ + Name: "cluster-init", + Hidden: hideDqlite, + Usage: "(experimental/cluster) Initialize new cluster master", + EnvVar: "K3S_CLUSTER_INIT", + Destination: &ServerConfig.ClusterInit, + }, + cli.BoolFlag{ + Name: "cluster-reset", + Hidden: hideDqlite, + Usage: "(experimental/cluster) Forget all peers and become a single cluster new cluster master", + EnvVar: "K3S_CLUSTER_RESET", + Destination: &ServerConfig.ClusterReset, + }, // Hidden/Deprecated flags below From 0ae20eb7a35da9edd0e75b673070b1c01d3ec11a Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Mon, 11 Nov 2019 22:18:26 +0000 Subject: [PATCH 07/12] Support both http and db based bootstrap --- pkg/cluster/cluster.go | 55 +++++++++++---- pkg/cluster/dqlite.go | 21 +++--- pkg/cluster/encrypt.go | 81 ++++++++++++++++++++++ pkg/cluster/join.go | 54 ++++++++++++--- pkg/cluster/nocluster.go | 4 ++ pkg/cluster/storage.go | 50 +++++++++++++ pkg/daemons/config/types.go | 9 +-- pkg/daemons/control/server.go | 23 ++---- pkg/dqlite/controller/client/controller.go | 17 +++-- pkg/server/router.go | 4 +- 10 files changed, 256 insertions(+), 62 deletions(-) create mode 100644 pkg/cluster/encrypt.go create mode 100644 pkg/cluster/storage.go diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 73eeac70ab..580d40d75d 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -2,36 +2,33 @@ package cluster import ( "context" + "strings" "github.com/rancher/k3s/pkg/clientaccess" "github.com/rancher/k3s/pkg/daemons/config" + "github.com/rancher/kine/pkg/client" + "github.com/rancher/kine/pkg/endpoint" ) type Cluster struct { - token string clientAccessInfo *clientaccess.Info config *config.Control runtime *config.ControlRuntime db interface{} + runJoin bool + storageStarted bool + etcdConfig endpoint.ETCDConfig + joining bool + saveBootstrap bool + storageClient client.Client } func (c *Cluster) Start(ctx context.Context) error { - join, err := c.shouldJoin() - if err != nil { - return err - } - - if join { - if err := c.join(); err != nil { - return err - } - } - if err := c.startClusterAndHTTPS(ctx); err != nil { return err } - if join { + if c.runJoin { if err := c.postJoin(ctx); err != nil { return err } @@ -41,7 +38,37 @@ func (c *Cluster) Start(ctx context.Context) error { return err } - return c.joined() + if c.saveBootstrap { + if err := c.save(ctx); err != nil { + return err + } + } + + if c.runJoin { + if err := c.joined(); err != nil { + return err + } + } + + return c.startStorage(ctx) +} + +func (c *Cluster) startStorage(ctx context.Context) error { + if c.storageStarted { + return nil + } + c.storageStarted = true + + etcdConfig, err := endpoint.Listen(ctx, c.config.Storage) + if err != nil { + return err + } + + c.etcdConfig = etcdConfig + c.config.Storage.Config = etcdConfig.TLSConfig + c.config.Storage.Endpoint = strings.Join(etcdConfig.Endpoints, ",") + c.config.NoLeaderElect = !etcdConfig.LeaderElect + return nil } func New(config *config.Control) *Cluster { diff --git a/pkg/cluster/dqlite.go b/pkg/cluster/dqlite.go index ab2d05cb0a..faeb80f994 100644 --- a/pkg/cluster/dqlite.go +++ b/pkg/cluster/dqlite.go @@ -24,7 +24,7 @@ import ( ) func (c *Cluster) testClusterDB(ctx context.Context) error { - if !c.enabled() { + if !c.dqliteEnabled() { return nil } @@ -45,7 +45,7 @@ func (c *Cluster) testClusterDB(ctx context.Context) error { } func (c *Cluster) initClusterDB(ctx context.Context, l net.Listener, handler http.Handler) (net.Listener, http.Handler, error) { - if !c.enabled() { + if !c.dqliteEnabled() { return l, handler, nil } @@ -61,17 +61,17 @@ func (c *Cluster) initClusterDB(ctx context.Context, l net.Listener, handler htt return nil, nil, err } - handler, err = dqlite.Start(ctx, c.config.ClusterInit, certs, handler) + handler, err = dqlite.Start(ctx, c.config.ClusterInit, c.config.ClusterReset, certs, handler) if err != nil { return nil, nil, err } if c.config.ClusterReset { if err := dqlite.Reset(ctx); err == nil { - logrus.Info("Cluster reset") + logrus.Info("Cluster reset successful, now rejoin members") os.Exit(0) } else { - logrus.Fatal("Cluster reset failed: %v", err) + logrus.Fatalf("Cluster reset failed: %v", err) } } @@ -85,17 +85,22 @@ func (c *Cluster) initClusterDB(ctx context.Context, l net.Listener, handler htt return l, handler, err } -func (c *Cluster) enabled() bool { +func (c *Cluster) dqliteEnabled() bool { stamp := filepath.Join(c.config.DataDir, "db", "state.dqlite") if _, err := os.Stat(stamp); err == nil { return true } - return c.config.Storage.Endpoint == "" && (c.config.ClusterInit || c.runtime.Cluster.Join) + driver, _ := endpoint.ParseStorageEndpoint(c.config.Storage.Endpoint) + if driver == endpoint.DQLiteBackend { + return true + } + + return c.config.Storage.Endpoint == "" && (c.config.ClusterInit || (c.config.Token != "" && c.config.JoinURL != "")) } func (c *Cluster) postJoin(ctx context.Context) error { - if !c.enabled() { + if !c.dqliteEnabled() { return nil } diff --git a/pkg/cluster/encrypt.go b/pkg/cluster/encrypt.go new file mode 100644 index 0000000000..be44ff2363 --- /dev/null +++ b/pkg/cluster/encrypt.go @@ -0,0 +1,81 @@ +package cluster + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "crypto/sha1" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "fmt" + "io" + "strings" + + "github.com/rancher/k3s/pkg/token" + "golang.org/x/crypto/pbkdf2" +) + +func storageKey(passphrase string) string { + d := sha256.New() + d.Write([]byte(passphrase)) + return "/bootstrap/" + hex.EncodeToString(d.Sum(nil)[:])[:12] +} + +func keyHash(passphrase string) string { + d := sha256.New() + d.Write([]byte(passphrase)) + return hex.EncodeToString(d.Sum(nil)[:])[:12] +} + +func encrypt(passphrase string, plaintext []byte) ([]byte, error) { + salt, err := token.Random(8) + if err != nil { + return nil, err + } + + clearKey := pbkdf2.Key([]byte(passphrase), []byte(salt), 4096, 32, sha1.New) + key, err := aes.NewCipher(clearKey) + if err != nil { + return nil, err + } + + gcm, err := cipher.NewGCM(key) + if err != nil { + return nil, err + } + + nonce := make([]byte, gcm.NonceSize()) + _, err = io.ReadFull(rand.Reader, nonce) + if err != nil { + return nil, err + } + + sealed := gcm.Seal(nonce, nonce, plaintext, nil) + return []byte(salt + ":" + base64.StdEncoding.EncodeToString(sealed)), nil +} + +func decrypt(passphrase string, ciphertext []byte) ([]byte, error) { + parts := strings.SplitN(string(ciphertext), ":", 2) + if len(parts) != 2 { + return nil, fmt.Errorf("invalid cipher text, not : delimited") + } + + clearKey := pbkdf2.Key([]byte(passphrase), []byte(parts[0]), 4096, 32, sha1.New) + key, err := aes.NewCipher(clearKey) + if err != nil { + return nil, err + } + + gcm, err := cipher.NewGCM(key) + if err != nil { + return nil, err + } + + data, err := base64.StdEncoding.DecodeString(parts[1]) + if err != nil { + return nil, err + } + + return gcm.Open(nil, data[:gcm.NonceSize()], data[gcm.NonceSize():], nil) +} diff --git a/pkg/cluster/join.go b/pkg/cluster/join.go index 99d1bdecdf..ecb85e2b7a 100644 --- a/pkg/cluster/join.go +++ b/pkg/cluster/join.go @@ -2,6 +2,7 @@ package cluster import ( "bytes" + "context" "fmt" "os" "path/filepath" @@ -11,18 +12,38 @@ import ( "github.com/sirupsen/logrus" ) +func (c *Cluster) Join(ctx context.Context) error { + runJoin, err := c.shouldJoin() + if err != nil { + return err + } + c.runJoin = runJoin + + if runJoin { + if err := c.join(ctx); err != nil { + return err + } + } + + return nil +} + func (c *Cluster) shouldJoin() (bool, error) { - if c.config.JoinURL == "" { - return false, nil + dqlite := c.dqliteEnabled() + if dqlite { + c.runtime.HTTPBootstrap = true + if c.config.JoinURL == "" { + return false, nil + } } - stamp := filepath.Join(c.config.DataDir, "db/joined") + stamp := c.joinStamp() if _, err := os.Stat(stamp); err == nil { - logrus.Info("Already joined to cluster, not rejoining") + logrus.Info("Cluster bootstrap already complete") return false, nil } - if c.config.Token == "" { + if dqlite && c.config.Token == "" { return false, fmt.Errorf("K3S_TOKEN is required to join a cluster") } @@ -46,14 +67,11 @@ func (c *Cluster) joined() error { return f.Close() } -func (c *Cluster) join() error { - c.runtime.Cluster.Join = true - +func (c *Cluster) httpJoin() error { token, err := clientaccess.NormalizeAndValidateTokenForUser(c.config.JoinURL, c.config.Token, "server") if err != nil { return err } - c.token = token info, err := clientaccess.ParseAndValidateToken(c.config.JoinURL, token) if err != nil { @@ -69,6 +87,20 @@ func (c *Cluster) join() error { return bootstrap.Read(bytes.NewBuffer(content), &c.runtime.ControlRuntimeBootstrap) } -func (c *Cluster) joinStamp() string { - return filepath.Join(c.config.DataDir, "db/joined") +func (c *Cluster) join(ctx context.Context) error { + c.joining = true + + if c.runtime.HTTPBootstrap { + return c.httpJoin() + } + + if err := c.storageJoin(ctx); err != nil { + return err + } + + return nil +} + +func (c *Cluster) joinStamp() string { + return filepath.Join(c.config.DataDir, "db/joined-"+keyHash(c.config.Token)) } diff --git a/pkg/cluster/nocluster.go b/pkg/cluster/nocluster.go index a7ed7be3d4..19f5728c2a 100644 --- a/pkg/cluster/nocluster.go +++ b/pkg/cluster/nocluster.go @@ -19,3 +19,7 @@ func (c *Cluster) initClusterDB(ctx context.Context, l net.Listener, handler htt func (c *Cluster) postJoin(ctx context.Context) error { return nil } + +func (c *Cluster) dqliteEnabled() bool { + return false +} diff --git a/pkg/cluster/storage.go b/pkg/cluster/storage.go new file mode 100644 index 0000000000..dc3d776de6 --- /dev/null +++ b/pkg/cluster/storage.go @@ -0,0 +1,50 @@ +package cluster + +import ( + "bytes" + "context" + + "github.com/rancher/k3s/pkg/bootstrap" + "github.com/rancher/kine/pkg/client" +) + +func (c *Cluster) save(ctx context.Context) error { + buf := &bytes.Buffer{} + if err := bootstrap.Write(buf, &c.runtime.ControlRuntimeBootstrap); err != nil { + return err + } + + data, err := encrypt(c.config.Token, buf.Bytes()) + if err != nil { + return err + } + + return c.storageClient.Create(ctx, storageKey(c.config.Token), data) +} + +func (c *Cluster) storageJoin(ctx context.Context) error { + if err := c.startStorage(ctx); err != nil { + return err + } + + storageClient, err := client.New(c.etcdConfig) + if err != nil { + return err + } + c.storageClient = storageClient + + value, err := storageClient.Get(ctx, storageKey(c.config.Token)) + if err == client.ErrNotFound { + c.saveBootstrap = true + return nil + } else if err != nil { + return err + } + + data, err := decrypt(c.config.Token, value.Data) + if err != nil { + return err + } + + return bootstrap.Read(bytes.NewBuffer(data), &c.runtime.ControlRuntimeBootstrap) +} diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index 8c41740c33..c3750ffa84 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -132,6 +132,8 @@ type ControlRuntimeBootstrap struct { type ControlRuntime struct { ControlRuntimeBootstrap + HTTPBootstrap bool + ClientKubeAPICert string ClientKubeAPIKey string NodePasswdFile string @@ -169,12 +171,7 @@ type ControlRuntime struct { ClientK3sControllerCert string ClientK3sControllerKey string - Cluster ClusterConfig - Core *core.Factory -} - -type ClusterConfig struct { - Join bool + Core *core.Factory } type ArgString []string diff --git a/pkg/daemons/control/server.go b/pkg/daemons/control/server.go index 9990ddd3f9..b3ff90acd1 100644 --- a/pkg/daemons/control/server.go +++ b/pkg/daemons/control/server.go @@ -26,7 +26,6 @@ import ( "github.com/rancher/k3s/pkg/daemons/config" "github.com/rancher/k3s/pkg/passwd" "github.com/rancher/k3s/pkg/token" - "github.com/rancher/kine/pkg/endpoint" "github.com/rancher/wrangler-api/pkg/generated/controllers/rbac" "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -317,11 +316,13 @@ func prepare(ctx context.Context, config *config.Control, runtime *config.Contro runtime.ClientAuthProxyCert = path.Join(config.DataDir, "tls", "client-auth-proxy.crt") runtime.ClientAuthProxyKey = path.Join(config.DataDir, "tls", "client-auth-proxy.key") - if err := genCerts(config, runtime); err != nil { + cluster := cluster.New(config) + + if err := cluster.Join(ctx); err != nil { return err } - if err := cluster.New(config).Start(ctx); err != nil { + if err := genCerts(config, runtime); err != nil { return err } @@ -337,23 +338,11 @@ func prepare(ctx context.Context, config *config.Control, runtime *config.Contro return err } - if err := prepareStorageBackend(ctx, config); err != nil { + if err := readTokens(runtime); err != nil { return err } - return readTokens(runtime) -} - -func prepareStorageBackend(ctx context.Context, config *config.Control) error { - etcdConfig, err := endpoint.Listen(ctx, config.Storage) - if err != nil { - return err - } - - config.Storage.Config = etcdConfig.TLSConfig - config.Storage.Endpoint = strings.Join(etcdConfig.Endpoints, ",") - config.NoLeaderElect = !etcdConfig.LeaderElect - return nil + return cluster.Start(ctx) } func readTokens(runtime *config.ControlRuntime) error { diff --git a/pkg/dqlite/controller/client/controller.go b/pkg/dqlite/controller/client/controller.go index 9df5780457..25027a7d3c 100644 --- a/pkg/dqlite/controller/client/controller.go +++ b/pkg/dqlite/controller/client/controller.go @@ -117,7 +117,11 @@ func (h *handler) updateNodeStore() error { return err } - var nodeInfos []client.NodeInfo + var ( + nodeInfos []client.NodeInfo + seen = map[string]bool{} + ) + for _, node := range nodes { address, ok := node.Annotations[nodeAddress] if !ok { @@ -135,10 +139,13 @@ func (h *handler) updateNodeStore() error { continue } - nodeInfos = append(nodeInfos, client.NodeInfo{ - ID: id, - Address: address, - }) + if !seen[address] { + nodeInfos = append(nodeInfos, client.NodeInfo{ + ID: id, + Address: address, + }) + seen[address] = true + } } if len(nodeInfos) == 0 { diff --git a/pkg/server/router.go b/pkg/server/router.go index 9909d13288..b9908b6dce 100644 --- a/pkg/server/router.go +++ b/pkg/server/router.go @@ -50,7 +50,9 @@ func router(serverConfig *config.Control, tunnel http.Handler, ca []byte) http.H serverAuthed.Use(authMiddleware(serverConfig, "k3s:server")) serverAuthed.NotFoundHandler = nodeAuthed serverAuthed.Path("/db/info").Handler(nodeAuthed) - serverAuthed.Path("/v1-k3s/server-bootstrap").Handler(bootstrap.Handler(&serverConfig.Runtime.ControlRuntimeBootstrap)) + if serverConfig.Runtime.HTTPBootstrap { + serverAuthed.Path("/v1-k3s/server-bootstrap").Handler(bootstrap.Handler(&serverConfig.Runtime.ControlRuntimeBootstrap)) + } staticDir := filepath.Join(serverConfig.DataDir, "static") router := mux.NewRouter() From b2439788d7293b95b30c6941cd5ca4e7631807f7 Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Mon, 11 Nov 2019 22:18:43 +0000 Subject: [PATCH 08/12] Reduce logging in dqlite --- pkg/dqlite/log.go | 8 +++++++- pkg/dqlite/pipe/pipe.go | 4 ++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/pkg/dqlite/log.go b/pkg/dqlite/log.go index e7185b7797..bbd07c8017 100644 --- a/pkg/dqlite/log.go +++ b/pkg/dqlite/log.go @@ -1,6 +1,8 @@ package dqlite import ( + "strings" + "github.com/canonical/go-dqlite/client" "github.com/sirupsen/logrus" ) @@ -13,7 +15,11 @@ func log() client.LogFunc { case client.LogError: logrus.Errorf(s, i...) case client.LogInfo: - logrus.Infof(s, i...) + if strings.HasPrefix(s, "connected") { + logrus.Debugf(s, i...) + } else { + logrus.Infof(s, i...) + } case client.LogWarn: logrus.Warnf(s, i...) } diff --git a/pkg/dqlite/pipe/pipe.go b/pkg/dqlite/pipe/pipe.go index cf8d6113c0..fc00031318 100644 --- a/pkg/dqlite/pipe/pipe.go +++ b/pkg/dqlite/pipe/pipe.go @@ -29,7 +29,7 @@ func Connect(src net.Conn, dst net.Conn) { go func() { _, err := io.Copy(eagain.Writer{Writer: dst}, eagain.Reader{Reader: src}) if err != nil && err != io.EOF { - logrus.Warnf("copy pipe src->dst closed: %v", err) + logrus.Debugf("copy pipe src->dst closed: %v", err) } src.Close() dst.Close() @@ -38,7 +38,7 @@ func Connect(src net.Conn, dst net.Conn) { go func() { _, err := io.Copy(eagain.Writer{Writer: src}, eagain.Reader{Reader: dst}) if err != nil { - logrus.Warnf("copy pipe dst->src closed: %v", err) + logrus.Debugf("copy pipe dst->src closed: %v", err) } src.Close() dst.Close() From 668fcf7e8338e153f5a8b0b183fdbf33f30f4cc2 Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Mon, 11 Nov 2019 22:19:00 +0000 Subject: [PATCH 09/12] Fix broken --cluster-reset --- pkg/dqlite/reset.go | 33 ++++----------------------------- pkg/dqlite/server.go | 11 +++++++++-- 2 files changed, 13 insertions(+), 31 deletions(-) diff --git a/pkg/dqlite/reset.go b/pkg/dqlite/reset.go index 95b263e55d..d9a379897f 100644 --- a/pkg/dqlite/reset.go +++ b/pkg/dqlite/reset.go @@ -2,39 +2,14 @@ package dqlite import ( "context" - "fmt" "github.com/canonical/go-dqlite/client" "github.com/sirupsen/logrus" ) func (d *DQLite) Reset(ctx context.Context) error { - dqClient, err := client.New(ctx, d.getBindAddress(), client.WithLogFunc(log())) - if err != nil { - return err - } - - current, err := dqClient.Cluster(ctx) - if err != nil { - return err - } - - // There's a chance our ID and the ID the server has doesn't match so find the ID - var surviving []client.NodeInfo - for _, testNode := range current { - if testNode.Address == d.NodeInfo.Address && testNode.ID == d.NodeInfo.ID { - surviving = append(surviving, testNode) - continue - } - if err := dqClient.Remove(ctx, testNode.ID); err != nil { - return err - } - } - - if len(surviving) != 1 { - return fmt.Errorf("failed to find %s in the current node, can not reset", d.NodeInfo.Address) - } - - logrus.Infof("Resetting cluster to single master, please rejoin members") - return d.node.Recover(surviving) + logrus.Infof("Resetting cluster to single master") + return d.node.Recover([]client.NodeInfo{ + d.NodeInfo, + }) } diff --git a/pkg/dqlite/server.go b/pkg/dqlite/server.go index b5e6b04a4a..9fab9c8e74 100644 --- a/pkg/dqlite/server.go +++ b/pkg/dqlite/server.go @@ -60,7 +60,7 @@ func New(dataDir, advertiseIP string, advertisePort int, getter NodeControllerGe } } -func (d *DQLite) Start(ctx context.Context, initCluster bool, certs *Certs, next http.Handler) (http.Handler, error) { +func (d *DQLite) Start(ctx context.Context, initCluster, resetCluster bool, certs *Certs, next http.Handler) (http.Handler, error) { bindAddress := d.getBindAddress() clientTLSConfig, err := getClientTLSConfig(certs.ClientCert, certs.ServerTrust) @@ -89,6 +89,7 @@ func (d *DQLite) Start(ctx context.Context, initCluster bool, certs *Certs, next } d.NodeInfo = nodeInfo + d.node = node go func() { <-ctx.Done() @@ -101,7 +102,13 @@ func (d *DQLite) Start(ctx context.Context, initCluster bool, certs *Certs, next go d.startController(ctx) - return router(ctx, next, nodeInfo, certs.ClientTrust, "kube-apiserver", bindAddress), node.Start() + if !resetCluster { + if err := node.Start(); err != nil { + return nil, err + } + } + + return router(ctx, next, nodeInfo, certs.ClientTrust, "kube-apiserver", bindAddress), nil } func (d *DQLite) startController(ctx context.Context) { From c3cb09cbdce1ad7fa4ba19445951a18ecfa0d871 Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Mon, 11 Nov 2019 22:19:15 +0000 Subject: [PATCH 10/12] Don't build hyperkube anymore --- scripts/build | 4 ++-- scripts/package-cli | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/build b/scripts/build index bb0bc07b03..eb83bdc588 100755 --- a/scripts/build +++ b/scripts/build @@ -83,8 +83,8 @@ ln -s containerd ./bin/k3s-server ln -s containerd ./bin/kubectl ln -s containerd ./bin/crictl ln -s containerd ./bin/ctr -echo Building hyperkube -CGO_ENABLED=1 go build -tags "$TAGS" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC_SQLITE" -o bin/hyperkube ./vendor/k8s.io/kubernetes/cmd/hyperkube/ +#echo Building hyperkube +#CGO_ENABLED=1 go build -tags "$TAGS" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC_SQLITE" -o bin/hyperkube ./vendor/k8s.io/kubernetes/cmd/hyperkube/ #echo Building ctr #CGO_ENABLED=1 go build -tags "$TAGS" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC_SQLITE" -o bin/ctr ./cmd/ctr/main.go # echo Building containerd diff --git a/scripts/package-cli b/scripts/package-cli index e8b6f20a54..19c253a9ec 100755 --- a/scripts/package-cli +++ b/scripts/package-cli @@ -34,7 +34,6 @@ elif [ ${ARCH} = arm ]; then BIN_SUFFIX="-armhf" fi -cp -f ./bin/hyperkube dist/artifacts/hyperkube${BIN_SUFFIX} CMD_NAME=dist/artifacts/k3s${BIN_SUFFIX} go generate From dca6a22f3f4ee4b3e6ea04ecfd213d5f4a754880 Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Mon, 11 Nov 2019 22:19:39 +0000 Subject: [PATCH 11/12] Don't build cni on each ./scripts/build invocation --- scripts/build | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/build b/scripts/build index eb83bdc588..45dc22ac94 100755 --- a/scripts/build +++ b/scripts/build @@ -48,7 +48,6 @@ rm -f \ bin/k3s-agent \ bin/hyperkube \ bin/containerd \ - bin/cni \ bin/runc \ bin/containerd-shim \ bin/containerd-shim-runc-v1 \ @@ -65,6 +64,7 @@ cleanup() { } INSTALLBIN=$(pwd)/bin +if [ ! -x ${INSTALLBIN}/cni ]; then ( echo Building cni TMPDIR=$(mktemp -d) @@ -74,6 +74,7 @@ INSTALLBIN=$(pwd)/bin cd $WORKDIR GOPATH=$TMPDIR CGO_ENABLED=0 go build -tags "$TAGS" -ldflags "$LDFLAGS $STATIC" -o $INSTALLBIN/cni ) +fi # echo Building agent # CGO_ENABLED=1 go build -tags "$TAGS" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/k3s-agent ./cmd/agent/main.go echo Building server From 840c5911aca577640183e9f793a85fc1385c9c3e Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Mon, 11 Nov 2019 22:21:21 +0000 Subject: [PATCH 12/12] Update to kine v0.2.1 --- go.mod | 4 +- go.sum | 2 + .../rancher/kine/pkg/client/client.go | 112 ++++++++++++++++++ .../kine/pkg/drivers/generic/generic.go | 4 +- vendor/modules.txt | 9 +- 5 files changed, 123 insertions(+), 8 deletions(-) create mode 100644 vendor/github.com/rancher/kine/pkg/client/client.go diff --git a/go.mod b/go.mod index 84c5eb744d..fe08ebc873 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,6 @@ replace ( github.com/prometheus/client_model => github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 github.com/prometheus/common => github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 github.com/prometheus/procfs => github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a - github.com/rancher/kine => ../kine k8s.io/api => github.com/rancher/kubernetes/staging/src/k8s.io/api v1.16.2-k3s.1 k8s.io/apiextensions-apiserver => github.com/rancher/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.16.2-k3s.1 k8s.io/apimachinery => github.com/rancher/kubernetes/staging/src/k8s.io/apimachinery v1.16.2-k3s.1 @@ -102,7 +101,7 @@ require ( github.com/rakelkar/gonetsh v0.0.0-20190719023240-501daadcadf8 // indirect github.com/rancher/dynamiclistener v0.1.1-0.20191110035254-aaa5bc0d2a07 github.com/rancher/helm-controller v0.2.2 - github.com/rancher/kine v0.2.0 + github.com/rancher/kine v0.2.1 github.com/rancher/remotedialer v0.2.0 github.com/rancher/wrangler v0.2.0 github.com/rancher/wrangler-api v0.2.0 @@ -112,6 +111,7 @@ require ( github.com/tchap/go-patricia v2.3.0+incompatible // indirect github.com/theckman/go-flock v0.7.1 // indirect github.com/urfave/cli v1.21.0 + golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8 golang.org/x/net v0.0.0-20190812203447-cdfb69ac37fc golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3 google.golang.org/grpc v1.23.0 diff --git a/go.sum b/go.sum index 2a24578703..5ee1c087ba 100644 --- a/go.sum +++ b/go.sum @@ -590,6 +590,8 @@ github.com/rancher/flannel v0.11.0-k3s.1 h1:mIwnfWDafjzQgFkZeJ1AkFrrAT3EdBaA1giE github.com/rancher/flannel v0.11.0-k3s.1/go.mod h1:Hn4ZV+eq0LhLZP63xZnxdGwXEoRSxs5sxELxu27M3UA= github.com/rancher/helm-controller v0.2.2 h1:MUqisy53/Ay1EYOF2uTCYBbGpgtZLNKKrI01BdxIbQo= github.com/rancher/helm-controller v0.2.2/go.mod h1:0JkL0UjxddNbT4FmLoESarD4Mz8xzA5YlejqJ/U4g+8= +github.com/rancher/kine v0.2.1 h1:pK7QJUVA+/oU6esxKa/LIlBbeLl2HGWIwmu8xrROukQ= +github.com/rancher/kine v0.2.1/go.mod h1:SdBUuE7e3XyrJvdBxCl9TMMapF+wyZnMZSP/H59OqNE= github.com/rancher/kubernetes v1.16.2-k3s.1 h1:+oJEecXgQDkEOD/X8z2YUdYVonbXZtGzXsmtKDPYesg= github.com/rancher/kubernetes v1.16.2-k3s.1/go.mod h1:SmhGgKfQ30imqjFVj8AI+iW+zSyFsswNErKYeTfgoH0= github.com/rancher/kubernetes/staging/src/k8s.io/api v1.16.2-k3s.1 h1:2kK5KD6MU86txBYKG+tM6j5zbey02DaIDtwpG5JsfnI= diff --git a/vendor/github.com/rancher/kine/pkg/client/client.go b/vendor/github.com/rancher/kine/pkg/client/client.go new file mode 100644 index 0000000000..d227090a91 --- /dev/null +++ b/vendor/github.com/rancher/kine/pkg/client/client.go @@ -0,0 +1,112 @@ +package client + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/coreos/etcd/clientv3" + "github.com/rancher/kine/pkg/endpoint" +) + +type Value struct { + Data []byte + Modified int64 +} + +var ( + ErrNotFound = errors.New("etcdwrapper: key not found") +) + +type Client interface { + Get(ctx context.Context, key string) (Value, error) + Put(ctx context.Context, key string, value []byte) error + Create(ctx context.Context, key string, value []byte) error + Update(ctx context.Context, key string, revision int64, value []byte) error + Close() error +} + +type client struct { + c *clientv3.Client +} + +func New(config endpoint.ETCDConfig) (Client, error) { + tlsConfig, err := config.TLSConfig.ClientConfig() + if err != nil { + return nil, err + } + + c, err := clientv3.New(clientv3.Config{ + Endpoints: config.Endpoints, + DialTimeout: 5 * time.Second, + TLS: tlsConfig, + }) + if err != nil { + return nil, err + } + + return &client{ + c: c, + }, nil +} + +func (c *client) Get(ctx context.Context, key string) (Value, error) { + resp, err := c.c.Get(ctx, key) + if err != nil { + return Value{}, err + } + + if len(resp.Kvs) == 1 { + return Value{ + Data: resp.Kvs[0].Value, + Modified: resp.Kvs[0].ModRevision, + }, nil + } + + return Value{}, ErrNotFound +} + +func (c *client) Put(ctx context.Context, key string, value []byte) error { + val, err := c.Get(ctx, key) + if err != nil { + return err + } + if val.Modified == 0 { + return c.Create(ctx, key, value) + } + return c.Update(ctx, key, val.Modified, value) +} + +func (c *client) Create(ctx context.Context, key string, value []byte) error { + resp, err := c.c.Txn(ctx). + If(clientv3.Compare(clientv3.ModRevision(key), "=", 0)). + Then(clientv3.OpPut(key, string(value))). + Commit() + if err != nil { + return err + } + if !resp.Succeeded { + return fmt.Errorf("key exists") + } + return nil +} + +func (c *client) Update(ctx context.Context, key string, revision int64, value []byte) error { + resp, err := c.c.Txn(ctx). + If(clientv3.Compare(clientv3.ModRevision(key), "=", revision)). + Then(clientv3.OpPut(key, string(value))). + Else(clientv3.OpGet(key)). + Commit() + if err != nil { + return err + } + if !resp.Succeeded { + return fmt.Errorf("revision %d doesnt match", revision) + } + return nil +} + +func (c *client) Close() error { + return c.c.Close() +} diff --git a/vendor/github.com/rancher/kine/pkg/drivers/generic/generic.go b/vendor/github.com/rancher/kine/pkg/drivers/generic/generic.go index 66a8bb5831..d6c9c5ab3c 100644 --- a/vendor/github.com/rancher/kine/pkg/drivers/generic/generic.go +++ b/vendor/github.com/rancher/kine/pkg/drivers/generic/generic.go @@ -199,9 +199,9 @@ func (d *Generic) execute(ctx context.Context, sql string, args ...interface{}) wait := strategy.Backoff(backoff.Linear(100 + time.Millisecond)) for i := uint(0); i < 20; i++ { if i > 2 { - logrus.Infof("EXEC (%d) %v : %s", i, args, Stripped(sql)) + logrus.Debugf("EXEC (try: %d) %v : %s", i, args, Stripped(sql)) } else { - logrus.Tracef("EXEC (%d) %v : %s", i, args, Stripped(sql)) + logrus.Tracef("EXEC (try: %d) %v : %s", i, args, Stripped(sql)) } result, err = d.DB.ExecContext(ctx, sql, args...) if err != nil && d.Retry != nil && d.Retry(err) { diff --git a/vendor/modules.txt b/vendor/modules.txt index 3049b1f5d9..d99ded53e6 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -343,14 +343,14 @@ github.com/containernetworking/plugins/pkg/ns # github.com/coreos/etcd v3.3.15+incompatible github.com/coreos/etcd/clientv3 github.com/coreos/etcd/pkg/transport -github.com/coreos/etcd/etcdserver/etcdserverpb -github.com/coreos/etcd/mvcc/mvccpb github.com/coreos/etcd/auth/authpb github.com/coreos/etcd/clientv3/balancer github.com/coreos/etcd/clientv3/balancer/picker github.com/coreos/etcd/clientv3/balancer/resolver/endpoint github.com/coreos/etcd/clientv3/credentials github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes +github.com/coreos/etcd/etcdserver/etcdserverpb +github.com/coreos/etcd/mvcc/mvccpb github.com/coreos/etcd/pkg/logutil github.com/coreos/etcd/pkg/types github.com/coreos/etcd/pkg/tlsutil @@ -769,7 +769,8 @@ github.com/rancher/helm-controller/pkg/generated/informers/externalversions/helm github.com/rancher/helm-controller/pkg/generated/listers/helm.cattle.io/v1 github.com/rancher/helm-controller/pkg/generated/informers/externalversions/internalinterfaces github.com/rancher/helm-controller/pkg/apis/helm.cattle.io -# github.com/rancher/kine v0.2.0 => ../kine +# github.com/rancher/kine v0.2.1 +github.com/rancher/kine/pkg/client github.com/rancher/kine/pkg/endpoint github.com/rancher/kine/pkg/drivers/dqlite github.com/rancher/kine/pkg/drivers/mysql @@ -920,10 +921,10 @@ go.uber.org/zap/buffer go.uber.org/zap/internal/color go.uber.org/zap/internal/exit # golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8 +golang.org/x/crypto/pbkdf2 golang.org/x/crypto/ssh/terminal golang.org/x/crypto/ssh golang.org/x/crypto/ed25519 -golang.org/x/crypto/pbkdf2 golang.org/x/crypto/ocsp golang.org/x/crypto/pkcs12 golang.org/x/crypto/curve25519