Add Querycheck: a serviec to check the plan and execution time of Postgres queries

cmd/querycheck/main.go

···
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"os/signal"
+	"sync"
+	"syscall"
+	"time"
+
+	"net/http"
+	_ "net/http/pprof"
+
+	"github.com/bluesky-social/indigo/querycheck"
+	"github.com/bluesky-social/indigo/util/tracing"
+	"github.com/labstack/echo-contrib/pprof"
+	"github.com/labstack/echo/v4"
+
+	"github.com/labstack/echo/v4/middleware"
+
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"go.opentelemetry.io/otel/trace"
+	"go.uber.org/zap"
+
+	"github.com/urfave/cli/v2"
+)
+
+func main() {
+	app := cli.App{
+		Name:    "querycheck",
+		Usage:   "a postgresql query plan checker",
+		Version: "0.0.1",
+	}
+
+	app.Flags = []cli.Flag{
+		&cli.StringFlag{
+			Name:    "postgres-url",
+			Usage:   "postgres url for storing events",
+			Value:   "postgres://postgres:postgres@localhost:5432/postgres?sslmode=disable",
+			EnvVars: []string{"POSTGRES_URL"},
+		},
+		&cli.IntFlag{
+			Name:    "port",
+			Usage:   "port to serve metrics on",
+			Value:   8080,
+			EnvVars: []string{"PORT"},
+		},
+		&cli.StringFlag{
+			Name:    "auth-token",
+			Usage:   "auth token for accessing the querycheck api",
+			Value:   "localdev",
+			EnvVars: []string{"AUTH_TOKEN"},
+		},
+	}
+
+	app.Action = Querycheck
+
+	err := app.Run(os.Args)
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
+var tracer trace.Tracer
+
+// Querycheck is the main function for querycheck
+func Querycheck(cctx *cli.Context) error {
+	ctx := cctx.Context
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	// Trap SIGINT to trigger a shutdown.
+	signals := make(chan os.Signal, 1)
+	signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
+
+	rawlog, err := zap.NewDevelopment()
+	if err != nil {
+		log.Fatalf("failed to create logger: %+v\n", err)
+	}
+	defer func() {
+		log.Printf("main function teardown\n")
+		err := rawlog.Sync()
+		if err != nil {
+			log.Printf("failed to sync logger on teardown: %+v", err.Error())
+		}
+	}()
+
+	log := rawlog.Sugar().With("source", "querycheck_main")
+
+	log.Info("starting querycheck")
+
+	// Registers a tracer Provider globally if the exporter endpoint is set
+	if os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT") != "" {
+		log.Info("initializing tracer...")
+		shutdown, err := tracing.InstallExportPipeline(ctx, "Querycheck", 1)
+		if err != nil {
+			log.Fatal(err)
+		}
+		defer func() {
+			if err := shutdown(ctx); err != nil {
+				log.Fatal(err)
+			}
+		}()
+	}
+
+	wg := sync.WaitGroup{}
+
+	// HTTP Server setup and Middleware Plumbing
+	e := echo.New()
+	e.HideBanner = true
+	e.HidePort = true
+	pprof.Register(e)
+	e.GET("/metrics", echo.WrapHandler(promhttp.Handler()))
+	e.Use(middleware.LoggerWithConfig(middleware.DefaultLoggerConfig))
+
+	// Start the query checker
+	querychecker, err := querycheck.NewQuerychecker(ctx, cctx.String("postgres-url"))
+	if err != nil {
+		log.Fatalf("failed to create querychecker: %+v\n", err)
+	}
+
+	getLikesQuery := `SELECT *
+FROM likes
+WHERE subject_actor_did = 'did:plc:q6gjnaw2blty4crticxkmujt'
+	AND subject_namespace = 'app.bsky.feed.post'
+	AND subject_rkey = '3k3jf5lgbsw24'
+ORDER BY created_at DESC
+LIMIT 50;`
+
+	querychecker.AddQuery(ctx, "get_likes", getLikesQuery, time.Second*30)
+
+	getLikeCountQuery := `SELECT *
+FROM like_counts
+WHERE actor_did = 'did:plc:q6gjnaw2blty4crticxkmujt'
+	AND ns = 'app.bsky.feed.post'
+	AND rkey = '3k3jf5lgbsw24'
+LIMIT 1;`
+
+	querychecker.AddQuery(ctx, "get_like_count", getLikeCountQuery, time.Second*20)
+
+	err = querychecker.Start()
+	if err != nil {
+		log.Fatalf("failed to start querychecker: %+v\n", err)
+	}
+
+	e.Use(func(next echo.HandlerFunc) echo.HandlerFunc {
+		return func(c echo.Context) error {
+			if c.Request().Header.Get("Authorization") != cctx.String("auth-token") {
+				return c.String(http.StatusUnauthorized, "unauthorized")
+			}
+			return next(c)
+		}
+	})
+
+	e.GET("/query", querychecker.HandleGetQuery)
+	e.GET("/queries", querychecker.HandleGetQueries)
+	e.POST("/query", querychecker.HandleAddQuery)
+	e.PUT("/query", querychecker.HandleUpdateQuery)
+	e.DELETE("/query", querychecker.HandleDeleteQuery)
+
+	// Start the metrics server
+	wg.Add(1)
+	go func() {
+		log.Infof("starting metrics server on port %d", cctx.Int("port"))
+		if err := e.Start(fmt.Sprintf(":%d", cctx.Int("port"))); err != nil {
+			log.Errorf("failed to start metrics server: %+v\n", err)
+		}
+		wg.Done()
+	}()
+
+	select {
+	case <-signals:
+		cancel()
+		fmt.Println("shutting down on signal")
+	case <-ctx.Done():
+		fmt.Println("shutting down on context done")
+	}
+
+	log.Info("shutting down, waiting for workers to clean up...")
+
+	if err := e.Shutdown(ctx); err != nil {
+		log.Errorf("failed to shut down metrics server: %+v\n", err)
+		wg.Done()
+	}
+
+	querychecker.Stop()
+
+	wg.Wait()
+	log.Info("shut down successfully")
+
+	return nil
+}

go.mod

···
 	github.com/ipfs/go-log/v2 v2.5.1
 	github.com/ipld/go-car v0.6.1-0.20230509095817-92d28eb23ba4
 	github.com/ipld/go-car/v2 v2.9.0
+	github.com/jackc/pgx/v5 v5.3.0
 	github.com/joho/godotenv v1.5.1
 	github.com/labstack/echo-contrib v0.15.0
 	github.com/labstack/echo/v4 v4.10.2
···
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.42.0
 	go.opentelemetry.io/otel v1.16.0
 	go.opentelemetry.io/otel/exporters/jaeger v1.14.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.16.0
 	go.opentelemetry.io/otel/sdk v1.16.0
+	go.opentelemetry.io/otel/trace v1.16.0
 	go.uber.org/zap v1.24.0
 	golang.org/x/crypto v0.11.0
 	golang.org/x/exp v0.0.0-20230321023759-10a507213a29
···
 	github.com/ipld/go-ipld-prime v0.20.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
-	github.com/jackc/pgx/v5 v5.3.0 // indirect
 	github.com/jbenet/goprocess v0.1.4 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
···
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 // indirect
 	go.opentelemetry.io/otel/metric v1.16.0 // indirect
-	go.opentelemetry.io/otel/trace v1.16.0 // indirect
 	go.opentelemetry.io/proto/otlp v0.19.0 // indirect
 	go.uber.org/atomic v1.10.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect

querycheck/check.go

···
+package querycheck
+
+import (
+	"context"
+	"log"
+	"math"
+	"sync"
+	"time"
+
+	"github.com/jackc/pgx/v5"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.uber.org/zap"
+)
+
+var tracer = otel.Tracer("querycheck")
+
+// Query is a query to check
+type Query struct {
+	Name        string
+	Query       string
+	LastPlan    *QueryPlan
+	LastChecked time.Time
+	LastError   error
+	CheckEvery  time.Duration
+
+	lk  sync.RWMutex
+	in  chan struct{}
+	out chan struct{}
+}
+
+// Querychecker is a query checker meta object
+type Querychecker struct {
+	Queries []*Query
+	Logger  *zap.SugaredLogger
+
+	connectionURL string
+	lk            sync.RWMutex
+}
+
+// NewQuerychecker creates a new querychecker
+func NewQuerychecker(ctx context.Context, connectionURL string) (*Querychecker, error) {
+	logger, err := zap.NewDevelopment()
+	if err != nil {
+		return nil, err
+	}
+	l := logger.Sugar().With("source", "querychecker_manager")
+
+	return &Querychecker{
+		connectionURL: connectionURL,
+		Logger:        l,
+		Queries:       []*Query{},
+	}, nil
+}
+
+// AddQuery adds a query to the checker
+func (q *Querychecker) AddQuery(ctx context.Context, name, query string, checkEvery time.Duration) {
+	ctx, span := tracer.Start(ctx, "AddQuery")
+	defer span.End()
+
+	span.SetAttributes(attribute.String("name", name))
+	span.SetAttributes(attribute.String("query", query))
+	span.SetAttributes(attribute.String("checkEvery", checkEvery.String()))
+
+	q.lk.Lock()
+	q.Queries = append(q.Queries, &Query{
+		Name:       name,
+		Query:      query,
+		CheckEvery: checkEvery,
+
+		in:  make(chan struct{}),
+		out: make(chan struct{}),
+	})
+	q.lk.Unlock()
+}
+
+// RemoveQuery removes a query from the checker
+func (q *Querychecker) RemoveQuery(ctx context.Context, name string) {
+	ctx, span := tracer.Start(ctx, "RemoveQuery")
+	defer span.End()
+
+	span.SetAttributes(attribute.String("name", name))
+
+	q.lk.Lock()
+	defer q.lk.Unlock()
+	for i, qu := range q.Queries {
+		if qu.Name == name {
+			q.Queries = append(q.Queries[:i], q.Queries[i+1:]...)
+			return
+		}
+	}
+}
+
+// GetQuery returns a copy of the query
+func (q *Querychecker) GetQuery(ctx context.Context, name string) *Query {
+	ctx, span := tracer.Start(ctx, "GetQuery")
+	defer span.End()
+
+	span.SetAttributes(attribute.String("name", name))
+
+	q.lk.RLock()
+	defer q.lk.RUnlock()
+	for _, qu := range q.Queries {
+		if qu.Name == name {
+			return &Query{
+				Name:        qu.Name,
+				Query:       qu.Query,
+				LastPlan:    qu.LastPlan,
+				LastChecked: qu.LastChecked,
+				LastError:   qu.LastError,
+				CheckEvery:  qu.CheckEvery,
+			}
+		}
+	}
+	return nil
+}
+
+// GetQueries returns a copy of the queries
+func (q *Querychecker) GetQueries(ctx context.Context) []*Query {
+	ctx, span := tracer.Start(ctx, "GetQueries")
+	defer span.End()
+
+	q.lk.RLock()
+	defer q.lk.RUnlock()
+	queries := make([]*Query, len(q.Queries))
+	for i, qu := range q.Queries {
+		queries[i] = &Query{
+			Name:        qu.Name,
+			Query:       qu.Query,
+			LastPlan:    qu.LastPlan,
+			LastChecked: qu.LastChecked,
+			LastError:   qu.LastError,
+			CheckEvery:  qu.CheckEvery,
+		}
+	}
+
+	return queries
+}
+
+// UpdateQuery updates a query
+func (q *Querychecker) UpdateQuery(ctx context.Context, name, query string, checkEvery time.Duration) {
+	ctx, span := tracer.Start(ctx, "UpdateQuery")
+	defer span.End()
+
+	span.SetAttributes(attribute.String("name", name))
+	span.SetAttributes(attribute.String("query", query))
+	span.SetAttributes(attribute.String("checkEvery", checkEvery.String()))
+
+	for _, qu := range q.Queries {
+		if qu.Name == name {
+			qu.lk.Lock()
+			qu.Query = query
+			qu.CheckEvery = checkEvery
+			qu.lk.Unlock()
+			return
+		}
+	}
+}
+
+// Start starts the query checker routines
+func (q *Querychecker) Start() error {
+	ctx, span := tracer.Start(context.Background(), "Start")
+	defer span.End()
+
+	for _, qu := range q.Queries {
+		go func(query *Query) {
+			rawlog, err := zap.NewDevelopment()
+			if err != nil {
+				log.Fatalf("failed to create logger: %+v\n", err)
+			}
+			log := rawlog.Sugar().With("source", "query_checker_routine", "query", query.Name)
+
+			log.Infof("query checker routine started for query: %s\n", query.Name)
+			log.Infof("Query: \n%s\n", query.Query)
+
+			// Check the query plan every CheckEvery duration
+			ticker := time.NewTicker(query.CheckEvery)
+			defer ticker.Stop()
+
+			query.LastPlan, err = q.CheckQueryPlan(ctx, query.Query)
+			if err != nil {
+				log.Errorf("failed to check query plan: %+v\n", err)
+			}
+
+			if query.LastPlan != nil {
+				log.Infof("Initial plan:\n%+v\n", query.LastPlan.String())
+				query.RecordPlanMetrics(*query.LastPlan)
+				query.LastChecked = time.Now()
+			}
+
+			for {
+				select {
+				case <-ticker.C:
+					log.Info("checking query plan")
+
+					query.lk.RLock()
+					queryString := query.Query
+					query.lk.RUnlock()
+
+					qp, err := q.CheckQueryPlan(ctx, queryString)
+
+					query.lk.Lock()
+					query.LastChecked = time.Now()
+					query.LastError = err
+					query.lk.Unlock()
+
+					execCounter.WithLabelValues(query.Name).Inc()
+
+					if err != nil || qp == nil {
+						if qp == nil {
+							log.Errorf("query plan is nil")
+						}
+						log.Errorf("failed to check query plan: %+v\n", err)
+						errorCounter.WithLabelValues(query.Name).Inc()
+						continue
+					}
+
+					query.lk.RLock()
+					lastPlan := *query.LastPlan
+					query.lk.RUnlock()
+
+					query.RecordPlanMetrics(*qp)
+
+					if !qp.HasSameStructureAs(lastPlan) {
+						sign := "+"
+						diff := math.Abs(lastPlan.Plan.ActualTotalTime - qp.Plan.ActualTotalTime)
+						if lastPlan.Plan.ActualTotalTime > qp.Plan.ActualTotalTime {
+							sign = "-"
+						}
+
+						log.Infof("query plan has changed (%s%.03fms): \n%+v\n", sign, diff, qp.String())
+
+						query.lk.Lock()
+						query.LastPlan = qp
+						query.lk.Unlock()
+					}
+				case <-query.in:
+					log.Info("shutting down query checker routine")
+					query.out <- struct{}{}
+					return
+				}
+			}
+		}(qu)
+	}
+
+	return nil
+}
+
+// Stop stops the query checker routines
+func (q *Querychecker) Stop() {
+	_, span := tracer.Start(context.Background(), "Stop")
+	defer span.End()
+
+	q.Logger.Info("stopping query checker")
+
+	for _, qu := range q.Queries {
+		qu.in <- struct{}{}
+	}
+
+	for _, qu := range q.Queries {
+		<-qu.out
+	}
+
+	q.Logger.Info("query checker stopped")
+}
+
+// CheckQueryPlan checks the query plan for a given query
+func (q *Querychecker) CheckQueryPlan(ctx context.Context, query string) (*QueryPlan, error) {
+	ctx, span := tracer.Start(ctx, "CheckQueryPlan")
+	defer span.End()
+
+	conn, err := pgx.Connect(ctx, q.connectionURL)
+	if err != nil {
+		return nil, err
+	}
+	defer conn.Close(ctx)
+
+	rows, err := conn.Query(ctx, "EXPLAIN (ANALYZE, COSTS, VERBOSE, BUFFERS, FORMAT JSON) "+query)
+	if err != nil {
+		return nil, err
+	}
+
+	var plan QueryPlan
+
+	for rows.Next() {
+		var plans QueryPlans
+		err := rows.Scan(&plans)
+		if err != nil {
+			return nil, err
+		}
+		for _, p := range plans {
+			plan = p
+		}
+	}
+
+	return &plan, nil
+}
+
+// RecordPlanMetrics records the query plan metrics
+func (qu *Query) RecordPlanMetrics(qp QueryPlan) {
+	execDurationCounter.WithLabelValues(qu.Name).Add(qp.Plan.ActualTotalTime)
+	blocksHitCounter.WithLabelValues(qu.Name).Add(float64(qp.Plan.SharedHitBlocks))
+	blocksReadCounter.WithLabelValues(qu.Name).Add(float64(qp.Plan.SharedReadBlocks))
+	blocksWrittenCounter.WithLabelValues(qu.Name).Add(float64(qp.Plan.SharedWrittenBlocks))
+	blocksDirtyCounter.WithLabelValues(qu.Name).Add(float64(qp.Plan.SharedDirtiedBlocks))
+	ioReadTimeCounter.WithLabelValues(qu.Name).Add(qp.Plan.IOReadTime)
+	ioWriteTimeCounter.WithLabelValues(qu.Name).Add(qp.Plan.IOWriteTime)
+	tempWrittenBlocksCounter.WithLabelValues(qu.Name).Add(float64(qp.Plan.TempWrittenBlocks))
+
+	qu.RecordPlanNode(qp.Plan)
+}
+
+// RecordPlanNode records the query plan node metrics
+func (qu *Query) RecordPlanNode(p Plan) {
+	planNodeCounter.WithLabelValues(qu.Name, p.NodeType).Inc()
+	for _, n := range p.Plans {
+		qu.RecordPlanNode(n)
+	}
+}

querycheck/handlers.go

···
+package querycheck
+
+import (
+	"time"
+
+	"github.com/labstack/echo/v4"
+)
+
+type RetQuery struct {
+	Name        string     `json:"name"`
+	Query       string     `json:"query"`
+	LastPlan    *QueryPlan `json:"last_plan"`
+	LastChecked time.Time  `json:"last_checked"`
+	LastError   error      `json:"last_error"`
+	CheckEvery  string     `json:"check_every"`
+}
+
+func (q *Querychecker) HandleGetQueries(c echo.Context) error {
+	queries := q.GetQueries(c.Request().Context())
+	retQueries := []RetQuery{}
+	for _, query := range queries {
+		retQueries = append(retQueries, RetQuery{
+			Name:        query.Name,
+			Query:       query.Query,
+			LastPlan:    query.LastPlan,
+			LastChecked: query.LastChecked,
+			LastError:   query.LastError,
+			CheckEvery:  query.CheckEvery.String(),
+		})
+	}
+
+	return c.JSON(200, retQueries)
+}
+
+func (q *Querychecker) HandleGetQuery(c echo.Context) error {
+	query := q.GetQuery(c.Request().Context(), c.Param("name"))
+	if query == nil {
+		return c.JSON(404, echo.Map{
+			"message": "not found",
+		})
+	}
+
+	retQuery := RetQuery{
+		Name:        query.Name,
+		Query:       query.Query,
+		LastPlan:    query.LastPlan,
+		LastChecked: query.LastChecked,
+		LastError:   query.LastError,
+		CheckEvery:  query.CheckEvery.String(),
+	}
+
+	return c.JSON(200, retQuery)
+}
+
+func (q *Querychecker) HandleDeleteQuery(c echo.Context) error {
+	q.RemoveQuery(c.Request().Context(), c.Param("name"))
+	return c.JSON(200, echo.Map{
+		"message": "success",
+	})
+}
+
+type AddQueryRequest struct {
+	Name       string `json:"name"`
+	Query      string `json:"query"`
+	CheckEvery int64  `json:"check_every_ms"`
+}
+
+func (q *Querychecker) HandleAddQuery(c echo.Context) error {
+	var req AddQueryRequest
+	c.Bind(&req)
+	q.AddQuery(c.Request().Context(), req.Name, req.Query, time.Duration(req.CheckEvery)*time.Millisecond)
+	return c.JSON(200, echo.Map{
+		"name":           req.Name,
+		"query":          req.Query,
+		"check_every_ms": req.CheckEvery,
+		"message":        "success",
+	})
+}
+
+type UpdateQueryRequest struct {
+	Query      string `json:"query"`
+	CheckEvery int64  `json:"check_every_ms"`
+}
+
+func (q *Querychecker) HandleUpdateQuery(c echo.Context) error {
+	var req UpdateQueryRequest
+	c.Bind(&req)
+	q.UpdateQuery(c.Request().Context(), c.Param("name"), req.Query, time.Duration(req.CheckEvery)*time.Millisecond)
+	return c.JSON(200, echo.Map{
+		"name":           c.Param("name"),
+		"query":          req.Query,
+		"check_every_ms": req.CheckEvery,
+		"message":        "success",
+	})
+}

querycheck/metrics.go

···
+package querycheck
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var execCounter = promauto.NewCounterVec(prometheus.CounterOpts{
+	Name: "querycheck_exec_total",
+	Help: "total number of executions since starting the querycheck",
+}, []string{"query"})
+
+var execDurationCounter = promauto.NewCounterVec(prometheus.CounterOpts{
+	Name: "querycheck_exec_duration_ms_total",
+	Help: "total ms spent executing the query since starting the querycheck",
+}, []string{"query"})
+
+var errorCounter = promauto.NewCounterVec(prometheus.CounterOpts{
+	Name: "querycheck_errors_total",
+	Help: "number of errors encountered since starting the querycheck",
+}, []string{"query"})
+
+var blocksHitCounter = promauto.NewCounterVec(prometheus.CounterOpts{
+	Name: "querycheck_blocks_hit_total",
+	Help: "blocks hit total since starting the querycheck",
+}, []string{"query"})
+
+var blocksReadCounter = promauto.NewCounterVec(prometheus.CounterOpts{
+	Name: "querycheck_blocks_read_total",
+	Help: "blocks read total since starting the querycheck",
+}, []string{"query"})
+
+var blocksWrittenCounter = promauto.NewCounterVec(prometheus.CounterOpts{
+	Name: "querycheck_blocks_written_total",
+	Help: "blocks written total since starting the querycheck",
+}, []string{"query"})
+
+var blocksDirtyCounter = promauto.NewCounterVec(prometheus.CounterOpts{
+	Name: "querycheck_blocks_dirty_total",
+	Help: "blocks dirty total since starting the querycheck",
+}, []string{"query"})
+
+var ioReadTimeCounter = promauto.NewCounterVec(prometheus.CounterOpts{
+	Name: "querycheck_io_read_ms_total",
+	Help: "io read time (in ms) total since starting the querycheck",
+}, []string{"query"})
+
+var ioWriteTimeCounter = promauto.NewCounterVec(prometheus.CounterOpts{
+	Name: "querycheck_io_write_ms_total",
+	Help: "io write time (in ms) total since starting the querycheck",
+}, []string{"query"})
+
+var tempWrittenBlocksCounter = promauto.NewCounterVec(prometheus.CounterOpts{
+	Name: "querycheck_temp_written_blocks_total",
+	Help: "temp written blocks total since starting the querycheck",
+}, []string{"query"})
+
+var planNodeCounter = promauto.NewCounterVec(prometheus.CounterOpts{
+	Name: "querycheck_plan_node_count_total",
+	Help: "plan node count total since starting the querycheck",
+}, []string{"query", "node_type"})

querycheck/plan.go

···
+package querycheck
+
+import "fmt"
+
+type Plan struct {
+	NodeType            string   `json:"Node Type"`
+	ParallelAware       bool     `json:"Parallel Aware"`
+	AsyncCapable        bool     `json:"Async Capable"`
+	StartupCost         float64  `json:"Startup Cost"`
+	TotalCost           float64  `json:"Total Cost"`
+	PlanRows            int      `json:"Plan Rows"`
+	PlanWidth           int      `json:"Plan Width"`
+	ActualStartupTime   float64  `json:"Actual Startup Time"`
+	ActualTotalTime     float64  `json:"Actual Total Time"`
+	ActualRows          int      `json:"Actual Rows"`
+	ActualLoops         int      `json:"Actual Loops"`
+	Output              []string `json:"Output"`
+	SharedHitBlocks     int      `json:"Shared Hit Blocks"`
+	SharedReadBlocks    int      `json:"Shared Read Blocks"`
+	SharedDirtiedBlocks int      `json:"Shared Dirtied Blocks"`
+	SharedWrittenBlocks int      `json:"Shared Written Blocks"`
+	LocalHitBlocks      int      `json:"Local Hit Blocks"`
+	LocalReadBlocks     int      `json:"Local Read Blocks"`
+	LocalDirtiedBlocks  int      `json:"Local Dirtied Blocks"`
+	LocalWrittenBlocks  int      `json:"Local Written Blocks"`
+	TempReadBlocks      int      `json:"Temp Read Blocks"`
+	TempWrittenBlocks   int      `json:"Temp Written Blocks"`
+	IOReadTime          float64  `json:"I/O Read Time"`
+	IOWriteTime         float64  `json:"I/O Write Time"`
+	Plans               []Plan   `json:"Plans,omitempty"`
+	ParentRelationship  string   `json:"Parent Relationship,omitempty"`
+	SortKey             []string `json:"Sort Key,omitempty"`
+	SortMethod          string   `json:"Sort Method,omitempty"`
+	SortSpaceUsed       int      `json:"Sort Space Used,omitempty"`
+	SortSpaceType       string   `json:"Sort Space Type,omitempty"`
+	WorkersPlanned      int      `json:"Workers Planned,omitempty"`
+	WorkersLaunched     int      `json:"Workers Launched,omitempty"`
+	SingleCopy          bool     `json:"Single Copy,omitempty"`
+	RelationName        string   `json:"Relation Name,omitempty"`
+	Schema              string   `json:"Schema,omitempty"`
+	Alias               string   `json:"Alias,omitempty"`
+	Filter              string   `json:"Filter,omitempty"`
+	RowsRemovedByFilter int      `json:"Rows Removed by Filter,omitempty"`
+	Workers             []Worker `json:"Workers,omitempty"`
+}
+
+type Worker struct {
+	WorkerNumber        int     `json:"Worker Number"`
+	ActualStartupTime   float64 `json:"Actual Startup Time"`
+	ActualTotalTime     float64 `json:"Actual Total Time"`
+	ActualRows          int     `json:"Actual Rows"`
+	ActualLoops         int     `json:"Actual Loops"`
+	JIT                 JIT     `json:"JIT"`
+	SharedHitBlocks     int     `json:"Shared Hit Blocks"`
+	SharedReadBlocks    int     `json:"Shared Read Blocks"`
+	SharedDirtiedBlocks int     `json:"Shared Dirtied Blocks"`
+	SharedWrittenBlocks int     `json:"Shared Written Blocks"`
+	LocalHitBlocks      int     `json:"Local Hit Blocks"`
+	LocalReadBlocks     int     `json:"Local Read Blocks"`
+	LocalDirtiedBlocks  int     `json:"Local Dirtied Blocks"`
+	LocalWrittenBlocks  int     `json:"Local Written Blocks"`
+	TempReadBlocks      int     `json:"Temp Read Blocks"`
+	TempWrittenBlocks   int     `json:"Temp Written Blocks"`
+	IOReadTime          float64 `json:"I/O Read Time"`
+	IOWriteTime         float64 `json:"I/O Write Time"`
+}
+
+type JIT struct {
+	Functions int     `json:"Functions"`
+	Options   Options `json:"Options"`
+	Timing    Timing  `json:"Timing"`
+}
+
+type Options struct {
+	Inlining     bool `json:"Inlining"`
+	Optimization bool `json:"Optimization"`
+	Expressions  bool `json:"Expressions"`
+	Deforming    bool `json:"Deforming"`
+}
+
+type Timing struct {
+	Generation   float64 `json:"Generation"`
+	Inlining     float64 `json:"Inlining"`
+	Optimization float64 `json:"Optimization"`
+	Emission     float64 `json:"Emission"`
+	Total        float64 `json:"Total"`
+}
+
+type QueryPlan struct {
+	Plan            Plan     `json:"Plan"`
+	QueryIdentifier int64    `json:"Query Identifier"`
+	Planning        Planning `json:"Planning"`
+	PlanningTime    float64  `json:"Planning Time"`
+	Triggers        []string `json:"Triggers"`
+	JIT             JIT      `json:"JIT"`
+	ExecutionTime   float64  `json:"Execution Time"`
+}
+
+type Planning struct {
+	SharedHitBlocks     int     `json:"Shared Hit Blocks"`
+	SharedReadBlocks    int     `json:"Shared Read Blocks"`
+	SharedDirtiedBlocks int     `json:"Shared Dirtied Blocks"`
+	SharedWrittenBlocks int     `json:"Shared Written Blocks"`
+	LocalHitBlocks      int     `json:"Local Hit Blocks"`
+	LocalReadBlocks     int     `json:"Local Read Blocks"`
+	LocalDirtiedBlocks  int     `json:"Local Dirtied Blocks"`
+	LocalWrittenBlocks  int     `json:"Local Written Blocks"`
+	TempReadBlocks      int     `json:"Temp Read Blocks"`
+	TempWrittenBlocks   int     `json:"Temp Written Blocks"`
+	IOReadTime          float64 `json:"I/O Read Time"`
+	IOWriteTime         float64 `json:"I/O Write Time"`
+}
+
+type QueryPlans []QueryPlan
+
+func (q *QueryPlan) String() string {
+	ret := ""
+	ret += q.Plan.String(1)
+	return ret
+}
+
+func (p *Plan) String(i int) string {
+	ret := ""
+	ret += fmt.Sprintf("(%s) Timing: %fms | IO Read: %fms (H %d R %d D %d W %d) | IO Write: %fms",
+		p.NodeType,
+		p.ActualTotalTime,
+		p.IOReadTime,
+		p.SharedHitBlocks,
+		p.SharedReadBlocks,
+		p.SharedWrittenBlocks,
+		p.SharedDirtiedBlocks,
+		p.IOWriteTime,
+	)
+
+	for _, plan := range p.Plans {
+		ret += "\n"
+		for j := 0; j < i; j++ {
+			ret += "\t"
+		}
+		ret += plan.String(i + 1)
+	}
+
+	return ret
+}
+
+func (q *QueryPlan) HasSameStructureAs(other QueryPlan) bool {
+	return q.Plan.HasSameStructureAs(other.Plan)
+}
+
+func (p *Plan) HasSameStructureAs(other Plan) bool {
+	if p.NodeType != other.NodeType {
+		return false
+	}
+
+	if len(p.Plans) != len(other.Plans) {
+		return false
+	}
+
+	for i, plan := range p.Plans {
+		if !plan.HasSameStructureAs(other.Plans[i]) {
+			return false
+		}
+	}
+
+	return true
+}

util/tracing/tracing.go

···
+package tracing
+
+import (
+	"context"
+	"fmt"
+
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
+	"go.opentelemetry.io/otel/sdk/resource"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	semconv "go.opentelemetry.io/otel/semconv/v1.17.0"
+)
+
+func InstallExportPipeline(ctx context.Context, serviceName string, sampleRatio float64) (func(context.Context) error, error) {
+	client := otlptracehttp.NewClient()
+	exporter, err := otlptrace.New(ctx, client)
+	if err != nil {
+		return nil, fmt.Errorf("creating OTLP trace exporter: %w", err)
+	}
+
+	tracerProvider := newTraceProvider(exporter, serviceName, sampleRatio)
+	otel.SetTracerProvider(tracerProvider)
+
+	return tracerProvider.Shutdown, nil
+}
+
+func newTraceProvider(exp sdktrace.SpanExporter, serviceName string, sampleRatio float64) *sdktrace.TracerProvider {
+	// Ensure default SDK resources and the required service name are set.
+	r, err := resource.Merge(
+		resource.Default(),
+		resource.NewWithAttributes(
+			semconv.SchemaURL,
+			semconv.ServiceName(serviceName),
+		),
+	)
+
+	if err != nil {
+		panic(err)
+	}
+
+	// initialize the traceIDRatioBasedSampler
+	traceIDRatioBasedSampler := sdktrace.TraceIDRatioBased(sampleRatio)
+
+	return sdktrace.NewTracerProvider(
+		sdktrace.WithSampler(traceIDRatioBasedSampler),
+		sdktrace.WithBatcher(exp),
+		sdktrace.WithResource(r),
+	)
+}