livepeer: keep two segments in flight (and add metrics)

See merge request streamplace/streamplace!125

Changelog: feature

pkg/atproto/firehose.go

···
 
 		case repomgr.EvtKindDeleteRecord:
 			if collection.String() == constants.APP_BSKY_GRAPH_FOLLOW {
+				if r == nil {
+					log.Debug(ctx, "no repo found for follow", "userDID", evt.Repo, "subjectDID", rkey.String())
+					continue
+				}
 				log.Debug(ctx, "deleting follow", "userDID", evt.Repo, "subjectDID", rkey.String())
 				err := atsync.Model.DeleteFollow(ctx, evt.Repo, rkey.String())
 				if err != nil {
···
 				}
 			}
 
-			if err != nil {
-				return err
-			}
 		default:
 			log.Error(ctx, "unexpected record op kind")
 		}

pkg/director/stream_session.go

···
 			} else {
 				log.Log(ctx, "transcoded segment", "took", took)
 			}
+			spmetrics.QueuedTranscodeDuration.WithLabelValues(spseg.Creator).Set(float64(time.Since(start).Milliseconds()))
 		}()
 	}
 

pkg/livepeer/livepeer.go

···
 	"mime/multipart"
 	"net/http"
 	"strings"
-	"sync"
 	"time"
 
 	"golang.org/x/net/context/ctxhttp"
 	"stream.place/streamplace/pkg/aqhttp"
 	"stream.place/streamplace/pkg/log"
+	"stream.place/streamplace/pkg/spmetrics"
 	"stream.place/streamplace/pkg/streamplace"
 )
 
+const SEGMENTS_IN_FLIGHT = 2
+
 type LivepeerSession struct {
 	SessionID  string
 	Count      int
 	GatewayURL string
-	SegLock    sync.Mutex
+	Guard      chan struct{}
 }
 
 // borrowed from catalyst-api
···
 		SessionID:  fmt.Sprintf("%s-%s", did, sessionID),
 		Count:      0,
 		GatewayURL: gatewayURL,
+		Guard:      make(chan struct{}, SEGMENTS_IN_FLIGHT),
 	}, nil
 }
 
-func (ls *LivepeerSession) PostSegmentToGateway(ctx context.Context, buf []byte, seg *streamplace.Segment) ([][]byte, error) {
+func (ls *LivepeerSession) PostSegmentToGateway(ctx context.Context, buf []byte, spseg *streamplace.Segment) ([][]byte, error) {
 	ctx = log.WithLogValues(ctx, "func", "PostSegmentToGateway")
-	ls.SegLock.Lock()
+	ls.Guard <- struct{}{}
+	start := time.Now()
 	// check if context is done since we were waiting for the lock
 	if ctx.Err() != nil {
-		ls.SegLock.Unlock()
+		<-ls.Guard
 		return nil, ctx.Err()
 	}
 	ctx, cancel := context.WithTimeout(ctx, time.Minute*5)
···
 	url := fmt.Sprintf("%s/live/%s/%d.mp4", ls.GatewayURL, ls.SessionID, ls.Count)
 	ls.Count++
 
-	dur := time.Duration(*seg.Duration)
+	dur := time.Duration(*spseg.Duration)
 	durationMs := int(dur.Milliseconds())
 	log.Debug(ctx, "posting segment to livepeer gateway", "duration_ms", durationMs, "url", url)
 
-	vid := seg.Video[0]
+	vid := spseg.Video[0]
 	width := int(vid.Width)
 	height := int(vid.Height)
 
 	req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(buf))
 	if err != nil {
-		ls.SegLock.Unlock()
+		<-ls.Guard
 		return nil, fmt.Errorf("failed to create request: %w", err)
 	}
 	req.Header.Set("Accept", "multipart/mixed")
···
 
 	resp, err := ctxhttp.Do(ctx, &aqhttp.Client, req)
 	if err != nil {
-		ls.SegLock.Unlock()
+		<-ls.Guard
 		return nil, fmt.Errorf("failed to send segment to gateway: %w", err)
 	}
-	ls.SegLock.Unlock()
+	<-ls.Guard
 	defer resp.Body.Close()
 
 	if resp.StatusCode != http.StatusOK {
···
 			out = append(out, bs)
 		}
 	}
-
+	spmetrics.TranscodeDuration.WithLabelValues(spseg.Creator).Observe(float64(time.Since(start).Milliseconds()))
 	return out, nil
 }

pkg/spmetrics/spmetrics.go

···
 var Viewers = promauto.NewGaugeVec(prometheus.GaugeOpts{
 	Name: "streamplace_viewers",
 	Help: "number of current viewers per user",
-}, []string{"user"})
+}, []string{"streamer"})
 
 var ViewersTotal = promauto.NewGauge(prometheus.GaugeOpts{
 	Name: "streamplace_viewers_total",
···
 	Name: "streamplace_transcode_errors_total",
 	Help: "total number of transcode errors",
 })
+
+var TranscodeDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
+	Name:    "streamplace_transcode_duration_ms",
+	Help:    "duration of transcode in ms",
+	Buckets: []float64{0, 250, 500, 750, 1000, 1250, 1500, 2000, 2500, 3000, 3500, 4000, 4500, 5000, 10000},
+}, []string{"streamer"})
+
+var QueuedTranscodeDuration = promauto.NewGaugeVec(prometheus.GaugeOpts{
+	Name: "streamplace_queued_transcode_duration_ms",
+	Help: "duration of transcode in ms, including time spent waiting",
+}, []string{"streamer"})
 
 var Version = promauto.NewCounterVec(prometheus.CounterOpts{
 	Name: "streamplace_version",