fork of whitequark.org/git-pages with mods for tangled

Replace `s3GetObjectErrorsCount` metric with `*ResponseCount`.

The former metric was misnamed: it only counted NoSuchKey errors.
Also, it was applied *after* the cache, meaning it was just a count
of every request that got a successful 404 from the S3 backend.
Also, it pooled blob and manifest requests together.

The new metric is 1-to-1 correspondent to S3 requests and distinguishes
between different kinds of errors. Also, it distinguishes kinds of
requests. Example output:

git_pages_s3_get_object_responses_count{code="NoSuchKey",kind="manifest"} 1
git_pages_s3_get_object_responses_count{code="OK",kind="blob"} 1
git_pages_s3_get_object_responses_count{code="OK",kind="manifest"} 1

Changed files
+27 -9
src
+27 -9
src/backend_s3.go
··· 35 35 manifestCacheEvictionsCount prometheus.Counter 36 36 37 37 s3GetObjectDurationSeconds *prometheus.HistogramVec 38 - s3GetObjectErrorsCount *prometheus.CounterVec 38 + s3GetObjectResponseCount *prometheus.CounterVec 39 39 ) 40 40 41 41 func initS3BackendMetrics() { ··· 95 95 NativeHistogramMaxBucketNumber: 100, 96 96 NativeHistogramMinResetDuration: 10 * time.Minute, 97 97 }, []string{"kind"}) 98 - s3GetObjectErrorsCount = promauto.NewCounterVec(prometheus.CounterOpts{ 99 - Name: "git_pages_s3_get_object_errors_count", 100 - Help: "Count of s3:GetObject errors", 101 - }, []string{"object_kind"}) 98 + s3GetObjectResponseCount = promauto.NewCounterVec(prometheus.CounterOpts{ 99 + Name: "git_pages_s3_get_object_responses_count", 100 + Help: "Count of s3:GetObject responses", 101 + }, []string{"kind", "code"}) 102 102 } 103 103 104 104 // Blobs can be safely cached indefinitely. They only need to be evicted to preserve memory. ··· 296 296 return &CachedBlob{data, stat.LastModified}, nil 297 297 } 298 298 299 + observer := func(ctx context.Context, name string) (*CachedBlob, error) { 300 + cached, err := loader(ctx, name) 301 + var code = "OK" 302 + if resp, ok := err.(minio.ErrorResponse); ok { 303 + code = resp.Code 304 + } 305 + s3GetObjectResponseCount.With(prometheus.Labels{"kind": "blob", "code": code}).Inc() 306 + return cached, err 307 + } 308 + 299 309 var cached *CachedBlob 300 - cached, err = s3.blobCache.Get(ctx, name, otter.LoaderFunc[string, *CachedBlob](loader)) 310 + cached, err = s3.blobCache.Get(ctx, name, otter.LoaderFunc[string, *CachedBlob](observer)) 301 311 if err != nil { 302 312 if errResp := minio.ToErrorResponse(err); errResp.Code == "NoSuchKey" { 303 - s3GetObjectErrorsCount.With(prometheus.Labels{"object_kind": "blob"}).Inc() 304 313 err = fmt.Errorf("%w: %s", ErrObjectNotFound, errResp.Key) 305 314 } 306 315 } else { ··· 427 436 return &CachedManifest{manifest, uint32(len(data)), stat.LastModified, stat.ETag, nil}, nil 428 437 } 429 438 439 + observer := func() (*CachedManifest, error) { 440 + cached, err := loader() 441 + var code = "OK" 442 + if resp, ok := err.(minio.ErrorResponse); ok { 443 + code = resp.Code 444 + } 445 + s3GetObjectResponseCount.With(prometheus.Labels{"kind": "manifest", "code": code}).Inc() 446 + return cached, err 447 + } 448 + 430 449 startTime := time.Now() 431 - cached, err := loader() 450 + cached, err := observer() 432 451 s3GetObjectDurationSeconds. 433 452 With(prometheus.Labels{"kind": "manifest"}). 434 453 Observe(time.Since(startTime).Seconds()) ··· 436 455 if err != nil { 437 456 errResp := minio.ToErrorResponse(err) 438 457 if errResp.Code == "NoSuchKey" { 439 - s3GetObjectErrorsCount.With(prometheus.Labels{"object_kind": "manifest"}).Inc() 440 458 err = fmt.Errorf("%w: %s", ErrObjectNotFound, errResp.Key) 441 459 return &CachedManifest{nil, 1, time.Time{}, "", err}, nil 442 460 } else if errResp.StatusCode == http.StatusNotModified && oldManifest != nil {