package db import ( "context" "database/sql" "fmt" "time" "tangled.org/core/spindle/quota" ) type CacheEntry struct { ID string StorageKey string OwnerDID string RepoDID string Engine string CacheKey string CacheHash string Checksum string SizeBytes int64 RestoreCount int64 State string CreatedAt time.Time LastUsedAt time.Time } const cacheEntryColumns = ` id, storage_key, owner_did, repo_did, engine, cache_key, cache_hash, checksum, size_bytes, restore_count, state, created_at, last_used_at` func (d *DB) InsertCacheEntry(ctx context.Context, entry CacheEntry) error { _, err := d.ExecContext(ctx, ` insert into cache_entries ( id, storage_key, owner_did, repo_did, engine, cache_key, cache_hash, checksum, size_bytes, restore_count, state, created_at, last_used_at ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, entry.ID, entry.StorageKey, entry.OwnerDID, entry.RepoDID, entry.Engine, entry.CacheKey, entry.CacheHash, entry.Checksum, entry.SizeBytes, entry.RestoreCount, entry.State, entry.CreatedAt.UnixNano(), entry.LastUsedAt.UnixNano(), ) return err } func (d *DB) InsertCacheEntryWithinQuota(ctx context.Context, entry CacheEntry, maxEntries int64) (bool, error) { result, err := d.ExecContext(ctx, ` insert into cache_entries ( id, storage_key, owner_did, repo_did, engine, cache_key, cache_hash, checksum, size_bytes, restore_count, state, created_at, last_used_at ) select ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? where ? <= 0 or ( select count(*) from cache_entries where owner_did = ? and state in ('pending', 'ready', 'deleting') ) < ?`, entry.ID, entry.StorageKey, entry.OwnerDID, entry.RepoDID, entry.Engine, entry.CacheKey, entry.CacheHash, entry.Checksum, entry.SizeBytes, entry.RestoreCount, entry.State, entry.CreatedAt.UnixNano(), entry.LastUsedAt.UnixNano(), maxEntries, entry.OwnerDID, maxEntries, ) if err != nil { return false, err } inserted, err := result.RowsAffected() return inserted == 1, err } func (d *DB) MarkCacheEntryReady(ctx context.Context, id string, sizeBytes int64, qs *QuotaStore, now time.Time) ([]CacheEntry, error) { return d.MarkCacheEntryReadyWithChecksum(ctx, id, sizeBytes, "", qs, 0, now) } // a nil qs charges nothing func (d *DB) MarkCacheEntryReadyWithChecksum(ctx context.Context, id string, sizeBytes int64, checksum string, qs *QuotaStore, maxTotalBytes int64, now time.Time) ([]CacheEntry, error) { tx, err := d.BeginTx(ctx, nil) if err != nil { return nil, err } defer tx.Rollback() superseded, ready, err := markCacheEntryReady(ctx, tx, id, sizeBytes, checksum, qs, maxTotalBytes, now) if err != nil { return nil, err } if !ready { return nil, sql.ErrNoRows } if err := tx.Commit(); err != nil { return nil, err } return superseded, nil } type execQuerier interface { ExecContext(context.Context, string, ...any) (sql.Result, error) QueryRowContext(context.Context, string, ...any) *sql.Row QueryContext(context.Context, string, ...any) (*sql.Rows, error) } func markCacheEntryReady(ctx context.Context, tx execQuerier, id string, sizeBytes int64, checksum string, qs *QuotaStore, maxTotalBytes int64, now time.Time) ([]CacheEntry, bool, error) { current, err := scanCacheEntry(tx.QueryRowContext(ctx, `select `+cacheEntryColumns+` from cache_entries where id = ?`, id)) if err == sql.ErrNoRows { return nil, false, nil } if err != nil { return nil, false, err } if current.State != "pending" { if current.State == "ready" { if current.SizeBytes != sizeBytes { return nil, false, fmt.Errorf("cache entry %q size changed from %d to %d", id, current.SizeBytes, sizeBytes) } return nil, true, nil } return nil, false, nil } rows, err := tx.QueryContext(ctx, ` update cache_entries set state = 'deleting' where repo_did = ? and engine = ? and cache_key = ? and cache_hash = ? and state = 'ready' returning `+cacheEntryColumns, current.RepoDID, current.Engine, current.CacheKey, current.CacheHash) if err != nil { return nil, false, err } var superseded []CacheEntry for rows.Next() { entry, err := scanCacheEntry(rows) if err != nil { rows.Close() return nil, false, err } superseded = append(superseded, *entry) } if err := rows.Close(); err != nil { return nil, false, err } if err := rows.Err(); err != nil { return nil, false, err } if maxTotalBytes > 0 { evicted, fits, err := evictForTotal(ctx, tx, sizeBytes, maxTotalBytes) if err != nil { return nil, false, err } if !fits { return nil, false, unmarkCacheEntries(ctx, tx, superseded) } superseded = append(superseded, evicted...) } if qs != nil { // what this save replaces stops counting against it, and counts again if the charge is refused if _, err := tx.ExecContext(ctx, `savepoint replace_charge`); err != nil { return nil, false, err } for _, entry := range superseded { if err := unchargeCacheEntry(ctx, tx, entry.RepoDID, entry.ID); err != nil { return nil, false, err } } res, err := qs.ChargeTx(ctx, tx, quota.ReserveRequest{ Kind: quota.KindGenericCache, Key: current.ID, Identity: quota.Identity{OwnerDID: current.OwnerDID, RepoDID: current.RepoDID}, Resources: quota.Resources{quota.ResourceCacheStorageBytes: sizeBytes}, }) if err != nil { return nil, false, err } if !res.Allowed { if _, err := tx.ExecContext(ctx, `rollback to replace_charge`); err != nil { return nil, false, err } } if _, err := tx.ExecContext(ctx, `release replace_charge`); err != nil { return nil, false, err } if !res.Allowed { return nil, false, unmarkCacheEntries(ctx, tx, superseded) } } updated, err := tx.QueryContext(ctx, ` update cache_entries set state = 'ready', checksum = ?, size_bytes = ?, last_used_at = ? where id = ? and state = 'pending' returning id`, checksum, sizeBytes, now.UnixNano(), id) if err != nil { return nil, false, err } ready := updated.Next() updated.Close() return superseded, ready, nil } // a refused save still commits inside the mill's batch tx func unmarkCacheEntries(ctx context.Context, tx execQuerier, entries []CacheEntry) error { for _, entry := range entries { if _, err := tx.ExecContext(ctx, `update cache_entries set state = 'ready' where id = ? and state = 'deleting'`, entry.ID); err != nil { return err } } return nil } const ( readyCacheBytesQuery = `select coalesce(sum(size_bytes), 0) from cache_entries where state = 'ready'` // entries a restore holds stay out of eviction evictableCacheEntriesQuery = `select ` + cacheEntryColumns + ` from cache_entries where state = 'ready' and restore_count = 0 order by last_used_at, created_at` ) // evicts nothing when even every evictable entry would not make room func evictForTotal(ctx context.Context, tx execQuerier, sizeBytes, maxTotalBytes int64) ([]CacheEntry, bool, error) { var used int64 if err := tx.QueryRowContext(ctx, readyCacheBytesQuery).Scan(&used); err != nil { return nil, false, err } need := used + sizeBytes - maxTotalBytes if need <= 0 { return nil, true, nil } rows, err := tx.QueryContext(ctx, evictableCacheEntriesQuery) if err != nil { return nil, false, err } var evicted []CacheEntry for need > 0 && rows.Next() { entry, err := scanCacheEntry(rows) if err != nil { rows.Close() return nil, false, err } evicted = append(evicted, *entry) need -= entry.SizeBytes } if err := rows.Close(); err != nil { return nil, false, err } if err := rows.Err(); err != nil { return nil, false, err } if need > 0 { return nil, false, nil } for _, entry := range evicted { if _, err := tx.ExecContext(ctx, `update cache_entries set state = 'deleting' where id = ?`, entry.ID); err != nil { return nil, false, err } } return evicted, true, nil } func (tx *EventBatchTx) MarkCacheEntryReadyWithChecksum(ctx context.Context, id string, sizeBytes int64, checksum string, qs *QuotaStore, maxTotalBytes int64, now time.Time) ([]CacheEntry, bool, error) { return markCacheEntryReady(ctx, tx.tx, id, sizeBytes, checksum, qs, maxTotalBytes, now) } func (tx *EventBatchTx) TouchCacheEntry(ctx context.Context, id string, now time.Time) error { _, err := tx.tx.ExecContext(ctx, ` update cache_entries set last_used_at = max(last_used_at, ?) where id = ? and state = 'ready'`, now.UnixNano(), id) return err } func (tx *EventBatchTx) DiscardCacheEntry(ctx context.Context, id string, pendingOnly bool) (*CacheEntry, error) { stateFilter := "" if pendingOnly { stateFilter = " and state in ('pending', 'deleting')" } current, err := scanCacheEntry(tx.tx.QueryRowContext(ctx, ` update cache_entries set state = 'deleting' where id = ?`+stateFilter+` returning `+cacheEntryColumns, id)) if err == sql.ErrNoRows { return nil, nil } return current, err } func (d *DB) FindCacheEntry(ctx context.Context, repoDID, engine, key, hash string) (*CacheEntry, error) { return scanCacheEntry(d.QueryRowContext(ctx, ` select `+cacheEntryColumns+` from cache_entries where repo_did = ? and engine = ? and cache_key = ? and cache_hash = ? and state = 'ready' order by created_at desc limit 1`, repoDID, engine, key, hash)) } func (d *DB) FindFallbackCacheEntry(ctx context.Context, repoDID, engine, key, excludeHash string) (*CacheEntry, error) { return scanCacheEntry(d.QueryRowContext(ctx, ` select `+cacheEntryColumns+` from cache_entries where repo_did = ? and engine = ? and cache_key = ? and cache_hash <> ? and cache_hash <> '' and state = 'ready' order by created_at desc limit 1`, repoDID, engine, key, excludeHash)) } func (d *DB) BeginCacheRestore(ctx context.Context, id string) (bool, error) { result, err := d.ExecContext(ctx, `update cache_entries set restore_count = restore_count + 1 where id = ? and state = 'ready'`, id) if err != nil { return false, err } n, err := result.RowsAffected() return n == 1, err } func (d *DB) EndCacheRestore(ctx context.Context, id string) error { _, err := d.ExecContext(ctx, `update cache_entries set restore_count = restore_count - 1 where id = ? and restore_count > 0`, id) return err } func (d *DB) CacheEntryPinned(ctx context.Context, id string) (bool, error) { var count int64 err := d.QueryRowContext(ctx, `select restore_count from cache_entries where id = ?`, id).Scan(&count) if err == sql.ErrNoRows { return false, nil } return count > 0, err } func (d *DB) TouchCacheEntry(ctx context.Context, id string, now time.Time) error { _, err := d.ExecContext(ctx, ` update cache_entries set last_used_at = max(last_used_at, ?) where id = ? and state = 'ready'`, now.UnixNano(), id) return err } func (d *DB) ClaimPendingCacheEntry(ctx context.Context, id string) (bool, error) { result, err := d.ExecContext(ctx, ` update cache_entries set state = 'deleting' where id = ? and state = 'pending'`, id) if err != nil { return false, err } changed, err := result.RowsAffected() return changed == 1, err } func (d *DB) ClaimCacheEntry(ctx context.Context, id, expectedState string, expectedLastUsed time.Time) (bool, error) { result, err := d.ExecContext(ctx, ` update cache_entries set state = 'deleting' where id = ? and state = ? and last_used_at = ? and restore_count = 0`, id, expectedState, expectedLastUsed.UnixNano()) if err != nil { return false, err } changed, err := result.RowsAffected() return changed == 1, err } func (d *DB) ReadyCacheBytes(ctx context.Context) (int64, error) { var used int64 err := d.QueryRowContext(ctx, readyCacheBytesQuery).Scan(&used) return used, err } func (d *DB) EvictableCacheEntries(ctx context.Context, limit int) ([]CacheEntry, error) { return d.queryCacheEntries(ctx, evictableCacheEntriesQuery+` limit ?`, limit) } func (d *DB) EvictableCacheEntriesOf(ctx context.Context, repoDID, ownerDID string, limit int) ([]CacheEntry, error) { return d.queryCacheEntries(ctx, `select `+cacheEntryColumns+` from cache_entries where state = 'ready' and restore_count = 0 and (repo_did = ? or owner_did = ?) order by last_used_at, created_at limit ?`, repoDID, ownerDID, limit) } func (d *DB) GetCacheEntry(ctx context.Context, id string) (*CacheEntry, error) { return scanCacheEntry(d.QueryRowContext(ctx, `select `+cacheEntryColumns+` from cache_entries where id = ?`, id)) } func (d *DB) RestoreCacheEntryState(ctx context.Context, id, state string) error { _, err := d.ExecContext(ctx, ` update cache_entries set state = ? where id = ? and state = 'deleting'`, state, id) return err } func (d *DB) ExpiredCacheEntries(ctx context.Context, readyBefore, pendingBefore time.Time, limit int) ([]CacheEntry, error) { return d.queryCacheEntries(ctx, ` select `+cacheEntryColumns+` from cache_entries where (state = 'ready' and last_used_at < ?) or (state in ('pending', 'deleting') and last_used_at < ?) order by last_used_at limit ?`, readyBefore.UnixNano(), pendingBefore.UnixNano(), limit, ) } // discardPendingCacheEntriesForLease fences uploads that can arrive after a lease dies. // Their tombstones let the cache pruner remove a late object without metadata. func (d *DB) DiscardPendingCacheEntriesForLease(leaseID string, now time.Time) ([]string, error) { tx, err := d.Begin() if err != nil { return nil, err } defer tx.Rollback() rows, err := tx.Query(` select c.id, c.storage_key from cache_entries c join mill_cache_capabilities cap on cap.cache_id = c.id where cap.lease_id = ? and cap.action = 'save' and c.state = 'pending'`, leaseID) if err != nil { return nil, err } var keys []string var ids []string for rows.Next() { var id, key string if err := rows.Scan(&id, &key); err != nil { _ = rows.Close() return nil, err } ids = append(ids, id) keys = append(keys, key) } if err := rows.Err(); err != nil { _ = rows.Close() return nil, err } if err := rows.Close(); err != nil { return nil, err } for i, id := range ids { if _, err := tx.Exec(`update cache_entries set state = 'deleting' where id = ? and state = 'pending'`, id); err != nil { return nil, err } if _, err := tx.Exec(` insert into cache_object_deletions (storage_key, created_at) values (?, ?) on conflict(storage_key) do nothing`, keys[i], now.UnixNano()); err != nil { return nil, err } } if err := tx.Commit(); err != nil { return nil, err } return keys, nil } func (d *DB) DeleteCacheEntry(ctx context.Context, id string) error { return d.deleteCacheEntries(ctx, `id = ?`, id) } func (d *DB) DeleteCacheEntriesByStorageKey(ctx context.Context, key string) error { return d.deleteCacheEntries(ctx, `storage_key = ? and restore_count = 0`, key) } func (d *DB) deleteCacheEntries(ctx context.Context, where string, args ...any) error { tx, err := d.BeginTx(ctx, nil) if err != nil { return err } defer tx.Rollback() rows, err := tx.QueryContext(ctx, `delete from cache_entries where `+where+` returning repo_did, id`, args...) if err != nil { return err } var gone [][2]string for rows.Next() { var repo, id string if err := rows.Scan(&repo, &id); err != nil { rows.Close() return err } gone = append(gone, [2]string{repo, id}) } if err := rows.Close(); err != nil { return err } for _, entry := range gone { if err := unchargeCacheEntry(ctx, tx, entry[0], entry[1]); err != nil { return err } } return tx.Commit() } func unchargeCacheEntry(ctx context.Context, tx execQuerier, repoDID, id string) error { _, err := tx.ExecContext(ctx, ` delete from quota_allocations where repo_did = ? and kind = ? and key = ?`, repoDID, quota.KindGenericCache, id) return err } type MillCacheCapability struct { Action string CacheID string StorageKey string } func (d *DB) SaveMillCacheCapabilities(leaseID string, capabilities []MillCacheCapability) error { tx, err := d.Begin() if err != nil { return err } defer tx.Rollback() if _, err := tx.Exec(`delete from mill_cache_capabilities where lease_id = ?`, leaseID); err != nil { return err } for _, capability := range capabilities { if capability.Action != "restore" && capability.Action != "save" { return fmt.Errorf("cache capability has invalid action %q", capability.Action) } if capability.CacheID == "" || capability.StorageKey == "" { return fmt.Errorf("cache capability has empty identity") } if _, err := tx.Exec(` insert into mill_cache_capabilities ( lease_id, action, cache_id, storage_key ) values (?, ?, ?, ?)`, leaseID, capability.Action, capability.CacheID, capability.StorageKey, ); err != nil { return err } } return tx.Commit() } func (tx *EventBatchTx) ConsumeMillCacheCapability(ctx context.Context, leaseID, action, cacheID string) (string, bool, error) { var storageKey string err := tx.tx.QueryRowContext(ctx, ` delete from mill_cache_capabilities where lease_id = ? and action = ? and cache_id = ? returning storage_key`, leaseID, action, cacheID, ).Scan(&storageKey) if err == sql.ErrNoRows { return "", false, nil } if err != nil { return "", false, err } return storageKey, true, nil } func (tx *EventBatchTx) QueueCacheObjectDeletion(ctx context.Context, storageKey string, now time.Time) error { _, err := tx.tx.ExecContext(ctx, ` insert into cache_object_deletions (storage_key, created_at) values (?, ?) on conflict(storage_key) do nothing`, storageKey, now.UnixNano(), ) return err } func (d *DB) PendingCacheObjectDeletions(ctx context.Context, limit int) ([]string, error) { rows, err := d.QueryContext(ctx, ` select storage_key from cache_object_deletions order by created_at limit ?`, limit) if err != nil { return nil, err } defer rows.Close() var keys []string for rows.Next() { var key string if err := rows.Scan(&key); err != nil { return nil, err } keys = append(keys, key) } return keys, rows.Err() } func (d *DB) CacheEntriesByStorageKey(ctx context.Context, key string) ([]CacheEntry, error) { return d.queryCacheEntries(ctx, `select `+cacheEntryColumns+` from cache_entries where storage_key = ?`, key) } func (d *DB) queryCacheEntries(ctx context.Context, query string, args ...any) ([]CacheEntry, error) { rows, err := d.QueryContext(ctx, query, args...) if err != nil { return nil, err } defer rows.Close() var entries []CacheEntry for rows.Next() { entry, err := scanCacheEntry(rows) if err != nil { return nil, err } entries = append(entries, *entry) } return entries, rows.Err() } func (d *DB) CompleteCacheObjectDeletion(ctx context.Context, storageKey string) error { _, err := d.ExecContext(ctx, `delete from cache_object_deletions where storage_key = ?`, storageKey) return err } type cacheEntryScanner interface { Scan(dest ...any) error } func scanCacheEntry(row cacheEntryScanner) (*CacheEntry, error) { var entry CacheEntry var createdAt, lastUsedAt int64 if err := row.Scan( &entry.ID, &entry.StorageKey, &entry.OwnerDID, &entry.RepoDID, &entry.Engine, &entry.CacheKey, &entry.CacheHash, &entry.Checksum, &entry.SizeBytes, &entry.RestoreCount, &entry.State, &createdAt, &lastUsedAt, ); err != nil { return nil, err } entry.CreatedAt = time.Unix(0, createdAt) entry.LastUsedAt = time.Unix(0, lastUsedAt) return &entry, nil }