hacky stopgap

pkg/feeds/fresh/feed.go

···
 type FeedType string
 
 const (
-	Rinds    FeedType = "rinds"
-	Random   FeedType = "random"
-	Mnine    FeedType = "mnine"
-	Reposts  FeedType = "reposts"
-	OReplies FeedType = "oreplies"
+	Rinds                     FeedType = "rinds"
+	Random                    FeedType = "random"
+	Mnine                     FeedType = "mnine"
+	Reposts                   FeedType = "reposts"
+	OReplies                  FeedType = "oreplies"
+	followingRefreshThreshold          = 12 * time.Hour
 )
 
 type StaticFeed struct {
···
 		hash = generateHash(userDID)
 	} else {
 		log.Println("Using existing hash")
+	}
+
+	// hacky stopgap following list refresh
+	if err := sf.ensureRefreshStatusTableExists(ctx); err != nil {
+		log.Printf("Could not ensure following_refresh_status table exists: %v", err)
+	} else {
+		sf.checkAndRefreshFollowers(ctx, userDID)
 	}
 
 	tableName := fmt.Sprintf("feedcache_%s_%s", userDID, hash)
···
 	return hex.EncodeToString(hash[:])[:5]
 }
 
-func getFollowers(ctx context.Context, db *sql.DB, userdid string) ([]string, error) {
-	unquotedTableName := "follows_" + userdid
-	tableName := pq.QuoteIdentifier(unquotedTableName)
-	fmt.Printf("Checking for table: %s\n", tableName) // Debug log
+func (sf *StaticFeed) ensureRefreshStatusTableExists(ctx context.Context) error {
+	query := `
+		CREATE TABLE IF NOT EXISTS following_refresh_status (
+			user_did TEXT PRIMARY KEY,
+			last_fetched TIMESTAMPTZ,
+			is_refreshing BOOLEAN DEFAULT FALSE
+		);
+	`
+	_, err := sf.DB.ExecContext(ctx, query)
+	if err != nil {
+		return fmt.Errorf("error creating following_refresh_status table: %w", err)
+	}
+	return nil
+}
 
-	// Check if cache table exists
-	var exists bool
-	query := "SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = $1)"
-	fmt.Printf("Executing query: %s with parameter: %s\n", query, unquotedTableName) // Debug log
-	err := db.QueryRowContext(ctx, query, unquotedTableName).Scan(&exists)
+func (sf *StaticFeed) checkAndRefreshFollowers(ctx context.Context, userDID string) {
+	_, err := sf.DB.ExecContext(ctx,
+		"INSERT INTO following_refresh_status (user_did) VALUES ($1) ON CONFLICT (user_did) DO NOTHING", userDID)
 	if err != nil {
-		// Check for specific errors
-		if err == sql.ErrNoRows {
-			return nil, fmt.Errorf("table existence check returned no rows: %w", err)
-		}
-		if pqErr, ok := err.(*pq.Error); ok {
-			return nil, fmt.Errorf("PostgreSQL error: %s, Code: %s", pqErr.Message, pqErr.Code)
-		}
-		return nil, fmt.Errorf("error checking cache table existence: %w", err)
+		log.Printf("Error ensuring refresh status entry for %s: %v", userDID, err)
+		return
+	}
+
+	var lastFetched sql.NullTime
+	var isRefreshing bool
+
+	err = sf.DB.QueryRowContext(ctx,
+		"SELECT last_fetched, is_refreshing FROM following_refresh_status WHERE user_did = $1", userDID).Scan(&lastFetched, &isRefreshing)
+	if err != nil {
+		log.Printf("Error getting refresh status for %s: %v", userDID, err)
+		return
+	}
+
+	if isRefreshing {
+		log.Printf("Follower refresh for %s is already in progress. Skipping.", userDID)
+		return
+	}
+
+	if !lastFetched.Valid || time.Since(lastFetched.Time) > followingRefreshThreshold {
+		log.Printf("Follower list for %s is stale or has never been fetched. Triggering background refresh.", userDID)
+		go sf.refreshFollowersAsync(userDID)
+		return
 	}
 
-	fmt.Printf("Table exists: %v\n", exists) // Debug log
+	log.Printf("Follower list for %s is fresh enough (last fetched %v ago). Skipping refresh.", userDID, time.Since(lastFetched.Time))
+}
 
-	if exists {
-		rows, err := db.QueryContext(ctx, fmt.Sprintf("SELECT follow FROM %s", tableName))
+func (sf *StaticFeed) refreshFollowersAsync(userDID string) {
+	ctx := context.Background()
+
+	_, err := sf.DB.ExecContext(ctx, "UPDATE following_refresh_status SET is_refreshing = TRUE WHERE user_did = $1", userDID)
+	if err != nil {
+		log.Printf("Error setting refresh lock for %s: %v", userDID, err)
+		return
+	}
+
+	defer func() {
+		_, err := sf.DB.ExecContext(ctx, "UPDATE following_refresh_status SET is_refreshing = FALSE WHERE user_did = $1", userDID)
 		if err != nil {
-			return nil, fmt.Errorf("error querying cache table: %w", err)
+			log.Printf("CRITICAL: Error releasing refresh lock for %s: %v", userDID, err)
 		}
-		defer rows.Close()
+	}()
 
-		var cachedFollows []string
-		for rows.Next() {
-			var follow string
-			if err := rows.Scan(&follow); err != nil {
-				return nil, fmt.Errorf("error scanning cached follow: %w", err)
-			}
-			cachedFollows = append(cachedFollows, follow)
-		}
+	err = fetchAndCacheFollowersFromAPI(ctx, sf.DB, userDID)
+	if err != nil {
+		log.Printf("Error refreshing followers for %s: %v", userDID, err)
+		return
+	}
 
-		if err := rows.Err(); err != nil {
-			return nil, fmt.Errorf("error iterating cached follows: %w", err)
-		}
+	_, err = sf.DB.ExecContext(ctx, "UPDATE following_refresh_status SET last_fetched = NOW() WHERE user_did = $1", userDID)
+	if err != nil {
+		log.Printf("Error updating last_fetched for %s: %v", userDID, err)
+	}
 
-		log.Println("Returning cached followers")
-		return cachedFollows, nil
-	}
+	log.Printf("Successfully refreshed followers for %s.", userDID)
+}
 
-	log.Println("Fetching followers from API")
+func fetchAndCacheFollowersFromAPI(ctx context.Context, db *sql.DB, userdid string) error {
+	log.Printf("Fetching followers from API for %s", userdid)
 	var allDIDs []string
 	cursor := ""
 
···
 		apiURL := fmt.Sprintf("https://public.api.bsky.app/xrpc/app.bsky.graph.getFollows?actor=%s&cursor=%s", userdid, cursor)
 		resp, err := http.Get(apiURL)
 		if err != nil {
-			log.Printf("Error making request: %v\n", err)
-			return nil, fmt.Errorf("failed to make request: %v", err)
+			return fmt.Errorf("failed to make request: %w", err)
 		}
 		defer resp.Body.Close()
 
 		body, err := ioutil.ReadAll(resp.Body)
 		if err != nil {
-			log.Printf("Error reading response: %v\n", err)
-			return nil, fmt.Errorf("failed to read response: %v", err)
+			return fmt.Errorf("failed to read response: %w", err)
 		}
 
 		var response Response
 		if err := json.Unmarshal(body, &response); err != nil {
-			log.Printf("Error unmarshalling JSON: %v\n", err)
-			return nil, fmt.Errorf("failed to unmarshal JSON: %v", err)
+			return fmt.Errorf("failed to unmarshal JSON: %w", err)
 		}
 
 		for _, follow := range response.Follows {
···
 		cursor = response.Cursor
 	}
 
-	// Drop the existing table if it exists
-	log.Println("Dropping existing followers table if it exists")
-	_, err = db.ExecContext(ctx, fmt.Sprintf("DROP TABLE IF EXISTS %s", tableName))
+	unquotedTableName := "follows_" + userdid
+	tableName := pq.QuoteIdentifier(unquotedTableName)
+	tempTableName := "follows_temp_" + userdid
+	quotedTempTableName := pq.QuoteIdentifier(tempTableName)
+
+	log.Println("Dropping existing temporary followers table if it exists")
+	_, err := db.ExecContext(ctx, fmt.Sprintf("DROP TABLE IF EXISTS %s", quotedTempTableName))
 	if err != nil {
-		return nil, fmt.Errorf("error dropping existing cache table: %w", err)
+		return fmt.Errorf("error dropping existing temp cache table: %w", err)
 	}
 
-	// Cache the results in the database
-	log.Println("Caching followers in the database")
-	_, err = db.ExecContext(ctx, fmt.Sprintf(`
-		CREATE TABLE %s (
-			follow TEXT UNIQUE
-		)
-	`, tableName))
+	log.Println("Creating temporary followers table")
+	_, err = db.ExecContext(ctx, fmt.Sprintf(`CREATE TABLE %s (follow TEXT UNIQUE)`, quotedTempTableName))
 	if err != nil {
-		return nil, fmt.Errorf("error creating cache table: %w", err)
+		return fmt.Errorf("error creating temp cache table: %w", err)
 	}
 
-	// Use a map to track unique follows
-	followMap := make(map[string]struct{})
-	for _, follow := range allDIDs {
-		if _, exists := followMap[follow]; !exists {
-			followMap[follow] = struct{}{}
-			_, err := db.ExecContext(ctx, fmt.Sprintf(`
-				INSERT INTO %s (follow)
-				VALUES ($1)
-				ON CONFLICT (follow) DO NOTHING
-			`, tableName), follow)
+	if len(allDIDs) > 0 {
+		txn, err := db.BeginTx(ctx, nil)
+		if err != nil {
+			return fmt.Errorf("failed to begin transaction: %w", err)
+		}
+
+		stmt, err := txn.Prepare(pq.CopyIn(tempTableName, "follow"))
+		if err != nil {
+			return fmt.Errorf("failed to prepare copy in: %w", err)
+		}
+
+		for _, did := range allDIDs {
+			_, err = stmt.Exec(did)
 			if err != nil {
-				return nil, fmt.Errorf("error inserting into cache table: %w", err)
+				return fmt.Errorf("failed to exec copy in: %w", err)
 			}
 		}
+
+		_, err = stmt.Exec()
+		if err != nil {
+			return fmt.Errorf("failed to finalize copy in: %w", err)
+		}
+
+		err = stmt.Close()
+		if err != nil {
+			return fmt.Errorf("failed to close statement: %w", err)
+		}
+
+		err = txn.Commit()
+		if err != nil {
+			return fmt.Errorf("failed to commit transaction: %w", err)
+		}
 	}
 
-	log.Println("Returning fetched followers")
-	return allDIDs, nil
+	log.Println("Atomically replacing old followers table with new one")
+	tx, err := db.Begin()
+	if err != nil {
+		return fmt.Errorf("error beginning transaction for table rename: %w", err)
+	}
+	_, err = tx.Exec(fmt.Sprintf("DROP TABLE IF EXISTS %s", tableName))
+	if err != nil {
+		tx.Rollback()
+		return fmt.Errorf("error dropping old follows table in transaction: %w", err)
+	}
+	_, err = tx.Exec(fmt.Sprintf("ALTER TABLE %s RENAME TO %s", quotedTempTableName, pq.QuoteIdentifier(unquotedTableName)))
+	if err != nil {
+		tx.Rollback()
+		return fmt.Errorf("error renaming temp table in transaction: %w", err)
+	}
+	err = tx.Commit()
+	if err != nil {
+		return fmt.Errorf("error committing transaction for table rename: %w", err)
+	}
+
+	log.Printf("Successfully cached %d followers for %s", len(allDIDs), userdid)
+	return nil
+}
+
+func getFollowers(ctx context.Context, db *sql.DB, userdid string) ([]string, error) {
+	unquotedTableName := "follows_" + userdid
+	tableName := pq.QuoteIdentifier(unquotedTableName)
+
+	var exists bool
+	err := db.QueryRowContext(ctx, "SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = $1)", unquotedTableName).Scan(&exists)
+	if err != nil {
+		return nil, fmt.Errorf("error checking follows cache table existence for %s: %w", userdid, err)
+	}
+
+	if !exists {
+		log.Printf("Follows cache table for %s does not exist. Waiting for background refresh to create it.", userdid)
+		return []string{}, nil
+	}
+
+	rows, err := db.QueryContext(ctx, fmt.Sprintf("SELECT follow FROM %s", tableName))
+	if err != nil {
+		return nil, fmt.Errorf("error querying cache table: %w", err)
+	}
+	defer rows.Close()
+
+	var cachedFollows []string
+	for rows.Next() {
+		var follow string
+		if err := rows.Scan(&follow); err != nil {
+			return nil, fmt.Errorf("error scanning cached follow: %w", err)
+		}
+		cachedFollows = append(cachedFollows, follow)
+	}
+
+	if err := rows.Err(); err != nil {
+		return nil, fmt.Errorf("error iterating cached follows: %w", err)
+	}
+
+	log.Printf("Returning %d cached followers for %s", len(cachedFollows), userdid)
+	return cachedFollows, nil
 }
 
 func markPostsAsViewed(ctx context.Context, db *sql.DB, userDID string, posts []PostWithDate, smartReportingEnabled bool) error {