Merge pull request #297 from ri72miieop/main

EPI-136 - Add username snapshot to archive_upload table

scripts/import_from_files_to_db.ts

···
 }
 
 
-const GLOBAL_ARCHIVE_PATH = process.env.ARCHIVE_PATH || './data/downloads/archives'
+const GLOBAL_ARCHIVE_PATH = process.env.ARCHIVE_PATH || '../data/downloads/archives'
 console.log('Using ARCHIVE_PATH:', GLOBAL_ARCHIVE_PATH)
 
 // Parse command line arguments for limiting archives
···
   }
   async function Upsert_Archive_Upload(data: any, fileRoot:string) {
     const accountId = data.account[0].account.accountId;
+    const username = data.account[0].account.username;
     const tweetsDate=data.tweets.map((t:any)=>new Date(t.tweet.created_at)).sort((a:any, b:any) => a - b);;
     const start_date = tweetsDate[0];
     const end_date = tweetsDate[tweetsDate.length-1];
 
-    const archive_upload : InsertArchiveUpload ={
-      account_id:accountId, 
+    const archive_upload = {
+      account_id:accountId,
+      username:username,
       start_date:start_date.toISOString(),
       end_date:end_date.toISOString(),
       archive_at:new Date().toISOString(),
-      keep_private:false}
+      keep_private:false} as InsertArchiveUpload
 
      let getItem = (item: any) => {
        return item as InsertArchiveUpload;

services/process_archive/process_archive_upload.ts

···
     logger.info('Fetching archive_upload records ready for processing...')
     
     const ready = await sql`
-      SELECT au.id, au.account_id, a.username, au.archive_at
-      FROM public.archive_upload au left join all_account a on au.account_id=a.account_id
+      SELECT au.id, au.account_id, au.username, au.archive_at
+      FROM public.archive_upload au
       WHERE upload_phase IN ('ready_for_commit')
       ORDER BY archive_at ASC
     `

sql/functions/archive/04_insert_temp_archive_upload.sql

···
 CREATE OR REPLACE FUNCTION public.insert_temp_archive_upload(
     p_account_id TEXT,
+    p_username TEXT,
     p_archive_at timestamp with time zone,
     p_keep_private BOOLEAN,
     p_upload_likes BOOLEAN,
···
     EXECUTE format('
         INSERT INTO temp.archive_upload_%s (
             account_id,
+            username,
             archive_at,
             keep_private,
             upload_likes,
             start_date,
             end_date
         )
-        VALUES ($1, $2, $3, $4, $5, $6)
+        VALUES ($1, $2, $3, $4, $5, $6, $7)
         RETURNING id
     ', p_suffix)
     USING
         p_account_id,
+        p_username,
         p_archive_at,
         p_keep_private,
         p_upload_likes,

src/lib/db_insert.ts

···
   select('id,archive_at').eq('account_id', accountId).in('upload_phase', ['uploading', 'ready_for_commit'])
   .order('created_at', { ascending: false }).limit(1).maybeSingle()
 
+  const username = archiveData.account[0].account.username;
+
   let supabaseUpsertQuery;
   if (lastUploadedArchive) {
      supabaseUpsertQuery = supabase
       .from('archive_upload')
       .update({ 
+        username: username,
         archive_at: latestTweetDate,
         keep_private: uploadOptions.keepPrivate,
         upload_likes: uploadOptions.uploadLikes,
···
         .from('archive_upload')
         .insert({
           account_id: accountId,
+          username: username,
           archive_at: latestTweetDate,
           keep_private: uploadOptions.keepPrivate,
           upload_likes: uploadOptions.uploadLikes,
           start_date: uploadOptions.startDate,
           end_date: uploadOptions.endDate,
-          upload_phase: 'uploading',
+          upload_phase: 'uploading'
         })
         .select('id')
         .single()

supabase/migrations/20251009160144_add_username_to_archive_upload.sql

···
+-- Add username column to archive_upload table to store the handle at time of upload
+-- This prevents issues when users change their handle after uploading archives
+
+ALTER TABLE "public"."archive_upload" 
+ADD COLUMN "username" text;
+
+-- Add a comment to explain the purpose
+COMMENT ON COLUMN "public"."archive_upload"."username" IS 'Username/handle at the time of archive upload - used to locate archive files in storage';
+
+-- Populate username column in existing archive_upload records
+-- This migration updates existing records to have the username from all_account table
+
+UPDATE "public"."archive_upload" 
+SET "username" = "all_account"."username"
+FROM "public"."all_account"
+WHERE "archive_upload"."account_id" = "all_account"."account_id"
+  AND "archive_upload"."username" IS NULL;

supabase/migrations/20251023163841_add_indexes_for_performance.sql

···
+CREATE INDEX IF NOT EXISTS idx_archive_upload_username ON public.archive_upload USING btree (username);
+
+CREATE INDEX IF NOT EXISTS idx_tweets_account_created ON public.tweets USING btree (account_id, created_at);

supabase/schemas/020_tables.sql

···
     "upload_likes" boolean DEFAULT true,
     "start_date" "date",
     "end_date" "date",
-    "upload_phase" "public"."upload_phase_enum" DEFAULT 'uploading'::"public"."upload_phase_enum"
+    "upload_phase" "public"."upload_phase_enum" DEFAULT 'uploading'::"public"."upload_phase_enum",
+    "username" "text"
 );
 ALTER TABLE "public"."archive_upload" OWNER TO "postgres";
 
···
     quoted_tweet_id TEXT NOT NULL,
     
     -- Composite primary key
-    PRIMARY KEY (tweet_id, quoted_tweet_id),
-    
-    -- Foreign key constraints
-    CONSTRAINT fk_quote_tweets_tweet_id FOREIGN KEY (tweet_id) REFERENCES public.tweets (tweet_id) ON DELETE CASCADE
+    PRIMARY KEY (tweet_id, quoted_tweet_id)
 );
 
 ALTER TABLE "public"."quote_tweets" OWNER TO "postgres";
 
 CREATE TABLE IF NOT EXISTS public.retweets (
     tweet_id TEXT NOT NULL PRIMARY KEY,
-    retweeted_tweet_id TEXT NULL,
-       
-    CONSTRAINT fk_retweets_tweet_id FOREIGN KEY (tweet_id) REFERENCES public.tweets (tweet_id) ON DELETE CASCADE,
-    CONSTRAINT fk_retweets_retweeted_tweet_id FOREIGN KEY (retweeted_tweet_id) REFERENCES public.tweets (tweet_id) ON DELETE SET NULL
+    retweeted_tweet_id TEXT NULL
 );
 
 ALTER TABLE "public"."retweets" OWNER TO "postgres";

supabase/schemas/030_indexes.sql

···
 
 -- public.archive_upload
 CREATE INDEX "idx_archive_upload_account_id" ON "public"."archive_upload" USING "btree" ("account_id");
+CREATE INDEX "idx_archive_upload_username" ON "public"."archive_upload" USING "btree" ("username");
 
 -- public.conversations
 CREATE INDEX "idx_conversation_id" ON "public"."conversations" USING "btree" ("conversation_id");
···
 CREATE INDEX "text_fts" ON "public"."tweets" USING "gin" ("fts");
 CREATE INDEX "tweets_account_id_favorite_idx" ON "public"."tweets" USING "btree" ("account_id", "favorite_count" DESC);
 CREATE INDEX "tweets_account_id_retweet_idx" ON "public"."tweets" USING "btree" ("account_id", "retweet_count" DESC);
+CREATE INDEX "idx_tweets_account_created" ON public.tweets (account_id, created_at);
 
 -- public.user_mentions
 CREATE INDEX "idx_user_mentions_mentioned_user_id" ON "public"."user_mentions" USING "btree" ("mentioned_user_id");

supabase/schemas/040_views.sql

···
 
 -- public.account moved to 032_views_prereq.sql
 
--- public.quote_tweets
-CREATE OR REPLACE VIEW "public"."quote_tweets" AS
- SELECT "t"."tweet_id",
-    "substring"("tu"."expanded_url", 'status/([0-9]+)'::"text") AS "quoted_tweet_id",
-    "substring"("tu"."expanded_url", 'https?://(?:www\\.)?twitter\\.com/([^/]+)/status/'::"text") AS "quoted_tweet_username"
-   FROM ("public"."tweet_urls" "tu"
-     JOIN "public"."tweets" "t" ON (("tu"."tweet_id" = "t"."tweet_id")))
-  WHERE (("tu"."expanded_url" ~~ 'https://twitter.com/%/status/%'::"text") OR ("tu"."expanded_url" ~~ 'https://x.com/%/status/%'::"text"));
-ALTER TABLE "public"."quote_tweets" OWNER TO "postgres";
-
 -- public.enriched_tweets
 CREATE OR REPLACE VIEW "public"."enriched_tweets" AS
  SELECT "t"."tweet_id",

supabase/schemas/050_constraints.sql

···
     ADD CONSTRAINT "optin_username_key" UNIQUE ("username");
 ALTER TABLE ONLY "public"."optin"
     ADD CONSTRAINT "optin_user_id_fkey" FOREIGN KEY ("user_id") REFERENCES "auth"."users"("id") ON DELETE CASCADE;
+
+
+-- public.quote_tweets foreign keys
+ALTER TABLE ONLY "public"."quote_tweets"
+    ADD CONSTRAINT "fk_quote_tweets_tweet_id" FOREIGN KEY ("tweet_id") REFERENCES "public"."tweets"("tweet_id") ON DELETE CASCADE;
+ALTER TABLE ONLY "public"."quote_tweets"
+    ADD CONSTRAINT "fk_quote_tweets_quoted_tweet_id" FOREIGN KEY ("quoted_tweet_id") REFERENCES "public"."tweets"("tweet_id") ON DELETE CASCADE;
+
+-- public.retweets foreign keys
+ALTER TABLE ONLY "public"."retweets"
+    ADD CONSTRAINT "fk_retweets_tweet_id" FOREIGN KEY ("tweet_id") REFERENCES "public"."tweets"("tweet_id") ON DELETE CASCADE;
+ALTER TABLE ONLY "public"."retweets"
+    ADD CONSTRAINT "fk_retweets_retweeted_tweet_id" FOREIGN KEY ("retweeted_tweet_id") REFERENCES "public"."tweets"("tweet_id") ON DELETE SET NULL;

supabase/schemas/070_functions.sql

···
 DECLARE
     v_archive_upload_id BIGINT;
     v_account_id TEXT;
+    v_username TEXT;
     v_archive_at TIMESTAMP WITH TIME ZONE;
     v_keep_private BOOLEAN;
     v_upload_likes BOOLEAN;
···
     v_phase_start := clock_timestamp();
     RAISE NOTICE 'Phase 2: Getting archive upload data';
     -- Get the archive upload that's ready for commit
-    SELECT id, archive_at, keep_private, upload_likes, start_date, end_date
-    INTO v_archive_upload_id, v_archive_at, v_keep_private, v_upload_likes, v_start_date, v_end_date
+    SELECT id, archive_at, keep_private, upload_likes, start_date, end_date, username
+    INTO v_archive_upload_id, v_archive_at, v_keep_private, v_upload_likes, v_start_date, v_end_date, v_username
     FROM public.archive_upload
     WHERE account_id = v_account_id
     AND upload_phase = 'ready_for_commit'

supabase/schemas/prod.sql

···
 DECLARE
     v_archive_upload_id BIGINT;
     v_account_id TEXT;
+    v_username TEXT;
     v_archive_at TIMESTAMP WITH TIME ZONE;
     v_keep_private BOOLEAN;
     v_upload_likes BOOLEAN;
···
     RAISE NOTICE 'Phase 2: Getting archive upload data';
     -- 2. Get the latest archive upload data from temp.archive_upload
     EXECUTE format('
-        SELECT archive_at, keep_private, upload_likes, start_date, end_date
+        SELECT archive_at, keep_private, upload_likes, start_date, end_date, username
         FROM temp.archive_upload_%s
         ORDER BY archive_at DESC
         LIMIT 1
-    ', p_suffix) INTO v_archive_at, v_keep_private, v_upload_likes, v_start_date, v_end_date;
+    ', p_suffix) INTO v_archive_at, v_keep_private, v_upload_likes, v_start_date, v_end_date, v_username;
 
     RAISE NOTICE 'Phase 3: Inserting archive upload data';
     -- 3. Insert or update archive_upload and get the ID
     INSERT INTO public.archive_upload (
         account_id, 
+        username,
         archive_at, 
         created_at, 
         keep_private, 
···
     )
     VALUES (
         v_account_id, 
+        v_username,
         v_archive_at, 
         CURRENT_TIMESTAMP, 
         v_keep_private, 
···
     ON CONFLICT (account_id, archive_at)
     DO UPDATE SET
         account_id = EXCLUDED.account_id,
+        username = EXCLUDED.username,
         created_at = CURRENT_TIMESTAMP,
         keep_private = EXCLUDED.keep_private,
         upload_likes = EXCLUDED.upload_likes,

tests/db-insertion/db-insertion.test.ts

···
     .from('archive_upload')
     .insert({
       account_id: accountId,
+      username: username,
       archive_at: latestTweetDate,
       keep_private: uploadOptions.keepPrivate,
       upload_likes: uploadOptions.uploadLikes,