this repo has no description
0
fork

Configure Feed

Select the types of activity you want to include in your feed.

overwrite archive_upload_id with the latest value when processing a new upload of the archive

+670 -29
+12 -7
services/process_archive/process_archive_upload.ts
··· 114 114 all_profile: { 115 115 columns: ['account_id', 'avatar_media_url', 'header_media_url', 'bio', 'location', 'website', 'archive_upload_id'], 116 116 conflict: 'account_id', 117 - updates: ['avatar_media_url', 'header_media_url', 'bio', 'location', 'website'] 117 + updates: ['avatar_media_url', 'header_media_url', 'bio', 'location', 'website', 'archive_upload_id'] 118 118 }, 119 119 tweets: { 120 120 columns: ['tweet_id', 'account_id', 'created_at', 'full_text', 'favorite_count', 'retweet_count', 'reply_to_tweet_id', 'reply_to_user_id', 'reply_to_username', 'archive_upload_id'], 121 121 conflict: 'tweet_id', 122 - updates: ['favorite_count', 'retweet_count'] 122 + updates: ['favorite_count', 'retweet_count', 'archive_upload_id'] 123 123 }, 124 124 mentioned_users: { 125 125 columns: ['user_id', 'name', 'screen_name'], ··· 137 137 tweet_media: { 138 138 columns: ['tweet_id', 'media_id', 'media_url', 'media_type', 'width', 'height', 'archive_upload_id'], 139 139 conflict: 'media_id', 140 - updates: ['media_url', 'width', 'height'] 140 + updates: ['media_url', 'width', 'height', 'archive_upload_id'] 141 141 }, 142 142 quote_tweets: { 143 143 columns: ['tweet_id', 'quoted_tweet_id'], ··· 149 149 }, 150 150 likes: { 151 151 columns: ['account_id', 'liked_tweet_id', 'archive_upload_id'], 152 - conflict: ['account_id', 'liked_tweet_id'] 152 + conflict: ['account_id', 'liked_tweet_id'], 153 + updates: ['archive_upload_id'] 153 154 }, 154 155 liked_tweets: { 155 156 columns: ['tweet_id', 'full_text'], ··· 157 158 }, 158 159 following: { 159 160 columns: ['account_id', 'following_account_id', 'archive_upload_id'], 160 - conflict: ['account_id', 'following_account_id'] 161 + conflict: ['account_id', 'following_account_id'], 162 + updates: ['archive_upload_id'] 161 163 }, 162 164 followers: { 163 165 columns: ['account_id', 'follower_account_id', 'archive_upload_id'], 164 - conflict: ['account_id', 'follower_account_id'] 166 + conflict: ['account_id', 'follower_account_id'], 167 + updates: ['archive_upload_id'] 165 168 } 166 169 } 167 170 ··· 939 942 const startTime = new Date() 940 943 logger.info(`Found ${ready.length} record(s) to process.`) 941 944 logger.debug(`Start time: ${startTime.toISOString()}`) 945 + let archives_processed = 0 942 946 943 947 for (const row of ready) { 944 948 const { id: archiveUploadId, account_id, username } = row ··· 968 972 WHERE id = ${archiveUploadId} 969 973 RETURNING id 970 974 ` 975 + archives_processed++ 971 976 972 977 if (!completeResult.length) { 973 978 throw new Error('Failed to mark as completed') ··· 1003 1008 logger.info(`Final memory usage: ${getMemoryUsageMB()}MB`) 1004 1009 1005 1010 1006 - if(CONFIG.PROCESS_RETWEETS) { 1011 + if(CONFIG.PROCESS_RETWEETS && archives_processed > 0) { 1007 1012 1008 1013 let startTimeRetweet = new Date() 1009 1014 logger.debug(`Starting retweet processing at: ${startTimeRetweet.toISOString()}`)
+658 -22
tests/db-insertion/db-insertion.test.ts
··· 107 107 endDate: null, 108 108 } 109 109 110 - // First try to update existing row 111 - const { data: existingUpload, error: existingError } = await supabase 112 - .from('archive_upload') 113 - .update({ upload_phase: 'uploading' }) 114 - .eq('account_id', accountId) 115 - .select('id') 116 - .maybeSingle() 117 - 118 110 let latestTweetDate = archive.tweets.reduce( 119 111 (latest: string, tweet: any) => { 120 112 const tweetDate = new Date(tweet.tweet.created_at) ··· 125 117 : tweetDate.toISOString() 126 118 },new Date().toISOString()); 127 119 128 - // If no existing row, create new one 129 - const { data: archiveUploadIdData, error: uploadError } = existingUpload 130 - ? { data: existingUpload, error: existingError } 131 - : await supabase 132 - .from('archive_upload') 133 - .insert({ 134 - account_id: accountId, 135 - archive_at: latestTweetDate, 136 - keep_private: uploadOptions.keepPrivate, 137 - upload_likes: uploadOptions.uploadLikes, 138 - upload_phase: 'ready_for_commit' 139 - }) 140 - .select('id') 141 - .single() 120 + // Always create a new archive upload record for testing (don't reuse existing ones) 121 + const { data: archiveUploadIdData, error: uploadError } = await supabase 122 + .from('archive_upload') 123 + .insert({ 124 + account_id: accountId, 125 + archive_at: latestTweetDate, 126 + keep_private: uploadOptions.keepPrivate, 127 + upload_likes: uploadOptions.uploadLikes, 128 + upload_phase: 'ready_for_commit' 129 + }) 130 + .select('id') 131 + .single() 142 132 143 133 144 134 ··· 776 766 }) 777 767 778 768 769 + 770 + describe('Archive Upload ID Upsert Tests', () => { 771 + it('should update archive_upload_id when upserting tweets', async () => { 772 + // Create test tweet 773 + const testTweet = { 774 + tweet: { 775 + id: '9999', 776 + id_str: '9999', 777 + created_at: '2023-01-01 00:00:00 +0000', 778 + full_text: 'Test tweet for archive_upload_id upsert', 779 + favorite_count: 5, 780 + retweet_count: 2, 781 + favorited: false, 782 + truncated: false, 783 + source: 'web', 784 + entities: { 785 + user_mentions: [], 786 + hashtags: [], 787 + symbols: [], 788 + urls: [] 789 + } 790 + } 791 + } 792 + 793 + // First archive upload 794 + testArchive = { 795 + account: [{ 796 + account: { 797 + accountId: testAccountId, 798 + username: `test_${Date.now()}`, 799 + createdVia: 'web', 800 + createdAt: '2010-01-01T00:00:00.000Z', 801 + accountDisplayName: 'Test User' 802 + } 803 + }], 804 + profile: [{ 805 + profile: { 806 + description: { bio: 'Test bio', website: '', location: '' }, 807 + avatarMediaUrl: '', 808 + headerMediaUrl: '' 809 + } 810 + }], 811 + tweets: [testTweet], 812 + 'note-tweet': [], 813 + like: [], 814 + follower: [], 815 + following: [], 816 + 'community-tweet': [] 817 + } 818 + 819 + tracker.addAccountId(testAccountId) 820 + tracker.addTweetId('9999') 821 + 822 + // Insert first archive 823 + await insertArchiveDirectly(supabase, testArchive) 824 + 825 + // Get the first archive_upload_id 826 + const { data: firstArchive } = await supabase 827 + .from('archive_upload') 828 + .select('id') 829 + .eq('account_id', testAccountId) 830 + .order('created_at', { ascending: false }) 831 + .limit(1) 832 + .single() 833 + 834 + const firstArchiveUploadId = firstArchive?.id 835 + expect(firstArchiveUploadId).toBeDefined() 836 + 837 + // Verify tweet has first archive_upload_id 838 + const { data: firstTweet } = await supabase 839 + .from('tweets') 840 + .select('archive_upload_id, favorite_count, retweet_count') 841 + .eq('tweet_id', '9999') 842 + .single() 843 + 844 + expect(firstTweet?.archive_upload_id).toBe(firstArchiveUploadId) 845 + expect(firstTweet?.favorite_count).toBe(5) 846 + expect(firstTweet?.retweet_count).toBe(2) 847 + 848 + // Create second archive upload with updated tweet data 849 + const updatedTweet = { 850 + ...testTweet, 851 + tweet: { 852 + ...testTweet.tweet, 853 + favorite_count: 10, // Updated count 854 + retweet_count: 5 // Updated count 855 + } 856 + } 857 + 858 + const secondTestArchive = { 859 + ...testArchive, 860 + tweets: [updatedTweet] 861 + } 862 + 863 + // Wait a moment to ensure different timestamps 864 + await new Promise(resolve => setTimeout(resolve, 100)) 865 + 866 + // Insert second archive (should upsert the tweet) 867 + await insertArchiveDirectly(supabase, secondTestArchive) 868 + 869 + // Get the second archive_upload_id 870 + const { data: secondArchive } = await supabase 871 + .from('archive_upload') 872 + .select('id') 873 + .eq('account_id', testAccountId) 874 + .order('created_at', { ascending: false }) 875 + .limit(1) 876 + .single() 877 + 878 + const secondArchiveUploadId = secondArchive?.id 879 + expect(secondArchiveUploadId).toBeDefined() 880 + expect(secondArchiveUploadId).not.toBe(firstArchiveUploadId) 881 + 882 + // Verify tweet now has second archive_upload_id and updated counts 883 + const { data: updatedTweetData } = await supabase 884 + .from('tweets') 885 + .select('archive_upload_id, favorite_count, retweet_count') 886 + .eq('tweet_id', '9999') 887 + .single() 888 + 889 + expect(updatedTweetData?.archive_upload_id).toBe(secondArchiveUploadId) 890 + expect(updatedTweetData?.favorite_count).toBe(10) 891 + expect(updatedTweetData?.retweet_count).toBe(5) 892 + }) 893 + 894 + it('should update archive_upload_id when upserting profile data', async () => { 895 + // First archive upload 896 + testArchive = { 897 + account: [{ 898 + account: { 899 + accountId: testAccountId, 900 + username: `test_${Date.now()}`, 901 + createdVia: 'web', 902 + createdAt: '2010-01-01T00:00:00.000Z', 903 + accountDisplayName: 'Test User' 904 + } 905 + }], 906 + profile: [{ 907 + profile: { 908 + description: { 909 + bio: 'Original bio', 910 + website: 'https://original.com', 911 + location: 'Original Location' 912 + }, 913 + avatarMediaUrl: 'https://original.com/avatar.jpg', 914 + headerMediaUrl: 'https://original.com/header.jpg' 915 + } 916 + }], 917 + tweets: [], 918 + 'note-tweet': [], 919 + like: [], 920 + follower: [], 921 + following: [], 922 + 'community-tweet': [] 923 + } 924 + 925 + tracker.addAccountId(testAccountId) 926 + 927 + // Insert first archive 928 + await insertArchiveDirectly(supabase, testArchive) 929 + 930 + // Get the first archive_upload_id 931 + const { data: firstArchive } = await supabase 932 + .from('archive_upload') 933 + .select('id') 934 + .eq('account_id', testAccountId) 935 + .order('created_at', { ascending: false }) 936 + .limit(1) 937 + .single() 938 + 939 + const firstArchiveUploadId = firstArchive?.id 940 + 941 + // Verify profile has first archive_upload_id 942 + const { data: firstProfile } = await supabase 943 + .from('all_profile') 944 + .select('archive_upload_id, bio, website') 945 + .eq('account_id', testAccountId) 946 + .single() 947 + 948 + expect(firstProfile?.archive_upload_id).toBe(firstArchiveUploadId) 949 + expect(firstProfile?.bio).toBe('Original bio') 950 + expect(firstProfile?.website).toBe('https://original.com') 951 + 952 + // Create second archive upload with updated profile 953 + const updatedTestArchive = { 954 + ...testArchive, 955 + profile: [{ 956 + profile: { 957 + description: { 958 + bio: 'Updated bio', 959 + website: 'https://updated.com', 960 + location: 'Updated Location' 961 + }, 962 + avatarMediaUrl: 'https://updated.com/avatar.jpg', 963 + headerMediaUrl: 'https://updated.com/header.jpg' 964 + } 965 + }] 966 + } 967 + 968 + // Wait a moment to ensure different timestamps 969 + await new Promise(resolve => setTimeout(resolve, 100)) 970 + 971 + // Insert second archive 972 + await insertArchiveDirectly(supabase, updatedTestArchive) 973 + 974 + // Get the second archive_upload_id 975 + const { data: secondArchive } = await supabase 976 + .from('archive_upload') 977 + .select('id') 978 + .eq('account_id', testAccountId) 979 + .order('created_at', { ascending: false }) 980 + .limit(1) 981 + .single() 982 + 983 + const secondArchiveUploadId = secondArchive?.id 984 + expect(secondArchiveUploadId).not.toBe(firstArchiveUploadId) 985 + 986 + // Verify profile now has second archive_upload_id and updated data 987 + const { data: updatedProfile } = await supabase 988 + .from('all_profile') 989 + .select('archive_upload_id, bio, website') 990 + .eq('account_id', testAccountId) 991 + .single() 992 + 993 + expect(updatedProfile?.archive_upload_id).toBe(secondArchiveUploadId) 994 + expect(updatedProfile?.bio).toBe('Updated bio') 995 + expect(updatedProfile?.website).toBe('https://updated.com') 996 + }) 997 + 998 + it('should update archive_upload_id when upserting tweet media', async () => { 999 + const testTweetWithMedia = { 1000 + tweet: { 1001 + id: '8888', 1002 + id_str: '8888', 1003 + created_at: '2023-01-01 00:00:00 +0000', 1004 + full_text: 'Tweet with media', 1005 + favorite_count: 0, 1006 + retweet_count: 0, 1007 + favorited: false, 1008 + truncated: false, 1009 + source: 'web', 1010 + entities: { 1011 + user_mentions: [], 1012 + hashtags: [], 1013 + symbols: [], 1014 + urls: [], 1015 + media: [{ 1016 + id_str: '123456789', 1017 + media_url_https: 'https://example.com/media1.jpg', 1018 + type: 'photo', 1019 + sizes: { 1020 + large: { w: 1024, h: 768 } 1021 + } 1022 + }] 1023 + } 1024 + } 1025 + } 1026 + 1027 + // First archive upload 1028 + testArchive = { 1029 + account: [{ 1030 + account: { 1031 + accountId: testAccountId, 1032 + username: `test_${Date.now()}`, 1033 + createdVia: 'web', 1034 + createdAt: '2010-01-01T00:00:00.000Z', 1035 + accountDisplayName: 'Test User' 1036 + } 1037 + }], 1038 + profile: [{ 1039 + profile: { 1040 + description: { bio: '', website: '', location: '' }, 1041 + avatarMediaUrl: '', 1042 + headerMediaUrl: '' 1043 + } 1044 + }], 1045 + tweets: [testTweetWithMedia], 1046 + 'note-tweet': [], 1047 + like: [], 1048 + follower: [], 1049 + following: [], 1050 + 'community-tweet': [] 1051 + } 1052 + 1053 + tracker.addAccountId(testAccountId) 1054 + tracker.addTweetId('8888') 1055 + 1056 + // Insert first archive 1057 + await insertArchiveDirectly(supabase, testArchive) 1058 + 1059 + // Get the first archive_upload_id 1060 + const { data: firstArchive } = await supabase 1061 + .from('archive_upload') 1062 + .select('id') 1063 + .eq('account_id', testAccountId) 1064 + .order('created_at', { ascending: false }) 1065 + .limit(1) 1066 + .single() 1067 + 1068 + const firstArchiveUploadId = firstArchive?.id 1069 + 1070 + // Verify media has first archive_upload_id 1071 + const { data: firstMedia } = await supabase 1072 + .from('tweet_media') 1073 + .select('archive_upload_id, media_url, width, height') 1074 + .eq('media_id', 123456789) 1075 + .single() 1076 + 1077 + expect(firstMedia?.archive_upload_id).toBe(firstArchiveUploadId) 1078 + expect(firstMedia?.width).toBe(1024) 1079 + expect(firstMedia?.height).toBe(768) 1080 + 1081 + // Create second archive with updated media dimensions 1082 + const updatedTweetWithMedia = { 1083 + ...testTweetWithMedia, 1084 + tweet: { 1085 + ...testTweetWithMedia.tweet, 1086 + entities: { 1087 + ...testTweetWithMedia.tweet.entities, 1088 + media: [{ 1089 + ...testTweetWithMedia.tweet.entities.media[0], 1090 + media_url_https: 'https://example.com/media1_updated.jpg', 1091 + sizes: { 1092 + large: { w: 2048, h: 1536 } // Updated dimensions 1093 + } 1094 + }] 1095 + } 1096 + } 1097 + } 1098 + 1099 + const secondTestArchive = { 1100 + ...testArchive, 1101 + tweets: [updatedTweetWithMedia] 1102 + } 1103 + 1104 + // Wait a moment to ensure different timestamps 1105 + await new Promise(resolve => setTimeout(resolve, 100)) 1106 + 1107 + // Insert second archive 1108 + await insertArchiveDirectly(supabase, secondTestArchive) 1109 + 1110 + // Get the second archive_upload_id 1111 + const { data: secondArchive } = await supabase 1112 + .from('archive_upload') 1113 + .select('id') 1114 + .eq('account_id', testAccountId) 1115 + .order('created_at', { ascending: false }) 1116 + .limit(1) 1117 + .single() 1118 + 1119 + const secondArchiveUploadId = secondArchive?.id 1120 + expect(secondArchiveUploadId).not.toBe(firstArchiveUploadId) 1121 + 1122 + // Verify media now has second archive_upload_id and updated data 1123 + const { data: updatedMedia } = await supabase 1124 + .from('tweet_media') 1125 + .select('archive_upload_id, media_url, width, height') 1126 + .eq('media_id', 123456789) 1127 + .single() 1128 + 1129 + expect(updatedMedia?.archive_upload_id).toBe(secondArchiveUploadId) 1130 + expect(updatedMedia?.media_url).toBe('https://example.com/media1_updated.jpg') 1131 + expect(updatedMedia?.width).toBe(2048) 1132 + expect(updatedMedia?.height).toBe(1536) 1133 + }) 1134 + 1135 + it('should update archive_upload_id when upserting likes, following, and followers', async () => { 1136 + // First archive upload 1137 + testArchive = { 1138 + account: [{ 1139 + account: { 1140 + accountId: testAccountId, 1141 + username: `test_${Date.now()}`, 1142 + createdVia: 'web', 1143 + createdAt: '2010-01-01T00:00:00.000Z', 1144 + accountDisplayName: 'Test User' 1145 + } 1146 + }], 1147 + profile: [{ 1148 + profile: { 1149 + description: { bio: '', website: '', location: '' }, 1150 + avatarMediaUrl: '', 1151 + headerMediaUrl: '' 1152 + } 1153 + }], 1154 + tweets: [], 1155 + 'note-tweet': [], 1156 + like: [ 1157 + { like: { tweetId: '1234567890123456789', fullText: 'First liked tweet' } } 1158 + ], 1159 + follower: [ 1160 + { follower: { accountId: '1234567890', userLink: 'https://twitter.com/follower_1' } } 1161 + ], 1162 + following: [ 1163 + { following: { accountId: '9876543210', userLink: 'https://twitter.com/following_1' } } 1164 + ], 1165 + 'community-tweet': [] 1166 + } 1167 + 1168 + tracker.addAccountId(testAccountId) 1169 + tracker.addLikedTweetId('1234567890123456789') 1170 + 1171 + // Insert first archive 1172 + await insertArchiveDirectly(supabase, testArchive) 1173 + 1174 + // Get the first archive_upload_id 1175 + const { data: firstArchive } = await supabase 1176 + .from('archive_upload') 1177 + .select('id') 1178 + .eq('account_id', testAccountId) 1179 + .order('created_at', { ascending: false }) 1180 + .limit(1) 1181 + .single() 1182 + 1183 + const firstArchiveUploadId = firstArchive?.id 1184 + 1185 + // Verify likes, following, followers have first archive_upload_id 1186 + const { data: firstLike } = await supabase 1187 + .from('likes') 1188 + .select('archive_upload_id') 1189 + .eq('account_id', testAccountId) 1190 + .eq('liked_tweet_id', '1234567890123456789') 1191 + .single() 1192 + 1193 + const { data: firstFollowing } = await supabase 1194 + .from('following') 1195 + .select('archive_upload_id') 1196 + .eq('account_id', testAccountId) 1197 + .eq('following_account_id', '9876543210') 1198 + .single() 1199 + 1200 + const { data: firstFollower } = await supabase 1201 + .from('followers') 1202 + .select('archive_upload_id') 1203 + .eq('account_id', testAccountId) 1204 + .eq('follower_account_id', '1234567890') 1205 + .single() 1206 + 1207 + expect(firstLike?.archive_upload_id).toBe(firstArchiveUploadId) 1208 + expect(firstFollowing?.archive_upload_id).toBe(firstArchiveUploadId) 1209 + expect(firstFollower?.archive_upload_id).toBe(firstArchiveUploadId) 1210 + 1211 + // Wait a moment to ensure different timestamps 1212 + await new Promise(resolve => setTimeout(resolve, 100)) 1213 + 1214 + // Insert second archive (should upsert the same relationships) 1215 + await insertArchiveDirectly(supabase, testArchive) 1216 + 1217 + // Get the second archive_upload_id 1218 + const { data: secondArchive } = await supabase 1219 + .from('archive_upload') 1220 + .select('id') 1221 + .eq('account_id', testAccountId) 1222 + .order('created_at', { ascending: false }) 1223 + .limit(1) 1224 + .single() 1225 + 1226 + const secondArchiveUploadId = secondArchive?.id 1227 + expect(secondArchiveUploadId).not.toBe(firstArchiveUploadId) 1228 + 1229 + // Verify all relationships now have second archive_upload_id 1230 + const { data: updatedLike } = await supabase 1231 + .from('likes') 1232 + .select('archive_upload_id') 1233 + .eq('account_id', testAccountId) 1234 + .eq('liked_tweet_id', '1234567890123456789') 1235 + .single() 1236 + 1237 + const { data: updatedFollowing } = await supabase 1238 + .from('following') 1239 + .select('archive_upload_id') 1240 + .eq('account_id', testAccountId) 1241 + .eq('following_account_id', '9876543210') 1242 + .single() 1243 + 1244 + const { data: updatedFollower } = await supabase 1245 + .from('followers') 1246 + .select('archive_upload_id') 1247 + .eq('account_id', testAccountId) 1248 + .eq('follower_account_id', '1234567890') 1249 + .single() 1250 + 1251 + expect(updatedLike?.archive_upload_id).toBe(secondArchiveUploadId) 1252 + expect(updatedFollowing?.archive_upload_id).toBe(secondArchiveUploadId) 1253 + expect(updatedFollower?.archive_upload_id).toBe(secondArchiveUploadId) 1254 + }) 1255 + 1256 + it('should handle multiple archive uploads with different data sets', async () => { 1257 + // First archive with some tweets 1258 + const firstArchive = { 1259 + account: [{ 1260 + account: { 1261 + accountId: testAccountId, 1262 + username: `test_${Date.now()}`, 1263 + createdVia: 'web', 1264 + createdAt: '2010-01-01T00:00:00.000Z', 1265 + accountDisplayName: 'Test User' 1266 + } 1267 + }], 1268 + profile: [{ 1269 + profile: { 1270 + description: { bio: 'First bio', website: '', location: '' }, 1271 + avatarMediaUrl: '', 1272 + headerMediaUrl: '' 1273 + } 1274 + }], 1275 + tweets: [ 1276 + { 1277 + tweet: { 1278 + id: '7777', 1279 + id_str: '7777', 1280 + created_at: '2023-01-01 00:00:00 +0000', 1281 + full_text: 'First archive tweet', 1282 + favorite_count: 1, 1283 + retweet_count: 0, 1284 + favorited: false, 1285 + truncated: false, 1286 + source: 'web', 1287 + entities: { user_mentions: [], hashtags: [], symbols: [], urls: [] } 1288 + } 1289 + } 1290 + ], 1291 + 'note-tweet': [], 1292 + like: [ 1293 + { like: { tweetId: '1111111111111111111', fullText: 'Liked in first archive' } } 1294 + ], 1295 + follower: [], 1296 + following: [], 1297 + 'community-tweet': [] 1298 + } 1299 + 1300 + tracker.addAccountId(testAccountId) 1301 + tracker.addTweetId('7777') 1302 + tracker.addLikedTweetId('1111111111111111111') 1303 + 1304 + // Insert first archive 1305 + await insertArchiveDirectly(supabase, firstArchive) 1306 + 1307 + // Second archive with overlapping and new data 1308 + const secondArchive = { 1309 + ...firstArchive, 1310 + profile: [{ 1311 + profile: { 1312 + description: { bio: 'Updated bio', website: 'https://updated.com', location: '' }, 1313 + avatarMediaUrl: '', 1314 + headerMediaUrl: '' 1315 + } 1316 + }], 1317 + tweets: [ 1318 + { 1319 + tweet: { 1320 + id: '7777', // Same tweet, updated counts 1321 + id_str: '7777', 1322 + created_at: '2023-01-01 00:00:00 +0000', 1323 + full_text: 'First archive tweet', 1324 + favorite_count: 5, // Updated 1325 + retweet_count: 2, // Updated 1326 + favorited: false, 1327 + truncated: false, 1328 + source: 'web', 1329 + entities: { user_mentions: [], hashtags: [], symbols: [], urls: [] } 1330 + } 1331 + }, 1332 + { 1333 + tweet: { 1334 + id: '6666', // New tweet 1335 + id_str: '6666', 1336 + created_at: '2023-01-02 00:00:00 +0000', 1337 + full_text: 'Second archive tweet', 1338 + favorite_count: 3, 1339 + retweet_count: 1, 1340 + favorited: false, 1341 + truncated: false, 1342 + source: 'web', 1343 + entities: { user_mentions: [], hashtags: [], symbols: [], urls: [] } 1344 + } 1345 + } 1346 + ], 1347 + like: [ 1348 + { like: { tweetId: '1111111111111111111', fullText: 'Liked in first archive' } }, // Same like 1349 + { like: { tweetId: '2222222222222222222', fullText: 'New like in second archive' } } // New like 1350 + ] 1351 + } 1352 + 1353 + tracker.addTweetId('6666') 1354 + tracker.addLikedTweetId('2222222222222222222') 1355 + 1356 + // Wait a moment to ensure different timestamps 1357 + await new Promise(resolve => setTimeout(resolve, 100)) 1358 + 1359 + // Insert second archive 1360 + await insertArchiveDirectly(supabase, secondArchive) 1361 + 1362 + // Get both archive upload IDs 1363 + const { data: archives } = await supabase 1364 + .from('archive_upload') 1365 + .select('id') 1366 + .eq('account_id', testAccountId) 1367 + .order('created_at', { ascending: true }) 1368 + 1369 + expect(archives?.length).toBe(2) 1370 + const [firstArchiveUploadId, secondArchiveUploadId] = archives!.map(a => a.id) 1371 + 1372 + // Verify tweet 7777 has updated archive_upload_id and counts 1373 + const { data: updatedTweet } = await supabase 1374 + .from('tweets') 1375 + .select('archive_upload_id, favorite_count, retweet_count') 1376 + .eq('tweet_id', '7777') 1377 + .single() 1378 + 1379 + expect(updatedTweet?.archive_upload_id).toBe(secondArchiveUploadId) 1380 + expect(updatedTweet?.favorite_count).toBe(5) 1381 + expect(updatedTweet?.retweet_count).toBe(2) 1382 + 1383 + // Verify tweet 6666 has second archive_upload_id 1384 + const { data: newTweet } = await supabase 1385 + .from('tweets') 1386 + .select('archive_upload_id') 1387 + .eq('tweet_id', '6666') 1388 + .single() 1389 + 1390 + expect(newTweet?.archive_upload_id).toBe(secondArchiveUploadId) 1391 + 1392 + // Verify profile has updated archive_upload_id 1393 + const { data: profile } = await supabase 1394 + .from('all_profile') 1395 + .select('archive_upload_id, bio, website') 1396 + .eq('account_id', testAccountId) 1397 + .single() 1398 + 1399 + expect(profile?.archive_upload_id).toBe(secondArchiveUploadId) 1400 + expect(profile?.bio).toBe('Updated bio') 1401 + expect(profile?.website).toBe('https://updated.com') 1402 + 1403 + // Verify both likes have second archive_upload_id 1404 + const { data: likes } = await supabase 1405 + .from('likes') 1406 + .select('archive_upload_id, liked_tweet_id') 1407 + .eq('account_id', testAccountId) 1408 + .order('liked_tweet_id') 1409 + 1410 + expect(likes?.length).toBe(2) 1411 + expect(likes?.[0].archive_upload_id).toBe(secondArchiveUploadId) // 1111111111111111111 1412 + expect(likes?.[1].archive_upload_id).toBe(secondArchiveUploadId) // 2222222222222222222 1413 + }) 1414 + }) 779 1415 780 1416 describe('Finalize archive_upload record', () => { 781 1417 it('should update archive_upload record to completed', async () => {