Live video on the AT Protocol
79
fork

Configure Feed

Select the types of activity you want to include in your feed.

app: custom live thumbnail + client-side thumbnail generation

authored by 10xcrazy.horse and committed by

Natalie B 7cec3033 fe0dcba9

+350 -19
+22 -1
js/app/components/create-livestream.tsx
··· 8 8 } from "features/bluesky/blueskySlice"; 9 9 import { useAppDispatch, useAppSelector } from "store/hooks"; 10 10 import { useLiveUser } from "hooks/useLiveUser"; 11 + import ThumbnailSelector from "components/thumbnail-selector/thumbnail-selector"; 11 12 12 13 const Left = ({ children }: { children: React.ReactNode }) => { 13 14 return ( ··· 31 32 const userIsLive = useLiveUser(); 32 33 const [title, setTitle] = useState(""); 33 34 const [loading, setLoading] = useState(false); 35 + const [customThumbnail, setCustomThumbnail] = useState<Blob | undefined>( 36 + undefined, 37 + ); 34 38 const profile = useAppSelector(selectUserProfile); 35 39 const newLivestream = useAppSelector(selectNewLivestream); 36 40 useEffect(() => { ··· 39 43 message: newLivestream.record.title, 40 44 }); 41 45 setTitle(""); 46 + setCustomThumbnail(undefined); 42 47 } 43 48 }, [newLivestream?.record]); 44 49 useEffect(() => { ··· 86 91 </Right> 87 92 </View> 88 93 </Label> 94 + <Label asChild={true}> 95 + <View flexDirection="row"> 96 + <Left> 97 + <Paragraph pb="$2">Thumbnail</Paragraph> 98 + </Left> 99 + <Right> 100 + <ThumbnailSelector onThumbnailSelected={setCustomThumbnail} /> 101 + </Right> 102 + </View> 103 + </Label> 89 104 <View gap="$2" w="100%"> 90 105 <Button 91 106 disabled={disabled} ··· 93 108 w="100%" 94 109 size="$4" 95 110 onPress={() => { 96 - dispatch(createLivestreamRecord({ title })); 111 + setLoading(true); 112 + dispatch( 113 + createLivestreamRecord({ 114 + title, 115 + customThumbnail, 116 + }), 117 + ).finally(() => setLoading(false)); 97 118 }} 98 119 > 99 120 {buttonText(loading, userIsLive)}
+160
js/app/components/thumbnail-selector/thumbnail-selector.tsx
··· 1 + import { useCallback, useEffect, useState } from "react"; 2 + import { Button, Image, Text, View } from "tamagui"; 3 + import { isWeb } from "tamagui"; 4 + import { Platform } from "react-native"; 5 + import * as ImagePicker from "expo-image-picker"; 6 + import { Camera, Image as ImageIcon, X } from "@tamagui/lucide-icons"; 7 + import { captureVideoFrame, findVideoElement } from "utils/videoCapture"; 8 + 9 + interface ThumbnailSelectorProps { 10 + onThumbnailSelected: (blob: Blob | undefined) => void; 11 + thumbnailUrl?: string; 12 + } 13 + 14 + export default function ThumbnailSelector({ 15 + onThumbnailSelected, 16 + thumbnailUrl, 17 + }: ThumbnailSelectorProps) { 18 + const [selectedImage, setSelectedImage] = useState<string | null>( 19 + thumbnailUrl || null, 20 + ); 21 + const [loading, setLoading] = useState(false); 22 + 23 + useEffect(() => { 24 + if (thumbnailUrl) { 25 + setSelectedImage(thumbnailUrl); 26 + } 27 + }, [thumbnailUrl]); 28 + 29 + const pickImage = useCallback(async () => { 30 + setLoading(true); 31 + try { 32 + // Request permissions first 33 + if (Platform.OS !== "web") { 34 + const { status } = 35 + await ImagePicker.requestMediaLibraryPermissionsAsync(); 36 + if (status !== "granted") { 37 + alert("Sorry, we need camera roll permissions to make this work!"); 38 + return; 39 + } 40 + } 41 + 42 + // Launch image picker 43 + const result = await ImagePicker.launchImageLibraryAsync({ 44 + mediaTypes: ImagePicker.MediaTypeOptions.Images, 45 + allowsEditing: true, 46 + aspect: [16, 9], 47 + quality: 0.8, 48 + }); 49 + 50 + if (!result.canceled) { 51 + const imageUri = result.assets[0].uri; 52 + setSelectedImage(imageUri); 53 + 54 + const response = await fetch(imageUri); 55 + const blob = await response.blob(); 56 + onThumbnailSelected(blob); 57 + } else { 58 + // User canceled 59 + console.log("Image selection canceled"); 60 + } 61 + } catch (error) { 62 + console.error("Error picking image:", error); 63 + } finally { 64 + setLoading(false); 65 + } 66 + }, [onThumbnailSelected]); 67 + 68 + // Capture a frame from the video stream 69 + const captureFrame = useCallback(async () => { 70 + if (!isWeb) { 71 + alert("Capturing from stream is only available on web"); 72 + return; 73 + } 74 + 75 + setLoading(true); 76 + try { 77 + const videoElement = findVideoElement(); 78 + if (!videoElement) { 79 + alert("No video stream found"); 80 + return; 81 + } 82 + 83 + const blob = await captureVideoFrame(videoElement, 1280, 0.85); 84 + const imageUrl = URL.createObjectURL(blob); 85 + setSelectedImage(imageUrl); 86 + onThumbnailSelected(blob); 87 + } catch (error) { 88 + console.error("Error capturing frame:", error); 89 + alert("Failed to capture frame from video"); 90 + } finally { 91 + setLoading(false); 92 + } 93 + }, [onThumbnailSelected]); 94 + 95 + // Clear the selected thumbnail 96 + const clearThumbnail = useCallback(() => { 97 + setSelectedImage(null); 98 + onThumbnailSelected(undefined); 99 + }, [onThumbnailSelected]); 100 + 101 + return ( 102 + <View> 103 + <Text pb="$2">Thumbnail (optional)</Text> 104 + 105 + {selectedImage ? ( 106 + <View position="relative"> 107 + <Image 108 + source={{ uri: selectedImage }} 109 + width="100%" 110 + height={150} 111 + objectFit="cover" 112 + borderRadius="$2" 113 + /> 114 + <Button 115 + position="absolute" 116 + top={5} 117 + right={5} 118 + size="$2" 119 + circular 120 + icon={<X size={16} />} 121 + onPress={clearThumbnail} 122 + /> 123 + </View> 124 + ) : ( 125 + <View 126 + height={150} 127 + width="100%" 128 + backgroundColor="$backgroundHover" 129 + borderRadius="$2" 130 + justifyContent="center" 131 + alignItems="center" 132 + > 133 + <Text color="$color">No thumbnail selected</Text> 134 + </View> 135 + )} 136 + 137 + <View flexDirection="row" gap="$2" mt="$2"> 138 + <Button 139 + flex={1} 140 + icon={<ImageIcon size={16} />} 141 + onPress={pickImage} 142 + disabled={loading} 143 + > 144 + {loading ? "Loading..." : "Choose Image"} 145 + </Button> 146 + {/* TODO: Re-enable this when we have camera working */} 147 + {/* {isWeb && ( 148 + <Button 149 + flex={1} 150 + icon={<Camera size={16} />} 151 + onPress={captureFrame} 152 + disabled={loading} 153 + > 154 + Capture Frame 155 + </Button> 156 + )} */} 157 + </View> 158 + </View> 159 + ); 160 + }
+77 -18
js/app/features/bluesky/blueskySlice.tsx
··· 26 26 import { createAppSlice } from "../../hooks/createSlice"; 27 27 import { BlueskyState } from "./blueskyTypes"; 28 28 import createOAuthClient from "./oauthClient"; 29 + import { captureVideoFrame, findVideoElement } from "utils/videoCapture"; 29 30 30 31 const initialState: BlueskyState = { 31 32 status: "start", ··· 58 59 u: URL, 59 60 pdsAgent: Agent, 60 61 profile: ProfileViewDetailed, 62 + customThumbnail?: Blob, 61 63 ) => { 62 - // download the thumbnail image and upload it to the pds IF POSSIBLE 63 - const thumbnailRes = await fetch( 64 - `${u.protocol}//${u.host}/api/playback/${profile.handle}/stream.png`, 65 - ); 66 - if (!thumbnailRes.ok) { 67 - throw new Error(`failed to fetch thumbnail (http ${thumbnailRes.status})`); 64 + if (customThumbnail) { 65 + try { 66 + const thumbnail = await pdsAgent.uploadBlob(customThumbnail); 67 + if (thumbnail.success) { 68 + console.log("Successfully uploaded custom thumbnail"); 69 + return thumbnail.data.blob; 70 + } 71 + } catch (e) { 72 + console.error("Error uploading custom thumbnail:", e); 73 + } 68 74 } 69 - const thumbnailBlob = await thumbnailRes.blob(); 70 - const thumbnail = await pdsAgent.uploadBlob(thumbnailBlob); 71 - if (!thumbnail.success) { 72 - throw new Error("failed to upload thumbnail"); 75 + // Capture a frame from the video element on web 76 + if (isWeb) { 77 + try { 78 + const videoElement = findVideoElement(); 79 + if (videoElement) { 80 + const thumbnailBlob = await captureVideoFrame(videoElement, 1280, 0.85); 81 + const thumbnail = await pdsAgent.uploadBlob(thumbnailBlob); 82 + if (thumbnail.success) { 83 + return thumbnail.data.blob; 84 + } 85 + } 86 + } catch (e) { 87 + console.error("Error capturing client-side thumbnail:", e); 88 + } 89 + } 90 + 91 + // Fallback to server-side thumbnail if client-side capture fails or we're not on web 92 + try { 93 + const thumbnailRes = await fetch( 94 + `${u.protocol}//${u.host}/api/playback/${profile.handle}/stream.png`, 95 + ); 96 + if (!thumbnailRes.ok) { 97 + throw new Error( 98 + `failed to fetch thumbnail (http ${thumbnailRes.status})`, 99 + ); 100 + } 101 + const thumbnailBlob = await thumbnailRes.blob(); 102 + const thumbnail = await pdsAgent.uploadBlob(thumbnailBlob); 103 + if (!thumbnail.success) { 104 + throw new Error("failed to upload thumbnail"); 105 + } 106 + return thumbnail.data.blob; 107 + } catch (e) { 108 + console.error("Error fetching server-side thumbnail:", e); 109 + throw e; 73 110 } 74 - return thumbnail.data.blob; 75 111 }; 76 112 77 113 // clear atproto login query params from url ··· 340 376 341 377 golivePost: create.asyncThunk( 342 378 async ( 343 - { text, now }: { text: string; now: Date }, 379 + { 380 + text, 381 + now, 382 + customThumbnail, 383 + }: { text: string; now: Date; customThumbnail?: Blob }, 344 384 thunkAPI, 345 385 ): Promise<{ 346 386 uri: string; ··· 374 414 u, 375 415 bluesky.pdsAgent, 376 416 profile, 417 + customThumbnail, 377 418 ); 378 419 } catch (e) { 379 420 console.error("uploadThumbnail error", e); ··· 743 784 ), 744 785 745 786 createLivestreamRecord: create.asyncThunk( 746 - async ({ title }: { title }, thunkAPI) => { 787 + async ( 788 + { title, customThumbnail }: { title: string; customThumbnail?: Blob }, 789 + thunkAPI, 790 + ) => { 747 791 const now = new Date(); 748 792 const { bluesky, streamplace } = thunkAPI.getState() as { 749 793 bluesky: BlueskyState; ··· 760 804 if (!profile) { 761 805 throw new Error("No profile"); 762 806 } 763 - if (!did) { 764 - throw new Error("No DID"); 807 + 808 + const newPostAction = await thunkAPI.dispatch( 809 + golivePost({ text: title, now, customThumbnail }), 810 + ); 811 + 812 + if (!newPostAction || newPostAction.type.endsWith("/rejected")) { 813 + throw new Error( 814 + `Failed to create post: ${(newPostAction as any)?.error?.message || "Unknown error"}`, 815 + ); 765 816 } 766 - const newPost = (await thunkAPI.dispatch( 767 - golivePost({ text: title, now }), 768 - )) as { payload: { uri: string; cid: string } }; 817 + 818 + const newPost = newPostAction as { 819 + payload: { uri: string; cid: string }; 820 + }; 821 + 822 + if (!newPost.payload?.uri || !newPost.payload?.cid) { 823 + throw new Error( 824 + "Cannot read properties of undefined (reading 'uri' or 'cid')", 825 + ); 826 + } 827 + 769 828 const record: PlaceStreamLivestream.Record = { 770 829 title: title, 771 830 url: streamplace.url,
+1
js/app/package.json
··· 58 58 "expo-dev-client": "~5.0.3", 59 59 "expo-file-system": "~18.0.4", 60 60 "expo-font": "~13.0.1", 61 + "expo-image-picker": "^16.1.4", 61 62 "expo-keep-awake": "~14.0.2", 62 63 "expo-linking": "~7.0.3", 63 64 "expo-notifications": "~0.29.8",
+69
js/app/utils/videoCapture.ts
··· 1 + /** 2 + * Utility functions for capturing and compressing video frames 3 + */ 4 + 5 + /** 6 + * Captures a frame from a video element and returns it as a compressed PNG blob 7 + * 8 + * @param videoElement The video element to capture from 9 + * @param maxWidth Maximum width of the output image (maintains aspect ratio) 10 + * @param quality JPEG quality (0-1), lower means smaller file size 11 + * @returns A Promise that resolves to a Blob of the compressed image 12 + */ 13 + export const captureVideoFrame = async ( 14 + videoElement: HTMLVideoElement, 15 + maxWidth = 1280, 16 + quality = 0.85, 17 + ): Promise<Blob> => { 18 + return new Promise((resolve, reject) => { 19 + try { 20 + const canvas = document.createElement("canvas"); 21 + const ctx = canvas.getContext("2d"); 22 + 23 + if (!ctx) { 24 + reject(new Error("Could not get canvas context")); 25 + return; 26 + } 27 + 28 + const videoWidth = videoElement.videoWidth; 29 + const videoHeight = videoElement.videoHeight; 30 + 31 + if (videoWidth === 0 || videoHeight === 0) { 32 + reject(new Error("Video has no dimensions, may not be playing yet")); 33 + return; 34 + } 35 + 36 + let width = videoWidth; 37 + let height = videoHeight; 38 + 39 + if (width > maxWidth) { 40 + const ratio = maxWidth / width; 41 + width = maxWidth; 42 + height = height * ratio; 43 + } 44 + 45 + canvas.width = width; 46 + canvas.height = height; 47 + 48 + ctx.drawImage(videoElement, 0, 0, width, height); 49 + 50 + canvas.toBlob( 51 + (blob) => { 52 + if (blob) { 53 + resolve(blob); 54 + } else { 55 + reject(new Error("Failed to create blob from canvas")); 56 + } 57 + }, 58 + "image/jpeg", 59 + quality, 60 + ); 61 + } catch (error) { 62 + reject(error); 63 + } 64 + }); 65 + }; 66 + 67 + export const findVideoElement = (): HTMLVideoElement | null => { 68 + return document.querySelector("video"); 69 + };
+21
yarn.lock
··· 17306 17306 languageName: node 17307 17307 linkType: hard 17308 17308 17309 + "expo-image-loader@npm:~5.1.0": 17310 + version: 5.1.0 17311 + resolution: "expo-image-loader@npm:5.1.0" 17312 + peerDependencies: 17313 + expo: "*" 17314 + checksum: 10/7c99e534dff619090f39f753e460471e4a84cc868ea21269e3c060f348e216c3ad1f0762512a048885d88380e4c4b763a77ac9d4688a85d31b0ee61834613968 17315 + languageName: node 17316 + linkType: hard 17317 + 17318 + "expo-image-picker@npm:^16.1.4": 17319 + version: 16.1.4 17320 + resolution: "expo-image-picker@npm:16.1.4" 17321 + dependencies: 17322 + expo-image-loader: "npm:~5.1.0" 17323 + peerDependencies: 17324 + expo: "*" 17325 + checksum: 10/a6006614f6c2f42177747010f35901d85cbb673031cfeb1acac55b8c23c8bd92f432191df5759555bf4401ed3c2eee8fc96ea97500a2d75e0a1b95d048aadc7a 17326 + languageName: node 17327 + linkType: hard 17328 + 17309 17329 "expo-json-utils@npm:~0.14.0": 17310 17330 version: 0.14.0 17311 17331 resolution: "expo-json-utils@npm:0.14.0" ··· 29615 29635 expo-dev-client: "npm:~5.0.3" 29616 29636 expo-file-system: "npm:~18.0.4" 29617 29637 expo-font: "npm:~13.0.1" 29638 + expo-image-picker: "npm:^16.1.4" 29618 29639 expo-keep-awake: "npm:~14.0.2" 29619 29640 expo-linking: "npm:~7.0.3" 29620 29641 expo-notifications: "npm:~0.29.8"