aesthetic.computer
Monorepo for Aesthetic.Computer
aesthetic.computer
1// recorder.c — On-device MP4 recording (H.264 + AAC in fragmented MP4)
2// Encoder runs in a dedicated thread so the main 60fps loop is never blocked.
3
4#ifdef HAVE_AVCODEC
5
6#include "recorder.h"
7#include <stdio.h>
8#include <stdlib.h>
9#include <string.h>
10#include <pthread.h>
11#include <stdatomic.h>
12#include <time.h>
13#include <sys/stat.h>
14
15#include <libavcodec/avcodec.h>
16#include <libavformat/avformat.h>
17#include <libavutil/opt.h>
18#include <libavutil/imgutils.h>
19#include <libavutil/channel_layout.h>
20#include <libswscale/swscale.h>
21#include <libswresample/swresample.h>
22
23// ── Ring buffer for audio ──
24#define AUDIO_RING_SAMPLES (192000 * 2) // ~1 second at 192kHz stereo (interleaved)
25#define AUDIO_RING_MASK (AUDIO_RING_SAMPLES - 1)
26// Ensure power of 2 (192000*2=384000 is not power of 2, so we use modulo instead)
27
28// ── Triple-buffer for video frames ──
29#define VIDEO_SLOTS 3
30
31extern void ac_log(const char *fmt, ...);
32
33struct ACRecorder {
34 // Config
35 int width, height, fps;
36 unsigned int audio_src_rate;
37
38 // State
39 volatile int recording;
40 volatile int stopping;
41 struct timespec start_time;
42
43 // Video triple-buffer
44 uint32_t *video_buf[VIDEO_SLOTS];
45 int video_stride;
46 atomic_int video_write_idx; // main thread writes here (mod VIDEO_SLOTS)
47 atomic_int video_read_idx; // encoder reads here (mod VIDEO_SLOTS)
48 atomic_int video_ready; // number of frames ready to encode
49
50 // Audio ring buffer (interleaved int16 stereo at source rate)
51 int16_t *audio_ring;
52 int audio_ring_size;
53 atomic_int audio_write_pos; // monotonic write position
54 atomic_int audio_read_pos; // encoder thread read position
55
56 // Encoder thread
57 pthread_t thread;
58 volatile int thread_running;
59
60 // ffmpeg contexts
61 AVFormatContext *fmt_ctx;
62 AVCodecContext *video_enc;
63 AVCodecContext *audio_enc;
64 AVStream *video_st;
65 AVStream *audio_st;
66 struct SwsContext *sws;
67 SwrContext *swr;
68 AVFrame *video_frame; // YUV420P frame for encoder
69 AVFrame *audio_frame; // float planar frame for AAC encoder
70 int64_t video_pts;
71 int64_t audio_pts;
72
73 // Audio resampler output buffer
74 int16_t *audio_resample_buf;
75 int audio_resample_buf_size;
76};
77
78// ── Forward declarations ──
79static void *encoder_thread(void *arg);
80static int setup_video_stream(ACRecorder *rec);
81static int setup_audio_stream(ACRecorder *rec);
82static void encode_video_frame(ACRecorder *rec, const uint32_t *pixels, int stride);
83static void encode_audio_chunk(ACRecorder *rec);
84static void flush_encoders(ACRecorder *rec);
85
86// ── Public API ──
87
88ACRecorder *recorder_create(int width, int height, int fps, unsigned int audio_rate) {
89 ACRecorder *rec = calloc(1, sizeof(ACRecorder));
90 if (!rec) return NULL;
91
92 rec->width = width;
93 rec->height = height;
94 rec->fps = fps;
95 rec->audio_src_rate = audio_rate;
96
97 // Allocate video triple-buffer
98 size_t frame_bytes = (size_t)width * height * sizeof(uint32_t);
99 for (int i = 0; i < VIDEO_SLOTS; i++) {
100 rec->video_buf[i] = malloc(frame_bytes);
101 if (!rec->video_buf[i]) {
102 for (int j = 0; j < i; j++) free(rec->video_buf[j]);
103 free(rec);
104 return NULL;
105 }
106 }
107 rec->video_stride = width;
108
109 // Allocate audio ring buffer
110 rec->audio_ring_size = AUDIO_RING_SAMPLES;
111 rec->audio_ring = calloc(rec->audio_ring_size, sizeof(int16_t));
112 if (!rec->audio_ring) {
113 for (int i = 0; i < VIDEO_SLOTS; i++) free(rec->video_buf[i]);
114 free(rec);
115 return NULL;
116 }
117
118 return rec;
119}
120
121int recorder_start(ACRecorder *rec, const char *path) {
122 if (!rec || rec->recording) return -1;
123
124 ac_log("[recorder] starting: %s (%dx%d @ %dfps, audio %uHz→48kHz)\n",
125 path, rec->width, rec->height, rec->fps, rec->audio_src_rate);
126
127 // Create output format context (fragmented MP4)
128 int ret = avformat_alloc_output_context2(&rec->fmt_ctx, NULL, "mp4", path);
129 if (ret < 0 || !rec->fmt_ctx) {
130 ac_log("[recorder] failed to create output context\n");
131 return -1;
132 }
133
134 // Setup streams
135 if (setup_video_stream(rec) < 0) {
136 avformat_free_context(rec->fmt_ctx);
137 rec->fmt_ctx = NULL;
138 return -1;
139 }
140 if (setup_audio_stream(rec) < 0) {
141 avcodec_free_context(&rec->video_enc);
142 avformat_free_context(rec->fmt_ctx);
143 rec->fmt_ctx = NULL;
144 return -1;
145 }
146
147 // Open output file
148 if (!(rec->fmt_ctx->oformat->flags & AVFMT_NOFILE)) {
149 ret = avio_open(&rec->fmt_ctx->pb, path, AVIO_FLAG_WRITE);
150 if (ret < 0) {
151 ac_log("[recorder] failed to open output file: %s\n", path);
152 avcodec_free_context(&rec->video_enc);
153 avcodec_free_context(&rec->audio_enc);
154 avformat_free_context(rec->fmt_ctx);
155 rec->fmt_ctx = NULL;
156 return -1;
157 }
158 }
159
160 // Set fragmented MP4 options (crash-safe)
161 AVDictionary *opts = NULL;
162 av_dict_set(&opts, "movflags", "frag_keyframe+empty_moov+default_base_moof", 0);
163
164 ret = avformat_write_header(rec->fmt_ctx, &opts);
165 av_dict_free(&opts);
166 if (ret < 0) {
167 ac_log("[recorder] failed to write header\n");
168 avio_closep(&rec->fmt_ctx->pb);
169 avcodec_free_context(&rec->video_enc);
170 avcodec_free_context(&rec->audio_enc);
171 avformat_free_context(rec->fmt_ctx);
172 rec->fmt_ctx = NULL;
173 return -1;
174 }
175
176 // Reset state
177 rec->video_pts = 0;
178 rec->audio_pts = 0;
179 atomic_store(&rec->video_write_idx, 0);
180 atomic_store(&rec->video_read_idx, 0);
181 atomic_store(&rec->video_ready, 0);
182 atomic_store(&rec->audio_write_pos, 0);
183 atomic_store(&rec->audio_read_pos, 0);
184 rec->stopping = 0;
185
186 clock_gettime(CLOCK_MONOTONIC, &rec->start_time);
187 rec->recording = 1;
188
189 // Start encoder thread
190 rec->thread_running = 1;
191 if (pthread_create(&rec->thread, NULL, encoder_thread, rec) != 0) {
192 ac_log("[recorder] failed to create encoder thread\n");
193 rec->recording = 0;
194 rec->thread_running = 0;
195 av_write_trailer(rec->fmt_ctx);
196 avio_closep(&rec->fmt_ctx->pb);
197 avcodec_free_context(&rec->video_enc);
198 avcodec_free_context(&rec->audio_enc);
199 avformat_free_context(rec->fmt_ctx);
200 rec->fmt_ctx = NULL;
201 return -1;
202 }
203
204 ac_log("[recorder] recording started\n");
205 return 0;
206}
207
208void recorder_submit_video(ACRecorder *rec, const uint32_t *pixels, int stride) {
209 if (!rec || !rec->recording) return;
210
211 // Copy into next write slot
212 int slot = atomic_load(&rec->video_write_idx) % VIDEO_SLOTS;
213
214 // Copy row by row in case stride differs
215 for (int y = 0; y < rec->height; y++) {
216 memcpy(rec->video_buf[slot] + y * rec->width,
217 pixels + y * stride,
218 rec->width * sizeof(uint32_t));
219 }
220
221 atomic_fetch_add(&rec->video_write_idx, 1);
222 atomic_fetch_add(&rec->video_ready, 1);
223}
224
225void recorder_submit_audio(ACRecorder *rec, const int16_t *pcm, int frames) {
226 if (!rec || !rec->recording) return;
227
228 int samples = frames * 2; // stereo interleaved
229 int wp = atomic_load(&rec->audio_write_pos);
230
231 for (int i = 0; i < samples; i++) {
232 rec->audio_ring[(wp + i) % rec->audio_ring_size] = pcm[i];
233 }
234 atomic_fetch_add(&rec->audio_write_pos, samples);
235}
236
237void recorder_stop(ACRecorder *rec) {
238 if (!rec || !rec->recording) return;
239
240 ac_log("[recorder] stopping...\n");
241 rec->stopping = 1;
242 rec->recording = 0;
243
244 // Wait for encoder thread to finish
245 if (rec->thread_running) {
246 pthread_join(rec->thread, NULL);
247 rec->thread_running = 0;
248 }
249
250 // Flush remaining frames
251 flush_encoders(rec);
252
253 // Finalize MP4
254 if (rec->fmt_ctx) {
255 av_write_trailer(rec->fmt_ctx);
256 if (!(rec->fmt_ctx->oformat->flags & AVFMT_NOFILE))
257 avio_closep(&rec->fmt_ctx->pb);
258 }
259
260 // Free encoder resources
261 if (rec->sws) { sws_freeContext(rec->sws); rec->sws = NULL; }
262 if (rec->swr) { swr_free(&rec->swr); }
263 if (rec->video_frame) { av_frame_free(&rec->video_frame); }
264 if (rec->audio_frame) { av_frame_free(&rec->audio_frame); }
265 if (rec->video_enc) { avcodec_free_context(&rec->video_enc); }
266 if (rec->audio_enc) { avcodec_free_context(&rec->audio_enc); }
267 if (rec->fmt_ctx) { avformat_free_context(rec->fmt_ctx); rec->fmt_ctx = NULL; }
268 if (rec->audio_resample_buf) { free(rec->audio_resample_buf); rec->audio_resample_buf = NULL; }
269
270 ac_log("[recorder] stopped, file finalized\n");
271}
272
273int recorder_is_recording(ACRecorder *rec) {
274 return rec ? rec->recording : 0;
275}
276
277double recorder_elapsed(ACRecorder *rec) {
278 if (!rec || !rec->recording) return 0.0;
279 struct timespec now;
280 clock_gettime(CLOCK_MONOTONIC, &now);
281 return (now.tv_sec - rec->start_time.tv_sec) +
282 (now.tv_nsec - rec->start_time.tv_nsec) / 1e9;
283}
284
285void recorder_destroy(ACRecorder *rec) {
286 if (!rec) return;
287 if (rec->recording) recorder_stop(rec);
288
289 for (int i = 0; i < VIDEO_SLOTS; i++) free(rec->video_buf[i]);
290 free(rec->audio_ring);
291 free(rec);
292}
293
294// ── Internal: stream setup ──
295
296static int setup_video_stream(ACRecorder *rec) {
297 const AVCodec *codec = avcodec_find_encoder(AV_CODEC_ID_H264);
298 if (!codec) {
299 // Fallback to MPEG4 if H.264 not available (ffmpeg-free on Fedora)
300 codec = avcodec_find_encoder(AV_CODEC_ID_MPEG4);
301 if (!codec) {
302 ac_log("[recorder] no video encoder found\n");
303 return -1;
304 }
305 ac_log("[recorder] H.264 not available, using MPEG-4\n");
306 }
307
308 rec->video_st = avformat_new_stream(rec->fmt_ctx, NULL);
309 if (!rec->video_st) return -1;
310
311 rec->video_enc = avcodec_alloc_context3(codec);
312 if (!rec->video_enc) return -1;
313
314 rec->video_enc->width = rec->width;
315 rec->video_enc->height = rec->height;
316 rec->video_enc->time_base = (AVRational){1, rec->fps};
317 rec->video_enc->framerate = (AVRational){rec->fps, 1};
318 rec->video_enc->pix_fmt = AV_PIX_FMT_YUV420P;
319 rec->video_enc->gop_size = rec->fps; // Keyframe every second
320 rec->video_enc->max_b_frames = 0; // No B-frames for low latency
321
322 // Encoder-specific options
323 if (codec->id == AV_CODEC_ID_H264) {
324 av_opt_set(rec->video_enc->priv_data, "preset", "ultrafast", 0);
325 av_opt_set(rec->video_enc->priv_data, "tune", "zerolatency", 0);
326 rec->video_enc->bit_rate = 4000000; // 4 Mbps for crisp pixel art
327 } else {
328 rec->video_enc->bit_rate = 4000000;
329 }
330
331 if (rec->fmt_ctx->oformat->flags & AVFMT_GLOBALHEADER)
332 rec->video_enc->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
333
334 int ret = avcodec_open2(rec->video_enc, codec, NULL);
335 if (ret < 0) {
336 ac_log("[recorder] failed to open video encoder\n");
337 return -1;
338 }
339
340 ret = avcodec_parameters_from_context(rec->video_st->codecpar, rec->video_enc);
341 if (ret < 0) return -1;
342 rec->video_st->time_base = rec->video_enc->time_base;
343
344 // Allocate YUV frame
345 rec->video_frame = av_frame_alloc();
346 rec->video_frame->format = AV_PIX_FMT_YUV420P;
347 rec->video_frame->width = rec->width;
348 rec->video_frame->height = rec->height;
349 av_frame_get_buffer(rec->video_frame, 0);
350
351 // Setup color space converter (ARGB → YUV420P)
352 // Note: our ARGB32 is stored as 0xAARRGGBB in memory, which on little-endian
353 // is byte order B, G, R, A → AV_PIX_FMT_BGRA
354 rec->sws = sws_getContext(
355 rec->width, rec->height, AV_PIX_FMT_BGRA,
356 rec->width, rec->height, AV_PIX_FMT_YUV420P,
357 SWS_FAST_BILINEAR, NULL, NULL, NULL);
358 if (!rec->sws) {
359 ac_log("[recorder] failed to create sws context\n");
360 return -1;
361 }
362
363 ac_log("[recorder] video: %s %dx%d @ %dfps\n",
364 codec->name, rec->width, rec->height, rec->fps);
365 return 0;
366}
367
368static int setup_audio_stream(ACRecorder *rec) {
369 const AVCodec *codec = avcodec_find_encoder(AV_CODEC_ID_AAC);
370 if (!codec) {
371 ac_log("[recorder] AAC encoder not found, trying mp2\n");
372 codec = avcodec_find_encoder(AV_CODEC_ID_MP2);
373 if (!codec) {
374 ac_log("[recorder] no audio encoder found\n");
375 return -1;
376 }
377 }
378
379 rec->audio_st = avformat_new_stream(rec->fmt_ctx, NULL);
380 if (!rec->audio_st) return -1;
381
382 rec->audio_enc = avcodec_alloc_context3(codec);
383 if (!rec->audio_enc) return -1;
384
385 rec->audio_enc->sample_rate = 48000;
386 rec->audio_enc->bit_rate = 128000;
387 AVChannelLayout stereo = AV_CHANNEL_LAYOUT_STEREO;
388 av_channel_layout_copy(&rec->audio_enc->ch_layout, &stereo);
389 rec->audio_enc->sample_fmt = codec->sample_fmts ? codec->sample_fmts[0] : AV_SAMPLE_FMT_FLTP;
390 rec->audio_enc->time_base = (AVRational){1, 48000};
391
392 if (rec->fmt_ctx->oformat->flags & AVFMT_GLOBALHEADER)
393 rec->audio_enc->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
394
395 int ret = avcodec_open2(rec->audio_enc, codec, NULL);
396 if (ret < 0) {
397 ac_log("[recorder] failed to open audio encoder\n");
398 return -1;
399 }
400
401 ret = avcodec_parameters_from_context(rec->audio_st->codecpar, rec->audio_enc);
402 if (ret < 0) return -1;
403 rec->audio_st->time_base = rec->audio_enc->time_base;
404
405 // Allocate audio frame
406 rec->audio_frame = av_frame_alloc();
407 rec->audio_frame->format = rec->audio_enc->sample_fmt;
408 av_channel_layout_copy(&rec->audio_frame->ch_layout, &rec->audio_enc->ch_layout);
409 rec->audio_frame->sample_rate = 48000;
410 rec->audio_frame->nb_samples = rec->audio_enc->frame_size;
411 if (rec->audio_frame->nb_samples == 0)
412 rec->audio_frame->nb_samples = 1024;
413 av_frame_get_buffer(rec->audio_frame, 0);
414
415 // Setup resampler: source rate stereo int16 → 48kHz stereo float planar
416 ret = swr_alloc_set_opts2(&rec->swr,
417 &stereo, rec->audio_enc->sample_fmt, 48000,
418 &stereo, AV_SAMPLE_FMT_S16, rec->audio_src_rate,
419 0, NULL);
420 if (ret < 0 || !rec->swr) {
421 ac_log("[recorder] failed to create resampler\n");
422 return -1;
423 }
424 ret = swr_init(rec->swr);
425 if (ret < 0) {
426 ac_log("[recorder] failed to init resampler\n");
427 return -1;
428 }
429
430 ac_log("[recorder] audio: %s %uHz→48kHz, %d-sample frames\n",
431 codec->name, rec->audio_src_rate, rec->audio_frame->nb_samples);
432 return 0;
433}
434
435// ── Internal: encoder thread ──
436
437static void *encoder_thread(void *arg) {
438 ACRecorder *rec = (ACRecorder *)arg;
439
440 while (rec->recording || atomic_load(&rec->video_ready) > 0) {
441 int did_work = 0;
442
443 // Encode pending video frames
444 while (atomic_load(&rec->video_ready) > 0) {
445 int slot = atomic_load(&rec->video_read_idx) % VIDEO_SLOTS;
446 encode_video_frame(rec, rec->video_buf[slot], rec->width);
447 atomic_fetch_add(&rec->video_read_idx, 1);
448 atomic_fetch_sub(&rec->video_ready, 1);
449 did_work = 1;
450 }
451
452 // Encode pending audio
453 int avail = atomic_load(&rec->audio_write_pos) - atomic_load(&rec->audio_read_pos);
454 if (avail >= rec->audio_frame->nb_samples * 2) { // *2 for stereo
455 encode_audio_chunk(rec);
456 did_work = 1;
457 }
458
459 if (!did_work) {
460 // Sleep ~2ms to avoid busy-waiting
461 struct timespec ts = {0, 2000000};
462 nanosleep(&ts, NULL);
463 }
464 }
465
466 rec->thread_running = 0;
467 return NULL;
468}
469
470static void encode_video_frame(ACRecorder *rec, const uint32_t *pixels, int stride) {
471 // Convert ARGB32 → YUV420P
472 const uint8_t *src_data[1] = { (const uint8_t *)pixels };
473 int src_linesize[1] = { stride * 4 };
474
475 av_frame_make_writable(rec->video_frame);
476 sws_scale(rec->sws, src_data, src_linesize, 0, rec->height,
477 rec->video_frame->data, rec->video_frame->linesize);
478
479 rec->video_frame->pts = rec->video_pts++;
480
481 // Send frame to encoder
482 int ret = avcodec_send_frame(rec->video_enc, rec->video_frame);
483 if (ret < 0) return;
484
485 // Read all available packets
486 AVPacket *pkt = av_packet_alloc();
487 while (avcodec_receive_packet(rec->video_enc, pkt) == 0) {
488 av_packet_rescale_ts(pkt, rec->video_enc->time_base, rec->video_st->time_base);
489 pkt->stream_index = rec->video_st->index;
490 av_interleaved_write_frame(rec->fmt_ctx, pkt);
491 av_packet_unref(pkt);
492 }
493 av_packet_free(&pkt);
494}
495
496static void encode_audio_chunk(ACRecorder *rec) {
497 int frame_samples = rec->audio_frame->nb_samples;
498 int src_samples_needed = frame_samples * 2; // stereo interleaved
499
500 // How many source samples do we need for one output frame?
501 // At 192kHz→48kHz that's a 4:1 ratio, so we need 4x the output frame size
502 int ratio = (rec->audio_src_rate + 47999) / 48000; // ceil
503 int src_needed = frame_samples * ratio * 2; // stereo interleaved
504
505 int rp = atomic_load(&rec->audio_read_pos);
506 int avail = atomic_load(&rec->audio_write_pos) - rp;
507 if (avail < src_needed) return;
508
509 // Copy source samples from ring buffer into a contiguous buffer
510 if (!rec->audio_resample_buf || rec->audio_resample_buf_size < src_needed) {
511 free(rec->audio_resample_buf);
512 rec->audio_resample_buf_size = src_needed * 2; // over-allocate
513 rec->audio_resample_buf = malloc(rec->audio_resample_buf_size * sizeof(int16_t));
514 }
515
516 for (int i = 0; i < src_needed; i++) {
517 rec->audio_resample_buf[i] = rec->audio_ring[(rp + i) % rec->audio_ring_size];
518 }
519 atomic_fetch_add(&rec->audio_read_pos, src_needed);
520
521 // Resample and encode
522 av_frame_make_writable(rec->audio_frame);
523
524 const uint8_t *in_data[1] = { (const uint8_t *)rec->audio_resample_buf };
525 int in_samples = src_needed / 2; // frames (not samples)
526
527 int out_samples = swr_convert(rec->swr,
528 rec->audio_frame->data, frame_samples,
529 in_data, in_samples);
530
531 if (out_samples <= 0) return;
532
533 rec->audio_frame->nb_samples = out_samples;
534 rec->audio_frame->pts = rec->audio_pts;
535 rec->audio_pts += out_samples;
536
537 int ret = avcodec_send_frame(rec->audio_enc, rec->audio_frame);
538 if (ret < 0) return;
539
540 AVPacket *pkt = av_packet_alloc();
541 while (avcodec_receive_packet(rec->audio_enc, pkt) == 0) {
542 av_packet_rescale_ts(pkt, rec->audio_enc->time_base, rec->audio_st->time_base);
543 pkt->stream_index = rec->audio_st->index;
544 av_interleaved_write_frame(rec->fmt_ctx, pkt);
545 av_packet_unref(pkt);
546 }
547 av_packet_free(&pkt);
548}
549
550static void flush_encoders(ACRecorder *rec) {
551 AVPacket *pkt = av_packet_alloc();
552
553 // Flush video encoder
554 avcodec_send_frame(rec->video_enc, NULL);
555 while (avcodec_receive_packet(rec->video_enc, pkt) == 0) {
556 av_packet_rescale_ts(pkt, rec->video_enc->time_base, rec->video_st->time_base);
557 pkt->stream_index = rec->video_st->index;
558 av_interleaved_write_frame(rec->fmt_ctx, pkt);
559 av_packet_unref(pkt);
560 }
561
562 // Flush remaining audio through resampler
563 if (rec->swr) {
564 av_frame_make_writable(rec->audio_frame);
565 int flushed = swr_convert(rec->swr,
566 rec->audio_frame->data, rec->audio_frame->nb_samples,
567 NULL, 0);
568 if (flushed > 0) {
569 rec->audio_frame->nb_samples = flushed;
570 rec->audio_frame->pts = rec->audio_pts;
571 rec->audio_pts += flushed;
572 avcodec_send_frame(rec->audio_enc, rec->audio_frame);
573 }
574 }
575
576 // Flush audio encoder
577 avcodec_send_frame(rec->audio_enc, NULL);
578 while (avcodec_receive_packet(rec->audio_enc, pkt) == 0) {
579 av_packet_rescale_ts(pkt, rec->audio_enc->time_base, rec->audio_st->time_base);
580 pkt->stream_index = rec->audio_st->index;
581 av_interleaved_write_frame(rec->fmt_ctx, pkt);
582 av_packet_unref(pkt);
583 }
584
585 av_packet_free(&pkt);
586}
587
588#endif /* HAVE_AVCODEC */