comparing modelDebug and master on nateholland.bsky.social/PoseDetection

+1 -1

posedetection/build.gradle.kts

··· 6 7 mavenPublishing { 8 publishToMavenCentral(SonatypeHost.CENTRAL_PORTAL) 9 - coordinates("com.performancecoachlab.posedetection", "posedetection-compose", "4.3.0") 10 11 pom { 12 name.set("Pose Detection")

··· 6 7 mavenPublishing { 8 publishToMavenCentral(SonatypeHost.CENTRAL_PORTAL) 9 + coordinates("com.performancecoachlab.posedetection", "posedetection-compose", "4.4.0") 10 11 pom { 12 name.set("Pose Detection")

+20 -65

posedetection/src/androidMain/kotlin/com.performancecoachlab/posedetection/camera/CameraView.android.kt

··· 55 import kotlinx.coroutines.launch 56 import org.tensorflow.lite.support.image.TensorImage 57 import org.tensorflow.lite.task.vision.detector.ObjectDetector 58 - import android.graphics.Paint 59 - import android.graphics.RectF 60 - import android.graphics.Color as AndroidColor 61 62 // Data class to hold recording state for each recording ID 63 data class RecordingSlot( ··· 211 val objectClient = if (currentDetectMode.doObject()) objectDetector?.getDetector() else null 212 imageProxy.process( 213 objectClient, poseClient, timestamp, area 214 - ) { analysisResult, _bitmap -> 215 - val androidBmp = _bitmap.copy(android.graphics.Bitmap.Config.ARGB_8888, true) 216 - val canvas = android.graphics.Canvas(androidBmp) 217 - 218 - val drawableObjects: List<DrawableObject> = drawObjects?.invoke(analysisResult.objects) ?: emptyList() 219 - 220 - val paint = Paint().apply { 221 - style = Paint.Style.STROKE 222 - strokeWidth = 4f // you can adjust thickness here 223 - color = AndroidColor.RED 224 - isAntiAlias = true 225 - } 226 - 227 - drawableObjects.forEach { dobj -> 228 - val box = dobj.obj.boundingBox 229 - val label = dobj.obj.labels.first().text 230 - val confidence = dobj.obj.labels.first().confidence 231 - 232 - val left = box.left 233 - val top = box.top 234 - val right = box.left + box.width 235 - val bottom = box.top + box.height 236 - val rect = RectF(left, top, right, bottom) 237 - 238 - val textPaint = Paint().apply { 239 - color = AndroidColor.RED 240 - textSize = 36f 241 - isAntiAlias = true 242 - style = Paint.Style.FILL 243 - setShadowLayer(4f, 2f, 2f, AndroidColor.BLACK) 244 - } 245 - 246 - val labelText = "$label ${(confidence * 100).toInt()}%" 247 - val textX = left + 8f 248 - val textY = top - 10f 249 - 250 - val textBounds = android.graphics.Rect() 251 - textPaint.getTextBounds(labelText, 0, labelText.length, textBounds) 252 - 253 - canvas.drawRect(rect, paint) 254 - canvas.drawText(labelText, textX, textY, textPaint) 255 } 256 - 257 - 258 - val annotatedImageBitmap: ImageBitmap = androidBmp.asImageBitmap() 259 - 260 - controller?.setRequestDataProvider { 261 - CameraViewData( 262 - width = annotatedImageBitmap.width.toFloat(), 263 - height = annotatedImageBitmap.height.toFloat(), 264 - rotation = when (imageProxy.imageInfo.rotationDegrees) { 265 - 0 -> SensorRotation.ROTATION_0 266 - 90 -> SensorRotation.ROTATION_90 267 - 180 -> SensorRotation.ROTATION_180 268 - 270 -> SensorRotation.ROTATION_270 269 - else -> SensorRotation.ROTATION_0 270 } 271 - ) 272 - } 273 - 274 - addFrameToActiveRecordings(annotatedImageBitmap, timestamp) 275 - 276 - bitmap = annotatedImageBitmap 277 - 278 imageProxy.close() 279 } 280 }

··· 55 import kotlinx.coroutines.launch 56 import org.tensorflow.lite.support.image.TensorImage 57 import org.tensorflow.lite.task.vision.detector.ObjectDetector 58 59 // Data class to hold recording state for each recording ID 60 data class RecordingSlot( ··· 208 val objectClient = if (currentDetectMode.doObject()) objectDetector?.getDetector() else null 209 imageProxy.process( 210 objectClient, poseClient, timestamp, area 211 + ){ analysisResult, _bitmap -> 212 + customObjectRepository.updateCustomObject(analysisResult.objects) 213 + analysisResult.skeleton?.let { skel -> 214 + skeletonRepository.updateSkeleton(skel) 215 } 216 + bitmap = _bitmap.asImageBitmap().let { inbmp -> 217 + controller?.setRequestDataProvider { 218 + CameraViewData( 219 + width = inbmp.width.toFloat(), 220 + height = inbmp.height.toFloat(), 221 + rotation = when (imageProxy.imageInfo.rotationDegrees) { 222 + 0 -> SensorRotation.ROTATION_0 223 + 90 -> SensorRotation.ROTATION_90 224 + 180 -> SensorRotation.ROTATION_180 225 + 270 -> SensorRotation.ROTATION_270 226 + else -> SensorRotation.ROTATION_0 227 + } 228 + ) 229 + } 230 + addFrameToActiveRecordings(inbmp, timestamp) 231 + inbmp.drawResults(if(drawSkeleton) analysisResult.skeleton else null, drawObjects?.invoke(analysisResult.objects)?: emptyList()) 232 } 233 imageProxy.close() 234 } 235 }

+94 -59

posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraEngine.kt

··· 10 import androidx.compose.ui.graphics.toComposeImageBitmap 11 import androidx.compose.ui.unit.Density 12 import androidx.compose.ui.unit.LayoutDirection 13 import com.performancecoachlab.posedetection.custom.CustomObjectRespository 14 import com.performancecoachlab.posedetection.custom.ObjectModel 15 import com.performancecoachlab.posedetection.recording.AnalysisObject ··· 88 import platform.darwin.dispatch_get_global_queue 89 import platform.darwin.dispatch_get_main_queue 90 import platform.darwin.dispatch_queue_create 91 import platform.posix.memcpy 92 import kotlin.math.abs 93 import kotlin.native.runtime.NativeRuntimeApi ··· 159 160 @OptIn(ExperimentalForeignApi::class) 161 private fun setupCamera() { 162 cameraController.setupSession() 163 cameraController.setupPreviewLayer(view) 164 startSession() ··· 214 cameraController.switchCamera() 215 } 216 217 - private fun startSession() { 218 MemoryManager.clearBufferPools() 219 cameraController.startSession() 220 } ··· 288 var drawSkeleton: Boolean = true 289 var drawObjects: ((List<AnalysisObject>) -> List<DrawableObject>)? = null 290 291 sealed class CameraException : Exception() { 292 class DeviceNotAvailable : CameraException() 293 class ConfigurationError(message: String) : CameraException() ··· 295 } 296 297 fun startRecording(): String? { 298 - if (movieFileOutput == null) { 299 - movieFileOutput = AVCaptureMovieFileOutput() 300 - captureSession?.addOutput(movieFileOutput!!) 301 } 302 val outputURL = generateSegmentURL() 303 movieFileOutput?.connections?.firstOrNull()?.let { connection -> 304 (connection as AVCaptureConnection).let { avConnection -> ··· 344 } 345 346 fun setupSession() { 347 - try { 348 - captureSession = AVCaptureSession() 349 - captureSession?.beginConfiguration() 350 - captureSession?.sessionPreset = AVCaptureSessionPresetMedium 351 352 - if (!setupInputs()) { 353 - throw CameraException.DeviceNotAvailable() 354 - } 355 - setupVideoOutput() 356 - // Add movie file output for recording 357 - if (movieFileOutput == null) { 358 - movieFileOutput = AVCaptureMovieFileOutput() 359 - captureSession?.addOutput(movieFileOutput!!) 360 } 361 - captureSession?.commitConfiguration() 362 - startRecording() 363 - } catch (e: CameraException) { 364 - cleanupSession() 365 - onError?.invoke(e) 366 } 367 } 368 369 private fun setupVideoOutput() { ··· 415 } 416 417 fun startSession() { 418 - if (captureSession?.isRunning() == false) { 419 - dispatch_async( 420 - dispatch_get_global_queue( 421 - DISPATCH_QUEUE_PRIORITY_HIGH.toLong(), 0u 422 - ) 423 - ) { 424 captureSession?.startRunning() 425 } 426 } 427 } 428 429 fun stopSession() { 430 - if (captureSession?.isRunning() == true) { 431 - captureSession?.stopRunning() 432 } 433 } 434 ··· 479 fun switchCamera() { 480 guard(captureSession != null) { return@guard } 481 482 - captureSession?.beginConfiguration() 483 484 - try { 485 - captureSession?.inputs?.firstOrNull()?.let { input -> 486 - captureSession?.removeInput(input as AVCaptureInput) 487 - } 488 489 - isUsingFrontCamera = !isUsingFrontCamera 490 - currentCamera = if (isUsingFrontCamera) frontCamera else backCamera 491 492 - val newCamera = currentCamera ?: throw CameraException.DeviceNotAvailable() 493 494 - val newInput = AVCaptureDeviceInput.deviceInputWithDevice( 495 - newCamera, null 496 - ) ?: throw CameraException.ConfigurationError("Failed to create input") 497 498 - if (captureSession?.canAddInput(newInput) == true) { 499 - captureSession?.addInput(newInput) 500 - } else { 501 - throw CameraException.ConfigurationError("Cannot add input") 502 - } 503 504 - cameraPreviewLayer?.connection?.let { connection -> 505 - if (connection.isVideoMirroringSupported()) { 506 - connection.automaticallyAdjustsVideoMirroring = false 507 - connection.setVideoMirrored(isUsingFrontCamera) 508 } 509 } 510 - 511 - captureSession?.commitConfiguration() 512 - } catch (e: CameraException) { 513 - captureSession?.commitConfiguration() 514 - onError?.invoke(e) 515 - } catch (e: Exception) { 516 - captureSession?.commitConfiguration() 517 - onError?.invoke(CameraException.ConfigurationError(e.message ?: "Unknown error")) 518 } 519 } 520 ··· 572 ).let { FrameSize(it.width.toInt(), it.height.toInt()) } 573 }) 574 } 575 - previewObjects?.also { objects -> 576 customObjectRepository?.updateCustomObject(objects) 577 } 578 preview.bounds.useContents { ··· 606 } 607 } 608 } catch (e: Exception) { 609 - //println(e.message ?: "Unknown error in frame processing") 610 } 611 } 612 }

··· 10 import androidx.compose.ui.graphics.toComposeImageBitmap 11 import androidx.compose.ui.unit.Density 12 import androidx.compose.ui.unit.LayoutDirection 13 + import co.touchlab.kermit.Logger 14 import com.performancecoachlab.posedetection.custom.CustomObjectRespository 15 import com.performancecoachlab.posedetection.custom.ObjectModel 16 import com.performancecoachlab.posedetection.recording.AnalysisObject ··· 89 import platform.darwin.dispatch_get_global_queue 90 import platform.darwin.dispatch_get_main_queue 91 import platform.darwin.dispatch_queue_create 92 + import platform.darwin.dispatch_sync 93 + import platform.darwin.dispatch_time 94 import platform.posix.memcpy 95 import kotlin.math.abs 96 import kotlin.native.runtime.NativeRuntimeApi ··· 162 163 @OptIn(ExperimentalForeignApi::class) 164 private fun setupCamera() { 165 + // Run session setup synchronously so callers can safely start the session afterwards 166 cameraController.setupSession() 167 cameraController.setupPreviewLayer(view) 168 startSession() ··· 218 cameraController.switchCamera() 219 } 220 221 + fun startSession() { 222 MemoryManager.clearBufferPools() 223 cameraController.startSession() 224 } ··· 292 var drawSkeleton: Boolean = true 293 var drawObjects: ((List<AnalysisObject>) -> List<DrawableObject>)? = null 294 295 + // Serial queue to serialize session configuration and start/stop calls 296 + private val sessionQueue = dispatch_queue_create("com.performancecoachlab.captureSessionQueue", null) 297 + 298 sealed class CameraException : Exception() { 299 class DeviceNotAvailable : CameraException() 300 class ConfigurationError(message: String) : CameraException() ··· 302 } 303 304 fun startRecording(): String? { 305 + // Ensure we add outputs on the session queue to avoid races 306 + dispatch_sync(sessionQueue) { 307 + if (movieFileOutput == null) { 308 + val candidate = AVCaptureMovieFileOutput() 309 + movieFileOutput = candidate 310 + if (captureSession?.canAddOutput(candidate) == true) { 311 + captureSession?.addOutput(movieFileOutput!!) 312 + } 313 + } 314 } 315 + 316 val outputURL = generateSegmentURL() 317 movieFileOutput?.connections?.firstOrNull()?.let { connection -> 318 (connection as AVCaptureConnection).let { avConnection -> ··· 358 } 359 360 fun setupSession() { 361 + // Run synchronously on sessionQueue so callers (like setupCamera) can safely call startSession afterwards 362 + dispatch_sync(sessionQueue) { 363 + var configurationBegan = false 364 + try { 365 + captureSession = AVCaptureSession() 366 + captureSession?.beginConfiguration() 367 + configurationBegan = true 368 + captureSession?.sessionPreset = AVCaptureSessionPresetMedium 369 + 370 + if (!setupInputs()) { 371 + throw CameraException.DeviceNotAvailable() 372 + } 373 + setupVideoOutput() 374 375 + // Add movie file output for recording only if it can be added 376 + if (movieFileOutput == null) { 377 + val candidate = AVCaptureMovieFileOutput() 378 + if (captureSession?.canAddOutput(candidate) == true) { 379 + movieFileOutput = candidate 380 + captureSession?.addOutput(movieFileOutput!!) 381 + } else { 382 + // keep the instance (not added) so it can be added later when appropriate 383 + movieFileOutput = candidate 384 + } 385 + } 386 + } catch (e: CameraException) { 387 + cleanupSession() 388 + onError?.invoke(e) 389 + return@dispatch_sync 390 + } finally { 391 + if (configurationBegan) { 392 + captureSession?.commitConfiguration() 393 + } 394 } 395 } 396 + // Do not start recording here; startRecording() should be called explicitly after session is running 397 } 398 399 private fun setupVideoOutput() { ··· 445 } 446 447 fun startSession() { 448 + // Run on sessionQueue to avoid overlapping with configuration 449 + dispatch_async(sessionQueue) { 450 + if (captureSession?.isRunning() == false) { 451 captureSession?.startRunning() 452 } 453 } 454 } 455 456 fun stopSession() { 457 + dispatch_async(sessionQueue) { 458 + if (captureSession?.isRunning() == true) { 459 + captureSession?.stopRunning() 460 + } 461 } 462 } 463 ··· 508 fun switchCamera() { 509 guard(captureSession != null) { return@guard } 510 511 + // Run camera switch on sessionQueue to serialize with setupSession/startSession 512 + dispatch_sync(sessionQueue) { 513 + var configurationBegan = false 514 + try { 515 + captureSession?.beginConfiguration() 516 + configurationBegan = true 517 518 + captureSession?.inputs?.firstOrNull()?.let { input -> 519 + captureSession?.removeInput(input as AVCaptureInput) 520 + } 521 522 + isUsingFrontCamera = !isUsingFrontCamera 523 + currentCamera = if (isUsingFrontCamera) frontCamera else backCamera 524 525 + val newCamera = currentCamera ?: throw CameraException.DeviceNotAvailable() 526 527 + val newInput = AVCaptureDeviceInput.deviceInputWithDevice( 528 + newCamera, null 529 + ) ?: throw CameraException.ConfigurationError("Failed to create input") 530 531 + if (captureSession?.canAddInput(newInput) == true) { 532 + captureSession?.addInput(newInput) 533 + } else { 534 + throw CameraException.ConfigurationError("Cannot add input") 535 + } 536 537 + cameraPreviewLayer?.connection?.let { connection -> 538 + if (connection.isVideoMirroringSupported()) { 539 + connection.automaticallyAdjustsVideoMirroring = false 540 + connection.setVideoMirrored(isUsingFrontCamera) 541 + } 542 + } 543 + 544 + } catch (e: CameraException) { 545 + onError?.invoke(e) 546 + } catch (e: Exception) { 547 + onError?.invoke(CameraException.ConfigurationError(e.message ?: "Unknown error")) 548 + } finally { 549 + if (configurationBegan) { 550 + captureSession?.commitConfiguration() 551 } 552 } 553 } 554 } 555 ··· 607 ).let { FrameSize(it.width.toInt(), it.height.toInt()) } 608 }) 609 } 610 + previewObjects.also { objects -> 611 customObjectRepository?.updateCustomObject(objects) 612 } 613 preview.bounds.useContents { ··· 641 } 642 } 643 } catch (e: Exception) { 644 + // ignore frame processing errors 645 } 646 } 647 }

+5

posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraPreview.kt

··· 21 CameraEngine() 22 } 23 LifecycleEventEffect(Lifecycle.Event.ON_STOP) { 24 cameraEngine.stopSession() 25 } 26 27 LaunchedEffect(cameraEngine) {

··· 21 CameraEngine() 22 } 23 LifecycleEventEffect(Lifecycle.Event.ON_STOP) { 24 + // Stops camera when going to background 25 cameraEngine.stopSession() 26 + } 27 + LifecycleEventEffect(Lifecycle.Event.ON_RESUME) { 28 + // Resumes the camera when coming back from background 29 + cameraEngine.startSession() 30 } 31 32 LaunchedEffect(cameraEngine) {

sample/composeApp/src/androidMain/assets/detect_float32_metadata.tflite