feat: add automated image moderation via Claude vision (#687)

adds automated scanning of uploaded images (track covers, album covers)
for policy violations using Claude Sonnet vision capabilities.

## moderation service (rust)
- new `/scan-image` endpoint accepts multipart form with image + image_id
- `claude.rs`: Claude API client with vision support
- `image_scans` table for cost tracking and audit trail
- auto-flags unsafe images to `sensitive_images` table

## backend (python)
- `ModerationClient.scan_image()` method for calling the new endpoint
- integration in `upload_track_image()` and album cover upload
- `image_moderation_enabled` setting (default: true)

moderation is best-effort - failures are logged but don't block uploads.
flagged images are blurred in the UI (existing sensitive_images behavior).

closes #166

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-authored-by: Claude Opus 4.5 <[e-mail]>

backend/src/backend/_internal/moderation_client.py

···
     urls: list[str]
 
 
+@dataclass
+class ScanImageResult:
+    """result from scanning an image for policy violations."""
+
+    is_safe: bool
+    reason: str | None
+    severity: str
+    violated_categories: list[str]
+
+
 class ModerationClient:
     """client for the plyr.fm moderation service.
 
···
                 image_ids=data.get("image_ids", []),
                 urls=data.get("urls", []),
             )
+
+    async def scan_image(
+        self, image_bytes: bytes, image_id: str, content_type: str = "image/png"
+    ) -> ScanImageResult:
+        """scan an image for policy violations using claude vision.
+
+        the moderation service will:
+        - analyze the image using claude
+        - store the scan result for cost tracking
+        - automatically flag the image if it violates policy
+
+        args:
+            image_bytes: raw image bytes
+            image_id: identifier for tracking (e.g., r2 file id)
+            content_type: mime type of the image
+
+        returns:
+            ScanImageResult with moderation decision
+
+        raises:
+            httpx.HTTPStatusError: on non-2xx response
+            httpx.TimeoutException: on timeout
+        """
+        # use a longer timeout for image moderation (claude API call)
+        timeout = httpx.Timeout(30.0)
+
+        async with httpx.AsyncClient(timeout=timeout) as client:
+            response = await client.post(
+                f"{self.labeler_url}/scan-image",
+                files={"image": (image_id, image_bytes, content_type)},
+                data={"image_id": image_id},
+                headers=self._headers(),
+            )
+            response.raise_for_status()
+            data = response.json()
+
+            result = ScanImageResult(
+                is_safe=data.get("is_safe", True),
+                reason=data.get("reason"),
+                severity=data.get("severity", "safe"),
+                violated_categories=data.get("violated_categories", []),
+            )
+
+            logfire.info(
+                "image scan completed",
+                image_id=image_id,
+                is_safe=result.is_safe,
+                severity=result.severity,
+            )
+
+            return result
 
     async def get_active_labels(self, uris: list[str]) -> set[str]:
         """check which URIs have active (non-negated) copyright-violation labels.

backend/src/backend/api/albums.py

···
 from backend._internal import Session as AuthSession
 from backend._internal import require_artist_profile
 from backend._internal.auth import get_session
+from backend._internal.moderation_client import get_moderation_client
+from backend.config import settings
 from backend.models import Album, Artist, Track, TrackLike, get_db
 from backend.schemas import TrackResponse
 from backend.storage import storage
···
 
         # construct R2 URL directly (images are stored under images/ prefix)
         image_url = f"{storage.public_image_bucket_url}/images/{image_id}{ext}"
+
+        # scan image for policy violations (non-blocking)
+        if settings.moderation.image_moderation_enabled:
+            try:
+                client = get_moderation_client()
+                content_type = {
+                    ".jpg": "image/jpeg",
+                    ".jpeg": "image/jpeg",
+                    ".png": "image/png",
+                    ".webp": "image/webp",
+                }.get(ext, "image/png")
+                await client.scan_image(bytes(image_data), image_id, content_type)
+                # if image is flagged, it's automatically added to sensitive_images
+                # by the moderation service. the image is still saved and returned.
+            except Exception as e:
+                # log but don't block upload - moderation is best-effort
+                logger.warning("image moderation failed for %s: %s", image_id, e)
 
         # delete old image if exists (prevent R2 object leaks)
         if album.image_id:

backend/src/backend/api/tracks/metadata_service.py

···
 
 import asyncio
 import json
+import logging
 from io import BytesIO
 from typing import TYPE_CHECKING, Any
 
···
 
 from backend._internal.atproto.handles import resolve_handle
 from backend._internal.image import ImageFormat
+from backend._internal.moderation_client import get_moderation_client
+from backend.config import settings
 from backend.models import Track
 from backend.storage import storage
 
 from .constants import MAX_FEATURES
 from .services import get_or_create_album
+
+logger = logging.getLogger(__name__)
 
 MAX_IMAGE_SIZE_BYTES = 20 * 1024 * 1024  # 20MB
 
···
     if not image.filename:
         raise HTTPException(status_code=400, detail="image filename missing")
 
-    _image_format, is_valid = ImageFormat.validate_and_extract(image.filename)
+    image_format, is_valid = ImageFormat.validate_and_extract(image.filename)
     if not is_valid:
         raise HTTPException(
             status_code=400,
···
     image_obj = BytesIO(image_data)
     image_id = await storage.save(image_obj, f"images/{image.filename}")
     image_url = await storage.get_url(image_id, file_type="image")
+
+    # scan image for policy violations (non-blocking)
+    if settings.moderation.image_moderation_enabled:
+        try:
+            client = get_moderation_client()
+            content_type = image_format.mime_type if image_format else "image/png"
+            await client.scan_image(image_data, image_id, content_type)
+            # note: if image is flagged, it's automatically added to sensitive_images
+            # by the moderation service. the image is still saved and returned -
+            # sensitive images are just blurred in the UI, not rejected.
+        except Exception as e:
+            # log but don't block upload - moderation is best-effort
+            logger.warning("image moderation failed for %s: %s", image_id, e)
 
     return image_id, image_url

backend/src/backend/config.py

···
         default=300,
         description="TTL in seconds for cached copyright label status (default 5 min)",
     )
+    image_moderation_enabled: bool = Field(
+        default=True,
+        description="Enable image moderation via Claude vision on upload",
+    )
 
 
 class DocketSettings(AppSettingsSection):

moderation/Cargo.lock

···
  "matchit",
  "memchr",
  "mime",
+ "multer",
  "percent-encoding",
  "pin-project-lite",
  "rustversion",
···
  "sec1",
  "subtle",
  "zeroize",
+]
+
+[[package]]
+name = "encoding_rs"
+version = "0.8.35"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3"
+dependencies = [
+ "cfg-if",
 ]
 
 [[package]]
···
 dependencies = [
  "anyhow",
  "axum",
+ "base64",
  "bytes",
  "chrono",
  "futures",
···
  "tower-http",
  "tracing",
  "tracing-subscriber",
+]
+
+[[package]]
+name = "multer"
+version = "3.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b"
+dependencies = [
+ "bytes",
+ "encoding_rs",
+ "futures-util",
+ "http",
+ "httparse",
+ "memchr",
+ "mime",
+ "spin",
+ "version_check",
 ]
 
 [[package]]

moderation/Cargo.toml

···
 
 [dependencies]
 anyhow = "1.0"
-axum = { version = "0.7", features = ["macros", "json", "ws"] }
+axum = { version = "0.7", features = ["macros", "json", "ws", "multipart"] }
+base64 = "0.22"
 rand = "0.8"
 bytes = "1.0"
 chrono = { version = "0.4", features = ["serde"] }

moderation/src/claude.rs

···
+//! Claude API client for image moderation using structured outputs.
+
+use base64::{engine::general_purpose::STANDARD, Engine};
+use serde::{Deserialize, Serialize};
+use tracing::info;
+
+const CLAUDE_API_URL: &str = "https://api.anthropic.com/v1/messages";
+const ANTHROPIC_VERSION: &str = "2023-06-01";
+const STRUCTURED_OUTPUTS_BETA: &str = "structured-outputs-2025-11-13";
+
+/// Result of image moderation analysis.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ModerationResult {
+    pub is_safe: bool,
+    pub violated_categories: Vec<String>,
+    pub severity: String,
+    pub explanation: String,
+}
+
+/// Claude API client for image moderation.
+pub struct ClaudeClient {
+    api_key: String,
+    model: String,
+    http: reqwest::Client,
+}
+
+impl ClaudeClient {
+    pub fn new(api_key: String, model: Option<String>) -> Self {
+        Self {
+            api_key,
+            model: model.unwrap_or_else(|| "claude-sonnet-4-5-20250514".to_string()),
+            http: reqwest::Client::new(),
+        }
+    }
+
+    /// Analyze an image for policy violations using structured outputs.
+    pub async fn analyze_image(
+        &self,
+        image_bytes: &[u8],
+        media_type: &str,
+    ) -> anyhow::Result<ModerationResult> {
+        let b64 = STANDARD.encode(image_bytes);
+
+        // Build request with structured output schema
+        let request = serde_json::json!({
+            "model": self.model,
+            "max_tokens": 1024,
+            "messages": [{
+                "role": "user",
+                "content": [
+                    {
+                        "type": "text",
+                        "text": MODERATION_PROMPT
+                    },
+                    {
+                        "type": "image",
+                        "source": {
+                            "type": "base64",
+                            "media_type": media_type,
+                            "data": b64
+                        }
+                    }
+                ]
+            }],
+            // Structured output schema - guarantees valid JSON matching this schema
+            "output_format": {
+                "type": "json_schema",
+                "schema": {
+                    "type": "object",
+                    "properties": {
+                        "is_safe": {
+                            "type": "boolean",
+                            "description": "Whether the image passes moderation"
+                        },
+                        "violated_categories": {
+                            "type": "array",
+                            "items": { "type": "string" },
+                            "description": "List of policy categories violated, empty if safe"
+                        },
+                        "severity": {
+                            "type": "string",
+                            "enum": ["safe", "low", "medium", "high"],
+                            "description": "Severity level of the violation"
+                        },
+                        "explanation": {
+                            "type": "string",
+                            "description": "Brief explanation of the moderation decision"
+                        }
+                    },
+                    "required": ["is_safe", "violated_categories", "severity", "explanation"],
+                    "additionalProperties": false
+                }
+            }
+        });
+
+        info!(model = %self.model, "analyzing image with structured outputs");
+
+        let response = self
+            .http
+            .post(CLAUDE_API_URL)
+            .header("x-api-key", &self.api_key)
+            .header("anthropic-version", ANTHROPIC_VERSION)
+            .header("anthropic-beta", STRUCTURED_OUTPUTS_BETA)
+            .header("content-type", "application/json")
+            .json(&request)
+            .send()
+            .await?;
+
+        if !response.status().is_success() {
+            let status = response.status();
+            let body = response.text().await.unwrap_or_default();
+            anyhow::bail!("claude API error {status}: {body}");
+        }
+
+        let response: ClaudeResponse = response.json().await?;
+
+        // Check for refusal
+        if response.stop_reason == Some("refusal".to_string()) {
+            anyhow::bail!("claude refused to analyze the image");
+        }
+
+        // Check for max_tokens cutoff
+        if response.stop_reason == Some("max_tokens".to_string()) {
+            anyhow::bail!("response was cut off due to max_tokens limit");
+        }
+
+        // Extract text content - guaranteed to be valid JSON matching our schema
+        let text = response
+            .content
+            .into_iter()
+            .find_map(|block| {
+                if block.content_type == "text" {
+                    block.text
+                } else {
+                    None
+                }
+            })
+            .ok_or_else(|| anyhow::anyhow!("no text content in response"))?;
+
+        // Direct JSON parse - no string manipulation needed thanks to structured outputs
+        serde_json::from_str(&text)
+            .map_err(|e| anyhow::anyhow!("failed to parse structured output: {e}"))
+    }
+}
+
+#[derive(Debug, Deserialize)]
+struct ClaudeResponse {
+    content: Vec<ContentBlock>,
+    stop_reason: Option<String>,
+}
+
+#[derive(Debug, Deserialize)]
+struct ContentBlock {
+    #[serde(rename = "type")]
+    content_type: String,
+    text: Option<String>,
+}
+
+const MODERATION_PROMPT: &str = r#"You are a content moderator for a music streaming platform. Analyze the provided image (album/track cover art) for policy violations.
+
+Check for:
+1. Explicit sexual content (nudity, pornography)
+2. Extreme violence or gore
+3. Hate symbols or content
+4. Illegal content
+5. Graphic drug use imagery
+
+Note: Artistic nudity in album art (like classic rock covers) may be acceptable if not explicit/pornographic.
+
+Analyze the image and provide your moderation decision."#;
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_parse_safe_response() {
+        let response = r#"{"is_safe": true, "violated_categories": [], "severity": "safe", "explanation": "Normal album artwork"}"#;
+        let result: ModerationResult = serde_json::from_str(response).unwrap();
+        assert!(result.is_safe);
+        assert!(result.violated_categories.is_empty());
+        assert_eq!(result.severity, "safe");
+    }
+
+    #[test]
+    fn test_parse_unsafe_response() {
+        let response = r#"{"is_safe": false, "violated_categories": ["explicit_sexual"], "severity": "high", "explanation": "Contains explicit nudity"}"#;
+        let result: ModerationResult = serde_json::from_str(response).unwrap();
+        assert!(!result.is_safe);
+        assert_eq!(result.violated_categories, vec!["explicit_sexual"]);
+        assert_eq!(result.severity, "high");
+    }
+}

moderation/src/config.rs

···
     pub database_url: Option<String>,
     pub labeler_did: Option<String>,
     pub labeler_signing_key: Option<String>,
+    /// Anthropic API key for Claude image moderation
+    pub claude_api_key: Option<String>,
+    /// Claude model to use (default: claude-sonnet-4-5-20250514)
+    pub claude_model: String,
 }
 
 impl Config {
···
             database_url: env::var("MODERATION_DATABASE_URL").ok(),
             labeler_did: env::var("MODERATION_LABELER_DID").ok(),
             labeler_signing_key: env::var("MODERATION_LABELER_SIGNING_KEY").ok(),
+            claude_api_key: env::var("ANTHROPIC_API_KEY").ok(),
+            claude_model: env::var("MODERATION_CLAUDE_MODEL")
+                .unwrap_or_else(|_| "claude-sonnet-4-5-20250514".to_string()),
         })
+    }
+
+    /// Check if Claude image moderation is enabled.
+    pub fn claude_enabled(&self) -> bool {
+        self.claude_api_key.is_some() && self.database_url.is_some()
     }
 
     /// Check if labeler is fully configured.

moderation/src/db.rs

···
             .execute(&self.pool)
             .await?;
 
+        // Image scans table for tracking automated moderation
+        sqlx::query(
+            r#"
+            CREATE TABLE IF NOT EXISTS image_scans (
+                id BIGSERIAL PRIMARY KEY,
+                image_id TEXT NOT NULL,
+                is_safe BOOLEAN NOT NULL,
+                violated_categories JSONB,
+                severity TEXT,
+                explanation TEXT,
+                scanned_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+                model TEXT
+            )
+            "#,
+        )
+        .execute(&self.pool)
+        .await?;
+
+        sqlx::query("CREATE INDEX IF NOT EXISTS idx_image_scans_image_id ON image_scans(image_id)")
+            .execute(&self.pool)
+            .await?;
+        sqlx::query("CREATE INDEX IF NOT EXISTS idx_image_scans_is_safe ON image_scans(is_safe)")
+            .execute(&self.pool)
+            .await?;
+
         // Review batches for mobile-friendly flag review
         sqlx::query(
             r#"
···
             .await?;
         Ok(result.rows_affected() > 0)
     }
+
+    // -------------------------------------------------------------------------
+    // Image scans
+    // -------------------------------------------------------------------------
+
+    /// Store an image scan result.
+    pub async fn store_image_scan(
+        &self,
+        image_id: &str,
+        is_safe: bool,
+        violated_categories: &[String],
+        severity: &str,
+        explanation: &str,
+        model: &str,
+    ) -> Result<i64, sqlx::Error> {
+        let categories_json = serde_json::to_value(violated_categories).unwrap_or_default();
+        sqlx::query_scalar::<_, i64>(
+            r#"
+            INSERT INTO image_scans (image_id, is_safe, violated_categories, severity, explanation, model)
+            VALUES ($1, $2, $3, $4, $5, $6)
+            RETURNING id
+            "#,
+        )
+        .bind(image_id)
+        .bind(is_safe)
+        .bind(categories_json)
+        .bind(severity)
+        .bind(explanation)
+        .bind(model)
+        .fetch_one(&self.pool)
+        .await
+    }
+
+    /// Get image scan stats for cost tracking.
+    pub async fn get_image_scan_stats(&self) -> Result<ImageScanStats, sqlx::Error> {
+        let row: (i64, i64, i64) = sqlx::query_as(
+            r#"
+            SELECT
+                COUNT(*) as total,
+                COUNT(*) FILTER (WHERE is_safe = true) as safe,
+                COUNT(*) FILTER (WHERE is_safe = false) as flagged
+            FROM image_scans
+            "#,
+        )
+        .fetch_one(&self.pool)
+        .await?;
+
+        Ok(ImageScanStats {
+            total: row.0,
+            safe: row.1,
+            flagged: row.2,
+        })
+    }
+}
+
+/// Statistics for image scans.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ImageScanStats {
+    pub total: i64,
+    pub safe: i64,
+    pub flagged: i64,
 }
 
 impl LabelRow {

moderation/src/handlers.rs

···
 //! HTTP request handlers for core endpoints.
 
-use axum::{extract::State, response::Html, Json};
+use axum::{
+    extract::{Multipart, State},
+    response::Html,
+    Json,
+};
 use serde::{Deserialize, Serialize};
 use tracing::info;
 
···
     let urls: Vec<String> = images.iter().filter_map(|i| i.url.clone()).collect();
 
     Ok(Json(SensitiveImagesResponse { image_ids, urls }))
+}
+
+// --- image moderation ---
+
+/// Response from image scanning endpoint.
+#[derive(Debug, Serialize)]
+pub struct ScanImageResponse {
+    pub is_safe: bool,
+    pub reason: Option<String>,
+    pub severity: String,
+    pub violated_categories: Vec<String>,
+}
+
+/// Scan an image for policy violations using Claude vision.
+///
+/// Accepts multipart form with:
+/// - `image`: the image file to scan
+/// - `image_id`: identifier for tracking (e.g., R2 file ID)
+///
+/// Returns moderation result. If image is not safe, it's automatically
+/// added to the sensitive_images table.
+pub async fn scan_image(
+    State(state): State<AppState>,
+    mut multipart: Multipart,
+) -> Result<Json<ScanImageResponse>, AppError> {
+    let claude = state
+        .claude
+        .as_ref()
+        .ok_or(AppError::ImageModerationNotConfigured)?;
+    let db = state
+        .db
+        .as_ref()
+        .ok_or(AppError::ImageModerationNotConfigured)?;
+
+    let mut image_bytes: Option<Vec<u8>> = None;
+    let mut image_id: Option<String> = None;
+    let mut media_type = "image/png".to_string();
+
+    // Parse multipart form
+    while let Some(field) = multipart
+        .next_field()
+        .await
+        .map_err(|e| AppError::BadRequest(format!("multipart error: {e}")))?
+    {
+        let name = field.name().unwrap_or_default().to_string();
+
+        match name.as_str() {
+            "image" => {
+                // Get content type from field
+                if let Some(ct) = field.content_type() {
+                    media_type = ct.to_string();
+                }
+                image_bytes = Some(
+                    field
+                        .bytes()
+                        .await
+                        .map_err(|e| AppError::BadRequest(format!("failed to read image: {e}")))?
+                        .to_vec(),
+                );
+            }
+            "image_id" => {
+                image_id = Some(
+                    field
+                        .text()
+                        .await
+                        .map_err(|e| AppError::BadRequest(format!("failed to read image_id: {e}")))?,
+                );
+            }
+            _ => {}
+        }
+    }
+
+    let image_bytes =
+        image_bytes.ok_or_else(|| AppError::BadRequest("missing 'image' field".to_string()))?;
+    let image_id =
+        image_id.ok_or_else(|| AppError::BadRequest("missing 'image_id' field".to_string()))?;
+
+    info!(image_id = %image_id, size = image_bytes.len(), "scanning image");
+
+    // Call Claude for analysis
+    let result = claude
+        .analyze_image(&image_bytes, &media_type)
+        .await
+        .map_err(|e| AppError::Claude(e.to_string()))?;
+
+    // Store scan result for cost tracking
+    db.store_image_scan(
+        &image_id,
+        result.is_safe,
+        &result.violated_categories,
+        &result.severity,
+        &result.explanation,
+        "claude-sonnet-4-5-20250514", // TODO: get from client
+    )
+    .await?;
+
+    // If not safe, add to sensitive images
+    if !result.is_safe {
+        info!(image_id = %image_id, severity = %result.severity, "flagging sensitive image");
+        db.add_sensitive_image(
+            Some(&image_id),
+            None,
+            Some(&result.explanation),
+            Some("claude-auto"),
+        )
+        .await?;
+    }
+
+    Ok(Json(ScanImageResponse {
+        is_safe: result.is_safe,
+        reason: if result.is_safe {
+            None
+        } else {
+            Some(result.explanation)
+        },
+        severity: result.severity,
+        violated_categories: result.violated_categories,
+    }))
 }
 
 #[cfg(test)]

moderation/src/main.rs

···
 mod admin;
 mod audd;
 mod auth;
+mod claude;
 mod config;
 mod db;
 mod handlers;
···
         (None, None, None)
     };
 
+    // Initialize Claude client for image moderation if configured
+    let claude_client = if config.claude_enabled() {
+        let client = claude::ClaudeClient::new(
+            config.claude_api_key.clone().unwrap(),
+            Some(config.claude_model.clone()),
+        );
+        info!(model = %config.claude_model, "claude image moderation enabled");
+        Some(client)
+    } else {
+        warn!("claude not configured - /scan-image endpoint will return 503");
+        None
+    };
+
     let state = AppState {
         audd_api_token: config.audd_api_token,
         audd_api_url: config.audd_api_url,
         db: db.map(Arc::new),
         signer: signer.map(Arc::new),
         label_tx,
+        claude: claude_client.map(Arc::new),
     };
 
     let app = Router::new()
···
         .route("/sensitive-images", get(handlers::get_sensitive_images))
         // AuDD scanning
         .route("/scan", post(audd::scan))
+        // Image moderation via Claude
+        .route("/scan-image", post(handlers::scan_image))
         // Label emission (internal API)
         .route("/emit-label", post(handlers::emit_label))
         // Admin UI and API

moderation/src/state.rs

···
 use tokio::sync::broadcast;
 use tracing::error;
 
+use crate::claude::ClaudeClient;
 use crate::db::LabelDb;
 use crate::labels::{Label, LabelError, LabelSigner};
 
···
     pub db: Option<Arc<LabelDb>>,
     pub signer: Option<Arc<LabelSigner>>,
     pub label_tx: Option<broadcast::Sender<(i64, Label)>>,
+    /// Claude client for image moderation (if configured)
+    pub claude: Option<Arc<ClaudeClient>>,
 }
 
 /// Application error type.
···
     #[error("audd error: {0}")]
     Audd(String),
 
+    #[error("claude error: {0}")]
+    Claude(String),
+
+    #[error("image moderation not configured")]
+    ImageModerationNotConfigured,
+
     #[error("labeler not configured")]
     LabelerNotConfigured,
 
···
         error!(error = %self, "request failed");
         let (status, error_type) = match &self {
             AppError::Audd(_) => (StatusCode::BAD_GATEWAY, "AuddError"),
+            AppError::Claude(_) => (StatusCode::BAD_GATEWAY, "ClaudeError"),
+            AppError::ImageModerationNotConfigured => {
+                (StatusCode::SERVICE_UNAVAILABLE, "ImageModerationNotConfigured")
+            }
             AppError::LabelerNotConfigured => {
                 (StatusCode::SERVICE_UNAVAILABLE, "LabelerNotConfigured")
             }