//! How much each mod is used: totals of downloads and page views per mod per //! day, and nothing about who downloaded or viewed it. //! //! The same visitor downloading or viewing a mod again the same day counts //! once. Telling them apart takes a salted hash of where they connected from, //! held only in memory and only for the day: the salt is thrown away at //! midnight UTC, so not even this server can match one day's visitors with //! another's. Crawlers and other bots don't count at all, and only browsers //! count as viewing a mod's page. use std::collections::HashSet; use std::net::SocketAddr; use std::sync::Mutex; use axum::http::{header, HeaderMap}; use chrono::{NaiveDate, Utc}; use rand::RngCore as _; use sha2::{Digest, Sha256}; use sqlx::SqlitePool; use crate::atproto::id::{Did, Rkey}; use crate::page; use crate::signup::client_ip; /// Words in a user agent that say it's a program rather than a person. const BOT_WORDS: &[&str] = &[ "bot", "crawl", "spider", "slurp", "preview", "fetch", "curl", "wget", "python", "go-http-client", "java/", "okhttp", "headlesschrome", "lighthouse", "facebookexternalhit", "embedly", "monitor", // AI assistants fetching a page for someone, which don't all say "bot". "chatgpt-user", "claude-user", "perplexity-user", "mistralai-user", ]; /// Who's asking, as [`Seen`] tells visitors apart, or None for a bot, which /// shouldn't count: one whose user agent says so, or that asks for Markdown, /// which browsers never do. /// /// Anything else counts, such as an app downloading a mod from the URL its /// version record gives. pub fn visitor(headers: &HeaderMap, peer: SocketAddr) -> Option { let user_agent = headers .get(header::USER_AGENT) .and_then(|value| value.to_str().ok()) .unwrap_or_default(); if is_bot(user_agent) || page::accepts_markdown(headers) { return None; } Some(format!("{} {user_agent}", client_ip(headers, peer))) } /// [`visitor`], but only a browser, since people read a mod's page in one. /// /// Browsers send `Sec-Fetch-Mode` with every request, and other clients /// almost never do, so a bot that passes for a browser in its user agent is /// still caught without anyone having to list it. pub fn page_visitor(headers: &HeaderMap, peer: SocketAddr) -> Option { if !headers.contains_key("sec-fetch-mode") { return None; } visitor(headers, peer) } fn is_bot(user_agent: &str) -> bool { let user_agent = user_agent.to_lowercase(); user_agent.is_empty() || BOT_WORDS.iter().any(|word| user_agent.contains(word)) } /// How many visits [`Seen`] remembers before it forgets them all, so a flood /// of made-up visitors costs a few megabytes at most. Forgetting early only /// means a repeat visit can count twice. const MAX_SEEN: usize = 100_000; /// Who did what today, as salted hashes. pub struct Seen(Mutex); struct SeenToday { day: NaiveDate, salt: [u8; 32], hashes: HashSet<[u8; 32]>, } impl SeenToday { fn new(day: NaiveDate) -> Self { let mut salt = [0; 32]; rand::rng().fill_bytes(&mut salt); Self { day, salt, hashes: HashSet::new(), } } } impl Default for Seen { fn default() -> Self { Self(Mutex::new(SeenToday::new(Utc::now().date_naive()))) } } impl Seen { /// Whether this is the first time today that `visitor` (such as an /// address and user agent) did `what` (such as downloading one mod). pub fn first_today(&self, visitor: &str, what: &str) -> bool { self.first_on(Utc::now().date_naive(), visitor, what) } fn first_on(&self, day: NaiveDate, visitor: &str, what: &str) -> bool { let mut seen = self.0.lock().unwrap(); if seen.day != day || seen.hashes.len() >= MAX_SEEN { *seen = SeenToday::new(day); } let hash = Sha256::new() .chain_update(seen.salt) .chain_update(visitor.len().to_le_bytes()) .chain_update(visitor) .chain_update(what) .finalize(); seen.hashes.insert(hash.into()) } } /// Counts one download of one of `did`'s mods, today. pub async fn record_download(db: &SqlitePool, did: &Did, rkey: &Rkey) -> sqlx::Result<()> { sqlx::query!( "INSERT INTO mod_stats_day (mod_did, mod_rkey, day, downloads) VALUES (?, ?, date('now'), 1) ON CONFLICT (mod_did, mod_rkey, day) DO UPDATE SET downloads = downloads + 1", did, rkey, ) .execute(db) .await?; Ok(()) } /// Counts one view of the page of one of `did`'s mods, today. pub async fn record_view(db: &SqlitePool, did: &Did, rkey: &Rkey) -> sqlx::Result<()> { sqlx::query!( "INSERT INTO mod_stats_day (mod_did, mod_rkey, day, views) VALUES (?, ?, date('now'), 1) ON CONFLICT (mod_did, mod_rkey, day) DO UPDATE SET views = views + 1", did, rkey, ) .execute(db) .await?; Ok(()) } /// How many times a mod was downloaded and its page viewed, ever. pub struct Totals { pub downloads: i64, pub views: i64, } /// The totals of one of `did`'s mods. pub async fn totals(db: &SqlitePool, did: &Did, rkey: &Rkey) -> sqlx::Result { let row = sqlx::query!( r#"SELECT coalesce(sum(downloads), 0) AS "downloads!: i64", coalesce(sum(views), 0) AS "views!: i64" FROM mod_stats_day WHERE mod_did = ? AND mod_rkey = ?"#, did, rkey ) .fetch_one(db) .await?; Ok(Totals { downloads: row.downloads, views: row.views, }) } /// A count short enough to sit in a small space, always rounded down: "987", /// "1.2k", "4.9k", then "8k", "34k", "1.2M", "5M". pub fn compact(count: i64) -> String { fn scaled(count: i64, unit: i64, suffix: &str) -> String { let tenths = count * 10 / unit; if count / unit >= 5 || tenths % 10 == 0 { format!("{}{suffix}", count / unit) } else { format!("{}.{}{suffix}", tenths / 10, tenths % 10) } } match count { ..1_000 => count.to_string(), 1_000..1_000_000 => scaled(count, 1_000, "k"), _ => scaled(count, 1_000_000, "M"), } } /// Forgets the counts of one of `did`'s mods, once it's deleted. pub async fn forget(db: &SqlitePool, did: &Did, rkey: &Rkey) -> sqlx::Result<()> { sqlx::query!( "DELETE FROM mod_stats_day WHERE mod_did = ? AND mod_rkey = ?", did, rkey ) .execute(db) .await?; Ok(()) } #[cfg(test)] mod tests { use super::*; fn day(n: u32) -> NaiveDate { NaiveDate::from_ymd_opt(2026, 10, n).unwrap() } #[test] fn a_visitor_counts_once_a_day_for_each_thing() { let seen = Seen::default(); assert!(seen.first_on(day(1), "1.2.3.4 firefox", "mod a")); assert!(!seen.first_on(day(1), "1.2.3.4 firefox", "mod a")); assert!(seen.first_on(day(1), "1.2.3.4 firefox", "mod b")); assert!(seen.first_on(day(1), "5.6.7.8 firefox", "mod a")); // The next day, they're a stranger again. assert!(seen.first_on(day(2), "1.2.3.4 firefox", "mod a")); } #[test] fn bots_say_so_in_their_user_agents() { assert!(is_bot("Mozilla/5.0 (compatible; Googlebot/2.1)")); assert!(is_bot("Discordbot/2.0")); assert!(is_bot("curl/8.7.1")); assert!(is_bot("python-requests/2.32")); assert!(is_bot("")); assert!(is_bot("Claude-User (claude-code/2.1)")); assert!(!is_bot( "Mozilla/5.0 (X11; Linux x86_64; rv:131.0) Gecko/20100101 Firefox/131.0" )); } #[test] fn only_a_browser_views_a_page() { let firefox = "Mozilla/5.0 (X11; Linux x86_64; rv:131.0) Gecko/20100101 Firefox/131.0"; let peer = "1.2.3.4:5678".parse().unwrap(); let mut headers = HeaderMap::new(); headers.insert(header::USER_AGENT, firefox.parse().unwrap()); // A script claiming to be Firefox, which can still download. assert!(page_visitor(&headers, peer).is_none()); assert!(visitor(&headers, peer).is_some()); headers.insert("sec-fetch-mode", "navigate".parse().unwrap()); assert!(page_visitor(&headers, peer).is_some()); headers.insert(header::ACCEPT, "text/markdown".parse().unwrap()); assert!(page_visitor(&headers, peer).is_none()); assert!(visitor(&headers, peer).is_none()); } #[test] fn a_visitor_cannot_pass_for_another_by_moving_text_between_the_two() { let seen = Seen::default(); assert!(seen.first_on(day(1), "ab", "c")); assert!(seen.first_on(day(1), "a", "bc")); } async fn test_db() -> SqlitePool { let db = SqlitePool::connect("sqlite::memory:").await.unwrap(); sqlx::migrate!().run(&db).await.unwrap(); db } #[tokio::test] async fn downloads_add_up_by_day() { let db = test_db().await; let did = Did::new("did:plc:lily"); let rkey = Rkey::from_verified("3lwafzkjqm22o"); record_download(&db, &did, &rkey).await.unwrap(); record_download(&db, &did, &rkey).await.unwrap(); record_view(&db, &did, &rkey).await.unwrap(); let counts = sqlx::query!("SELECT downloads, views FROM mod_stats_day") .fetch_all(&db) .await .unwrap(); assert_eq!(counts.len(), 1); assert_eq!((counts[0].downloads, counts[0].views), (2, 1)); let totals = totals(&db, &did, &rkey).await.unwrap(); assert_eq!((totals.downloads, totals.views), (2, 1)); let nothing = super::totals(&db, &did, &Rkey::from_verified("3lwafzkjqm23o")) .await .unwrap(); assert_eq!((nothing.downloads, nothing.views), (0, 0)); forget(&db, &did, &rkey).await.unwrap(); let rows = sqlx::query_scalar!("SELECT downloads FROM mod_stats_day") .fetch_all(&db) .await .unwrap(); assert!(rows.is_empty()); } #[test] fn counts_are_shortened_to_fit() { for (count, shown) in [ (0, "0"), (987, "987"), (1_000, "1k"), (1_234, "1.2k"), (4_999, "4.9k"), (5_000, "5k"), (8_900, "8k"), (10_000, "10k"), (34_567, "34k"), (999_999, "999k"), (1_000_000, "1M"), (1_500_000, "1.5M"), (4_999_999, "4.9M"), (5_500_000, "5M"), ] { assert_eq!(compact(count), shown, "{count}"); } } }