Something went wrong. Try again.
Paper Mario mod hosting site - AT Protocol appview starhaven.dev
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340//! How much each mod is used: totals of downloads and page views per mod per//! day, and nothing about who downloaded or viewed it.//!//! The same visitor downloading or viewing a mod again the same day counts//! once. Telling them apart takes a salted hash of where they connected from,//! held only in memory and only for the day: the salt is thrown away at//! midnight UTC, so not even this server can match one day's visitors with//! another's. Crawlers and other bots don't count at all, and only browsers//! count as viewing a mod's page.
use std::collections::HashSet;use std::net::SocketAddr;use std::sync::Mutex;
use axum::http::{header, HeaderMap};use chrono::{NaiveDate, Utc};use rand::RngCore as _;use sha2::{Digest, Sha256};use sqlx::SqlitePool;
use crate::atproto::id::{Did, Rkey};use crate::page;use crate::signup::client_ip;
/// Words in a user agent that say it's a program rather than a person.const BOT_WORDS: &[&str] = &[ "bot", "crawl", "spider", "slurp", "preview", "fetch", "curl", "wget", "python", "go-http-client", "java/", "okhttp", "headlesschrome", "lighthouse", "facebookexternalhit", "embedly", "monitor", // AI assistants fetching a page for someone, which don't all say "bot". "chatgpt-user", "claude-user", "perplexity-user", "mistralai-user",];
/// Who's asking, as [`Seen`] tells visitors apart, or None for a bot, which/// shouldn't count: one whose user agent says so, or that asks for Markdown,/// which browsers never do.////// Anything else counts, such as an app downloading a mod from the URL its/// version record gives.pub fn visitor(headers: &HeaderMap, peer: SocketAddr) -> Option<String> { let user_agent = headers .get(header::USER_AGENT) .and_then(|value| value.to_str().ok()) .unwrap_or_default(); if is_bot(user_agent) || page::accepts_markdown(headers) { return None; } Some(format!("{} {user_agent}", client_ip(headers, peer)))}
/// [`visitor`], but only a browser, since people read a mod's page in one.////// Browsers send `Sec-Fetch-Mode` with every request, and other clients/// almost never do, so a bot that passes for a browser in its user agent is/// still caught without anyone having to list it.pub fn page_visitor(headers: &HeaderMap, peer: SocketAddr) -> Option<String> { if !headers.contains_key("sec-fetch-mode") { return None; } visitor(headers, peer)}
fn is_bot(user_agent: &str) -> bool { let user_agent = user_agent.to_lowercase(); user_agent.is_empty() || BOT_WORDS.iter().any(|word| user_agent.contains(word))}
/// How many visits [`Seen`] remembers before it forgets them all, so a flood/// of made-up visitors costs a few megabytes at most. Forgetting early only/// means a repeat visit can count twice.const MAX_SEEN: usize = 100_000;
/// Who did what today, as salted hashes.pub struct Seen(Mutex<SeenToday>);
struct SeenToday { day: NaiveDate, salt: [u8; 32], hashes: HashSet<[u8; 32]>,}
impl SeenToday { fn new(day: NaiveDate) -> Self { let mut salt = [0; 32]; rand::rng().fill_bytes(&mut salt); Self { day, salt, hashes: HashSet::new(), } }}
impl Default for Seen { fn default() -> Self { Self(Mutex::new(SeenToday::new(Utc::now().date_naive()))) }}
impl Seen { /// Whether this is the first time today that `visitor` (such as an /// address and user agent) did `what` (such as downloading one mod). pub fn first_today(&self, visitor: &str, what: &str) -> bool { self.first_on(Utc::now().date_naive(), visitor, what) }
fn first_on(&self, day: NaiveDate, visitor: &str, what: &str) -> bool { let mut seen = self.0.lock().unwrap(); if seen.day != day || seen.hashes.len() >= MAX_SEEN { *seen = SeenToday::new(day); } let hash = Sha256::new() .chain_update(seen.salt) .chain_update(visitor.len().to_le_bytes()) .chain_update(visitor) .chain_update(what) .finalize(); seen.hashes.insert(hash.into()) }}
/// Counts one download of one of `did`'s mods, today.pub async fn record_download(db: &SqlitePool, did: &Did, rkey: &Rkey) -> sqlx::Result<()> { sqlx::query!( "INSERT INTO mod_stats_day (mod_did, mod_rkey, day, downloads) VALUES (?, ?, date('now'), 1) ON CONFLICT (mod_did, mod_rkey, day) DO UPDATE SET downloads = downloads + 1", did, rkey, ) .execute(db) .await?; Ok(())}
/// Counts one view of the page of one of `did`'s mods, today.pub async fn record_view(db: &SqlitePool, did: &Did, rkey: &Rkey) -> sqlx::Result<()> { sqlx::query!( "INSERT INTO mod_stats_day (mod_did, mod_rkey, day, views) VALUES (?, ?, date('now'), 1) ON CONFLICT (mod_did, mod_rkey, day) DO UPDATE SET views = views + 1", did, rkey, ) .execute(db) .await?; Ok(())}
/// How many times a mod was downloaded and its page viewed, ever.pub struct Totals { pub downloads: i64, pub views: i64,}
/// The totals of one of `did`'s mods.pub async fn totals(db: &SqlitePool, did: &Did, rkey: &Rkey) -> sqlx::Result<Totals> { let row = sqlx::query!( r#"SELECT coalesce(sum(downloads), 0) AS "downloads!: i64", coalesce(sum(views), 0) AS "views!: i64" FROM mod_stats_day WHERE mod_did = ? AND mod_rkey = ?"#, did, rkey ) .fetch_one(db) .await?; Ok(Totals { downloads: row.downloads, views: row.views, })}
/// A count short enough to sit in a small space, always rounded down: "987",/// "1.2k", "4.9k", then "8k", "34k", "1.2M", "5M".pub fn compact(count: i64) -> String { fn scaled(count: i64, unit: i64, suffix: &str) -> String { let tenths = count * 10 / unit; if count / unit >= 5 || tenths % 10 == 0 { format!("{}{suffix}", count / unit) } else { format!("{}.{}{suffix}", tenths / 10, tenths % 10) } } match count { ..1_000 => count.to_string(), 1_000..1_000_000 => scaled(count, 1_000, "k"), _ => scaled(count, 1_000_000, "M"), }}
/// Forgets the counts of one of `did`'s mods, once it's deleted.pub async fn forget(db: &SqlitePool, did: &Did, rkey: &Rkey) -> sqlx::Result<()> { sqlx::query!( "DELETE FROM mod_stats_day WHERE mod_did = ? AND mod_rkey = ?", did, rkey ) .execute(db) .await?; Ok(())}
#[cfg(test)]mod tests { use super::*;
fn day(n: u32) -> NaiveDate { NaiveDate::from_ymd_opt(2026, 10, n).unwrap() }
#[test] fn a_visitor_counts_once_a_day_for_each_thing() { let seen = Seen::default();
assert!(seen.first_on(day(1), "1.2.3.4 firefox", "mod a")); assert!(!seen.first_on(day(1), "1.2.3.4 firefox", "mod a")); assert!(seen.first_on(day(1), "1.2.3.4 firefox", "mod b")); assert!(seen.first_on(day(1), "5.6.7.8 firefox", "mod a")); // The next day, they're a stranger again. assert!(seen.first_on(day(2), "1.2.3.4 firefox", "mod a")); }
#[test] fn bots_say_so_in_their_user_agents() { assert!(is_bot("Mozilla/5.0 (compatible; Googlebot/2.1)")); assert!(is_bot("Discordbot/2.0")); assert!(is_bot("curl/8.7.1")); assert!(is_bot("python-requests/2.32")); assert!(is_bot("")); assert!(is_bot("Claude-User (claude-code/2.1)")); assert!(!is_bot( "Mozilla/5.0 (X11; Linux x86_64; rv:131.0) Gecko/20100101 Firefox/131.0" )); }
#[test] fn only_a_browser_views_a_page() { let firefox = "Mozilla/5.0 (X11; Linux x86_64; rv:131.0) Gecko/20100101 Firefox/131.0"; let peer = "1.2.3.4:5678".parse().unwrap(); let mut headers = HeaderMap::new(); headers.insert(header::USER_AGENT, firefox.parse().unwrap()); // A script claiming to be Firefox, which can still download. assert!(page_visitor(&headers, peer).is_none()); assert!(visitor(&headers, peer).is_some());
headers.insert("sec-fetch-mode", "navigate".parse().unwrap()); assert!(page_visitor(&headers, peer).is_some());
headers.insert(header::ACCEPT, "text/markdown".parse().unwrap()); assert!(page_visitor(&headers, peer).is_none()); assert!(visitor(&headers, peer).is_none()); }
#[test] fn a_visitor_cannot_pass_for_another_by_moving_text_between_the_two() { let seen = Seen::default();
assert!(seen.first_on(day(1), "ab", "c")); assert!(seen.first_on(day(1), "a", "bc")); }
async fn test_db() -> SqlitePool { let db = SqlitePool::connect("sqlite::memory:").await.unwrap(); sqlx::migrate!().run(&db).await.unwrap(); db }
#[tokio::test] async fn downloads_add_up_by_day() { let db = test_db().await; let did = Did::new("did:plc:lily"); let rkey = Rkey::from_verified("3lwafzkjqm22o");
record_download(&db, &did, &rkey).await.unwrap(); record_download(&db, &did, &rkey).await.unwrap();
record_view(&db, &did, &rkey).await.unwrap();
let counts = sqlx::query!("SELECT downloads, views FROM mod_stats_day") .fetch_all(&db) .await .unwrap(); assert_eq!(counts.len(), 1); assert_eq!((counts[0].downloads, counts[0].views), (2, 1));
let totals = totals(&db, &did, &rkey).await.unwrap(); assert_eq!((totals.downloads, totals.views), (2, 1)); let nothing = super::totals(&db, &did, &Rkey::from_verified("3lwafzkjqm23o")) .await .unwrap(); assert_eq!((nothing.downloads, nothing.views), (0, 0));
forget(&db, &did, &rkey).await.unwrap(); let rows = sqlx::query_scalar!("SELECT downloads FROM mod_stats_day") .fetch_all(&db) .await .unwrap(); assert!(rows.is_empty()); }
#[test] fn counts_are_shortened_to_fit() { for (count, shown) in [ (0, "0"), (987, "987"), (1_000, "1k"), (1_234, "1.2k"), (4_999, "4.9k"), (5_000, "5k"), (8_900, "8k"), (10_000, "10k"), (34_567, "34k"), (999_999, "999k"), (1_000_000, "1M"), (1_500_000, "1.5M"), (4_999_999, "4.9M"), (5_500_000, "5M"), ] { assert_eq!(compact(count), shown, "{count}"); } }}