an atproto pds written in F# (.NET 9) 🦒
pds fsharp giraffe dotnet atproto

feat: add SQLite WAL auto-checkpoint configuration

* S3-compatible blob storage support

Changed files
+45 -29
PDSharp
PDSharp.Core
PDSharp.Tests
+2
PDSharp.Core/Config.fs
··· 8 8 JwtSecret : string 9 9 /// Connection string for SQLite 10 10 SqliteConnectionString : string 11 + /// Disable SQLite WAL auto-checkpoint (for Litestream compatibility) 12 + DisableWalAutoCheckpoint : bool 11 13 /// Blob storage configuration 12 14 BlobStore : BlobStoreConfig 13 15 }
+7 -3
PDSharp.Core/SqliteStore.fs
··· 13 13 module SqliteStore = 14 14 15 15 /// Initialize the database schema 16 - let initialize (connectionString : string) = 17 - use conn = new SqliteConnection(connectionString) 18 - conn.Open() 16 + let initialize (config : AppConfig) = 17 + use conn = new SqliteConnection(config.SqliteConnectionString) 19 18 19 + conn.Open() 20 20 conn.Execute("PRAGMA journal_mode=WAL;") |> ignore 21 + 22 + if config.DisableWalAutoCheckpoint then 23 + conn.Execute("PRAGMA wal_autocheckpoint=0;") |> ignore 24 + 21 25 // TODO: fast, slightly less safe. Keep default (FULL) for now. 22 26 // conn.Execute("PRAGMA synchronous=NORMAL;") |> ignore 23 27
+2 -8
PDSharp.Tests/Handlers.Tests.fs
··· 68 68 member _.Deserialize<'T>(json : string) = JsonSerializer.Deserialize<'T> json 69 69 70 70 member _.Deserialize<'T>(bytes : byte[]) = 71 - JsonSerializer.Deserialize<'T>(ReadOnlySpan(bytes)) 71 + JsonSerializer.Deserialize<'T>(ReadOnlySpan bytes) 72 72 73 73 member _.DeserializeAsync<'T>(stream : Stream) = task { return! JsonSerializer.DeserializeAsync<'T>(stream) } 74 74 ··· 115 115 DidHost = "did:web:pds.example.com" 116 116 JwtSecret = "secret" 117 117 SqliteConnectionString = "" 118 + DisableWalAutoCheckpoint = false 118 119 BlobStore = Disk "blobs" 119 120 } 120 121 ··· 130 131 let body = JsonSerializer.Serialize req 131 132 let ctx = mockContext services body Map.empty 132 133 let next : HttpFunc = fun _ -> Task.FromResult(None) 133 - 134 134 let! result = PDSharp.Handlers.Auth.createAccountHandler next ctx 135 - 136 135 Assert.Equal(200, ctx.Response.StatusCode) 137 136 138 137 let store = accountStore :> IAccountStore ··· 144 143 let ``Server.indexHandler returns HTML`` () = task { 145 144 let ctx = new DefaultHttpContext() 146 145 let next : HttpFunc = fun _ -> Task.FromResult(None) 147 - 148 146 let! result = PDSharp.Handlers.Server.indexHandler next ctx 149 - 150 147 Assert.Equal(200, ctx.Response.StatusCode) 151 148 Assert.Equal("text/html", ctx.Response.ContentType) 152 149 } ··· 175 172 } 176 173 177 174 let body = JsonSerializer.Serialize(req) 178 - 179 175 let ctx = mockContext services body Map.empty 180 176 let next : HttpFunc = fun _ -> Task.FromResult(None) 181 - 182 177 let! result = PDSharp.Handlers.Repo.createRecordHandler next ctx 183 - 184 178 Assert.Equal(400, ctx.Response.StatusCode) 185 179 }
+4 -5
PDSharp.Tests/Tests.fs
··· 18 18 DidHost = "did:web:example.com" 19 19 JwtSecret = "test-secret-key-for-testing-only" 20 20 SqliteConnectionString = "Data Source=:memory:" 21 + DisableWalAutoCheckpoint = false 21 22 BlobStore = Disk "blobs" 22 23 } 23 24 ··· 63 64 let keyPair = generateKey P256 64 65 let data = Encoding.UTF8.GetBytes("test message") 65 66 let hash = sha256 data 66 - 67 67 let signature = sign keyPair hash 68 68 Assert.True(signature.Length = 64, "Signature should be 64 bytes (R|S)") 69 69 ··· 75 75 let keyPair = generateKey K256 76 76 let data = Encoding.UTF8.GetBytes("test k256") 77 77 let hash = sha256 data 78 - 79 78 let signature = sign keyPair hash 80 79 Assert.True(signature.Length = 64, "Signature should be 64 bytes") 81 80 ··· 117 116 [<Fact>] 118 117 let ``CID Generation from Hash`` () = 119 118 let hash = 120 - Hex.Decode("b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9") 119 + Hex.Decode "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" 121 120 122 121 let cid = Cid.FromHash hash 123 122 Assert.Equal<byte>(0x01uy, cid.Bytes.[0]) ··· 127 126 128 127 [<Fact>] 129 128 let ``DAG-CBOR Canonical Sorting`` () = 130 - let m = Map.ofList [ ("b", box 1); ("a", box 2) ] 129 + let m = Map.ofList [ "b", box 1; "a", box 2 ] 131 130 let encoded = DagCbor.encode m 132 131 let hex = Hex.ToHexString encoded 133 132 Assert.Equal("a2616102616201", hex) 134 133 135 134 [<Fact>] 136 135 let ``DAG-CBOR Sorting Length vs Bytes`` () = 137 - let m = Map.ofList [ ("aa", box 1); ("b", box 2) ] 136 + let m = Map.ofList [ "aa", box 1; "b", box 2 ] 138 137 let encoded = DagCbor.encode m 139 138 let hex = Hex.ToHexString encoded 140 139 Assert.Equal("a261620262616101", hex)
+27 -10
PDSharp/Program.fs
··· 21 21 let publicUrl = env "PDSHARP_PublicUrl" "http://localhost:5000" 22 22 let dbPath = env "PDSHARP_DbPath" "pdsharp.db" 23 23 24 + let disableWalAutoCheckpoint = 25 + env "PDSHARP_SQLITE_DISABLE_WAL_AUTO_CHECKPOINT" "false" |> bool.Parse 26 + 27 + let blobStoreConfig = 28 + match env "PDSHARP_BLOBSTORE_TYPE" "disk" with 29 + | "s3" -> 30 + S3 { 31 + Bucket = env "PDSHARP_S3_BUCKET" "pdsharp-blobs" 32 + Region = env "PDSHARP_S3_REGION" "us-east-1" 33 + AccessKey = Option.ofObj (Environment.GetEnvironmentVariable "PDSHARP_S3_ACCESS_KEY") 34 + SecretKey = Option.ofObj (Environment.GetEnvironmentVariable "PDSHARP_S3_SECRET_KEY") 35 + ServiceUrl = Option.ofObj (Environment.GetEnvironmentVariable "PDSHARP_S3_SERVICE_URL") 36 + ForcePathStyle = env "PDSHARP_S3_FORCE_PATH_STYLE" "false" |> bool.Parse 37 + } 38 + | _ -> Disk "blobs" 39 + 24 40 { 25 41 PublicUrl = publicUrl 26 42 DidHost = env "PDSHARP_DidHost" "did:web:localhost" 27 43 JwtSecret = env "PDSHARP_JwtSecret" "development-secret-do-not-use-in-prod" 28 44 SqliteConnectionString = $"Data Source={dbPath}" 29 - BlobStore = Disk "blobs" // Default to disk for now 45 + DisableWalAutoCheckpoint = disableWalAutoCheckpoint 46 + BlobStore = blobStoreConfig 30 47 } 31 48 32 49 let config = getConfig () 33 50 34 - SqliteStore.initialize config.SqliteConnectionString 51 + SqliteStore.initialize config 35 52 36 53 module App = 37 - let webApp = 54 + let appRouter = 38 55 choose [ 39 56 GET 40 57 >=> choose [ ··· 65 82 RequestErrors.NOT_FOUND "Not Found" 66 83 ] 67 84 68 - let configureApp (app : IApplicationBuilder) = 85 + let webApp (app : IApplicationBuilder) = 69 86 app.UseWebSockets() |> ignore 70 - app.UseGiraffe webApp 87 + app.UseGiraffe appRouter 71 88 72 89 let configureServices (config : AppConfig) (services : IServiceCollection) = 73 90 services.AddGiraffe() |> ignore ··· 77 94 let accountStore = new SqliteAccountStore(config.SqliteConnectionString) 78 95 let repoStore = new SqliteRepoStore(config.SqliteConnectionString) 79 96 80 - services.AddSingleton<IBlockStore>(blockStore) |> ignore 81 - services.AddSingleton<IAccountStore>(accountStore) |> ignore 82 - services.AddSingleton<IRepoStore>(repoStore) |> ignore 97 + services.AddSingleton<IBlockStore> blockStore |> ignore 98 + services.AddSingleton<IAccountStore> accountStore |> ignore 99 + services.AddSingleton<IRepoStore> repoStore |> ignore 83 100 84 101 let blobStore : IBlobStore = 85 102 match config.BlobStore with 86 103 | Disk path -> new DiskBlobStore(path) :> IBlobStore 87 104 | S3 s3Config -> new S3BlobStore(s3Config) :> IBlobStore 88 105 89 - services.AddSingleton<IBlobStore>(blobStore) |> ignore 106 + services.AddSingleton<IBlobStore> blobStore |> ignore 90 107 services.AddSingleton<FirehoseState>(new FirehoseState()) |> ignore 91 108 services.AddSingleton<SigningKeyStore>(new SigningKeyStore()) |> ignore 92 109 ··· 95 112 Host 96 113 .CreateDefaultBuilder(args) 97 114 .ConfigureWebHostDefaults(fun webHostBuilder -> 98 - webHostBuilder.Configure(configureApp).ConfigureServices(configureServices config) 115 + webHostBuilder.Configure(webApp).ConfigureServices(configureServices config) 99 116 |> ignore) 100 117 .Build() 101 118 .Run()
+3 -3
roadmap.txt
··· 61 61 -------------------------------------------------------------------------------- 62 62 Milestone J: Storage Backend Configuration 63 63 -------------------------------------------------------------------------------- 64 - - [ ] Configure SQLite WAL mode (PDS_SQLITE_DISABLE_WAL_AUTO_CHECKPOINT=true) 65 - - [ ] Implement S3-compatible blobstore adapter (optional via config) 66 - - [ ] Configure disk-based vs S3-based blob storage selection 64 + - [x] Configure SQLite WAL mode (PDS_SQLITE_DISABLE_WAL_AUTO_CHECKPOINT=true) 65 + - [x] Implement S3-compatible blobstore adapter (optional via config) 66 + - [x] Configure disk-based vs S3-based blob storage selection 67 67 DoD: PDS runs with S3 blobs (if configured) and SQLite passes Litestream checks 68 68 -------------------------------------------------------------------------------- 69 69 Milestone K: Backup Automation + Guardrails