+66 -293

Cargo.lock

··· 112 112 113 113 [[package]] 114 114 name = "anyhow" 115 115 - version = "1.0.100" 115 115 + version = "1.0.97" 116 116 source = "registry+https://github.com/rust-lang/crates.io-index" 117 117 - checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" 117 117 + checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f" 118 118 119 119 [[package]] 120 120 name = "arbitrary" ··· 127 127 version = "1.7.1" 128 128 source = "registry+https://github.com/rust-lang/crates.io-index" 129 129 checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" 130 130 - 131 131 - [[package]] 132 132 - name = "arrayref" 133 133 - version = "0.3.9" 134 134 - source = "registry+https://github.com/rust-lang/crates.io-index" 135 135 - checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" 136 130 137 131 [[package]] 138 132 name = "arrayvec" ··· 198 192 "nom", 199 193 "num-traits", 200 194 "rusticata-macros", 201 201 - "thiserror 2.0.17", 195 195 + "thiserror 2.0.16", 202 196 "time", 203 197 ] 204 198 ··· 650 644 "axum", 651 645 "handlebars", 652 646 "serde", 653 653 - "thiserror 2.0.17", 647 647 + "thiserror 2.0.16", 654 648 ] 655 649 656 650 [[package]] ··· 679 673 version = "0.2.0" 680 674 source = "registry+https://github.com/rust-lang/crates.io-index" 681 675 checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" 682 682 - 683 683 - [[package]] 684 684 - name = "base256emoji" 685 685 - version = "1.0.2" 686 686 - source = "registry+https://github.com/rust-lang/crates.io-index" 687 687 - checksum = "b5e9430d9a245a77c92176e649af6e275f20839a48389859d1661e9a128d077c" 688 688 - dependencies = [ 689 689 - "const-str", 690 690 - "match-lookup", 691 691 - ] 692 676 693 677 [[package]] 694 678 name = "base64" ··· 828 812 checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" 829 813 830 814 [[package]] 831 831 - name = "blake3" 832 832 - version = "1.8.2" 833 833 - source = "registry+https://github.com/rust-lang/crates.io-index" 834 834 - checksum = "3888aaa89e4b2a40fca9848e400f6a658a5a3978de7be858e209cafa8be9a4a0" 835 835 - dependencies = [ 836 836 - "arrayref", 837 837 - "arrayvec", 838 838 - "cc", 839 839 - "cfg-if", 840 840 - "constant_time_eq", 841 841 - ] 842 842 - 843 843 - [[package]] 844 815 name = "block-buffer" 845 816 version = "0.10.4" 846 817 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 868 839 checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" 869 840 870 841 [[package]] 871 871 - name = "byteorder-lite" 872 872 - version = "0.1.0" 873 873 - source = "registry+https://github.com/rust-lang/crates.io-index" 874 874 - checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" 875 875 - 876 876 - [[package]] 877 842 name = "bytes" 878 843 version = "1.10.1" 879 844 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 886 851 checksum = "6236364b88b9b6d0bc181ba374cf1ab55ba3ef97a1cb6f8cddad48a273767fb5" 887 852 888 853 [[package]] 889 889 - name = "byteview" 890 890 - version = "0.8.0" 891 891 - source = "registry+https://github.com/rust-lang/crates.io-index" 892 892 - checksum = "1e6b0e42e210b794e14b152c6fe1a55831e30ef4a0f5dc39d73d714fb5f1906c" 893 893 - 894 894 - [[package]] 895 854 name = "bzip2-sys" 896 855 version = "0.1.13+1.0.8" 897 856 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 933 892 ] 934 893 935 894 [[package]] 936 936 - name = "cbor4ii" 937 937 - version = "0.2.14" 938 938 - source = "registry+https://github.com/rust-lang/crates.io-index" 939 939 - checksum = "b544cf8c89359205f4f990d0e6f3828db42df85b5dac95d09157a250eb0749c4" 940 940 - dependencies = [ 941 941 - "serde", 942 942 - ] 943 943 - 944 944 - [[package]] 945 945 - name = "cbor4ii" 946 946 - version = "1.2.0" 947 947 - source = "registry+https://github.com/rust-lang/crates.io-index" 948 948 - checksum = "b28d2802395e3bccd95cc4ae984bff7444b6c1f5981da46a41360c42a2c7e2d9" 949 949 - 950 950 - [[package]] 951 895 name = "cc" 952 896 version = "1.2.18" 953 897 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1032 976 "multihash", 1033 977 "serde", 1034 978 "serde_bytes", 1035 1035 - "unsigned-varint 0.8.0", 979 979 + "unsigned-varint", 1036 980 ] 1037 981 1038 982 [[package]] ··· 1143 1087 checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" 1144 1088 1145 1089 [[package]] 1146 1146 - name = "const-str" 1147 1147 - version = "0.4.3" 1148 1148 - source = "registry+https://github.com/rust-lang/crates.io-index" 1149 1149 - checksum = "2f421161cb492475f1661ddc9815a745a1c894592070661180fdec3d4872e9c3" 1150 1150 - 1151 1151 - [[package]] 1152 1152 - name = "constant_time_eq" 1153 1153 - version = "0.3.1" 1154 1154 - source = "registry+https://github.com/rust-lang/crates.io-index" 1155 1155 - checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" 1156 1156 - 1157 1157 - [[package]] 1158 1090 name = "constellation" 1159 1091 version = "0.1.0" 1160 1092 dependencies = [ ··· 1421 1353 ] 1422 1354 1423 1355 [[package]] 1424 1424 - name = "dasl" 1425 1425 - version = "0.2.0" 1426 1426 - source = "registry+https://github.com/rust-lang/crates.io-index" 1427 1427 - checksum = "b59666035a4386b0fd272bd78da4cbc3ccb558941e97579ab00f0eb4639f2a49" 1428 1428 - dependencies = [ 1429 1429 - "blake3", 1430 1430 - "cbor4ii 1.2.0", 1431 1431 - "data-encoding", 1432 1432 - "data-encoding-macro", 1433 1433 - "scopeguard", 1434 1434 - "serde", 1435 1435 - "serde_bytes", 1436 1436 - "sha2", 1437 1437 - "thiserror 2.0.17", 1438 1438 - ] 1439 1439 - 1440 1440 - [[package]] 1441 1356 name = "data-encoding" 1442 1442 - version = "2.9.0" 1357 1357 + version = "2.8.0" 1443 1358 source = "registry+https://github.com/rust-lang/crates.io-index" 1444 1444 - checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476" 1359 1359 + checksum = "575f75dfd25738df5b91b8e43e14d44bda14637a58fae779fd2b064f8bf3e010" 1445 1360 1446 1361 [[package]] 1447 1362 name = "data-encoding-macro" 1448 1448 - version = "0.1.18" 1363 1363 + version = "0.1.17" 1449 1364 source = "registry+https://github.com/rust-lang/crates.io-index" 1450 1450 - checksum = "47ce6c96ea0102f01122a185683611bd5ac8d99e62bc59dd12e6bda344ee673d" 1365 1365 + checksum = "9f9724adfcf41f45bf652b3995837669d73c4d49a1b5ac1ff82905ac7d9b5558" 1451 1366 dependencies = [ 1452 1367 "data-encoding", 1453 1368 "data-encoding-macro-internal", ··· 1455 1370 1456 1371 [[package]] 1457 1372 name = "data-encoding-macro-internal" 1458 1458 - version = "0.1.16" 1373 1373 + version = "0.1.15" 1459 1374 source = "registry+https://github.com/rust-lang/crates.io-index" 1460 1460 - checksum = "8d162beedaa69905488a8da94f5ac3edb4dd4788b732fadb7bd120b2625c1976" 1375 1375 + checksum = "18e4fdb82bd54a12e42fb58a800dcae6b9e13982238ce2296dc3570b92148e1f" 1461 1376 dependencies = [ 1462 1377 "data-encoding", 1463 1378 "syn 2.0.106", ··· 1664 1579 "slog-bunyan", 1665 1580 "slog-json", 1666 1581 "slog-term", 1667 1667 - "thiserror 2.0.17", 1582 1582 + "thiserror 2.0.16", 1668 1583 "tokio", 1669 1584 "tokio-rustls 0.25.0", 1670 1585 "toml 0.9.7", ··· 1868 1783 checksum = "0b25ad44cd4360a0448a9b5a0a6f1c7a621101cca4578706d43c9a821418aebc" 1869 1784 dependencies = [ 1870 1785 "byteorder", 1871 1871 - "byteview 0.6.1", 1786 1786 + "byteview", 1872 1787 "dashmap", 1873 1788 "log", 1874 1874 - "lsm-tree 2.10.4", 1789 1789 + "lsm-tree", 1875 1790 "path-absolutize", 1876 1791 "std-semaphore", 1877 1792 "tempfile", ··· 1884 1799 source = "git+https://github.com/fjall-rs/fjall.git#42d811f7c8cc9004407d520d37d2a1d8d246c03d" 1885 1800 dependencies = [ 1886 1801 "byteorder", 1887 1887 - "byteview 0.6.1", 1802 1802 + "byteview", 1888 1803 "dashmap", 1889 1804 "log", 1890 1890 - "lsm-tree 2.10.4", 1805 1805 + "lsm-tree", 1891 1806 "path-absolutize", 1892 1892 - "std-semaphore", 1893 1893 - "tempfile", 1894 1894 - "xxhash-rust", 1895 1895 - ] 1896 1896 - 1897 1897 - [[package]] 1898 1898 - name = "fjall" 1899 1899 - version = "3.0.0-pre.0" 1900 1900 - source = "registry+https://github.com/rust-lang/crates.io-index" 1901 1901 - checksum = "467588c1f15d1cfa9e43f02a45cf55d82fa1f12a6ae961b848c520458525600c" 1902 1902 - dependencies = [ 1903 1903 - "byteorder-lite", 1904 1904 - "byteview 0.8.0", 1905 1905 - "dashmap", 1906 1906 - "log", 1907 1907 - "lsm-tree 3.0.0-pre.0", 1908 1807 "std-semaphore", 1909 1808 "tempfile", 1910 1809 "xxhash-rust", ··· 1992 1891 "mixtrics", 1993 1892 "pin-project", 1994 1893 "serde", 1995 1995 - "thiserror 2.0.17", 1894 1894 + "thiserror 2.0.16", 1996 1895 "tokio", 1997 1896 "tracing", 1998 1897 ] ··· 2012 1911 "parking_lot", 2013 1912 "pin-project", 2014 1913 "serde", 2015 2015 - "thiserror 2.0.17", 1914 1914 + "thiserror 2.0.16", 2016 1915 "tokio", 2017 1916 "twox-hash", 2018 1917 ] ··· 2045 1944 "parking_lot", 2046 1945 "pin-project", 2047 1946 "serde", 2048 2048 - "thiserror 2.0.17", 1947 1947 + "thiserror 2.0.16", 2049 1948 "tokio", 2050 1949 "tracing", 2051 1950 ] ··· 2077 1976 "pin-project", 2078 1977 "rand 0.9.1", 2079 1978 "serde", 2080 2080 - "thiserror 2.0.17", 1979 1979 + "thiserror 2.0.16", 2081 1980 "tokio", 2082 1981 "tracing", 2083 1982 "twox-hash", ··· 2321 2220 "pest_derive", 2322 2221 "serde", 2323 2222 "serde_json", 2324 2324 - "thiserror 2.0.17", 2223 2223 + "thiserror 2.0.16", 2325 2224 "walkdir", 2326 2225 ] 2327 2226 ··· 2446 2345 "once_cell", 2447 2346 "rand 0.9.1", 2448 2347 "ring", 2449 2449 - "thiserror 2.0.17", 2348 2348 + "thiserror 2.0.16", 2450 2349 "tinyvec", 2451 2350 "tokio", 2452 2351 "tracing", ··· 2469 2368 "rand 0.9.1", 2470 2369 "resolv-conf", 2471 2370 "smallvec", 2472 2472 - "thiserror 2.0.17", 2371 2371 + "thiserror 2.0.16", 2473 2372 "tokio", 2474 2373 "tracing", 2475 2374 ] ··· 2901 2800 ] 2902 2801 2903 2802 [[package]] 2904 2904 - name = "iroh-car" 2905 2905 - version = "0.5.1" 2906 2906 - source = "registry+https://github.com/rust-lang/crates.io-index" 2907 2907 - checksum = "cb7f8cd4cb9aa083fba8b52e921764252d0b4dcb1cd6d120b809dbfe1106e81a" 2908 2908 - dependencies = [ 2909 2909 - "anyhow", 2910 2910 - "cid", 2911 2911 - "futures", 2912 2912 - "serde", 2913 2913 - "serde_ipld_dagcbor", 2914 2914 - "thiserror 1.0.69", 2915 2915 - "tokio", 2916 2916 - "unsigned-varint 0.7.2", 2917 2917 - ] 2918 2918 - 2919 2919 - [[package]] 2920 2803 name = "is-terminal" 2921 2804 version = "0.4.16" 2922 2805 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 2980 2863 "metrics", 2981 2864 "serde", 2982 2865 "serde_json", 2983 2983 - "thiserror 2.0.17", 2866 2866 + "thiserror 2.0.16", 2984 2867 "tokio", 2985 2868 "tokio-tungstenite 0.26.2", 2986 2869 "url", ··· 3233 3116 version = "0.1.0" 3234 3117 dependencies = [ 3235 3118 "anyhow", 3236 3236 - "dasl", 3237 3119 "fluent-uri", 3238 3120 "nom", 3239 3239 - "serde", 3240 3240 - "thiserror 2.0.17", 3121 3121 + "thiserror 2.0.16", 3241 3122 "tinyjson", 3242 3123 ] 3243 3124 ··· 3305 3186 3306 3187 [[package]] 3307 3188 name = "lsm-tree" 3308 3308 - version = "2.10.4" 3189 3189 + version = "2.10.2" 3309 3190 source = "registry+https://github.com/rust-lang/crates.io-index" 3310 3310 - checksum = "799399117a2bfb37660e08be33f470958babb98386b04185288d829df362ea15" 3191 3191 + checksum = "55b6d7475a8dd22e749186968daacf8e2a77932b061b1bd263157987bbfc0c6c" 3311 3192 dependencies = [ 3312 3193 "byteorder", 3313 3194 "crossbeam-skiplist", ··· 3328 3209 ] 3329 3210 3330 3211 [[package]] 3331 3331 - name = "lsm-tree" 3332 3332 - version = "3.0.0-pre.0" 3333 3333 - source = "registry+https://github.com/rust-lang/crates.io-index" 3334 3334 - checksum = "be375d45e348328e78582dffbda4f1709dd52fca27c1a81c7bf6ca134e6335f7" 3335 3335 - dependencies = [ 3336 3336 - "byteorder-lite", 3337 3337 - "byteview 0.8.0", 3338 3338 - "crossbeam-skiplist", 3339 3339 - "enum_dispatch", 3340 3340 - "interval-heap", 3341 3341 - "log", 3342 3342 - "lz4_flex", 3343 3343 - "quick_cache", 3344 3344 - "rustc-hash 2.1.1", 3345 3345 - "self_cell", 3346 3346 - "sfa", 3347 3347 - "tempfile", 3348 3348 - "varint-rs", 3349 3349 - "xxhash-rust", 3350 3350 - ] 3351 3351 - 3352 3352 - [[package]] 3353 3212 name = "lz4" 3354 3213 version = "1.28.1" 3355 3214 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 3370 3229 3371 3230 [[package]] 3372 3231 name = "lz4_flex" 3373 3373 - version = "0.11.5" 3232 3232 + version = "0.11.3" 3374 3233 source = "registry+https://github.com/rust-lang/crates.io-index" 3375 3375 - checksum = "08ab2867e3eeeca90e844d1940eab391c9dc5228783db2ed999acbc0a9ed375a" 3234 3234 + checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5" 3376 3235 3377 3236 [[package]] 3378 3237 name = "mach2" ··· 3438 3297 ] 3439 3298 3440 3299 [[package]] 3441 3441 - name = "match-lookup" 3442 3442 - version = "0.1.1" 3443 3443 - source = "registry+https://github.com/rust-lang/crates.io-index" 3444 3444 - checksum = "1265724d8cb29dbbc2b0f06fffb8bf1a8c0cf73a78eede9ba73a4a66c52a981e" 3445 3445 - dependencies = [ 3446 3446 - "proc-macro2", 3447 3447 - "quote", 3448 3448 - "syn 1.0.109", 3449 3449 - ] 3450 3450 - 3451 3451 - [[package]] 3452 3300 name = "match_cfg" 3453 3301 version = "0.1.0" 3454 3302 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 3536 3384 "metrics", 3537 3385 "metrics-util 0.20.0", 3538 3386 "quanta", 3539 3539 - "thiserror 2.0.17", 3387 3387 + "thiserror 2.0.16", 3540 3388 "tokio", 3541 3389 "tracing", 3542 3390 ] ··· 3683 3531 3684 3532 [[package]] 3685 3533 name = "multibase" 3686 3686 - version = "0.9.2" 3534 3534 + version = "0.9.1" 3687 3535 source = "registry+https://github.com/rust-lang/crates.io-index" 3688 3688 - checksum = "8694bb4835f452b0e3bb06dbebb1d6fc5385b6ca1caf2e55fd165c042390ec77" 3536 3536 + checksum = "9b3539ec3c1f04ac9748a260728e855f261b4977f5c3406612c884564f329404" 3689 3537 dependencies = [ 3690 3538 "base-x", 3691 3691 - "base256emoji", 3692 3539 "data-encoding", 3693 3540 "data-encoding-macro", 3694 3541 ] ··· 3701 3548 dependencies = [ 3702 3549 "core2", 3703 3550 "serde", 3704 3704 - "unsigned-varint 0.8.0", 3551 3551 + "unsigned-varint", 3705 3552 ] 3706 3553 3707 3554 [[package]] ··· 4079 3926 checksum = "1db05f56d34358a8b1066f67cbb203ee3e7ed2ba674a6263a1d5ec6db2204323" 4080 3927 dependencies = [ 4081 3928 "memchr", 4082 4082 - "thiserror 2.0.17", 3929 3929 + "thiserror 2.0.16", 4083 3930 "ucd-trie", 4084 3931 ] 4085 3932 ··· 4189 4036 "rusqlite", 4190 4037 "serde", 4191 4038 "serde_json", 4192 4192 - "thiserror 2.0.17", 4039 4039 + "thiserror 2.0.16", 4193 4040 "tokio", 4194 4041 "tracing-subscriber", 4195 4042 ] ··· 4232 4079 "smallvec", 4233 4080 "sync_wrapper", 4234 4081 "tempfile", 4235 4235 - "thiserror 2.0.17", 4082 4082 + "thiserror 2.0.16", 4236 4083 "tokio", 4237 4084 "tokio-rustls 0.26.2", 4238 4085 "tokio-stream", ··· 4276 4123 "serde_json", 4277 4124 "serde_urlencoded", 4278 4125 "serde_yaml", 4279 4279 - "thiserror 2.0.17", 4126 4126 + "thiserror 2.0.16", 4280 4127 "tokio", 4281 4128 ] 4282 4129 ··· 4295 4142 "quote", 4296 4143 "regex", 4297 4144 "syn 2.0.106", 4298 4298 - "thiserror 2.0.17", 4145 4145 + "thiserror 2.0.16", 4299 4146 ] 4300 4147 4301 4148 [[package]] ··· 4422 4269 4423 4270 [[package]] 4424 4271 name = "quick_cache" 4425 4425 - version = "0.6.16" 4272 4272 + version = "0.6.12" 4426 4273 source = "registry+https://github.com/rust-lang/crates.io-index" 4427 4427 - checksum = "9ad6644cb07b7f3488b9f3d2fde3b4c0a7fa367cafefb39dff93a659f76eb786" 4274 4274 + checksum = "8f8ed0655cbaf18a26966142ad23b95d8ab47221c50c4f73a1db7d0d2d6e3da8" 4428 4275 dependencies = [ 4429 4276 "equivalent", 4430 4277 "hashbrown 0.15.2", ··· 4444 4291 "rustc-hash 2.1.1", 4445 4292 "rustls 0.23.31", 4446 4293 "socket2 0.5.9", 4447 4447 - "thiserror 2.0.17", 4294 4294 + "thiserror 2.0.16", 4448 4295 "tokio", 4449 4296 "tracing", 4450 4297 "web-time", ··· 4465 4312 "rustls 0.23.31", 4466 4313 "rustls-pki-types", 4467 4314 "slab", 4468 4468 - "thiserror 2.0.17", 4315 4315 + "thiserror 2.0.16", 4469 4316 "tinyvec", 4470 4317 "tracing", 4471 4318 "web-time", ··· 4691 4538 checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" 4692 4539 4693 4540 [[package]] 4694 4694 - name = "repo-stream" 4695 4695 - version = "0.2.2" 4696 4696 - source = "registry+https://github.com/rust-lang/crates.io-index" 4697 4697 - checksum = "093b48e604c138949bf3d4a1a9bc1165feb1db28a73af0101c84eb703d279f43" 4698 4698 - dependencies = [ 4699 4699 - "bincode 2.0.1", 4700 4700 - "futures", 4701 4701 - "futures-core", 4702 4702 - "ipld-core", 4703 4703 - "iroh-car", 4704 4704 - "log", 4705 4705 - "multibase", 4706 4706 - "rusqlite", 4707 4707 - "serde", 4708 4708 - "serde_bytes", 4709 4709 - "serde_ipld_dagcbor", 4710 4710 - "sha2", 4711 4711 - "thiserror 2.0.17", 4712 4712 - "tokio", 4713 4713 - ] 4714 4714 - 4715 4715 - [[package]] 4716 4541 name = "reqwest" 4717 4717 - version = "0.12.24" 4542 4542 + version = "0.12.23" 4718 4543 source = "registry+https://github.com/rust-lang/crates.io-index" 4719 4719 - checksum = "9d0946410b9f7b082a427e4ef5c8ff541a88b357bc6c637c40db3a68ac70a36f" 4544 4544 + checksum = "d429f34c8092b2d42c7c93cec323bb4adeb7c67698f70839adec842ec10c7ceb" 4720 4545 dependencies = [ 4721 4546 "async-compression", 4722 4547 "base64 0.22.1", ··· 4756 4581 "url", 4757 4582 "wasm-bindgen", 4758 4583 "wasm-bindgen-futures", 4759 4759 - "wasm-streams", 4760 4584 "web-sys", 4761 4585 ] 4762 4586 ··· 5138 4962 5139 4963 [[package]] 5140 4964 name = "self_cell" 5141 5141 - version = "1.2.0" 4965 4965 + version = "1.1.0" 5142 4966 source = "registry+https://github.com/rust-lang/crates.io-index" 5143 5143 - checksum = "0f7d95a54511e0c7be3f51e8867aa8cf35148d7b9445d44de2f943e2b206e749" 4967 4967 + checksum = "c2fdfc24bc566f839a2da4c4295b82db7d25a24253867d5c64355abb5799bdbe" 5144 4968 5145 4969 [[package]] 5146 4970 name = "semver" ··· 5160 4984 5161 4985 [[package]] 5162 4986 name = "serde_bytes" 5163 5163 - version = "0.11.19" 4987 4987 + version = "0.11.17" 5164 4988 source = "registry+https://github.com/rust-lang/crates.io-index" 5165 5165 - checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" 4989 4989 + checksum = "8437fd221bde2d4ca316d61b90e337e9e702b3820b87d63caa9ba6c02bd06d96" 5166 4990 dependencies = [ 5167 4991 "serde", 5168 5168 - "serde_core", 5169 4992 ] 5170 4993 5171 4994 [[package]] ··· 5213 5036 ] 5214 5037 5215 5038 [[package]] 5216 5216 - name = "serde_ipld_dagcbor" 5217 5217 - version = "0.6.4" 5218 5218 - source = "registry+https://github.com/rust-lang/crates.io-index" 5219 5219 - checksum = "46182f4f08349a02b45c998ba3215d3f9de826246ba02bb9dddfe9a2a2100778" 5220 5220 - dependencies = [ 5221 5221 - "cbor4ii 0.2.14", 5222 5222 - "ipld-core", 5223 5223 - "scopeguard", 5224 5224 - "serde", 5225 5225 - ] 5226 5226 - 5227 5227 - [[package]] 5228 5039 name = "serde_json" 5229 5040 version = "1.0.145" 5230 5041 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 5257 5068 "percent-encoding", 5258 5069 "ryu", 5259 5070 "serde", 5260 5260 - "thiserror 2.0.17", 5071 5071 + "thiserror 2.0.16", 5261 5072 ] 5262 5073 5263 5074 [[package]] ··· 5346 5157 ] 5347 5158 5348 5159 [[package]] 5349 5349 - name = "sfa" 5350 5350 - version = "0.0.1" 5351 5351 - source = "registry+https://github.com/rust-lang/crates.io-index" 5352 5352 - checksum = "e5f5f9dc21f55409f15103d5a7e7601b804935923c7fe4746dc806c3a422a038" 5353 5353 - dependencies = [ 5354 5354 - "byteorder-lite", 5355 5355 - "log", 5356 5356 - "xxhash-rust", 5357 5357 - ] 5358 5358 - 5359 5359 - [[package]] 5360 5160 name = "sha1" 5361 5161 version = "0.10.6" 5362 5162 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 5420 5220 dependencies = [ 5421 5221 "num-bigint", 5422 5222 "num-traits", 5423 5423 - "thiserror 2.0.17", 5223 5223 + "thiserror 2.0.16", 5424 5224 "time", 5425 5225 ] 5426 5226 ··· 5462 5262 "rustls 0.23.31", 5463 5263 "serde", 5464 5264 "serde_json", 5465 5465 - "thiserror 2.0.17", 5265 5265 + "thiserror 2.0.16", 5466 5266 "time", 5467 5267 "tokio", 5468 5268 "tokio-util", ··· 5555 5355 name = "spacedust" 5556 5356 version = "0.1.0" 5557 5357 dependencies = [ 5558 5558 - "anyhow", 5559 5559 - "async-channel", 5560 5358 "async-trait", 5561 5359 "clap", 5562 5360 "ctrlc", 5563 5563 - "dasl", 5564 5361 "dropshot", 5565 5362 "env_logger", 5566 5566 - "fjall 3.0.0-pre.0", 5567 5363 "futures", 5568 5364 "http", 5569 5569 - "ipld-core", 5570 5365 "jetstream", 5571 5366 "links", 5572 5367 "log", 5573 5368 "metrics", 5574 5369 "metrics-exporter-prometheus 0.17.2", 5575 5370 "rand 0.9.1", 5576 5576 - "repo-stream", 5577 5577 - "reqwest", 5578 5371 "schemars", 5579 5372 "semver", 5580 5373 "serde", 5581 5581 - "serde_ipld_dagcbor", 5582 5374 "serde_json", 5583 5375 "serde_qs", 5584 5584 - "thiserror 2.0.17", 5376 5376 + "thiserror 2.0.16", 5585 5377 "tinyjson", 5586 5378 "tokio", 5587 5379 "tokio-tungstenite 0.27.0", ··· 5714 5506 5715 5507 [[package]] 5716 5508 name = "tempfile" 5717 5717 - version = "3.23.0" 5509 5509 + version = "3.19.1" 5718 5510 source = "registry+https://github.com/rust-lang/crates.io-index" 5719 5719 - checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" 5511 5511 + checksum = "7437ac7763b9b123ccf33c338a5cc1bac6f69b45a136c19bdd8a65e3916435bf" 5720 5512 dependencies = [ 5721 5513 "fastrand", 5722 5514 "getrandom 0.3.3", ··· 5747 5539 5748 5540 [[package]] 5749 5541 name = "thiserror" 5750 5750 - version = "2.0.17" 5542 5542 + version = "2.0.16" 5751 5543 source = "registry+https://github.com/rust-lang/crates.io-index" 5752 5752 - checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" 5544 5544 + checksum = "3467d614147380f2e4e374161426ff399c91084acd2363eaf549172b3d5e60c0" 5753 5545 dependencies = [ 5754 5754 - "thiserror-impl 2.0.17", 5546 5546 + "thiserror-impl 2.0.16", 5755 5547 ] 5756 5548 5757 5549 [[package]] ··· 5767 5559 5768 5560 [[package]] 5769 5561 name = "thiserror-impl" 5770 5770 - version = "2.0.17" 5562 5562 + version = "2.0.16" 5771 5563 source = "registry+https://github.com/rust-lang/crates.io-index" 5772 5772 - checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" 5564 5564 + checksum = "6c5e1be1c48b9172ee610da68fd9cd2770e7a4056cb3fc98710ee6906f0c7960" 5773 5565 dependencies = [ 5774 5566 "proc-macro2", 5775 5567 "quote", ··· 6201 5993 "native-tls", 6202 5994 "rand 0.9.1", 6203 5995 "sha1", 6204 6204 - "thiserror 2.0.17", 5996 5996 + "thiserror 2.0.16", 6205 5997 "url", 6206 5998 "utf-8", 6207 5999 ] ··· 6219 6011 "log", 6220 6012 "rand 0.9.1", 6221 6013 "sha1", 6222 6222 - "thiserror 2.0.17", 6014 6014 + "thiserror 2.0.16", 6223 6015 "utf-8", 6224 6016 ] 6225 6017 ··· 6262 6054 "http", 6263 6055 "jetstream", 6264 6056 "log", 6265 6265 - "lsm-tree 2.10.4", 6057 6057 + "lsm-tree", 6266 6058 "metrics", 6267 6059 "metrics-exporter-prometheus 0.17.2", 6268 6060 "schemars", ··· 6272 6064 "serde_qs", 6273 6065 "sha2", 6274 6066 "tempfile", 6275 6275 - "thiserror 2.0.17", 6067 6067 + "thiserror 2.0.16", 6276 6068 "tikv-jemallocator", 6277 6069 "tokio", 6278 6070 "tokio-util", ··· 6325 6117 6326 6118 [[package]] 6327 6119 name = "unsigned-varint" 6328 6328 - version = "0.7.2" 6329 6329 - source = "registry+https://github.com/rust-lang/crates.io-index" 6330 6330 - checksum = "6889a77d49f1f013504cec6bf97a2c730394adedaeb1deb5ea08949a50541105" 6331 6331 - 6332 6332 - [[package]] 6333 6333 - name = "unsigned-varint" 6334 6120 version = "0.8.0" 6335 6121 source = "registry+https://github.com/rust-lang/crates.io-index" 6336 6122 checksum = "eb066959b24b5196ae73cb057f45598450d2c5f71460e98c49b738086eff9c06" ··· 6407 6193 checksum = "62fc7c4ce161f049607ecea654dca3f2d727da5371ae85e2e4f14ce2b98ed67c" 6408 6194 dependencies = [ 6409 6195 "byteorder", 6410 6410 - "byteview 0.6.1", 6196 6196 + "byteview", 6411 6197 "interval-heap", 6412 6198 "log", 6413 6199 "path-absolutize", ··· 6556 6342 ] 6557 6343 6558 6344 [[package]] 6559 6559 - name = "wasm-streams" 6560 6560 - version = "0.4.2" 6561 6561 - source = "registry+https://github.com/rust-lang/crates.io-index" 6562 6562 - checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" 6563 6563 - dependencies = [ 6564 6564 - "futures-util", 6565 6565 - "js-sys", 6566 6566 - "wasm-bindgen", 6567 6567 - "wasm-bindgen-futures", 6568 6568 - "web-sys", 6569 6569 - ] 6570 6570 - 6571 6571 - [[package]] 6572 6345 name = "web-sys" 6573 6346 version = "0.3.77" 6574 6347 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 6627 6400 "reqwest", 6628 6401 "serde", 6629 6402 "serde_json", 6630 6630 - "thiserror 2.0.17", 6403 6403 + "thiserror 2.0.16", 6631 6404 "tokio", 6632 6405 "tokio-util", 6633 6406 "url", ··· 6985 6758 "nom", 6986 6759 "oid-registry", 6987 6760 "rusticata-macros", 6988 6988 - "thiserror 2.0.17", 6761 6761 + "thiserror 2.0.16", 6989 6762 "time", 6990 6763 ] 6991 6764

+13 -6

constellation/src/consumer/jetstream.rs

··· 226 226 println!("jetstream closed the websocket cleanly."); 227 227 break; 228 228 } 229 229 - r => eprintln!("jetstream: close result after error: {r:?}"), 229 229 + Err(_) => { 230 230 + counter!("jetstream_read_fail", "url" => stream.clone(), "reason" => "dirty close").increment(1); 231 231 + println!("jetstream failed to close the websocket cleanly."); 232 232 + break; 233 233 + } 234 234 + Ok(r) => { 235 235 + eprintln!("jetstream: close result after error: {r:?}"); 236 236 + counter!("jetstream_read_fail", "url" => stream.clone(), "reason" => "read error") 237 237 + .increment(1); 238 238 + // if we didn't immediately get ConnectionClosed, we should keep polling read 239 239 + // until we get it. 240 240 + continue; 241 241 + } 230 242 } 231 231 - counter!("jetstream_read_fail", "url" => stream.clone(), "reason" => "read error") 232 232 - .increment(1); 233 233 - // if we didn't immediately get ConnectionClosed, we should keep polling read 234 234 - // until we get it. 235 235 - continue; 236 243 } 237 244 }; 238 245

-2

links/Cargo.toml

··· 5 5 6 6 [dependencies] 7 7 anyhow = "1.0.95" 8 8 - dasl = "0.2.0" 9 8 fluent-uri = "0.3.2" 10 9 nom = "7.1.3" 11 11 - serde = { version = "1.0.228", features = ["derive"] } 12 10 thiserror = "2.0.9" 13 11 tinyjson = "2.5.1"

+2 -3

links/src/lib.rs

··· 1 1 use fluent_uri::Uri; 2 2 - use serde::{Deserialize, Serialize}; 3 2 4 3 pub mod at_uri; 5 4 pub mod did; ··· 7 6 8 7 pub use record::collect_links; 9 8 10 10 - #[derive(Debug, Clone, Ord, Eq, PartialOrd, PartialEq, Serialize, Deserialize)] 9 9 + #[derive(Debug, Clone, Ord, Eq, PartialOrd, PartialEq)] 11 10 pub enum Link { 12 11 AtUri(String), 13 12 Uri(String), ··· 60 59 } 61 60 } 62 61 63 63 - #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] 62 62 + #[derive(Debug, PartialEq)] 64 63 pub struct CollectedLink { 65 64 pub path: String, 66 65 pub target: Link,

-41

links/src/record.rs

··· 1 1 - use dasl::drisl::Value as DrislValue; 2 1 use tinyjson::JsonValue; 3 2 4 3 use crate::{parse_any_link, CollectedLink}; ··· 37 36 } 38 37 } 39 38 40 40 - pub fn walk_drisl(path: &str, v: &DrislValue, found: &mut Vec<CollectedLink>) { 41 41 - match v { 42 42 - DrislValue::Map(o) => { 43 43 - for (key, child) in o { 44 44 - walk_drisl(&format!("{path}.{key}"), child, found) 45 45 - } 46 46 - } 47 47 - DrislValue::Array(a) => { 48 48 - for child in a { 49 49 - let child_p = match child { 50 50 - DrislValue::Map(o) => { 51 51 - if let Some(DrislValue::Text(t)) = o.get("$type") { 52 52 - format!("{path}[{t}]") 53 53 - } else { 54 54 - format!("{path}[]") 55 55 - } 56 56 - } 57 57 - _ => format!("{path}[]"), 58 58 - }; 59 59 - walk_drisl(&child_p, child, found) 60 60 - } 61 61 - } 62 62 - DrislValue::Text(s) => { 63 63 - if let Some(link) = parse_any_link(s) { 64 64 - found.push(CollectedLink { 65 65 - path: path.to_string(), 66 66 - target: link, 67 67 - }); 68 68 - } 69 69 - } 70 70 - _ => {} 71 71 - } 72 72 - } 73 73 - 74 39 pub fn collect_links(v: &JsonValue) -> Vec<CollectedLink> { 75 40 let mut found = vec![]; 76 41 walk_record("", v, &mut found); 77 77 - found 78 78 - } 79 79 - 80 80 - pub fn collect_links_drisl(v: &DrislValue) -> Vec<CollectedLink> { 81 81 - let mut found = vec![]; 82 82 - walk_drisl("", v, &mut found); 83 42 found 84 43 } 85 44

+4 -2

slingshot/src/firehose_cache.rs

··· 4 4 5 5 pub async fn firehose_cache( 6 6 cache_dir: impl AsRef<Path>, 7 7 + memory_mb: usize, 8 8 + disk_gb: usize, 7 9 ) -> Result<HybridCache<String, CachedRecord>, String> { 8 10 let cache = HybridCacheBuilder::new() 9 11 .with_name("firehose") 10 10 - .memory(64 * 2_usize.pow(20)) 12 12 + .memory(memory_mb * 2_usize.pow(20)) 11 13 .with_weighter(|k: &String, v| k.len() + std::mem::size_of_val(v)) 12 14 .storage(Engine::large()) 13 15 .with_device_options( 14 16 DirectFsDeviceOptions::new(cache_dir) 15 15 - .with_capacity(2_usize.pow(30)) // TODO: configurable (1GB to have something) 17 17 + .with_capacity(disk_gb * 2_usize.pow(30)) 16 18 .with_file_size(16 * 2_usize.pow(20)), // note: this does limit the max cached item size, warning jumbo records 17 19 ) 18 20 .build()

+26 -5

slingshot/src/main.rs

··· 25 25 /// where to keep disk caches 26 26 #[arg(long)] 27 27 cache_dir: PathBuf, 28 28 + /// memory cache size in MB 29 29 + #[arg(long, default_value_t = 64)] 30 30 + cache_memory_mb: usize, 31 31 + /// disk cache size in GB 32 32 + #[arg(long, default_value_t = 1)] 33 33 + cache_disk_gb: usize, 34 34 + /// host for HTTP server (when not using --domain) 35 35 + #[arg(long, default_value = "127.0.0.1")] 36 36 + host: String, 37 37 + /// port for HTTP server (when not using --domain) 38 38 + #[arg(long, default_value_t = 3000)] 39 39 + port: u16, 40 40 + /// port for metrics/prometheus server 41 41 + #[arg(long, default_value_t = 8765)] 42 42 + metrics_port: u16, 28 43 /// the domain pointing to this server 29 44 /// 30 45 /// if present: ··· 62 77 63 78 let args = Args::parse(); 64 79 65 65 - if let Err(e) = install_metrics_server() { 80 80 + if let Err(e) = install_metrics_server(args.metrics_port) { 66 81 log::error!("failed to install metrics server: {e:?}"); 67 82 } else { 68 68 - log::info!("metrics listening at http://0.0.0.0:8765"); 83 83 + log::info!("metrics listening at http://0.0.0.0:{}", args.metrics_port); 69 84 } 70 85 71 86 std::fs::create_dir_all(&args.cache_dir).map_err(|e| { ··· 83 98 log::info!("cache dir ready at at {cache_dir:?}."); 84 99 85 100 log::info!("setting up firehose cache..."); 86 86 - let cache = firehose_cache(cache_dir.join("./firehose")).await?; 101 101 + let cache = firehose_cache( 102 102 + cache_dir.join("./firehose"), 103 103 + args.cache_memory_mb, 104 104 + args.cache_disk_gb, 105 105 + ) 106 106 + .await?; 87 107 log::info!("firehose cache ready."); 88 108 89 109 let mut tasks: tokio::task::JoinSet<Result<(), MainTaskError>> = tokio::task::JoinSet::new(); ··· 112 132 args.domain, 113 133 args.acme_contact, 114 134 args.certs, 135 135 + args.host, 136 136 + args.port, 115 137 server_shutdown, 116 138 ) 117 139 .await?; ··· 172 194 Ok(()) 173 195 } 174 196 175 175 - fn install_metrics_server() -> Result<(), metrics_exporter_prometheus::BuildError> { 197 197 + fn install_metrics_server(port: u16) -> Result<(), metrics_exporter_prometheus::BuildError> { 176 198 log::info!("installing metrics server..."); 177 199 let host = [0, 0, 0, 0]; 178 178 - let port = 8765; 179 200 PrometheusBuilder::new() 180 201 .set_quantiles(&[0.5, 0.9, 0.99, 1.0])? 181 202 .set_bucket_duration(std::time::Duration::from_secs(300))?

+19 -12

slingshot/src/server.rs

··· 437 437 Ok(did) => did, 438 438 Err(_) => { 439 439 let Ok(alleged_handle) = Handle::new(identifier) else { 440 440 - return invalid("identifier was not a valid DID or handle"); 440 440 + return invalid("Identifier was not a valid DID or handle"); 441 441 }; 442 442 443 443 match self.identity.handle_to_did(alleged_handle.clone()).await { ··· 453 453 Err(e) => { 454 454 log::debug!("failed to resolve handle: {e}"); 455 455 // TODO: ServerError not BadRequest 456 456 - return invalid("errored while trying to resolve handle to DID"); 456 456 + return invalid("Errored while trying to resolve handle to DID"); 457 457 } 458 458 } 459 459 } 460 460 }; 461 461 let Ok(partial_doc) = self.identity.did_to_partial_mini_doc(&did).await else { 462 462 - return invalid("failed to get DID doc"); 462 462 + return invalid("Failed to get DID doc"); 463 463 }; 464 464 let Some(partial_doc) = partial_doc else { 465 465 - return invalid("failed to find DID doc"); 465 465 + return invalid("Failed to find DID doc"); 466 466 }; 467 467 468 468 // ok so here's where we're at: ··· 483 483 .handle_to_did(partial_doc.unverified_handle.clone()) 484 484 .await 485 485 else { 486 486 - return invalid("failed to get did doc's handle"); 486 486 + return invalid("Failed to get DID doc's handle"); 487 487 }; 488 488 let Some(handle_did) = handle_did else { 489 489 - return invalid("failed to resolve did doc's handle"); 489 489 + return invalid("Failed to resolve DID doc's handle"); 490 490 }; 491 491 if handle_did == did { 492 492 partial_doc.unverified_handle.to_string() ··· 516 516 let Ok(handle) = Handle::new(repo) else { 517 517 return GetRecordResponse::BadRequest(xrpc_error( 518 518 "InvalidRequest", 519 519 - "repo was not a valid DID or handle", 519 519 + "Repo was not a valid DID or handle", 520 520 )); 521 521 }; 522 522 match self.identity.handle_to_did(handle).await { ··· 534 534 log::debug!("handle resolution failed: {e}"); 535 535 return GetRecordResponse::ServerError(xrpc_error( 536 536 "ResolutionFailed", 537 537 - "errored while trying to resolve handle to DID", 537 537 + "Errored while trying to resolve handle to DID", 538 538 )); 539 539 } 540 540 } ··· 544 544 let Ok(collection) = Nsid::new(collection) else { 545 545 return GetRecordResponse::BadRequest(xrpc_error( 546 546 "InvalidRequest", 547 547 - "invalid NSID for collection", 547 547 + "Invalid NSID for collection", 548 548 )); 549 549 }; 550 550 551 551 let Ok(rkey) = RecordKey::new(rkey) else { 552 552 - return GetRecordResponse::BadRequest(xrpc_error("InvalidRequest", "invalid rkey")); 552 552 + return GetRecordResponse::BadRequest(xrpc_error("InvalidRequest", "Invalid rkey")); 553 553 }; 554 554 555 555 let cid: Option<Cid> = if let Some(cid) = cid { 556 556 let Ok(cid) = Cid::from_str(&cid) else { 557 557 - return GetRecordResponse::BadRequest(xrpc_error("InvalidRequest", "invalid CID")); 557 557 + return GetRecordResponse::BadRequest(xrpc_error("InvalidRequest", "Invalid CID")); 558 558 }; 559 559 Some(cid) 560 560 } else { ··· 694 694 domain: Option<String>, 695 695 acme_contact: Option<String>, 696 696 certs: Option<PathBuf>, 697 697 + host: String, 698 698 + port: u16, 697 699 shutdown: CancellationToken, 698 700 ) -> Result<(), ServerError> { 699 701 let repo = Arc::new(repo); ··· 752 754 ) 753 755 .await 754 756 } else { 755 755 - run(TcpListener::bind("127.0.0.1:3000"), app, shutdown).await 757 757 + run( 758 758 + TcpListener::bind(format!("{host}:{port}")), 759 759 + app, 760 760 + shutdown, 761 761 + ) 762 762 + .await 756 763 } 757 764 } 758 765

-8

spacedust/Cargo.toml

··· 4 4 edition = "2024" 5 5 6 6 [dependencies] 7 7 - anyhow = "1.0.100" 8 8 - async-channel = "2.5.0" 9 7 async-trait = "0.1.88" 10 8 clap = { version = "4.5.40", features = ["derive"] } 11 9 ctrlc = "3.4.7" 12 12 - dasl = "0.2.0" 13 10 dropshot = "0.16.2" 14 11 env_logger = "0.11.8" 15 15 - fjall = "3.0.0-pre.0" 16 12 futures = "0.3.31" 17 13 http = "1.3.1" 18 18 - ipld-core = { version = "0.4.2", features = ["serde"] } 19 14 jetstream = { path = "../jetstream", features = ["metrics"] } 20 15 links = { path = "../links" } 21 16 log = "0.4.27" 22 17 metrics = "0.24.2" 23 18 metrics-exporter-prometheus = { version = "0.17.1", features = ["http-listener"] } 24 19 rand = "0.9.1" 25 25 - repo-stream = "0.2.2" 26 26 - reqwest = { version = "0.12.24", features = ["json", "stream"] } 27 20 schemars = "0.8.22" 28 21 semver = "1.0.26" 29 22 serde = { version = "1.0.219", features = ["derive"] } 30 30 - serde_ipld_dagcbor = "0.6.4" 31 23 serde_json = "1.0.140" 32 24 serde_qs = "1.0.0-rc.3" 33 25 thiserror = "2.0.12"

-21

spacedust/src/bin/import_car_file.rs

··· 1 1 - use clap::Parser; 2 2 - use std::path::PathBuf; 3 3 - 4 4 - type Result<T> = std::result::Result<T, Box<dyn std::error::Error>>; 5 5 - 6 6 - #[derive(Debug, Parser)] 7 7 - struct Args { 8 8 - #[arg()] 9 9 - file: PathBuf, 10 10 - } 11 11 - 12 12 - #[tokio::main] 13 13 - async fn main() -> Result<()> { 14 14 - env_logger::init(); 15 15 - 16 16 - let Args { file } = Args::parse(); 17 17 - 18 18 - let _reader = tokio::fs::File::open(file).await?; 19 19 - 20 20 - Ok(()) 21 21 - }

-258

spacedust/src/bin/import_scraped.rs

··· 1 1 - use clap::Parser; 2 2 - use links::CollectedLink; 3 3 - use repo_stream::{ 4 4 - DiskBuilder, DiskStore, Driver, DriverBuilder, Processable, drive::DriverBuilderWithProcessor, 5 5 - drive::NeedDisk, 6 6 - }; 7 7 - use std::path::PathBuf; 8 8 - use std::sync::{ 9 9 - Arc, 10 10 - atomic::{AtomicUsize, Ordering}, 11 11 - }; 12 12 - use tokio::{io::AsyncRead, task::JoinSet}; 13 13 - 14 14 - type Result<T> = anyhow::Result<T>; //std::result::Result<T, Box<dyn std::error::Error>>; 15 15 - 16 16 - #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] 17 17 - struct CollectedProcessed(CollectedLink); 18 18 - 19 19 - impl Processable for CollectedProcessed { 20 20 - fn get_size(&self) -> usize { 21 21 - self.0.path.capacity() + self.0.target.as_str().len() 22 22 - } 23 23 - } 24 24 - 25 25 - #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] 26 26 - struct ErrString(String); 27 27 - 28 28 - impl Processable for ErrString { 29 29 - fn get_size(&self) -> usize { 30 30 - self.0.capacity() 31 31 - } 32 32 - } 33 33 - 34 34 - type Processed = std::result::Result<Vec<CollectedProcessed>, ErrString>; 35 35 - 36 36 - /// hacky for now: put errors in strings 🤷‍♀️ 37 37 - fn process(block: Vec<u8>) -> Processed { 38 38 - let value: dasl::drisl::Value = dasl::drisl::from_slice(&block) 39 39 - .map_err(|e| ErrString(format!("failed to parse block with drisl: {e:?}")))?; 40 40 - let links = links::record::collect_links_drisl(&value) 41 41 - .into_iter() 42 42 - .map(CollectedProcessed) 43 43 - .collect(); 44 44 - Ok(links) 45 45 - } 46 46 - 47 47 - #[derive(Debug, Parser)] 48 48 - struct Args { 49 49 - #[arg(long)] 50 50 - cars_folder: PathBuf, 51 51 - #[arg(long)] 52 52 - mem_workers: usize, 53 53 - #[arg(long)] 54 54 - disk_workers: usize, 55 55 - #[arg(long)] 56 56 - disk_folder: PathBuf, 57 57 - } 58 58 - 59 59 - async fn get_cars( 60 60 - cars_folder: PathBuf, 61 61 - tx: async_channel::Sender<tokio::io::BufReader<tokio::fs::File>>, 62 62 - ) -> Result<()> { 63 63 - let mut dir = tokio::fs::read_dir(cars_folder).await?; 64 64 - while let Some(entry) = dir.next_entry().await? { 65 65 - if !entry.file_type().await?.is_file() { 66 66 - continue; 67 67 - } 68 68 - let reader = tokio::fs::File::open(&entry.path()).await?; 69 69 - let reader = tokio::io::BufReader::new(reader); 70 70 - tx.send(reader).await?; 71 71 - } 72 72 - Ok(()) 73 73 - } 74 74 - 75 75 - async fn drive_mem<R: AsyncRead + Unpin + Send + Sync + 'static>( 76 76 - f: R, 77 77 - builder: &DriverBuilderWithProcessor<Processed>, 78 78 - disk_tx: &async_channel::Sender<NeedDisk<R, Processed>>, 79 79 - ) -> Result<Option<(usize, usize)>> { 80 80 - let mut n = 0; 81 81 - let mut n_records = 0; 82 82 - match builder.load_car(f).await? { 83 83 - Driver::Memory(_commit, mut driver) => { 84 84 - while let Some(chunk) = driver.next_chunk(512).await? { 85 85 - n_records += chunk.len(); 86 86 - for (_key, links) in chunk { 87 87 - match links { 88 88 - Ok(links) => n += links.len(), 89 89 - Err(e) => eprintln!("wat: {e:?}"), 90 90 - } 91 91 - } 92 92 - } 93 93 - Ok(Some((n, n_records))) 94 94 - } 95 95 - Driver::Disk(need_disk) => { 96 96 - disk_tx.send(need_disk).await?; 97 97 - Ok(None) 98 98 - } 99 99 - } 100 100 - } 101 101 - 102 102 - async fn mem_worker<R: AsyncRead + Unpin + Send + Sync + 'static>( 103 103 - car_rx: async_channel::Receiver<R>, 104 104 - disk_tx: async_channel::Sender<NeedDisk<R, Processed>>, 105 105 - n: Arc<AtomicUsize>, 106 106 - n_records: Arc<AtomicUsize>, 107 107 - ) -> Result<()> { 108 108 - let builder = DriverBuilder::new() 109 109 - .with_block_processor(process) // don't care just counting records 110 110 - .with_mem_limit_mb(128); 111 111 - while let Ok(f) = car_rx.recv().await { 112 112 - let driven = match drive_mem(f, &builder, &disk_tx).await { 113 113 - Ok(d) => d, 114 114 - Err(e) => { 115 115 - eprintln!("failed to drive mem: {e:?}. skipping..."); 116 116 - continue; 117 117 - } 118 118 - }; 119 119 - if let Some((drove, recs)) = driven { 120 120 - n.fetch_add(drove, Ordering::Relaxed); 121 121 - n_records.fetch_add(recs, Ordering::Relaxed); 122 122 - } 123 123 - } 124 124 - Ok(()) 125 125 - } 126 126 - 127 127 - async fn drive_disk<R: AsyncRead + Unpin>( 128 128 - needed: NeedDisk<R, Processed>, 129 129 - store: DiskStore, 130 130 - ) -> Result<(usize, usize, DiskStore)> { 131 131 - let (_commit, mut driver) = needed.finish_loading(store).await?; 132 132 - let mut n = 0; 133 133 - let mut n_records = 0; 134 134 - while let Some(chunk) = driver.next_chunk(512).await? { 135 135 - n_records += chunk.len(); 136 136 - for (_key, links) in chunk { 137 137 - match links { 138 138 - Ok(links) => n += links.len(), 139 139 - Err(e) => eprintln!("wat: {e:?}"), 140 140 - } 141 141 - } 142 142 - } 143 143 - let store = driver.reset_store().await?; 144 144 - Ok((n, n_records, store)) 145 145 - } 146 146 - 147 147 - async fn disk_worker<R: AsyncRead + Unpin>( 148 148 - worker_id: usize, 149 149 - disk_rx: async_channel::Receiver<NeedDisk<R, Processed>>, 150 150 - folder: PathBuf, 151 151 - n: Arc<AtomicUsize>, 152 152 - n_records: Arc<AtomicUsize>, 153 153 - disk_workers_active: Arc<AtomicUsize>, 154 154 - ) -> Result<()> { 155 155 - let mut file = folder; 156 156 - file.push(format!("disk-worker-{worker_id}.sqlite")); 157 157 - let builder = DiskBuilder::new().with_cache_size_mb(128); 158 158 - let mut store = builder.open(file.clone()).await?; 159 159 - while let Ok(needed) = disk_rx.recv().await { 160 160 - let active = disk_workers_active.fetch_add(1, Ordering::AcqRel); 161 161 - println!("-> disk workers active: {}", active + 1); 162 162 - let (drove, records) = match drive_disk(needed, store).await { 163 163 - Ok((d, r, s)) => { 164 164 - store = s; 165 165 - (d, r) 166 166 - } 167 167 - Err(e) => { 168 168 - eprintln!("failed to drive disk: {e:?}. skipping..."); 169 169 - store = builder.open(file.clone()).await?; 170 170 - continue; 171 171 - } 172 172 - }; 173 173 - n.fetch_add(drove, Ordering::Relaxed); 174 174 - n_records.fetch_add(records, Ordering::Relaxed); 175 175 - let were_active = disk_workers_active.fetch_sub(1, Ordering::AcqRel); 176 176 - println!("<- disk workers active: {}", were_active - 1); 177 177 - } 178 178 - Ok(()) 179 179 - } 180 180 - 181 181 - #[tokio::main] 182 182 - async fn main() -> Result<()> { 183 183 - env_logger::init(); 184 184 - 185 185 - let Args { 186 186 - cars_folder, 187 187 - disk_folder, 188 188 - disk_workers, 189 189 - mem_workers, 190 190 - } = Args::parse(); 191 191 - 192 192 - let mut set = JoinSet::<Result<()>>::new(); 193 193 - 194 194 - let (cars_tx, cars_rx) = async_channel::bounded(2); 195 195 - set.spawn(get_cars(cars_folder, cars_tx)); 196 196 - 197 197 - let n: Arc<AtomicUsize> = Arc::new(0.into()); 198 198 - let n_records: Arc<AtomicUsize> = Arc::new(0.into()); 199 199 - let disk_workers_active: Arc<AtomicUsize> = Arc::new(0.into()); 200 200 - 201 201 - set.spawn({ 202 202 - let n = n.clone(); 203 203 - let n_records = n_records.clone(); 204 204 - let mut interval = tokio::time::interval(std::time::Duration::from_secs(10)); 205 205 - async move { 206 206 - let mut last_n = n.load(Ordering::Relaxed); 207 207 - let mut last_n_records = n.load(Ordering::Relaxed); 208 208 - loop { 209 209 - interval.tick().await; 210 210 - let n = n.load(Ordering::Relaxed); 211 211 - let n_records = n_records.load(Ordering::Relaxed); 212 212 - let diff_n = n - last_n; 213 213 - let diff_records = n_records - last_n_records; 214 214 - println!("rate: {} rec/sec; {} n/sec", diff_records / 10, diff_n / 10); 215 215 - if n_records > 0 && diff_records == 0 { 216 216 - println!("zero encountered, stopping rate calculation polling."); 217 217 - break Ok(()); 218 218 - } 219 219 - last_n = n; 220 220 - last_n_records = n_records; 221 221 - } 222 222 - } 223 223 - }); 224 224 - 225 225 - let (needs_disk_tx, needs_disk_rx) = async_channel::bounded(disk_workers); 226 226 - 227 227 - for _ in 0..mem_workers { 228 228 - set.spawn(mem_worker( 229 229 - cars_rx.clone(), 230 230 - needs_disk_tx.clone(), 231 231 - n.clone(), 232 232 - n_records.clone(), 233 233 - )); 234 234 - } 235 235 - drop(cars_rx); 236 236 - drop(needs_disk_tx); 237 237 - 238 238 - tokio::fs::create_dir_all(disk_folder.clone()).await?; 239 239 - for id in 0..disk_workers { 240 240 - set.spawn(disk_worker( 241 241 - id, 242 242 - needs_disk_rx.clone(), 243 243 - disk_folder.clone(), 244 244 - n.clone(), 245 245 - n_records.clone(), 246 246 - disk_workers_active.clone(), 247 247 - )); 248 248 - } 249 249 - drop(needs_disk_rx); 250 250 - 251 251 - while let Some(res) = set.join_next().await { 252 252 - println!("task from set joined: {res:?}"); 253 253 - } 254 254 - 255 255 - eprintln!("total records processed: {n_records:?}; total n: {n:?}"); 256 256 - 257 257 - Ok(()) 258 258 - }

-137

spacedust/src/bin/scrape_pds.rs

··· 1 1 - use clap::Parser; 2 2 - use reqwest::Url; 3 3 - use serde::Deserialize; 4 4 - use std::path::PathBuf; 5 5 - use tokio::io::AsyncWriteExt; 6 6 - use tokio::{sync::mpsc, time}; 7 7 - 8 8 - type Result<T> = std::result::Result<T, Box<dyn std::error::Error>>; 9 9 - 10 10 - use futures::StreamExt; 11 11 - 12 12 - #[derive(Debug, Parser)] 13 13 - struct Args { 14 14 - #[arg(long)] 15 15 - pds: Url, 16 16 - #[arg(long)] 17 17 - throttle_ms: u64, // 100ms per pds? 18 18 - #[arg(long)] 19 19 - folder: PathBuf, 20 20 - } 21 21 - 22 22 - async fn download_repo( 23 23 - client: &reqwest::Client, 24 24 - mut pds: Url, 25 25 - did: String, 26 26 - mut path: PathBuf, 27 27 - ) -> Result<()> { 28 28 - path.push(format!("{did}.car")); 29 29 - let f = tokio::fs::File::create(path).await?; 30 30 - let mut w = tokio::io::BufWriter::new(f); 31 31 - 32 32 - pds.set_path("/xrpc/com.atproto.sync.getRepo"); 33 33 - pds.set_query(Some(&format!("did={did}"))); 34 34 - let mut byte_stream = client.get(pds).send().await?.bytes_stream(); 35 35 - 36 36 - while let Some(stuff) = byte_stream.next().await { 37 37 - tokio::io::copy(&mut stuff?.as_ref(), &mut w).await?; 38 38 - } 39 39 - w.flush().await?; 40 40 - 41 41 - Ok(()) 42 42 - } 43 43 - 44 44 - #[derive(Debug, Deserialize)] 45 45 - struct RepoInfo { 46 46 - did: String, 47 47 - active: bool, 48 48 - } 49 49 - 50 50 - #[derive(Debug, Deserialize)] 51 51 - struct ListReposResponse { 52 52 - cursor: Option<String>, 53 53 - repos: Vec<RepoInfo>, 54 54 - } 55 55 - 56 56 - fn get_pds_dids(client: reqwest::Client, mut pds: Url) -> mpsc::Receiver<String> { 57 57 - let (tx, rx) = mpsc::channel(2); 58 58 - tokio::task::spawn(async move { 59 59 - pds.set_path("/xrpc/com.atproto.sync.listRepos"); 60 60 - let mut cursor = None; 61 61 - 62 62 - loop { 63 63 - if let Some(c) = cursor { 64 64 - pds.set_query(Some(&format!("cursor={c}"))); 65 65 - } 66 66 - let res: ListReposResponse = client 67 67 - .get(pds.clone()) 68 68 - .send() 69 69 - .await 70 70 - .expect("to send request") 71 71 - .error_for_status() 72 72 - .expect("to be ok") 73 73 - .json() 74 74 - .await 75 75 - .expect("json response"); 76 76 - for repo in res.repos { 77 77 - if repo.active { 78 78 - tx.send(repo.did) 79 79 - .await 80 80 - .expect("to be able to send on the channel"); 81 81 - } 82 82 - } 83 83 - cursor = res.cursor; 84 84 - if cursor.is_none() { 85 85 - break; 86 86 - } 87 87 - } 88 88 - }); 89 89 - rx 90 90 - } 91 91 - 92 92 - #[tokio::main] 93 93 - async fn main() -> Result<()> { 94 94 - env_logger::init(); 95 95 - 96 96 - let Args { 97 97 - pds, 98 98 - throttle_ms, 99 99 - folder, 100 100 - } = Args::parse(); 101 101 - 102 102 - tokio::fs::create_dir_all(folder.clone()).await?; 103 103 - 104 104 - let client = reqwest::Client::builder() 105 105 - .user_agent("microcosm/spacedust-testing") 106 106 - .build()?; 107 107 - 108 108 - let mut dids = get_pds_dids(client.clone(), pds.clone()); 109 109 - 110 110 - let mut interval = time::interval(time::Duration::from_millis(throttle_ms)); 111 111 - let mut oks = 0; 112 112 - let mut single_fails = 0; 113 113 - let mut double_fails = 0; 114 114 - 115 115 - while let Some(did) = dids.recv().await { 116 116 - interval.tick().await; 117 117 - println!("did: {did:?}"); 118 118 - if let Err(e) = download_repo(&client, pds.clone(), did.clone(), folder.clone()).await { 119 119 - single_fails += 1; 120 120 - eprintln!("failed to download repo for did: {did:?}: {e:?}. retrying in a moment..."); 121 121 - tokio::time::sleep(time::Duration::from_secs(3)).await; 122 122 - interval.reset(); 123 123 - if let Err(e) = download_repo(&client, pds.clone(), did.clone(), folder.clone()).await { 124 124 - double_fails += 1; 125 125 - eprintln!("failed again: {e:?}. moving on in a moment..."); 126 126 - tokio::time::sleep(time::Duration::from_secs(1)).await; 127 127 - continue; 128 128 - } 129 129 - } 130 130 - oks += 1; 131 131 - println!(" -> done. did: {did:?}"); 132 132 - } 133 133 - 134 134 - eprintln!("got {oks} repos. single fails: {single_fails}; doubles: {double_fails}."); 135 135 - 136 136 - Ok(()) 137 137 - }

-1

spacedust/src/lib.rs

··· 3 3 pub mod error; 4 4 pub mod removable_delay_queue; 5 5 pub mod server; 6 6 - pub mod storage; 7 6 pub mod subscriber; 8 7 9 8 use jetstream::events::CommitEvent;

spacedust/src/storage/car/drive.rs

This is a binary file and will not be displayed.

-1

spacedust/src/storage/car/mod.rs

··· 1 1 -

spacedust/src/storage/car/walk.rs

This is a binary file and will not be displayed.

-9

spacedust/src/storage/fjall/mod.rs

··· 1 1 - use crate::storage::Storage; 2 2 - 3 3 - pub struct FjallStorage {} 4 4 - 5 5 - impl Storage for FjallStorage { 6 6 - fn import_car() { 7 7 - todo!() 8 8 - } 9 9 - }

-6

spacedust/src/storage/mod.rs

··· 1 1 - pub mod car; 2 2 - pub mod fjall; 3 3 - 4 4 - pub trait Storage { 5 5 - fn import_car() {} 6 6 - }