+3 -1

.gitignore

··· 1 1 /target 2 2 - .idea 2 2 + .idea 3 3 + .env 4 4 + .DS_Store

+696 -1

Cargo.lock

··· 77 77 ] 78 78 79 79 [[package]] 80 80 + name = "anyhow" 81 81 + version = "1.0.100" 82 82 + source = "registry+https://github.com/rust-lang/crates.io-index" 83 83 + checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" 84 84 + 85 85 + [[package]] 80 86 name = "async-compression" 81 87 version = "0.4.32" 82 88 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 85 91 "compression-codecs", 86 92 "compression-core", 87 93 "futures-core", 94 94 + "futures-io", 88 95 "pin-project-lite", 96 96 + "tokio", 97 97 + ] 98 98 + 99 99 + [[package]] 100 100 + name = "async-trait" 101 101 + version = "0.1.89" 102 102 + source = "registry+https://github.com/rust-lang/crates.io-index" 103 103 + checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" 104 104 + dependencies = [ 105 105 + "proc-macro2", 106 106 + "quote", 107 107 + "syn", 89 108 ] 90 109 91 110 [[package]] ··· 95 114 checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" 96 115 97 116 [[package]] 117 117 + name = "attohttpc" 118 118 + version = "0.30.1" 119 119 + source = "registry+https://github.com/rust-lang/crates.io-index" 120 120 + checksum = "16e2cdb6d5ed835199484bb92bb8b3edd526effe995c61732580439c1a67e2e9" 121 121 + dependencies = [ 122 122 + "base64", 123 123 + "http", 124 124 + "log", 125 125 + "native-tls", 126 126 + "serde", 127 127 + "serde_json", 128 128 + "url", 129 129 + ] 130 130 + 131 131 + [[package]] 132 132 + name = "aws-creds" 133 133 + version = "0.39.0" 134 134 + source = "registry+https://github.com/rust-lang/crates.io-index" 135 135 + checksum = "b13804829a843b3f26e151c97acbb315ee1177a2724690edfcd28f1894146200" 136 136 + dependencies = [ 137 137 + "attohttpc", 138 138 + "home", 139 139 + "log", 140 140 + "quick-xml", 141 141 + "rust-ini", 142 142 + "serde", 143 143 + "thiserror", 144 144 + "time", 145 145 + "url", 146 146 + ] 147 147 + 148 148 + [[package]] 149 149 + name = "aws-region" 150 150 + version = "0.28.0" 151 151 + source = "registry+https://github.com/rust-lang/crates.io-index" 152 152 + checksum = "5532f65342f789f9c1b7078ea9c9cd9293cd62dcc284fa99adc4a1c9ba43469c" 153 153 + dependencies = [ 154 154 + "thiserror", 155 155 + ] 156 156 + 157 157 + [[package]] 98 158 name = "backtrace" 99 159 version = "0.3.76" 100 160 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 122 182 checksum = "2261d10cca569e4643e526d8dc2e62e433cc8aba21ab764233731f8d369bf394" 123 183 124 184 [[package]] 185 185 + name = "block-buffer" 186 186 + version = "0.10.4" 187 187 + source = "registry+https://github.com/rust-lang/crates.io-index" 188 188 + checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" 189 189 + dependencies = [ 190 190 + "generic-array", 191 191 + ] 192 192 + 193 193 + [[package]] 125 194 name = "bumpalo" 126 195 version = "3.19.0" 127 196 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 134 203 checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" 135 204 136 205 [[package]] 206 206 + name = "castaway" 207 207 + version = "0.2.4" 208 208 + source = "registry+https://github.com/rust-lang/crates.io-index" 209 209 + checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" 210 210 + dependencies = [ 211 211 + "rustversion", 212 212 + ] 213 213 + 214 214 + [[package]] 137 215 name = "cc" 138 216 version = "1.2.41" 139 217 source = "registry+https://github.com/rust-lang/crates.io-index" 140 218 checksum = "ac9fe6cdbb24b6ade63616c0a0688e45bb56732262c158df3c0c4bea4ca47cb7" 141 219 dependencies = [ 142 220 "find-msvc-tools", 221 221 + "jobserver", 222 222 + "libc", 143 223 "shlex", 144 224 ] 145 225 ··· 196 276 checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" 197 277 198 278 [[package]] 279 279 + name = "compact_str" 280 280 + version = "0.7.1" 281 281 + source = "registry+https://github.com/rust-lang/crates.io-index" 282 282 + checksum = "f86b9c4c00838774a6d902ef931eff7470720c51d90c2e32cfe15dc304737b3f" 283 283 + dependencies = [ 284 284 + "castaway", 285 285 + "cfg-if", 286 286 + "itoa", 287 287 + "ryu", 288 288 + "static_assertions", 289 289 + ] 290 290 + 291 291 + [[package]] 199 292 name = "compression-codecs" 200 293 version = "0.4.31" 201 294 source = "registry+https://github.com/rust-lang/crates.io-index" 202 295 checksum = "ef8a506ec4b81c460798f572caead636d57d3d7e940f998160f52bd254bf2d23" 203 296 dependencies = [ 204 297 "compression-core", 298 298 + "zstd", 299 299 + "zstd-safe", 205 300 ] 206 301 207 302 [[package]] ··· 211 306 checksum = "e47641d3deaf41fb1538ac1f54735925e275eaf3bf4d55c81b137fba797e5cbb" 212 307 213 308 [[package]] 309 309 + name = "const-random" 310 310 + version = "0.1.18" 311 311 + source = "registry+https://github.com/rust-lang/crates.io-index" 312 312 + checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" 313 313 + dependencies = [ 314 314 + "const-random-macro", 315 315 + ] 316 316 + 317 317 + [[package]] 318 318 + name = "const-random-macro" 319 319 + version = "0.1.16" 320 320 + source = "registry+https://github.com/rust-lang/crates.io-index" 321 321 + checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" 322 322 + dependencies = [ 323 323 + "getrandom 0.2.16", 324 324 + "once_cell", 325 325 + "tiny-keccak", 326 326 + ] 327 327 + 328 328 + [[package]] 214 329 name = "core-foundation" 215 330 version = "0.9.4" 216 331 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 227 342 checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" 228 343 229 344 [[package]] 345 345 + name = "cpufeatures" 346 346 + version = "0.2.17" 347 347 + source = "registry+https://github.com/rust-lang/crates.io-index" 348 348 + checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" 349 349 + dependencies = [ 350 350 + "libc", 351 351 + ] 352 352 + 353 353 + [[package]] 354 354 + name = "crunchy" 355 355 + version = "0.2.4" 356 356 + source = "registry+https://github.com/rust-lang/crates.io-index" 357 357 + checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" 358 358 + 359 359 + [[package]] 360 360 + name = "crypto-common" 361 361 + version = "0.1.6" 362 362 + source = "registry+https://github.com/rust-lang/crates.io-index" 363 363 + checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" 364 364 + dependencies = [ 365 365 + "generic-array", 366 366 + "typenum", 367 367 + ] 368 368 + 369 369 + [[package]] 370 370 + name = "deranged" 371 371 + version = "0.5.4" 372 372 + source = "registry+https://github.com/rust-lang/crates.io-index" 373 373 + checksum = "a41953f86f8a05768a6cda24def994fd2f424b04ec5c719cf89989779f199071" 374 374 + dependencies = [ 375 375 + "powerfmt", 376 376 + "serde_core", 377 377 + ] 378 378 + 379 379 + [[package]] 380 380 + name = "digest" 381 381 + version = "0.10.7" 382 382 + source = "registry+https://github.com/rust-lang/crates.io-index" 383 383 + checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" 384 384 + dependencies = [ 385 385 + "block-buffer", 386 386 + "crypto-common", 387 387 + "subtle", 388 388 + ] 389 389 + 390 390 + [[package]] 230 391 name = "displaydoc" 231 392 version = "0.2.5" 232 393 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 235 396 "proc-macro2", 236 397 "quote", 237 398 "syn", 399 399 + ] 400 400 + 401 401 + [[package]] 402 402 + name = "dlv-list" 403 403 + version = "0.5.2" 404 404 + source = "registry+https://github.com/rust-lang/crates.io-index" 405 405 + checksum = "442039f5147480ba31067cb00ada1adae6892028e40e45fc5de7b7df6dcc1b5f" 406 406 + dependencies = [ 407 407 + "const-random", 238 408 ] 239 409 240 410 [[package]] ··· 334 504 ] 335 505 336 506 [[package]] 507 507 + name = "futures" 508 508 + version = "0.3.31" 509 509 + source = "registry+https://github.com/rust-lang/crates.io-index" 510 510 + checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" 511 511 + dependencies = [ 512 512 + "futures-channel", 513 513 + "futures-core", 514 514 + "futures-executor", 515 515 + "futures-io", 516 516 + "futures-sink", 517 517 + "futures-task", 518 518 + "futures-util", 519 519 + ] 520 520 + 521 521 + [[package]] 337 522 name = "futures-channel" 338 523 version = "0.3.31" 339 524 source = "registry+https://github.com/rust-lang/crates.io-index" 340 525 checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" 341 526 dependencies = [ 342 527 "futures-core", 528 528 + "futures-sink", 343 529 ] 344 530 345 531 [[package]] ··· 349 535 checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" 350 536 351 537 [[package]] 538 538 + name = "futures-executor" 539 539 + version = "0.3.31" 540 540 + source = "registry+https://github.com/rust-lang/crates.io-index" 541 541 + checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" 542 542 + dependencies = [ 543 543 + "futures-core", 544 544 + "futures-task", 545 545 + "futures-util", 546 546 + ] 547 547 + 548 548 + [[package]] 549 549 + name = "futures-io" 550 550 + version = "0.3.31" 551 551 + source = "registry+https://github.com/rust-lang/crates.io-index" 552 552 + checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" 553 553 + 554 554 + [[package]] 555 555 + name = "futures-macro" 556 556 + version = "0.3.31" 557 557 + source = "registry+https://github.com/rust-lang/crates.io-index" 558 558 + checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" 559 559 + dependencies = [ 560 560 + "proc-macro2", 561 561 + "quote", 562 562 + "syn", 563 563 + ] 564 564 + 565 565 + [[package]] 352 566 name = "futures-sink" 353 567 version = "0.3.31" 354 568 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 366 580 source = "registry+https://github.com/rust-lang/crates.io-index" 367 581 checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" 368 582 dependencies = [ 583 583 + "futures-channel", 369 584 "futures-core", 585 585 + "futures-io", 586 586 + "futures-macro", 587 587 + "futures-sink", 370 588 "futures-task", 589 589 + "memchr", 371 590 "pin-project-lite", 372 591 "pin-utils", 592 592 + "slab", 593 593 + ] 594 594 + 595 595 + [[package]] 596 596 + name = "generic-array" 597 597 + version = "0.14.7" 598 598 + source = "registry+https://github.com/rust-lang/crates.io-index" 599 599 + checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" 600 600 + dependencies = [ 601 601 + "typenum", 602 602 + "version_check", 373 603 ] 374 604 375 605 [[package]] ··· 422 652 423 653 [[package]] 424 654 name = "hashbrown" 655 655 + version = "0.14.5" 656 656 + source = "registry+https://github.com/rust-lang/crates.io-index" 657 657 + checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" 658 658 + 659 659 + [[package]] 660 660 + name = "hashbrown" 425 661 version = "0.16.0" 426 662 source = "registry+https://github.com/rust-lang/crates.io-index" 427 663 checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" ··· 431 667 version = "0.5.0" 432 668 source = "registry+https://github.com/rust-lang/crates.io-index" 433 669 checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" 670 670 + 671 671 + [[package]] 672 672 + name = "hex" 673 673 + version = "0.4.3" 674 674 + source = "registry+https://github.com/rust-lang/crates.io-index" 675 675 + checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" 676 676 + 677 677 + [[package]] 678 678 + name = "hmac" 679 679 + version = "0.12.1" 680 680 + source = "registry+https://github.com/rust-lang/crates.io-index" 681 681 + checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" 682 682 + dependencies = [ 683 683 + "digest", 684 684 + ] 685 685 + 686 686 + [[package]] 687 687 + name = "home" 688 688 + version = "0.5.11" 689 689 + source = "registry+https://github.com/rust-lang/crates.io-index" 690 690 + checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" 691 691 + dependencies = [ 692 692 + "windows-sys 0.59.0", 693 693 + ] 434 694 435 695 [[package]] 436 696 name = "http" ··· 666 926 checksum = "4b0f83760fb341a774ed326568e19f5a863af4a952def8c39f9ab92fd95b88e5" 667 927 dependencies = [ 668 928 "equivalent", 669 669 - "hashbrown", 929 929 + "hashbrown 0.16.0", 670 930 ] 671 931 672 932 [[package]] ··· 733 993 ] 734 994 735 995 [[package]] 996 996 + name = "jobserver" 997 997 + version = "0.1.34" 998 998 + source = "registry+https://github.com/rust-lang/crates.io-index" 999 999 + checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" 1000 1000 + dependencies = [ 1001 1001 + "getrandom 0.3.3", 1002 1002 + "libc", 1003 1003 + ] 1004 1004 + 1005 1005 + [[package]] 736 1006 name = "js-sys" 737 1007 version = "0.3.81" 738 1008 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 767 1037 checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" 768 1038 769 1039 [[package]] 1040 1040 + name = "maybe-async" 1041 1041 + version = "0.2.10" 1042 1042 + source = "registry+https://github.com/rust-lang/crates.io-index" 1043 1043 + checksum = "5cf92c10c7e361d6b99666ec1c6f9805b0bea2c3bd8c78dc6fe98ac5bd78db11" 1044 1044 + dependencies = [ 1045 1045 + "proc-macro2", 1046 1046 + "quote", 1047 1047 + "syn", 1048 1048 + ] 1049 1049 + 1050 1050 + [[package]] 1051 1051 + name = "md5" 1052 1052 + version = "0.8.0" 1053 1053 + source = "registry+https://github.com/rust-lang/crates.io-index" 1054 1054 + checksum = "ae960838283323069879657ca3de837e9f7bbb4c7bf6ea7f1b290d5e9476d2e0" 1055 1055 + 1056 1056 + [[package]] 770 1057 name = "memchr" 771 1058 version = "2.7.6" 772 1059 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 777 1064 version = "0.3.17" 778 1065 source = "registry+https://github.com/rust-lang/crates.io-index" 779 1066 checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" 1067 1067 + 1068 1068 + [[package]] 1069 1069 + name = "minidom" 1070 1070 + version = "0.16.0" 1071 1071 + source = "registry+https://github.com/rust-lang/crates.io-index" 1072 1072 + checksum = "e394a0e3c7ccc2daea3dffabe82f09857b6b510cb25af87d54bf3e910ac1642d" 1073 1073 + dependencies = [ 1074 1074 + "rxml", 1075 1075 + ] 780 1076 781 1077 [[package]] 782 1078 name = "miniz_oxide" ··· 816 1112 ] 817 1113 818 1114 [[package]] 1115 1115 + name = "ntapi" 1116 1116 + version = "0.4.1" 1117 1117 + source = "registry+https://github.com/rust-lang/crates.io-index" 1118 1118 + checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4" 1119 1119 + dependencies = [ 1120 1120 + "winapi", 1121 1121 + ] 1122 1122 + 1123 1123 + [[package]] 1124 1124 + name = "num-conv" 1125 1125 + version = "0.1.0" 1126 1126 + source = "registry+https://github.com/rust-lang/crates.io-index" 1127 1127 + checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" 1128 1128 + 1129 1129 + [[package]] 1130 1130 + name = "objc2-core-foundation" 1131 1131 + version = "0.3.2" 1132 1132 + source = "registry+https://github.com/rust-lang/crates.io-index" 1133 1133 + checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" 1134 1134 + dependencies = [ 1135 1135 + "bitflags", 1136 1136 + ] 1137 1137 + 1138 1138 + [[package]] 1139 1139 + name = "objc2-io-kit" 1140 1140 + version = "0.3.2" 1141 1141 + source = "registry+https://github.com/rust-lang/crates.io-index" 1142 1142 + checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" 1143 1143 + dependencies = [ 1144 1144 + "libc", 1145 1145 + "objc2-core-foundation", 1146 1146 + ] 1147 1147 + 1148 1148 + [[package]] 819 1149 name = "object" 820 1150 version = "0.37.3" 821 1151 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 881 1211 ] 882 1212 883 1213 [[package]] 1214 1214 + name = "ordered-multimap" 1215 1215 + version = "0.7.3" 1216 1216 + source = "registry+https://github.com/rust-lang/crates.io-index" 1217 1217 + checksum = "49203cdcae0030493bad186b28da2fa25645fa276a51b6fec8010d281e02ef79" 1218 1218 + dependencies = [ 1219 1219 + "dlv-list", 1220 1220 + "hashbrown 0.14.5", 1221 1221 + ] 1222 1222 + 1223 1223 + [[package]] 884 1224 name = "pds_whatsit_compress_test" 885 1225 version = "0.1.0" 886 1226 dependencies = [ 1227 1227 + "anyhow", 887 1228 "async-compression", 888 1229 "clap", 889 1230 "dotenvy", 890 1231 "env_logger", 1232 1232 + "futures", 891 1233 "log", 892 1234 "reqwest", 1235 1235 + "rust-s3", 1236 1236 + "serde", 1237 1237 + "serde_json", 893 1238 "tokio", 1239 1239 + "tokio-stream", 1240 1240 + "tokio-util", 894 1241 ] 895 1242 896 1243 [[package]] ··· 942 1289 ] 943 1290 944 1291 [[package]] 1292 1292 + name = "powerfmt" 1293 1293 + version = "0.2.0" 1294 1294 + source = "registry+https://github.com/rust-lang/crates.io-index" 1295 1295 + checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" 1296 1296 + 1297 1297 + [[package]] 945 1298 name = "proc-macro2" 946 1299 version = "1.0.101" 947 1300 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 951 1304 ] 952 1305 953 1306 [[package]] 1307 1307 + name = "quick-xml" 1308 1308 + version = "0.38.3" 1309 1309 + source = "registry+https://github.com/rust-lang/crates.io-index" 1310 1310 + checksum = "42a232e7487fc2ef313d96dde7948e7a3c05101870d8985e4fd8d26aedd27b89" 1311 1311 + dependencies = [ 1312 1312 + "memchr", 1313 1313 + "serde", 1314 1314 + ] 1315 1315 + 1316 1316 + [[package]] 954 1317 name = "quote" 955 1318 version = "1.0.41" 956 1319 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1004 1367 "bytes", 1005 1368 "encoding_rs", 1006 1369 "futures-core", 1370 1370 + "futures-util", 1007 1371 "h2", 1008 1372 "http", 1009 1373 "http-body", ··· 1025 1389 "sync_wrapper", 1026 1390 "tokio", 1027 1391 "tokio-native-tls", 1392 1392 + "tokio-util", 1028 1393 "tower", 1029 1394 "tower-http", 1030 1395 "tower-service", 1031 1396 "url", 1032 1397 "wasm-bindgen", 1033 1398 "wasm-bindgen-futures", 1399 1399 + "wasm-streams", 1034 1400 "web-sys", 1035 1401 ] 1036 1402 ··· 1049 1415 ] 1050 1416 1051 1417 [[package]] 1418 1418 + name = "rust-ini" 1419 1419 + version = "0.21.3" 1420 1420 + source = "registry+https://github.com/rust-lang/crates.io-index" 1421 1421 + checksum = "796e8d2b6696392a43bea58116b667fb4c29727dc5abd27d6acf338bb4f688c7" 1422 1422 + dependencies = [ 1423 1423 + "cfg-if", 1424 1424 + "ordered-multimap", 1425 1425 + ] 1426 1426 + 1427 1427 + [[package]] 1428 1428 + name = "rust-s3" 1429 1429 + version = "0.37.0" 1430 1430 + source = "registry+https://github.com/rust-lang/crates.io-index" 1431 1431 + checksum = "94f9b973bd4097f5bb47e5827dcb9fb5dc17e93879e46badc27d2a4e9a4e5588" 1432 1432 + dependencies = [ 1433 1433 + "async-trait", 1434 1434 + "aws-creds", 1435 1435 + "aws-region", 1436 1436 + "base64", 1437 1437 + "bytes", 1438 1438 + "cfg-if", 1439 1439 + "futures-util", 1440 1440 + "hex", 1441 1441 + "hmac", 1442 1442 + "http", 1443 1443 + "log", 1444 1444 + "maybe-async", 1445 1445 + "md5", 1446 1446 + "minidom", 1447 1447 + "percent-encoding", 1448 1448 + "quick-xml", 1449 1449 + "reqwest", 1450 1450 + "serde", 1451 1451 + "serde_derive", 1452 1452 + "serde_json", 1453 1453 + "sha2", 1454 1454 + "sysinfo", 1455 1455 + "thiserror", 1456 1456 + "time", 1457 1457 + "tokio", 1458 1458 + "tokio-stream", 1459 1459 + "url", 1460 1460 + ] 1461 1461 + 1462 1462 + [[package]] 1052 1463 name = "rustc-demangle" 1053 1464 version = "0.1.26" 1054 1465 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1107 1518 checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" 1108 1519 1109 1520 [[package]] 1521 1521 + name = "rxml" 1522 1522 + version = "0.11.1" 1523 1523 + source = "registry+https://github.com/rust-lang/crates.io-index" 1524 1524 + checksum = "65bc94b580d0f5a6b7a2d604e597513d3c673154b52ddeccd1d5c32360d945ee" 1525 1525 + dependencies = [ 1526 1526 + "bytes", 1527 1527 + "rxml_validation", 1528 1528 + ] 1529 1529 + 1530 1530 + [[package]] 1531 1531 + name = "rxml_validation" 1532 1532 + version = "0.11.0" 1533 1533 + source = "registry+https://github.com/rust-lang/crates.io-index" 1534 1534 + checksum = "826e80413b9a35e9d33217b3dcac04cf95f6559d15944b93887a08be5496c4a4" 1535 1535 + dependencies = [ 1536 1536 + "compact_str", 1537 1537 + ] 1538 1538 + 1539 1539 + [[package]] 1110 1540 name = "ryu" 1111 1541 version = "1.0.20" 1112 1542 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1200 1630 ] 1201 1631 1202 1632 [[package]] 1633 1633 + name = "sha2" 1634 1634 + version = "0.10.9" 1635 1635 + source = "registry+https://github.com/rust-lang/crates.io-index" 1636 1636 + checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" 1637 1637 + dependencies = [ 1638 1638 + "cfg-if", 1639 1639 + "cpufeatures", 1640 1640 + "digest", 1641 1641 + ] 1642 1642 + 1643 1643 + [[package]] 1203 1644 name = "shlex" 1204 1645 version = "1.3.0" 1205 1646 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1234 1675 checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" 1235 1676 1236 1677 [[package]] 1678 1678 + name = "static_assertions" 1679 1679 + version = "1.1.0" 1680 1680 + source = "registry+https://github.com/rust-lang/crates.io-index" 1681 1681 + checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" 1682 1682 + 1683 1683 + [[package]] 1237 1684 name = "strsim" 1238 1685 version = "0.11.1" 1239 1686 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1277 1724 ] 1278 1725 1279 1726 [[package]] 1727 1727 + name = "sysinfo" 1728 1728 + version = "0.37.2" 1729 1729 + source = "registry+https://github.com/rust-lang/crates.io-index" 1730 1730 + checksum = "16607d5caffd1c07ce073528f9ed972d88db15dd44023fa57142963be3feb11f" 1731 1731 + dependencies = [ 1732 1732 + "libc", 1733 1733 + "memchr", 1734 1734 + "ntapi", 1735 1735 + "objc2-core-foundation", 1736 1736 + "objc2-io-kit", 1737 1737 + "windows", 1738 1738 + ] 1739 1739 + 1740 1740 + [[package]] 1280 1741 name = "system-configuration" 1281 1742 version = "0.6.1" 1282 1743 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1308 1769 "once_cell", 1309 1770 "rustix", 1310 1771 "windows-sys 0.61.2", 1772 1772 + ] 1773 1773 + 1774 1774 + [[package]] 1775 1775 + name = "thiserror" 1776 1776 + version = "2.0.17" 1777 1777 + source = "registry+https://github.com/rust-lang/crates.io-index" 1778 1778 + checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" 1779 1779 + dependencies = [ 1780 1780 + "thiserror-impl", 1781 1781 + ] 1782 1782 + 1783 1783 + [[package]] 1784 1784 + name = "thiserror-impl" 1785 1785 + version = "2.0.17" 1786 1786 + source = "registry+https://github.com/rust-lang/crates.io-index" 1787 1787 + checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" 1788 1788 + dependencies = [ 1789 1789 + "proc-macro2", 1790 1790 + "quote", 1791 1791 + "syn", 1792 1792 + ] 1793 1793 + 1794 1794 + [[package]] 1795 1795 + name = "time" 1796 1796 + version = "0.3.44" 1797 1797 + source = "registry+https://github.com/rust-lang/crates.io-index" 1798 1798 + checksum = "91e7d9e3bb61134e77bde20dd4825b97c010155709965fedf0f49bb138e52a9d" 1799 1799 + dependencies = [ 1800 1800 + "deranged", 1801 1801 + "itoa", 1802 1802 + "num-conv", 1803 1803 + "powerfmt", 1804 1804 + "serde", 1805 1805 + "time-core", 1806 1806 + "time-macros", 1807 1807 + ] 1808 1808 + 1809 1809 + [[package]] 1810 1810 + name = "time-core" 1811 1811 + version = "0.1.6" 1812 1812 + source = "registry+https://github.com/rust-lang/crates.io-index" 1813 1813 + checksum = "40868e7c1d2f0b8d73e4a8c7f0ff63af4f6d19be117e90bd73eb1d62cf831c6b" 1814 1814 + 1815 1815 + [[package]] 1816 1816 + name = "time-macros" 1817 1817 + version = "0.2.24" 1818 1818 + source = "registry+https://github.com/rust-lang/crates.io-index" 1819 1819 + checksum = "30cfb0125f12d9c277f35663a0a33f8c30190f4e4574868a330595412d34ebf3" 1820 1820 + dependencies = [ 1821 1821 + "num-conv", 1822 1822 + "time-core", 1823 1823 + ] 1824 1824 + 1825 1825 + [[package]] 1826 1826 + name = "tiny-keccak" 1827 1827 + version = "2.0.2" 1828 1828 + source = "registry+https://github.com/rust-lang/crates.io-index" 1829 1829 + checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" 1830 1830 + dependencies = [ 1831 1831 + "crunchy", 1311 1832 ] 1312 1833 1313 1834 [[package]] ··· 1370 1891 ] 1371 1892 1372 1893 [[package]] 1894 1894 + name = "tokio-stream" 1895 1895 + version = "0.1.17" 1896 1896 + source = "registry+https://github.com/rust-lang/crates.io-index" 1897 1897 + checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" 1898 1898 + dependencies = [ 1899 1899 + "futures-core", 1900 1900 + "pin-project-lite", 1901 1901 + "tokio", 1902 1902 + ] 1903 1903 + 1904 1904 + [[package]] 1373 1905 name = "tokio-util" 1374 1906 version = "0.7.16" 1375 1907 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1377 1909 dependencies = [ 1378 1910 "bytes", 1379 1911 "futures-core", 1912 1912 + "futures-io", 1380 1913 "futures-sink", 1381 1914 "pin-project-lite", 1382 1915 "tokio", ··· 1453 1986 checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" 1454 1987 1455 1988 [[package]] 1989 1989 + name = "typenum" 1990 1990 + version = "1.19.0" 1991 1991 + source = "registry+https://github.com/rust-lang/crates.io-index" 1992 1992 + checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" 1993 1993 + 1994 1994 + [[package]] 1456 1995 name = "unicode-ident" 1457 1996 version = "1.0.19" 1458 1997 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1495 2034 checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" 1496 2035 1497 2036 [[package]] 2037 2037 + name = "version_check" 2038 2038 + version = "0.9.5" 2039 2039 + source = "registry+https://github.com/rust-lang/crates.io-index" 2040 2040 + checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" 2041 2041 + 2042 2042 + [[package]] 1498 2043 name = "want" 1499 2044 version = "0.3.1" 1500 2045 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1600 2145 ] 1601 2146 1602 2147 [[package]] 2148 2148 + name = "wasm-streams" 2149 2149 + version = "0.4.2" 2150 2150 + source = "registry+https://github.com/rust-lang/crates.io-index" 2151 2151 + checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" 2152 2152 + dependencies = [ 2153 2153 + "futures-util", 2154 2154 + "js-sys", 2155 2155 + "wasm-bindgen", 2156 2156 + "wasm-bindgen-futures", 2157 2157 + "web-sys", 2158 2158 + ] 2159 2159 + 2160 2160 + [[package]] 1603 2161 name = "web-sys" 1604 2162 version = "0.3.81" 1605 2163 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1610 2168 ] 1611 2169 1612 2170 [[package]] 2171 2171 + name = "winapi" 2172 2172 + version = "0.3.9" 2173 2173 + source = "registry+https://github.com/rust-lang/crates.io-index" 2174 2174 + checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" 2175 2175 + dependencies = [ 2176 2176 + "winapi-i686-pc-windows-gnu", 2177 2177 + "winapi-x86_64-pc-windows-gnu", 2178 2178 + ] 2179 2179 + 2180 2180 + [[package]] 2181 2181 + name = "winapi-i686-pc-windows-gnu" 2182 2182 + version = "0.4.0" 2183 2183 + source = "registry+https://github.com/rust-lang/crates.io-index" 2184 2184 + checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" 2185 2185 + 2186 2186 + [[package]] 2187 2187 + name = "winapi-x86_64-pc-windows-gnu" 2188 2188 + version = "0.4.0" 2189 2189 + source = "registry+https://github.com/rust-lang/crates.io-index" 2190 2190 + checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" 2191 2191 + 2192 2192 + [[package]] 2193 2193 + name = "windows" 2194 2194 + version = "0.61.3" 2195 2195 + source = "registry+https://github.com/rust-lang/crates.io-index" 2196 2196 + checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" 2197 2197 + dependencies = [ 2198 2198 + "windows-collections", 2199 2199 + "windows-core", 2200 2200 + "windows-future", 2201 2201 + "windows-link 0.1.3", 2202 2202 + "windows-numerics", 2203 2203 + ] 2204 2204 + 2205 2205 + [[package]] 2206 2206 + name = "windows-collections" 2207 2207 + version = "0.2.0" 2208 2208 + source = "registry+https://github.com/rust-lang/crates.io-index" 2209 2209 + checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8" 2210 2210 + dependencies = [ 2211 2211 + "windows-core", 2212 2212 + ] 2213 2213 + 2214 2214 + [[package]] 2215 2215 + name = "windows-core" 2216 2216 + version = "0.61.2" 2217 2217 + source = "registry+https://github.com/rust-lang/crates.io-index" 2218 2218 + checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" 2219 2219 + dependencies = [ 2220 2220 + "windows-implement", 2221 2221 + "windows-interface", 2222 2222 + "windows-link 0.1.3", 2223 2223 + "windows-result", 2224 2224 + "windows-strings", 2225 2225 + ] 2226 2226 + 2227 2227 + [[package]] 2228 2228 + name = "windows-future" 2229 2229 + version = "0.2.1" 2230 2230 + source = "registry+https://github.com/rust-lang/crates.io-index" 2231 2231 + checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" 2232 2232 + dependencies = [ 2233 2233 + "windows-core", 2234 2234 + "windows-link 0.1.3", 2235 2235 + "windows-threading", 2236 2236 + ] 2237 2237 + 2238 2238 + [[package]] 2239 2239 + name = "windows-implement" 2240 2240 + version = "0.60.2" 2241 2241 + source = "registry+https://github.com/rust-lang/crates.io-index" 2242 2242 + checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" 2243 2243 + dependencies = [ 2244 2244 + "proc-macro2", 2245 2245 + "quote", 2246 2246 + "syn", 2247 2247 + ] 2248 2248 + 2249 2249 + [[package]] 2250 2250 + name = "windows-interface" 2251 2251 + version = "0.59.3" 2252 2252 + source = "registry+https://github.com/rust-lang/crates.io-index" 2253 2253 + checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" 2254 2254 + dependencies = [ 2255 2255 + "proc-macro2", 2256 2256 + "quote", 2257 2257 + "syn", 2258 2258 + ] 2259 2259 + 2260 2260 + [[package]] 1613 2261 name = "windows-link" 1614 2262 version = "0.1.3" 1615 2263 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1622 2270 checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" 1623 2271 1624 2272 [[package]] 2273 2273 + name = "windows-numerics" 2274 2274 + version = "0.2.0" 2275 2275 + source = "registry+https://github.com/rust-lang/crates.io-index" 2276 2276 + checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1" 2277 2277 + dependencies = [ 2278 2278 + "windows-core", 2279 2279 + "windows-link 0.1.3", 2280 2280 + ] 2281 2281 + 2282 2282 + [[package]] 1625 2283 name = "windows-registry" 1626 2284 version = "0.5.3" 1627 2285 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1717 2375 "windows_x86_64_gnu 0.53.1", 1718 2376 "windows_x86_64_gnullvm 0.53.1", 1719 2377 "windows_x86_64_msvc 0.53.1", 2378 2378 + ] 2379 2379 + 2380 2380 + [[package]] 2381 2381 + name = "windows-threading" 2382 2382 + version = "0.1.0" 2383 2383 + source = "registry+https://github.com/rust-lang/crates.io-index" 2384 2384 + checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6" 2385 2385 + dependencies = [ 2386 2386 + "windows-link 0.1.3", 1720 2387 ] 1721 2388 1722 2389 [[package]] ··· 1910 2577 "quote", 1911 2578 "syn", 1912 2579 ] 2580 2580 + 2581 2581 + [[package]] 2582 2582 + name = "zstd" 2583 2583 + version = "0.13.3" 2584 2584 + source = "registry+https://github.com/rust-lang/crates.io-index" 2585 2585 + checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" 2586 2586 + dependencies = [ 2587 2587 + "zstd-safe", 2588 2588 + ] 2589 2589 + 2590 2590 + [[package]] 2591 2591 + name = "zstd-safe" 2592 2592 + version = "7.2.4" 2593 2593 + source = "registry+https://github.com/rust-lang/crates.io-index" 2594 2594 + checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" 2595 2595 + dependencies = [ 2596 2596 + "zstd-sys", 2597 2597 + ] 2598 2598 + 2599 2599 + [[package]] 2600 2600 + name = "zstd-sys" 2601 2601 + version = "2.0.16+zstd.1.5.7" 2602 2602 + source = "registry+https://github.com/rust-lang/crates.io-index" 2603 2603 + checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" 2604 2604 + dependencies = [ 2605 2605 + "cc", 2606 2606 + "pkg-config", 2607 2607 + ]

+9 -2

Cargo.toml

··· 4 4 edition = "2024" 5 5 6 6 [dependencies] 7 7 - async-compression = "0.4.32" 7 7 + anyhow = "1.0.100" 8 8 + async-compression = { version = "0.4.30", features = ["futures-io", "tokio", "zstd"] } 8 9 clap = { version = "4.5", features = ["derive"] } 9 10 dotenvy = "0.15" 10 11 env_logger = "0.11" 11 12 log = "0.4" 12 12 - reqwest = "0.12.23" 13 13 + reqwest = { version = "0.12.23", features = ["stream"] } 13 14 tokio = { version = "1", features = ["macros", "rt-multi-thread"] } 15 15 + futures = "0.3.31" 16 16 + tokio-util = { version = "0.7.16", features = ["compat"] } 17 17 + tokio-stream = { version = "0.1.17", features = ["io-util"] } 18 18 + rust-s3 = "0.37.0" 19 19 + serde = { version = "1", features = ["derive"] } 20 20 + serde_json = "1"

+219 -26

src/main.rs

··· 1 1 - use std::fs; 2 1 use std::path::PathBuf; 2 2 + use std::{env, fs}; 3 3 4 4 + use anyhow; 5 5 + use async_compression::futures::bufread::{ZstdDecoder, ZstdEncoder}; 4 6 use clap::{Parser, Subcommand}; 5 7 use dotenvy::dotenv; 8 8 + use futures::io::BufReader; 6 9 use log::{error, info}; 10 10 + use reqwest::header::{ACCEPT, ACCEPT_ENCODING}; 11 11 + use s3::creds::Credentials; 12 12 + use s3::{Bucket, Region}; 13 13 + use std::future::Future; 14 14 + use tokio::io::AsyncWriteExt; 15 15 + use tokio_stream::wrappers::LinesStream; 16 16 + use tokio_util::compat::{FuturesAsyncReadCompatExt, TokioAsyncReadCompatExt}; 17 17 + use tokio_util::io::StreamReader; 18 18 + use serde::Deserialize; 7 19 8 20 #[derive(Parser, Debug)] 9 21 #[command(author, version, about = "PDS Whatsit Compress Test CLI", long_about = None)] ··· 15 27 #[derive(Subcommand, Debug)] 16 28 enum Commands { 17 29 /// Export data from a PDS instance 18 18 - Export { 30 30 + Backup { 19 31 /// Output directory path (defaults to ./export) 20 20 - #[arg(short, long)] 21 21 - out: Option<PathBuf>, 32 32 + // #[arg(short, long)] 33 33 + // out: Option<PathBuf>, 22 34 23 35 /// The DID to export 24 36 #[arg(long)] ··· 28 40 #[arg(long, value_name = "URL")] 29 41 pds_url: String, 30 42 }, 43 43 + 44 44 + /// Import: download from S3, decompress zstd, and save locally 45 45 + Restore { 46 46 + /// The DID to import 47 47 + #[arg(long)] 48 48 + did: String, 49 49 + }, 31 50 } 32 51 33 52 fn init_logging() { ··· 45 64 fs::create_dir_all(parent)?; 46 65 } 47 66 } 48 48 - if !path.exists() { 49 49 - fs::create_dir_all(path)?; 50 50 - } 51 67 Ok(()) 52 68 } 53 69 54 70 #[tokio::main] 55 55 - async fn main() { 71 71 + async fn main() -> anyhow::Result<()> { 56 72 init_logging(); 57 73 58 74 let cli = Cli::parse(); 75 75 + // Custom region requires valid region name and endpoint 76 76 + let region_name = env::var("S3_REGION")?; 77 77 + let endpoint = env::var("S3_ENDPOINT")?; 78 78 + let region = Region::Custom { 79 79 + region: region_name, 80 80 + endpoint, 81 81 + }; 82 82 + 83 83 + let bucket = Bucket::new( 84 84 + env::var("S3_BUCKET_NAME")?.as_str(), 85 85 + region, 86 86 + // Credentials are collected from environment, config, profile or instance metadata 87 87 + Credentials::new( 88 88 + Some(env::var("S3_ACCESS_KEY")?.as_str()), 89 89 + Some(env::var("S3_SECRET_KEY")?.as_str()), 90 90 + None, 91 91 + None, 92 92 + None, 93 93 + )?, 94 94 + )?; 59 95 60 96 match cli.command { 61 61 - Commands::Export { out, did, pds_url } => { 62 62 - let out_dir = out.unwrap_or_else(|| PathBuf::from("export")); 97 97 + Commands::Backup { did, pds_url } => { 98 98 + info!("Export requested: did={}, pds_url={}", did, pds_url,); 99 99 + 100 100 + match do_backup(pds_url, did, bucket).await { 101 101 + Ok(_) => { 102 102 + info!("Export completed"); 103 103 + Ok(()) 104 104 + } 105 105 + Err(err) => { 106 106 + error!("Export failed: {}", err); 107 107 + Err(err) 108 108 + } 109 109 + } 110 110 + } 111 111 + Commands::Restore { did } => { 112 112 + info!("Import requested: did={}", did); 113 113 + match do_restore(did, bucket).await { 114 114 + Ok(path) => { 115 115 + info!("Import completed, wrote {}", path.display()); 116 116 + Ok(()) 117 117 + } 118 118 + Err(err) => { 119 119 + error!("Import failed: {}", err); 120 120 + Err(err) 121 121 + } 122 122 + } 123 123 + } 124 124 + } 125 125 + } 126 126 + 127 127 + async fn do_backup(pds_url: String, did: String, bucket: Box<Bucket>) -> anyhow::Result<()> { 128 128 + use futures::TryStreamExt; 129 129 + let atproto_client = reqwest::Client::new(); 130 130 + let back_up_path = format!("users/{did}"); 131 131 + 132 132 + // 1) Backup the full repo CAR (compressed) 133 133 + let response_reader = atproto_client 134 134 + .get(format!("{pds_url}/xrpc/com.atproto.sync.getRepo?did={did}")) 135 135 + .header(ACCEPT, "application/vnd.ipld.car") 136 136 + .send() 137 137 + .await? 138 138 + .error_for_status()? 139 139 + .bytes_stream() 140 140 + .map_err(|e| futures::io::Error::new(futures::io::ErrorKind::Other, e)) 141 141 + .into_async_read(); 142 142 + 143 143 + let buf_reader = BufReader::new(response_reader); 144 144 + let zstd_encoder = ZstdEncoder::new(buf_reader); 145 145 + let mut zstd_tokio_reader = zstd_encoder.compat(); 146 146 + 147 147 + bucket 148 148 + .put_object_stream_builder(format!("/{back_up_path}/{did}.car.zst").as_str()) 149 149 + .with_content_type("application/vnd.ipld.car") 150 150 + .with_content_encoding("zstd")? 151 151 + .execute_stream(&mut zstd_tokio_reader) 152 152 + .await?; 153 153 + 154 154 + // 2) Paginate listBlobs and upload each as zstd-compressed 155 155 + #[derive(Deserialize)] 156 156 + struct ListBlobsResponse { 157 157 + #[allow(dead_code)] 158 158 + cursor: Option<String>, 159 159 + cids: Vec<String>, 160 160 + } 161 161 + 162 162 + let mut cursor: Option<String> = None; 163 163 + let limit = 1000u32; 164 164 + 165 165 + loop { 166 166 + let mut url = format!( 167 167 + "{}/xrpc/com.atproto.sync.listBlobs?did={}&limit={}", 168 168 + pds_url, did, limit 169 169 + ); 170 170 + if let Some(ref c) = cursor { 171 171 + if !c.is_empty() { 172 172 + url.push_str("&cursor="); 173 173 + url.push_str(c); 174 174 + } 175 175 + } 176 176 + 177 177 + info!("Listing blobs: {}", url); 178 178 + let resp = atproto_client 179 179 + .get(url) 180 180 + .header(ACCEPT, "application/json") 181 181 + .send() 182 182 + .await? 183 183 + .error_for_status()?; 184 184 + let bytes = resp.bytes().await?; 185 185 + let page: ListBlobsResponse = serde_json::from_slice(&bytes)?; 63 186 64 64 - if let Err(e) = ensure_dir(&out_dir) { 65 65 - error!("Failed to create/export directory {}: {}", out_dir.display(), e); 66 66 - std::process::exit(1); 187 187 + if page.cids.is_empty() { 188 188 + if cursor.is_none() || cursor.as_deref() == Some("") { 189 189 + break; 67 190 } 191 191 + } 68 192 69 69 - info!( 70 70 - "Export requested: did={}, pds_url={}, out_dir={}", 71 71 - did, 72 72 - pds_url, 73 73 - out_dir.display() 193 193 + for cid in page.cids { 194 194 + let blob_url = format!( 195 195 + "{}/xrpc/com.atproto.sync.getBlob?did={}&cid={}", 196 196 + pds_url, did, cid 74 197 ); 198 198 + info!("Downloading blob {}", cid); 199 199 + let blob_reader = atproto_client 200 200 + .get(blob_url) 201 201 + .header(ACCEPT, "*/*") 202 202 + .send() 203 203 + .await? 204 204 + .error_for_status()? 205 205 + .bytes_stream() 206 206 + .map_err(|e| futures::io::Error::new(futures::io::ErrorKind::Other, e)) 207 207 + .into_async_read(); 75 208 76 76 - // Placeholder for actual export logic. 77 77 - // Implement your export functionality here. 78 78 - println!( 79 79 - "Export would run here with did={} pds_url={} out_dir={}", 80 80 - did, 81 81 - pds_url, 82 82 - out_dir.display() 83 83 - ); 209 209 + let blob_buf = BufReader::new(blob_reader); 210 210 + let blob_zstd = ZstdEncoder::new(blob_buf); 211 211 + let mut blob_tokio_reader = blob_zstd.compat(); 212 212 + 213 213 + let object_key = format!("/{}/blobs/{}.zst", back_up_path, cid); 214 214 + bucket 215 215 + .put_object_stream_builder(&object_key) 216 216 + .with_content_type("application/octet-stream") 217 217 + .with_content_encoding("zstd")? 218 218 + .execute_stream(&mut blob_tokio_reader) 219 219 + .await?; 220 220 + } 221 221 + 222 222 + // Update or finish based on cursor 223 223 + match page.cursor { 224 224 + Some(c) if !c.is_empty() => { 225 225 + cursor = Some(c); 226 226 + } 227 227 + _ => break, 84 228 } 85 229 } 230 230 + 231 231 + Ok(()) 232 232 + } 233 233 + 234 234 + async fn do_restore(did: String, bucket: Box<Bucket>) -> anyhow::Result<PathBuf> { 235 235 + use futures::StreamExt; 236 236 + 237 237 + let back_up_path = format!("users/{did}"); 238 238 + 239 239 + // Stream download from S3 240 240 + let mut s3_stream = bucket 241 241 + .get_object_stream(format!("/{back_up_path}/{did}.car.zst")) 242 242 + .await?; 243 243 + 244 244 + // Convert the stream of Bytes into a tokio AsyncRead 245 245 + let byte_stream = s3_stream 246 246 + .bytes() 247 247 + .map(|res| res.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))); 248 248 + 249 249 + let tokio_reader = StreamReader::new(byte_stream); 250 250 + 251 251 + // Convert tokio AsyncRead -> futures AsyncRead, then buffer for decoder 252 252 + let futures_reader = tokio_reader.compat(); 253 253 + let futures_buf = futures::io::BufReader::new(futures_reader); 254 254 + 255 255 + // Zstd decode 256 256 + let decoder = ZstdDecoder::new(futures_buf); 257 257 + 258 258 + // Convert back to tokio AsyncRead for writing 259 259 + let mut decoded_tokio_reader = decoder.compat(); 260 260 + 261 261 + // Prepare local output path, labeled as decompressed 262 262 + let out_path: PathBuf = [ 263 263 + "export", 264 264 + "users", 265 265 + did.as_str(), 266 266 + &format!("{}-decompressed.car", did), 267 267 + ] 268 268 + .iter() 269 269 + .collect(); 270 270 + ensure_dir(&out_path)?; 271 271 + 272 272 + let mut out_file = tokio::fs::File::create(&out_path).await?; 273 273 + 274 274 + // Stream copy decoded content to file 275 275 + tokio::io::copy(&mut decoded_tokio_reader, &mut out_file).await?; 276 276 + out_file.flush().await?; 277 277 + 278 278 + Ok(out_path) 86 279 }