upload: form an index of all files to be uploaded

todo: detect mime type from file extension

upload/Cargo.lock

···
 version = 4
 
 [[package]]
+name = "aho-corasick"
+version = "1.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
 name = "anstream"
 version = "0.6.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2"
 dependencies = [
- "windows-sys",
+ "windows-sys 0.60.2",
 ]
 
 [[package]]
···
 dependencies = [
  "anstyle",
  "once_cell_polyfill",
- "windows-sys",
+ "windows-sys 0.60.2",
 ]
 
 [[package]]
···
 version = "0.1.0"
 dependencies = [
  "clap",
+ "ignore",
+ "regex",
+]
+
+[[package]]
+name = "bstr"
+version = "1.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4"
+dependencies = [
+ "memchr",
+ "serde",
 ]
 
 [[package]]
···
 checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75"
 
 [[package]]
+name = "crossbeam-deque"
+version = "0.8.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51"
+dependencies = [
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
+dependencies = [
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
+
+[[package]]
+name = "globset"
+version = "0.4.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "54a1028dfc5f5df5da8a56a73e6c153c9a9708ec57232470703592a3f18e49f5"
+dependencies = [
+ "aho-corasick",
+ "bstr",
+ "log",
+ "regex-automata",
+ "regex-syntax",
+]
+
+[[package]]
 name = "heck"
 version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
 
 [[package]]
+name = "ignore"
+version = "0.4.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d89fd380afde86567dfba715db065673989d6253f42b88179abd3eae47bda4b"
+dependencies = [
+ "crossbeam-deque",
+ "globset",
+ "log",
+ "memchr",
+ "regex-automata",
+ "same-file",
+ "walkdir",
+ "winapi-util",
+]
+
+[[package]]
 name = "is_terminal_polyfill"
 version = "1.70.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"
 
 [[package]]
+name = "log"
+version = "0.4.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432"
+
+[[package]]
+name = "memchr"
+version = "2.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273"
+
+[[package]]
 name = "once_cell_polyfill"
 version = "1.70.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 ]
 
 [[package]]
+name = "regex"
+version = "1.12.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-automata",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-automata"
+version = "0.4.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-syntax"
+version = "0.8.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58"
+
+[[package]]
+name = "same-file"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
+dependencies = [
+ "winapi-util",
+]
+
+[[package]]
+name = "serde"
+version = "1.0.228"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
+dependencies = [
+ "serde_core",
+]
+
+[[package]]
+name = "serde_core"
+version = "1.0.228"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.228"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
 name = "strsim"
 version = "0.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
 
 [[package]]
+name = "walkdir"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
+dependencies = [
+ "same-file",
+ "winapi-util",
+]
+
+[[package]]
+name = "winapi-util"
+version = "0.1.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
+dependencies = [
+ "windows-sys 0.61.2",
+]
+
+[[package]]
 name = "windows-link"
 version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb"
 dependencies = [
  "windows-targets",
+]
+
+[[package]]
+name = "windows-sys"
+version = "0.61.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
+dependencies = [
+ "windows-link",
 ]
 
 [[package]]

upload/Cargo.toml

···
 
 [dependencies]
 clap = { version = "4.5.49", features = ["derive"] }
+ignore = "0.4.23"
+regex = "1.12.2"

upload/src/main.rs

···
-use clap::Parser;
+use clap::{ArgAction, Parser};
 use std::path::PathBuf;
+
+mod sitemap;
 
 #[derive(Parser, Debug)]
 #[command(version, about, long_about = None)]
···
   #[arg(verbatim_doc_comment, short, long)]
   password: String,
 
+  /// Include dotfiles in upload
+  /// Default: false
+  #[arg(verbatim_doc_comment, short = 'a', long = "all")]
+  all_files: bool,
+
+  /// Respect gitignore files
+  /// Note: gitignore files are not uploaded unless --all is set
+  /// Default: true
+  #[arg(verbatim_doc_comment, short = 'g', long = "no-gitignore", action = ArgAction::SetFalse, )]
+  git_ignore: bool,
+
   /// Directory to upload
   #[arg(verbatim_doc_comment)]
   dir: PathBuf,
···
   println!(
     "user: {}
 pword: {}
-dir: {:?}",
+dir: {:?}
+all: {}
+gitignore: {}",
     config.user,
     config.password,
     config.dir.to_str(),
+    config.all_files,
+    config.git_ignore
   );
 
   // get local site info
+  let sitemap = sitemap::local_sitemap(config.dir, config.all_files, config.git_ignore);
 
   // create session
 

upload/src/sitemap.rs

···
+use ignore::WalkBuilder;
+use std::{collections::HashMap, error::Error, fmt, path::PathBuf};
+
+pub type Sitemap = HashMap<String, SitemapNode>;
+
+#[derive(Debug)]
+pub struct SitemapNode {
+  pub mime_type: String,
+  pub blob: BlobRef,
+}
+
+#[derive(Debug)]
+pub enum BlobRef {
+  Local(PathBuf),
+  Remote(String),
+}
+
+#[derive(Debug)]
+pub enum SitemapErr {
+  IoErr(std::io::Error),
+  IgnoreErr(ignore::Error),
+}
+
+impl From<std::io::Error> for SitemapErr {
+  fn from(value: std::io::Error) -> Self {
+    SitemapErr::IoErr(value)
+  }
+}
+
+impl From<ignore::Error> for SitemapErr {
+  fn from(value: ignore::Error) -> Self {
+    SitemapErr::IgnoreErr(value)
+  }
+}
+
+impl Error for SitemapErr {}
+
+impl fmt::Display for SitemapErr {
+  fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+    match self {
+      SitemapErr::IoErr(err) => write!(f, "IO Error: {}", err),
+      SitemapErr::IgnoreErr(err) => write!(f, "Ignore Error: {}", err),
+    }
+  }
+}
+
+pub fn local_sitemap<'a>(
+  dir: PathBuf,
+  include_dotfiles: bool,
+  use_gitignore: bool,
+) -> Result<Sitemap, SitemapErr> {
+  let res = WalkBuilder::new(dir)
+    .hidden(!include_dotfiles)
+    .git_ignore(use_gitignore)
+    .git_exclude(use_gitignore)
+    .git_global(use_gitignore)
+    .follow_links(true)
+    .build()
+    .filter_map(|x| {
+      let x = x
+        .or_else(|err| {
+          println!("Warning: {} ", err);
+          Err(())
+        })
+        .ok()?;
+
+      let metadata = x
+        .metadata()
+        .or_else(|err| {
+          println!("Warning on file {}:\n{} ", x.path().display(), err);
+          Err(())
+        })
+        .ok()?;
+
+      if !metadata.is_file() {
+        return None;
+      };
+
+      let key = x.path().to_str()?;
+      let key = String::from(key);
+
+      Some((
+        key,
+        SitemapNode {
+          mime_type: String::from("application/octet-stream"),
+          blob: BlobRef::Local(x.path().to_owned()),
+        },
+      ))
+    })
+    .collect::<HashMap<_, _>>();
+
+  println!(
+    "  - {}",
+    res
+      .iter()
+      .map(|(k, v)| format!("{}: {:?}", k, v))
+      .collect::<Vec<_>>()
+      .join("\n  - ")
+  );
+
+  Ok(res)
+}