Implement sitemap derivation

I think this structure will be able to
carry us to the glorious finish line.

src/file.rs

···
 
 use crate::frontmatter::FrontMatter;
 
-fn make_mdast(data: &str) -> anyhow::Result<mdast::Node> {
-    let options = {
-        let mut out = ParseOptions::gfm();
-        out.constructs.frontmatter = true;
-        out
-    };
-    let ast = to_mdast(data, &options).map_err(|e| anyhow!("failed to parse markdown: {e}"))?;
-    Ok(ast)
-}
-
-fn write_md_ast<'root>(writer: &mut impl io::Write, ast: &'root mdast::Node) -> anyhow::Result<()> {
-    enum Work<'a> {
-        Node(&'a mdast::Node),
-        Lit(&'static str),
-        Str(String),
-    }
-
-    let mut footnote_ids = Sequential::<&'root str>::default();
-    let mut footnote_defs =
-        Vec::<Option<(&'root str, &'root [mdast::Node])>>::with_capacity(1 << 6);
-
-    // Work contains unprocessed nodes.
-    //
-    // The capacity should be the largest amount of children or nesting we expect to see.
-    let mut q: Vec<Work<'_>> = Vec::with_capacity(1 << 8);
-    macro_rules! children {
-        ($x:expr) => {
-            q.extend($x.iter().rev().map(Work::Node))
-        };
-    }
-    macro_rules! lit {
-        ($x:expr) => {
-            q.push(Work::Lit($x))
-        };
-    }
-    macro_rules! fmt {
-        ($fmt:literal, $($xs:expr),*) => {
-            q.push(Work::Str(
-                format!($fmt $(, $xs)*)
-            ))
-        };
-    }
-    q.push(Work::Node(ast));
-    while let Some(work) = q.pop() {
-        let node = match work {
-            Work::Str(s) => {
-                writer.write_all(s.as_bytes())?;
-                continue;
-            }
-            Work::Lit(s) => {
-                writer.write_all(s.as_bytes())?;
-                continue;
-            }
-            Work::Node(node) => node,
-        };
-        use mdast::Node::*;
-        match node {
-            Root(n) => {
-                children!(n.children);
-            }
-            Paragraph(n) => {
-                lit!("</p>");
-                children!(n.children);
-                lit!("\n<p>");
-            }
-            Blockquote(n) => {
-                lit!("</blockquote>");
-                children!(n.children);
-                lit!("\n<blockquote>");
-            }
-            FootnoteDefinition(n) => {
-                let id = footnote_ids.value(&n.identifier);
-                let id_usize = id as usize;
-                footnote_defs.resize(footnote_defs.len().max(id_usize + 1), None);
-                footnote_defs[id_usize] = Some((n.identifier.as_str(), n.children.as_slice()));
-            }
-            List(n) => {
-                if n.ordered {
-                    lit!("</ol>");
-                    children!(n.children);
-                    lit!("\n<ol>");
-                } else {
-                    lit!("</ul>");
-                    children!(n.children);
-                    lit!("\n<ul>");
-                }
-            }
-            ListItem(n) => {
-                lit!("</li>");
-                match (n.spread, n.children.as_slice()) {
-                    (false, [Paragraph(inner)]) => {
-                        children!(inner.children);
-                    }
-                    (_, children) => {
-                        children!(children);
-                    }
-                }
-                lit!("<li>");
-            }
-            Yaml(_) => {
-                // Ignore front matter
-            }
-            Break(_) => {
-                lit!("\n<br/>");
-            }
-            InlineCode(n) => {
-                fmt!("<code>{}</code>", &n.value);
-            }
-            Delete(n) => {
-                lit!("</del>");
-                children!(n.children);
-                lit!("<del>");
-            }
-            Emphasis(n) => {
-                lit!("</em>");
-                children!(n.children);
-                lit!("<em>");
-            }
-            FootnoteReference(n) => {
-                let id = footnote_ids.value(&n.identifier);
-                fmt!(
-                    "<sup><a href=\"#fn-{}\">{}</a></sup>",
-                    &n.identifier,
-                    id + 1
-                );
-            }
-            Html(n) => {
-                fmt!("{}", n.value);
-            }
-            Image(n) => {
-                let title = n
-                    .title
-                    .as_ref()
-                    .map(|x| format!("title={x}"))
-                    .unwrap_or_default();
-                fmt!("\n<img src={} alt={} {}/>", n.url, n.alt, title);
-            }
-            Strong(n) => {
-                lit!("</strong>");
-                children!(n.children);
-                lit!("<strong>");
-            }
-            Link(n) => {
-                lit!("</a>");
-                children!(n.children);
-                fmt!("<a href={}>", n.url);
-            }
-            Code(n) => {
-                fmt!("\n<pre><code>{}</code></pre>", n.value);
-            }
-            InlineMath(n) => {
-                fmt!("<code>${}$</code>", n.value);
-            }
-            Math(n) => {
-                fmt!("\n<pre>\n<code>\n$$\n{}\n$$\n</code>\n</pre>", n.value);
-            }
-            Text(n) => {
-                writer.write_all(n.value.as_bytes())?;
-            }
-            ThematicBreak(_) => {
-                lit!("\n<hr />");
-            }
-            Table(n) => {
-                lit!("\n</table>");
-                children!(n.children);
-                lit!("\n<table>");
-            }
-            TableRow(n) => {
-                lit!("\n</tr>");
-                children!(n.children);
-                lit!("\n<tr>");
-            }
-            TableCell(n) => {
-                lit!("</th>");
-                children!(n.children);
-                lit!("\n<th>");
-            }
-            Heading(n) => {
-                fmt!("</h{}>", n.depth);
-                children!(n.children);
-                fmt!("\n<h{}>", n.depth);
-            }
-            MdxJsxFlowElement(_) => unimplemented!("MdxJsxFlowElement"),
-            MdxjsEsm(_) => unimplemented!("MdxjsEsm"),
-            Toml(_) => unimplemented!("Toml"),
-            MdxTextExpression(_) => unimplemented!("MdxTextExpression"),
-            ImageReference(_) => unimplemented!("ImageReference"),
-            MdxJsxTextElement(_) => unimplemented!("MdxJsxTextElement"),
-            LinkReference(_) => unimplemented!("LinkReference"),
-            MdxFlowExpression(_) => unimplemented!("MdxFlowExpression"),
-            Definition(_) => unimplemented!("Definition"),
-        }
-    }
-    write!(writer, "<section class=\"footnotes\">\n<ol>\n")?;
-    for def in footnote_defs.into_iter() {
-        match def {
-            None => {
-                write!(writer, "<li>???</li>\n")?;
-            }
-            Some((identifier, children)) => {
-                write!(writer, "<li id=\"fn-{identifier}\">")?;
-                for n in children {
-                    write_md_ast(writer, n)?;
-                }
-                write!(writer, "</li>\n")?;
-            }
-        }
-    }
-    write!(writer, "</ol>\n</section>")?;
-    write!(writer, "\n")?;
-    Ok(())
-}
-
-pub fn find_yaml_frontmatter<'root>(ast: &'root mdast::Node) -> Option<&'root str> {
-    let mut q = vec![ast];
-    while let Some(n) = q.pop() {
-        use mdast::Node::*;
-        match n {
-            Root(n) => {
-                q.extend(n.children.iter());
-            }
-            Yaml(n) => return Some(&n.value),
-            _ => {}
-        }
-    }
-    None
-}
-
 /// Represents a file we process in our blog engine.
 pub struct File<'a> {
     rel_path: &'a Path,

src/main.rs

···
-use anyhow::{Context, anyhow};
-use minijinja::Environment;
+use anyhow::anyhow;
+use minijinja::{Environment, context};
 use std::{
-    borrow::Cow,
     fs::{self},
+    io::{BufWriter, Write},
     path::PathBuf,
 };
 
 mod frontmatter;
-
-mod file;
-use file::File;
-
 mod fs_utils;
+mod markdown;
+mod sitemap;
+
 use fs_utils::copy_dir;
+use sitemap::SiteMap;
+
+use crate::markdown::{make_mdast, write_md_ast};
 
 /// A static string for usage errors.
 const USAGE: &str = "usage: clog <input_dir> <output_dir>";
···
         let env = Environment::new();
         let template_data = fs::read_to_string(self.template_dir.join("index.html"))?;
         let template = env.template_from_str(&template_data)?;
-        let mut dirs = vec![Cow::Borrowed(&self.content_dir)];
-        while let Some(dir) = dirs.pop() {
-            for entry in fs::read_dir(dir.as_path())? {
-                let entry = entry?;
-                let file_type = entry.file_type()?;
-                if !file_type.is_file() {
-                    if file_type.is_dir() {
-                        dirs.push(Cow::Owned(entry.path()));
-                    }
-                    continue;
-                }
-                let path = entry.path();
-                let Some(extension) = path.extension() else {
-                    continue;
-                };
-                // Copy any images "in place".
-                // Contains won't work because of the need to cast.
-                if ["png", "jpg"].into_iter().any(|x| x == extension) {
-                    let rel_path = path.strip_prefix(&self.content_dir)?;
-                    let out_path = self.output_dir.join(rel_path);
-                    if let Some(parent) = out_path.parent() {
-                        fs::create_dir_all(parent)?;
-                    }
-                    fs::copy(&path, &self.output_dir.join(rel_path))?;
-                    continue;
-                }
-                // Skip non-markdown files.
-                if !path.extension().map(|x| x == "md").unwrap_or(true) {
-                    continue;
-                }
-                File::read(&self.content_dir, &path)
-                    .with_context(|| format!("failed to read file: {:?}", &path))?
-                    .write(&self.output_dir, template.clone())
-                    .with_context(|| format!("failed to write file: {:?}", &path))?;
+        let site_map = SiteMap::build(&self.content_dir, &self.output_dir)?;
+        for file in site_map.statics() {
+            if let Some(parent) = file.out_path.parent() {
+                fs::create_dir_all(parent)?;
+            }
+            fs::copy(&file.in_path, &file.out_path)?;
+        }
+        let mut buf = Vec::with_capacity(1 << 14);
+        for page in site_map.pages() {
+            let content = fs::read_to_string(&page.in_path)?;
+            let md = make_mdast(&content)?;
+            let body = {
+                buf.clear();
+                write_md_ast(&mut buf, &md)?;
+                String::from_utf8_lossy(&buf)
+            };
+            if let Some(parent) = page.out_path.parent() {
+                fs::create_dir_all(parent)?;
             }
+            let file = fs::File::create(&page.out_path)?;
+            let mut writer = BufWriter::new(file);
+            let ctx = context! {
+              body => body,
+              title => page.front_matter.title,
+              date => page.front_matter.date,
+              authors => page.front_matter.authors,
+              published => page.front_matter.published,
+              link => page.front_matter.link,
+              tags => page.front_matter.tags,
+            };
+            template.render_to_write(ctx, &mut writer)?;
+            writer.flush()?;
         }
         Ok(())
     }

src/markdown.rs

···
+use anyhow::anyhow;
+use markdown::{ParseOptions, mdast, to_mdast};
+use std::io;
+
+mod counter;
+
+use counter::Sequential;
+
+pub fn make_mdast(data: &str) -> anyhow::Result<mdast::Node> {
+    let options = {
+        let mut out = ParseOptions::gfm();
+        out.constructs.frontmatter = true;
+        out
+    };
+    let ast = to_mdast(data, &options).map_err(|e| anyhow!("failed to parse markdown: {e}"))?;
+    Ok(ast)
+}
+
+pub fn write_md_ast<'root>(
+    writer: &mut impl io::Write,
+    ast: &'root mdast::Node,
+) -> anyhow::Result<()> {
+    enum Work<'a> {
+        Node(&'a mdast::Node),
+        Lit(&'static str),
+        Str(String),
+    }
+
+    let mut footnote_ids = Sequential::<&'root str>::default();
+    let mut footnote_defs =
+        Vec::<Option<(&'root str, &'root [mdast::Node])>>::with_capacity(1 << 6);
+
+    // Work contains unprocessed nodes.
+    //
+    // The capacity should be the largest amount of children or nesting we expect to see.
+    let mut q: Vec<Work<'_>> = Vec::with_capacity(1 << 8);
+    macro_rules! children {
+        ($x:expr) => {
+            q.extend($x.iter().rev().map(Work::Node))
+        };
+    }
+    macro_rules! lit {
+        ($x:expr) => {
+            q.push(Work::Lit($x))
+        };
+    }
+    macro_rules! fmt {
+        ($fmt:literal, $($xs:expr),*) => {
+            q.push(Work::Str(
+                format!($fmt $(, $xs)*)
+            ))
+        };
+    }
+    q.push(Work::Node(ast));
+    while let Some(work) = q.pop() {
+        let node = match work {
+            Work::Str(s) => {
+                writer.write_all(s.as_bytes())?;
+                continue;
+            }
+            Work::Lit(s) => {
+                writer.write_all(s.as_bytes())?;
+                continue;
+            }
+            Work::Node(node) => node,
+        };
+        use mdast::Node::*;
+        match node {
+            Root(n) => {
+                children!(n.children);
+            }
+            Paragraph(n) => {
+                lit!("</p>");
+                children!(n.children);
+                lit!("\n<p>");
+            }
+            Blockquote(n) => {
+                lit!("</blockquote>");
+                children!(n.children);
+                lit!("\n<blockquote>");
+            }
+            FootnoteDefinition(n) => {
+                let id = footnote_ids.value(&n.identifier);
+                let id_usize = id as usize;
+                footnote_defs.resize(footnote_defs.len().max(id_usize + 1), None);
+                footnote_defs[id_usize] = Some((n.identifier.as_str(), n.children.as_slice()));
+            }
+            List(n) => {
+                if n.ordered {
+                    lit!("</ol>");
+                    children!(n.children);
+                    lit!("\n<ol>");
+                } else {
+                    lit!("</ul>");
+                    children!(n.children);
+                    lit!("\n<ul>");
+                }
+            }
+            ListItem(n) => {
+                lit!("</li>");
+                match (n.spread, n.children.as_slice()) {
+                    (false, [Paragraph(inner)]) => {
+                        children!(inner.children);
+                    }
+                    (_, children) => {
+                        children!(children);
+                    }
+                }
+                lit!("<li>");
+            }
+            Yaml(_) => {
+                // Ignore front matter
+            }
+            Break(_) => {
+                lit!("\n<br/>");
+            }
+            InlineCode(n) => {
+                fmt!("<code>{}</code>", &n.value);
+            }
+            Delete(n) => {
+                lit!("</del>");
+                children!(n.children);
+                lit!("<del>");
+            }
+            Emphasis(n) => {
+                lit!("</em>");
+                children!(n.children);
+                lit!("<em>");
+            }
+            FootnoteReference(n) => {
+                let id = footnote_ids.value(&n.identifier);
+                fmt!(
+                    "<sup><a href=\"#fn-{}\">{}</a></sup>",
+                    &n.identifier,
+                    id + 1
+                );
+            }
+            Html(n) => {
+                fmt!("{}", n.value);
+            }
+            Image(n) => {
+                let title = n
+                    .title
+                    .as_ref()
+                    .map(|x| format!("title={x}"))
+                    .unwrap_or_default();
+                fmt!("\n<img src={} alt={} {}/>", n.url, n.alt, title);
+            }
+            Strong(n) => {
+                lit!("</strong>");
+                children!(n.children);
+                lit!("<strong>");
+            }
+            Link(n) => {
+                lit!("</a>");
+                children!(n.children);
+                fmt!("<a href={}>", n.url);
+            }
+            Code(n) => {
+                fmt!("\n<pre><code>{}</code></pre>", n.value);
+            }
+            InlineMath(n) => {
+                fmt!("<code>${}$</code>", n.value);
+            }
+            Math(n) => {
+                fmt!("\n<pre>\n<code>\n$$\n{}\n$$\n</code>\n</pre>", n.value);
+            }
+            Text(n) => {
+                writer.write_all(n.value.as_bytes())?;
+            }
+            ThematicBreak(_) => {
+                lit!("\n<hr />");
+            }
+            Table(n) => {
+                lit!("\n</table>");
+                children!(n.children);
+                lit!("\n<table>");
+            }
+            TableRow(n) => {
+                lit!("\n</tr>");
+                children!(n.children);
+                lit!("\n<tr>");
+            }
+            TableCell(n) => {
+                lit!("</th>");
+                children!(n.children);
+                lit!("\n<th>");
+            }
+            Heading(n) => {
+                fmt!("</h{}>", n.depth);
+                children!(n.children);
+                fmt!("\n<h{}>", n.depth);
+            }
+            MdxJsxFlowElement(_) => unimplemented!("MdxJsxFlowElement"),
+            MdxjsEsm(_) => unimplemented!("MdxjsEsm"),
+            Toml(_) => unimplemented!("Toml"),
+            MdxTextExpression(_) => unimplemented!("MdxTextExpression"),
+            ImageReference(_) => unimplemented!("ImageReference"),
+            MdxJsxTextElement(_) => unimplemented!("MdxJsxTextElement"),
+            LinkReference(_) => unimplemented!("LinkReference"),
+            MdxFlowExpression(_) => unimplemented!("MdxFlowExpression"),
+            Definition(_) => unimplemented!("Definition"),
+        }
+    }
+    write!(writer, "<section class=\"footnotes\">\n<ol>\n")?;
+    for def in footnote_defs.into_iter() {
+        match def {
+            None => {
+                write!(writer, "<li>???</li>\n")?;
+            }
+            Some((identifier, children)) => {
+                write!(writer, "<li id=\"fn-{identifier}\">")?;
+                for n in children {
+                    write_md_ast(writer, n)?;
+                }
+                write!(writer, "</li>\n")?;
+            }
+        }
+    }
+    write!(writer, "</ol>\n</section>")?;
+    write!(writer, "\n")?;
+    Ok(())
+}
+
+pub fn find_yaml_frontmatter<'root>(ast: &'root mdast::Node) -> Option<&'root str> {
+    let mut q = vec![ast];
+    while let Some(n) = q.pop() {
+        use mdast::Node::*;
+        match n {
+            Root(n) => {
+                q.extend(n.children.iter());
+            }
+            Yaml(n) => return Some(&n.value),
+            _ => {}
+        }
+    }
+    None
+}

src/sitemap.rs

···
+use crate::{
+    frontmatter::FrontMatter,
+    markdown::{find_yaml_frontmatter, make_mdast},
+};
+use anyhow::anyhow;
+use std::{
+    borrow::Cow,
+    cmp::Reverse,
+    collections::HashMap,
+    ffi::OsStr,
+    fs,
+    path::{Path, PathBuf},
+};
+
+const STATIC_EXTENSIONS: [&str; 2] = ["png", "jpg"];
+
+fn is_static_extension(e: &OsStr) -> bool {
+    STATIC_EXTENSIONS.iter().any(|&x| x == e)
+}
+
+/// Translate a path, moving it from having in_path as parent, to out_path.
+fn translate(in_path: &Path, out_path: &Path, path: &Path) -> anyhow::Result<PathBuf> {
+    Ok(out_path.join(path.strip_prefix(in_path)?))
+}
+
+fn read_front_matter(path: &Path) -> anyhow::Result<FrontMatter> {
+    let contents = fs::read_to_string(path)?;
+    let ast = make_mdast(&contents)?;
+    let yaml = find_yaml_frontmatter(&ast);
+    let fm = FrontMatter::try_from_yaml(&path, yaml)?;
+    Ok(fm)
+}
+
+/// A Static file, like an image.
+///
+/// This is still contained inside of the content folder.
+#[derive(Debug)]
+pub struct Static {
+    pub in_path: PathBuf,
+    pub out_path: PathBuf,
+}
+
+/// A page with actual markdown content.
+#[derive(Debug)]
+pub struct Page {
+    pub name: String,
+    pub link: String,
+    pub front_matter: FrontMatter,
+    pub in_path: PathBuf,
+    pub out_path: PathBuf,
+}
+
+impl Page {
+    pub fn folder(&self, base: &Path) -> anyhow::Result<Option<PathBuf>> {
+        let Some(parent) = self.in_path.parent() else {
+            return Ok(None);
+        };
+        Ok(Some(parent.strip_prefix(base)?.to_path_buf()))
+    }
+}
+
+type PageIndex = usize;
+
+fn sort_page_indices(pages: &[Page], indices: &mut [PageIndex]) {
+    indices.sort_by_key(|&i| {
+        (
+            Reverse(&pages[i].front_matter.date),
+            Reverse(&pages[i].front_matter.title),
+        )
+    });
+}
+
+#[derive(Debug)]
+pub struct SiteMap {
+    statics: Vec<Static>,
+    pages: Vec<Page>,
+    pages_by_name: HashMap<String, Vec<usize>>,
+    folders: HashMap<PathBuf, Vec<usize>>,
+}
+
+impl SiteMap {
+    pub fn build(in_path: &Path, out_path: &Path) -> anyhow::Result<Self> {
+        let mut statics: Vec<Static> = Vec::with_capacity(128);
+        let mut pages: Vec<Page> = Vec::with_capacity(1024);
+        let mut q = vec![Cow::Borrowed(in_path)];
+        while let Some(dir) = q.pop() {
+            for entry in fs::read_dir(dir)? {
+                let entry = entry?;
+                let file_type = entry.file_type()?;
+                if file_type.is_dir() {
+                    q.push(Cow::Owned(entry.path()));
+                    continue;
+                }
+                if !file_type.is_file() {
+                    continue;
+                }
+                let path = entry.path();
+                if path.to_str().is_none() {
+                    continue;
+                }
+                let Some(extension) = path.extension() else {
+                    continue;
+                };
+                if is_static_extension(extension) {
+                    statics.push(Static {
+                        out_path: translate(in_path, out_path, &path)?,
+                        in_path: path,
+                    });
+                    continue;
+                }
+                if extension != "md" {
+                    continue;
+                }
+                let front_matter = read_front_matter(&path)?;
+                let name = path
+                    .file_stem()
+                    .and_then(|x| x.to_str())
+                    .ok_or_else(|| anyhow!("failed to get file stem"))?
+                    .to_string();
+                let link = {
+                    let rel_path = path.strip_prefix(in_path)?.with_extension("html");
+                    let out_segment = rel_path.to_str().unwrap();
+                    let mut out = String::with_capacity(1 + out_segment.len());
+                    out.push('/');
+                    out.push_str(out_segment);
+                    out
+                };
+                pages.push(Page {
+                    name,
+                    link,
+                    front_matter,
+                    out_path: translate(in_path, out_path, &path.with_extension("html"))?,
+                    in_path: path,
+                });
+            }
+        }
+        let pages_by_name = {
+            let mut out = HashMap::<_, Vec<_>>::new();
+            for (i, page) in pages.iter().enumerate() {
+                out.entry(page.name.clone()).or_default().push(i);
+            }
+            out
+        };
+        // Generate warnings for duplicate names
+        for (name, indices) in &pages_by_name {
+            if indices.len() > 1 {
+                eprintln!("WARNING: `{name}` has conflicts");
+                for &i in indices {
+                    eprintln!("\t{}", pages[i].in_path.to_string_lossy());
+                }
+            }
+        }
+        let folders = {
+            let mut out = HashMap::<_, Vec<_>>::new();
+            for (i, page) in pages.iter().enumerate() {
+                if let Some(folder) = page.folder(in_path)? {
+                    out.entry(folder).or_default().push(i);
+                }
+            }
+            out
+        };
+        Ok(Self {
+            statics,
+            pages,
+            pages_by_name,
+            folders,
+        })
+    }
+
+    /// Iterate over all the static files in the content directory.
+    pub fn statics(&self) -> impl Iterator<Item = &Static> {
+        self.statics.iter()
+    }
+
+    /// Iterate over all of the pages.
+    pub fn pages(&self) -> impl Iterator<Item = &Page> {
+        self.pages.iter()
+    }
+}