add parser for robots.txt

Cargo.lock

···
 ]
 
 [[package]]
+name = "thiserror"
+version = "2.0.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8"
+dependencies = [
+ "thiserror-impl",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "2.0.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
 name = "tinystr"
 version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0"
 dependencies = [
  "pin-project-lite",
+ "tracing-attributes",
  "tracing-core",
+]
+
+[[package]]
+name = "tracing-attributes"
+version = "0.1.30"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
 ]
 
 [[package]]
···
 dependencies = [
  "reqwest",
  "scraper",
+ "thiserror",
+ "tracing",
 ]
 
 [[package]]

Cargo.toml

···
 [dependencies]
 reqwest = "0.12"
 scraper = "0.24"
+thiserror = "2.0"
+tracing = "0.1"

flake.nix

···
         ])
         clang
         pkg-config
+        cargo-tarpaulin
       ] ++ librarys;
 
       LD_LIBRARY_PATH = "${pkgs.lib.makeLibraryPath librarys}";

src/lib.rs

···
-mod robotstxt;
+#![allow(clippy::needless_return)]
+
+pub mod robotstxt;
+pub mod utils;
 
 pub static USER_AGENT: &str = concat!(
     env!("CARGO_PKG_NAME"),

src/main.rs

···
+use webring_mapper::robotstxt::RobotsTxt;
+
 fn main() {
-    println!("{}", webring_mapper::USER_AGENT);
+    let input = "
+        User-agent: Googlebot
+        Disallow: /
+
+        User-agent: *
+        Disallow:
+    ";
+
+    dbg!(RobotsTxt::parse(input));
 }

src/robotstxt.rs

···
+use thiserror::Error;
+
+use crate::utils;
+
 // simple parsing for robots.txt, no sitemap because lazy
 
-#[derive(Default, Debug)]
-struct RobotsTxt {
+#[derive(Default, Debug, PartialEq)]
+pub struct RobotsTxt {
     agents: std::collections::HashMap<String, Rules>,
 }
 
-#[derive(Default, Debug)]
-struct Rules {
+#[derive(Default, Debug, Clone, PartialEq)]
+pub struct Rules {
     allow: Vec<String>,
     disallow: Vec<String>,
 }
 
+#[derive(Error, Debug)]
+pub enum RobotsTxtError {
+    /// a line is missing a colon, all non comment lines in robots.txt should contain a colon
+    #[error("missing colon in line: {0}")]
+    MissingColon(String),
+
+    /// a `user-agent` directive was followed by an empty value, this is invalid
+    #[error("no user agent specified on line: {0}")]
+    EmptyUserAgent(String),
+
+    /// invlaid directive, the only valid ones are `user-agent`, `allow`, `disallow` and `sitemap`
+    #[error("unknown directive \"{directive}\" on line: {line}")]
+    UnknownDirective { directive: String, line: String },
+
+    /// a rule was found without a `user-agent` directive before it somewhere INVALID!
+    #[error("no user agent was specified before rule: {0}")]
+    NoUserAgent(String),
+}
+
 impl RobotsTxt {
-    fn parse(input: &str) -> Self {
-        todo!()
+    pub fn parse(input: &str) -> Result<Self, RobotsTxtError> {
+        let mut robots = Self::default();
+        let mut current_agent = (String::default(), Rules::default());
+
+        for line in input.lines() {
+            let line = line.trim();
+
+            if line.is_empty() || line.starts_with('#') {
+                continue;
+            }
+
+            let (key, value) = line
+                .split_once(':')
+                .map(|(k, v)| (k.trim().to_lowercase(), v.trim().to_string()))
+                .ok_or_else(|| RobotsTxtError::MissingColon(line.to_string()))?;
+
+            let handle_rule = |list: &mut Vec<String>| -> Result<(), RobotsTxtError> {
+                if current_agent.0.is_empty() {
+                    return Err(RobotsTxtError::NoUserAgent(line.to_string()));
+                }
+
+                if value.is_empty() {
+                    return Ok(());
+                }
+
+                list.push(value.to_string());
+                Ok(())
+            };
+
+            match key.as_str() {
+                "user-agent" => {
+                    println!("found user agent {value}");
+                    if value.is_empty() {
+                        return Err(RobotsTxtError::EmptyUserAgent(line.to_string()));
+                    }
+
+                    if !current_agent.0.is_empty() {
+                        robots
+                            .agents
+                            .insert(current_agent.0, current_agent.1.clone());
+                    }
+
+                    current_agent.0 = value.to_lowercase();
+                    current_agent.1.clear();
+                }
+                "allow" => handle_rule(&mut current_agent.1.allow)?,
+                "disallow" => handle_rule(&mut current_agent.1.disallow)?,
+                "sitemap" => (),
+                _ => {
+                    return Err(RobotsTxtError::UnknownDirective {
+                        directive: key,
+                        line: line.to_string(),
+                    });
+                }
+            };
+        }
+
+        if !current_agent.0.is_empty() {
+            robots
+                .agents
+                .insert(current_agent.0, current_agent.1.clone());
+        }
+
+        Ok(robots)
+    }
+
+    /// retrive the rules for a input user agent
+    /// case insensitive, will remove everything after the `/`, and everything in `()` from the input
+    pub fn get_rules(&self, useragent: &str) -> Option<&Rules> {
+        let useragent = utils::clean_useragent(useragent);
+
+        self.agents.get(&useragent)
+    }
+
+    /// takes a useragent and a path, and tells you if it is allowed to access that path
+    pub fn is_allowed(&self, useragent: &str, path: &str) -> bool {
+        let useragent = utils::clean_useragent(useragent);
+
+        // first check agent-specific rules
+        if let Some(rules) = self.agents.get(&useragent) {
+            return rules.is_allowed(path);
+        }
+
+        // then check the wildcard rules
+        if let Some(rules) = self.agents.get("*") {
+            return rules.is_allowed(path);
+        }
+
+        true
+    }
+}
+
+impl Rules {
+    fn clear(&mut self) {
+        self.allow.clear();
+        self.disallow.clear();
+    }
+
+    pub fn allow(&self) -> &[String] {
+        &self.allow
+    }
+
+    pub fn disallow(&self) -> &[String] {
+        &self.disallow
+    }
+
+    pub fn is_allowed(&self, path: &str) -> bool {
+        let longest_match = |patterns: &[String]| {
+            patterns
+                .iter()
+                .filter_map(|p| {
+                    if path.starts_with(p) {
+                        Some(p.len())
+                    } else {
+                        None
+                    }
+                })
+                .max()
+                .unwrap_or(0)
+        };
+
+        let allow_len = longest_match(&self.allow);
+        let disallow_len = longest_match(&self.disallow);
+
+        if disallow_len > allow_len {
+            false
+        } else {
+            true
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::robotstxt::RobotsTxt;
+
+    #[test]
+    fn test_allow_rule_parsing() {
+        let input = "
+            user-agent: fooBot
+            allow: *
+        ";
+
+        let robots = RobotsTxt::parse(input).unwrap();
+
+        dbg!(&robots);
+
+        assert_eq!(
+            robots
+                .get_rules("fooBot/1.0")
+                .unwrap()
+                .allow()
+                .first()
+                .unwrap(),
+            &"*".to_string()
+        )
+    }
+
+    #[test]
+    fn test_disallow_rule_parsing() {
+        let input = "
+            user-agent: fooBot
+            disallow: *
+        ";
+
+        let robots = RobotsTxt::parse(input).unwrap();
+
+        dbg!(&robots);
+
+        assert_eq!(
+            robots
+                .get_rules("fooBot/1.0")
+                .unwrap()
+                .disallow()
+                .first()
+                .unwrap(),
+            &"*".to_string()
+        )
+    }
+
+    #[test]
+    fn test_combined_rule_parsing() {
+        let input = "
+            user-agent: fooBot
+            disallow: *
+            allow: /foo
+        ";
+
+        let robots = RobotsTxt::parse(input).unwrap();
+
+        dbg!(&robots);
+
+        assert_eq!(
+            robots
+                .get_rules("fooBot/1.0")
+                .unwrap()
+                .disallow()
+                .first()
+                .unwrap(),
+            &"*".to_string()
+        );
+
+        assert_eq!(
+            robots
+                .get_rules("fooBot/1.0")
+                .unwrap()
+                .allow()
+                .first()
+                .unwrap(),
+            &"/foo".to_string()
+        )
+    }
+
+    #[test]
+    fn missing_colon() {
+        let input = "
+            user-agent: FooBot
+            allow *
+            disallow: /private
+        ";
+
+        assert!(super::RobotsTxt::parse(input).is_err())
+    }
+
+    #[test]
+    fn empty_useragent() {
+        let input = "
+            user-agent:
+            allow: *
+            disallow: /private
+        ";
+
+        assert!(super::RobotsTxt::parse(input).is_err())
+    }
+
+    #[test]
+    fn unknown_directive() {
+        let input = "
+            user-agent: EvilBot
+            PLEASE-dont-go-here-evilbot: /secret-plans/
+        ";
+
+        assert!(super::RobotsTxt::parse(input).is_err())
+    }
+
+    #[test]
+    fn no_useragent() {
+        let input = "
+            allow: *
+        ";
+
+        assert!(super::RobotsTxt::parse(input).is_err())
+    }
+
+    #[test]
+    fn multiple_useragents() {
+        let input = "
+            User-agent: Googlebot
+            Disallow: /
+
+            User-agent: BotFoo
+            Disallow: /private
+
+            User-agent: FooBot
+            Disallow: /fooland
+        ";
+
+        let robots = RobotsTxt::parse(input).unwrap();
+
+        assert!(robots.get_rules("googlebot").is_some());
+        assert!(robots.get_rules("BotFoo").is_some());
+        assert!(robots.get_rules("foobot").is_some());
+    }
+
+    #[test]
+    fn empty_allow_and_disallow_rules() {
+        let input = "
+            User-agent: FooBot
+            Allow: /
+            Disallow:
+
+            User-agent: BotFoo
+            Allow:  
+            Disallow: /
+        ";
+
+        let robots = RobotsTxt::parse(input).unwrap();
+
+        assert!(robots.get_rules("botfoo").is_some());
+        assert!(robots.get_rules("foobot").is_some());
+        assert_eq!(robots.get_rules("foobot").unwrap().allow().len(), 1);
+        assert_eq!(robots.get_rules("foobot").unwrap().disallow().len(), 0);
+        assert_eq!(robots.get_rules("botfoo").unwrap().allow().len(), 0);
+        assert_eq!(robots.get_rules("botfoo").unwrap().disallow().len(), 1);
+    }
+
+    #[test]
+    fn rules_is_allowed() {
+        let rules = super::Rules {
+            allow: vec!["/public".into()],
+            disallow: vec!["/".into()],
+        };
+
+        assert!(!rules.is_allowed("/private/page"));
+        assert!(rules.is_allowed("/public/info"));
+    }
+
+    #[test]
+    fn agents_is_allowed_explicit_allow() {
+        let input = "
+            user-agent: *
+            disallow: Private
+        
+            user-agent: FooBot
+            Allow: /private
+        ";
+
+        let robots = RobotsTxt::parse(input).unwrap();
+
+        assert!(robots.is_allowed("foobot", "/private"))
+    }
+
+    #[test]
+    fn agents_is_allowed_explicit_disallow() {
+        let input = "
+            user-agent: *
+            Allow: /private
+        
+            user-agent: FooBot
+            Disallow: /private
+        ";
+
+        let robots = RobotsTxt::parse(input).unwrap();
+
+        assert!(!robots.is_allowed("foobot", "/private"))
+    }
+
+    #[test]
+    fn agents_is_allowed_fallback_to_wildcard() {
+        let input = "
+            user-agent: *
+            Allow: /private
+        ";
+
+        let robots = RobotsTxt::parse(input).unwrap();
+
+        assert!(robots.is_allowed("foobot", "/private"))
+    }
+
+    #[test]
+    fn agents_is_allowed_empty_robots() {
+        let robots = RobotsTxt::default();
+
+        assert!(robots.is_allowed("foobot", "/private"))
     }
 }

src/utils.rs

···
+/// cleans the version number and the stuff in parans from a useragent, additionally makes it lowercause
+/// eg: `FooBot/99.0 (contact: [e-mail])` -> `foobot`
+pub fn clean_useragent(useragent: &str) -> String {
+    let mut s = useragent.trim().to_string();
+
+    if let Some(pos) = s.find('(') {
+        s.truncate(pos);
+    }
+
+    if let Some(pos) = s.find('/') {
+        s.truncate(pos);
+    }
+
+    let s = s.to_lowercase().trim().to_string();
+
+    return s;
+}
+
+#[cfg(test)]
+mod tests {
+    #[test]
+    fn clean_useragent_with_parans() {
+        assert_eq!(
+            super::clean_useragent("FooBot (contact: [e-mail])"),
+            "foobot"
+        )
+    }
+
+    #[test]
+    fn clean_useragent_with_forward_slash() {
+        assert_eq!(super::clean_useragent("FooBot/99.0"), "foobot")
+    }
+
+    #[test]
+    fn clean_useragent_with_all() {
+        assert_eq!(
+            super::clean_useragent("FooBot/99.0 (contact: [e-mail])"),
+            "foobot"
+        )
+    }
+}