src/main.rs at main · skoove.dev/webring-mapper

project to map out webrings

webring-mapper / src / main.rs

at main 209 B view raw

 1use webring_mapper::robotstxt::RobotsTxt;
 2
 3fn main() {
 4    let input = "
 5        User-agent: Googlebot
 6        Disallow: /
 7
 8        User-agent: *
 9        Disallow:
10    ";
11
12    dbg!(RobotsTxt::parse(input));
13}