appview/state: update robots.txt

Signed-off-by: Anirudh Oppiliappan <[e-mail]>

appview/state/state.go

···
 	w.Header().Set("Content-Type", "text/plain")
 	w.Header().Set("Cache-Control", "public, max-age=86400") // one day
 
-	robotsTxt := `User-agent: *
+	robotsTxt := `# Hello, Tanglers!
+User-agent: *
 Allow: /
+Disallow: /*/*/settings
+Disallow: /settings
+Disallow: /*/*/compare
+Disallow: /*/*/fork
+
+Crawl-delay: 1
 `
 	w.Write([]byte(robotsTxt))
 }