appview/state: add a permissive robots.txt

Maybe we load this from disk/elsewhere later?

Signed-off-by: Anirudh Oppiliappan <[e-mail]>

appview/state/router.go

···
 	router.Get("/favicon.svg", s.Favicon)
 	router.Get("/favicon.ico", s.Favicon)
 	router.Get("/pwa-manifest.json", s.PWAManifest)
+	router.Get("/robots.txt", s.RobotsTxt)
 
 	userRouter := s.UserRouter(&middleware)
 	standardRouter := s.StandardRouter(&middleware)

appview/state/state.go

···
 	s.pages.Favicon(w)
 }
 
+func (s *State) RobotsTxt(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "text/plain")
+	w.Header().Set("Cache-Control", "public, max-age=86400") // one day
+
+	robotsTxt := `User-agent: *
+Allow: /
+`
+	w.Write([]byte(robotsTxt))
+}
+
 // https://developer.mozilla.org/en-US/docs/Web/Progressive_web_apps/Manifest
 const manifestJson = `{
   "name": "tangled",