tangled.org
Signed-off-by: Anirudh Oppiliappan anirudh@tangled.org
+8
-1
appview/state/state.go
+8
-1
appview/state/state.go
···
207
207
w.Header().Set("Content-Type", "text/plain")
208
208
w.Header().Set("Cache-Control", "public, max-age=86400") // one day
209
209
210
-
robotsTxt := `User-agent: *
210
+
robotsTxt := `# Hello, Tanglers!
211
+
User-agent: *
211
212
Allow: /
213
+
Disallow: /*/*/settings
214
+
Disallow: /settings
215
+
Disallow: /*/*/compare
216
+
Disallow: /*/*/fork
217
+
218
+
Crawl-delay: 1
212
219
`
213
220
w.Write([]byte(robotsTxt))
214
221
}
History
3 rounds
1 comment
anirudh.fi
submitted
#2
1 commit
expand
collapse
appview/state: update robots.txt
Signed-off-by: Anirudh Oppiliappan <anirudh@tangled.org>
2/3 timeout, 1/3 success
expand
collapse
expand 1 comment
pull request successfully merged
anirudh.fi
submitted
#1
1 commit
expand
collapse
appview/state: update robots.txt
Signed-off-by: Anirudh Oppiliappan <anirudh@tangled.org>
2/3 timeout, 1/3 success
expand
collapse
expand 0 comments
anirudh.fi
submitted
#0
1 commit
expand
collapse
appview/state: update robots.txt
Signed-off-by: Anirudh Oppiliappan <anirudh@tangled.org>
lgtm, ++ to disallowing compare. my hunch is that bots that we can't block via nginx will probably not respect this anyway, but worth a shot!