commit d4c5d4e65b89fb3da83826edfdd5cfd5bf4a0efb Author: Cecylia Bocovich cohosh@torproject.org Date: Mon Apr 15 16:43:21 2019 -0400
Updated robots.txt handler on broker
Updated robots.txt file to disallow crawling as in ticket #29565. --- broker/broker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/broker/broker.go b/broker/broker.go index b3a255e..7e2da8d 100644 --- a/broker/broker.go +++ b/broker/broker.go @@ -218,7 +218,7 @@ func debugHandler(ctx *BrokerContext, w http.ResponseWriter, r *http.Request) {
func robotsTxtHandler(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/plain; charset=utf-8") - w.Write([]byte("User-agent: *\nDisallow:\n")) + w.Write([]byte("User-agent: *\nDisallow: /\n")) }
func main() {