add robots.txt

srv/content/router.hy

@@ -52,6 +52,16 @@
       :headers headers
       :body data)))
 
+  
+  (when (= path "/robots.txt")
+    (when (!= method "GET")
+      (return (error 405 "method not allowed")))
+
+    (return (dict
+      :code 200
+      :headers {"Content-Type" "text/plain"}
+      :body "user-agent: *\ndisallow: /\n")))
+
   (when (= path "/")
     (when (!= method "GET")
       (return (error 405 "method not allowed")))