Add robots.txt to prevent scraping of full Geminispace

2 files changed, 6 insertions, 0 deletions

diff --git a/build.py b/build.py
index 27166721..4d3ea887 100755
--- a/build.py
+++ b/build.py
@@ -108,6 +108,10 @@ def build():
     posts = load_posts()
     create_index(sorted(posts, key=lambda p: p.posted, reverse=True))
     create_individual_posts(posts)
+    for directory, _, files in os.walk("static/gmi"):
+        new_dir = directory.replace("static/gmi", "build/gmi")
+        for file in files:
+            shutil.copy(f"{directory}/{file}", f"{new_dir}/{file}")
 
 if __name__ == "__main__":
     build()
diff --git a/static/gmi/robots.txt b/static/gmi/robots.txt
new file mode 100644
index 00000000..da2aac59
--- /dev/null
+++ b/static/gmi/robots.txt
@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /x/