diff --git a/docs/robots.txt b/docs/robots.txt
new file mode 100644
index 0000000..4df526e
--- /dev/null
+++ b/docs/robots.txt
@@ -0,0 +1,4 @@
+User-agent: *
+Disallow: /cgi-bin/
+Disallow: /scrapers/
+Disallow: /api.php