From 26d13fc88a03f7b6c06013a4acc36c41460a5b26 Mon Sep 17 00:00:00 2001
From: "duncan.parkes" <duncan.parkes@cf68f80a-222c-0410-aabe-5f79d0504a29>
Date: Sun, 21 Oct 2007 22:39:22 +0000
Subject: [PATCH] Add a robots.txt file to stop search engines bothering our
 scrapers and api.

---
 docs/robots.txt | 4 ++++
 1 file changed, 4 insertions(+)
 create mode 100644 docs/robots.txt

diff --git a/docs/robots.txt b/docs/robots.txt
new file mode 100644
index 0000000..4df526e
--- /dev/null
+++ b/docs/robots.txt
@@ -0,0 +1,4 @@
+User-agent: *
+Disallow: /cgi-bin/
+Disallow: /scrapers/
+Disallow: /api.php