From c2bf89e634c64e8336329189adddfa0a5e3dc6f9 Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Sun, 26 Apr 2015 01:40:17 +0200 Subject: [PATCH] Added proper robots.txt --- public/robots.txt | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/public/robots.txt b/public/robots.txt index 1a3a5e4..26164f9 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -1,5 +1,4 @@ -# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file -# -# To ban all spiders from the entire site uncomment the next two lines: -# User-agent: * -# Disallow: / +User-agent: * +Disallow: /json +Disallow: /api +Disallow: