diff --git a/public/robots.txt b/public/robots.txt index ce950e7..4ecb74c 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -36,6 +36,8 @@ User-agent: Factset_spyderbot User-agent: FirecrawlAgent User-agent: FriendlyCrawler User-agent: Gemini-Deep-Research +User-agent: Google-CloudVertexBot +User-agent: GoogleOther User-agent: GPTBot User-agent: iaskspider/2.0 User-agent: ICC-Crawler