# robots.txt — animalsss.com # Generated by pages.seo_views.robots_txt; edit that, not a static file. User-agent: * Allow: / Disallow: /daudi/ Disallow: /emp/ Disallow: /ckeditor/ Disallow: /ckeditor_uploader/ # Read-only market data, deliberately open. These are the JSON endpoints # the Dataset markup on the market pages points at, so a crawler that # trusts robots.txt and a crawler that reads structured data agree. Allow: /mandi/api/ # The rest of the API is the app talking to itself: write endpoints, # per-user reads, and paginated views of pages that are already indexed. Disallow: /api/ # Private conversations and per-user pages. Login-gated, so there is # nothing here to index; the legacy /chat/ answered 500 to # Applebot and ClaudeBot until the view was hardened, and a route with # no indexable content should not cost crawl budget either way. Disallow: /chat Disallow: /cha/ Disallow: /send_msg Disallow: /marketplace/chat/ Disallow: /profile Disallow: /my-blog-dashboard/ # A filtered view of a table already indexed unfiltered is a lot of # requests for no additional indexable content. The one exception is the # line that follows: the archive index's pages are not filtered views of # one another, they are consecutive slices of a 1,531-row table, and page 2 # onwards is where 1,483 of those listings are linked from. Allow: /marketplace/archive/?page= # /mandi/?category=poultry was reported as "Blocked by robots.txt", which # it is and should be: mandi_home never read that parameter, so the URL # served the whole dashboard and canonicalised itself to /mandi/. The # answer was a real page rather than a hole in this line — the eight # categories live at /mandi/category//, are in their own sitemap and # are linked from the dashboard. The old address 301s there, which no # crawler will see, because this line stops it being fetched at all; that # redirect is for the bookmarks and the links that already exist. Disallow: /*?category= Disallow: /*?page= Disallow: /*_rates/ Disallow: /*_data/ Disallow: /*_commodities/ # Answer engines. Named individually because AI-use permission # is a separate grant from search indexing for several of these, # and an unnamed crawler cannot tell permission from oversight. User-agent: GPTBot Allow: / Allow: /mandi/api/ Allow: /marketplace/archive/?page= Disallow: /daudi/ Disallow: /emp/ Disallow: /ckeditor/ Disallow: /ckeditor_uploader/ Disallow: /api/ Disallow: /chat Disallow: /cha/ Disallow: /send_msg Disallow: /marketplace/chat/ Disallow: /profile Disallow: /my-blog-dashboard/ Disallow: /*?category= Disallow: /*?page= Disallow: /*_rates/ Disallow: /*_data/ Disallow: /*_commodities/ User-agent: OAI-SearchBot Allow: / Allow: /mandi/api/ Allow: /marketplace/archive/?page= Disallow: /daudi/ Disallow: /emp/ Disallow: /ckeditor/ Disallow: /ckeditor_uploader/ Disallow: /api/ Disallow: /chat Disallow: /cha/ Disallow: /send_msg Disallow: /marketplace/chat/ Disallow: /profile Disallow: /my-blog-dashboard/ Disallow: /*?category= Disallow: /*?page= Disallow: /*_rates/ Disallow: /*_data/ Disallow: /*_commodities/ User-agent: ChatGPT-User Allow: / Allow: /mandi/api/ Allow: /marketplace/archive/?page= Disallow: /daudi/ Disallow: /emp/ Disallow: /ckeditor/ Disallow: /ckeditor_uploader/ Disallow: /api/ Disallow: /chat Disallow: /cha/ Disallow: /send_msg Disallow: /marketplace/chat/ Disallow: /profile Disallow: /my-blog-dashboard/ Disallow: /*?category= Disallow: /*?page= Disallow: /*_rates/ Disallow: /*_data/ Disallow: /*_commodities/ User-agent: ClaudeBot Allow: / Allow: /mandi/api/ Allow: /marketplace/archive/?page= Disallow: /daudi/ Disallow: /emp/ Disallow: /ckeditor/ Disallow: /ckeditor_uploader/ Disallow: /api/ Disallow: /chat Disallow: /cha/ Disallow: /send_msg Disallow: /marketplace/chat/ Disallow: /profile Disallow: /my-blog-dashboard/ Disallow: /*?category= Disallow: /*?page= Disallow: /*_rates/ Disallow: /*_data/ Disallow: /*_commodities/ User-agent: Claude-User Allow: / Allow: /mandi/api/ Allow: /marketplace/archive/?page= Disallow: /daudi/ Disallow: /emp/ Disallow: /ckeditor/ Disallow: /ckeditor_uploader/ Disallow: /api/ Disallow: /chat Disallow: /cha/ Disallow: /send_msg Disallow: /marketplace/chat/ Disallow: /profile Disallow: /my-blog-dashboard/ Disallow: /*?category= Disallow: /*?page= Disallow: /*_rates/ Disallow: /*_data/ Disallow: /*_commodities/ User-agent: Claude-SearchBot Allow: / Allow: /mandi/api/ Allow: /marketplace/archive/?page= Disallow: /daudi/ Disallow: /emp/ Disallow: /ckeditor/ Disallow: /ckeditor_uploader/ Disallow: /api/ Disallow: /chat Disallow: /cha/ Disallow: /send_msg Disallow: /marketplace/chat/ Disallow: /profile Disallow: /my-blog-dashboard/ Disallow: /*?category= Disallow: /*?page= Disallow: /*_rates/ Disallow: /*_data/ Disallow: /*_commodities/ User-agent: anthropic-ai Allow: / Allow: /mandi/api/ Allow: /marketplace/archive/?page= Disallow: /daudi/ Disallow: /emp/ Disallow: /ckeditor/ Disallow: /ckeditor_uploader/ Disallow: /api/ Disallow: /chat Disallow: /cha/ Disallow: /send_msg Disallow: /marketplace/chat/ Disallow: /profile Disallow: /my-blog-dashboard/ Disallow: /*?category= Disallow: /*?page= Disallow: /*_rates/ Disallow: /*_data/ Disallow: /*_commodities/ User-agent: PerplexityBot Allow: / Allow: /mandi/api/ Allow: /marketplace/archive/?page= Disallow: /daudi/ Disallow: /emp/ Disallow: /ckeditor/ Disallow: /ckeditor_uploader/ Disallow: /api/ Disallow: /chat Disallow: /cha/ Disallow: /send_msg Disallow: /marketplace/chat/ Disallow: /profile Disallow: /my-blog-dashboard/ Disallow: /*?category= Disallow: /*?page= Disallow: /*_rates/ Disallow: /*_data/ Disallow: /*_commodities/ User-agent: Perplexity-User Allow: / Allow: /mandi/api/ Allow: /marketplace/archive/?page= Disallow: /daudi/ Disallow: /emp/ Disallow: /ckeditor/ Disallow: /ckeditor_uploader/ Disallow: /api/ Disallow: /chat Disallow: /cha/ Disallow: /send_msg Disallow: /marketplace/chat/ Disallow: /profile Disallow: /my-blog-dashboard/ Disallow: /*?category= Disallow: /*?page= Disallow: /*_rates/ Disallow: /*_data/ Disallow: /*_commodities/ User-agent: Google-Extended Allow: / Allow: /mandi/api/ Allow: /marketplace/archive/?page= Disallow: /daudi/ Disallow: /emp/ Disallow: /ckeditor/ Disallow: /ckeditor_uploader/ Disallow: /api/ Disallow: /chat Disallow: /cha/ Disallow: /send_msg Disallow: /marketplace/chat/ Disallow: /profile Disallow: /my-blog-dashboard/ Disallow: /*?category= Disallow: /*?page= Disallow: /*_rates/ Disallow: /*_data/ Disallow: /*_commodities/ User-agent: Applebot-Extended Allow: / Allow: /mandi/api/ Allow: /marketplace/archive/?page= Disallow: /daudi/ Disallow: /emp/ Disallow: /ckeditor/ Disallow: /ckeditor_uploader/ Disallow: /api/ Disallow: /chat Disallow: /cha/ Disallow: /send_msg Disallow: /marketplace/chat/ Disallow: /profile Disallow: /my-blog-dashboard/ Disallow: /*?category= Disallow: /*?page= Disallow: /*_rates/ Disallow: /*_data/ Disallow: /*_commodities/ User-agent: Bingbot Allow: / Allow: /mandi/api/ Allow: /marketplace/archive/?page= Disallow: /daudi/ Disallow: /emp/ Disallow: /ckeditor/ Disallow: /ckeditor_uploader/ Disallow: /api/ Disallow: /chat Disallow: /cha/ Disallow: /send_msg Disallow: /marketplace/chat/ Disallow: /profile Disallow: /my-blog-dashboard/ Disallow: /*?category= Disallow: /*?page= Disallow: /*_rates/ Disallow: /*_data/ Disallow: /*_commodities/ Crawl-delay: 2 User-agent: meta-externalagent Allow: / Allow: /mandi/api/ Allow: /marketplace/archive/?page= Disallow: /daudi/ Disallow: /emp/ Disallow: /ckeditor/ Disallow: /ckeditor_uploader/ Disallow: /api/ Disallow: /chat Disallow: /cha/ Disallow: /send_msg Disallow: /marketplace/chat/ Disallow: /profile Disallow: /my-blog-dashboard/ Disallow: /*?category= Disallow: /*?page= Disallow: /*_rates/ Disallow: /*_data/ Disallow: /*_commodities/ User-agent: Amazonbot Allow: / Allow: /mandi/api/ Allow: /marketplace/archive/?page= Disallow: /daudi/ Disallow: /emp/ Disallow: /ckeditor/ Disallow: /ckeditor_uploader/ Disallow: /api/ Disallow: /chat Disallow: /cha/ Disallow: /send_msg Disallow: /marketplace/chat/ Disallow: /profile Disallow: /my-blog-dashboard/ Disallow: /*?category= Disallow: /*?page= Disallow: /*_rates/ Disallow: /*_data/ Disallow: /*_commodities/ User-agent: DuckAssistBot Allow: / Allow: /mandi/api/ Allow: /marketplace/archive/?page= Disallow: /daudi/ Disallow: /emp/ Disallow: /ckeditor/ Disallow: /ckeditor_uploader/ Disallow: /api/ Disallow: /chat Disallow: /cha/ Disallow: /send_msg Disallow: /marketplace/chat/ Disallow: /profile Disallow: /my-blog-dashboard/ Disallow: /*?category= Disallow: /*?page= Disallow: /*_rates/ Disallow: /*_data/ Disallow: /*_commodities/ User-agent: cohere-ai Allow: / Allow: /mandi/api/ Allow: /marketplace/archive/?page= Disallow: /daudi/ Disallow: /emp/ Disallow: /ckeditor/ Disallow: /ckeditor_uploader/ Disallow: /api/ Disallow: /chat Disallow: /cha/ Disallow: /send_msg Disallow: /marketplace/chat/ Disallow: /profile Disallow: /my-blog-dashboard/ Disallow: /*?category= Disallow: /*?page= Disallow: /*_rates/ Disallow: /*_data/ Disallow: /*_commodities/ # Search crawlers that honour a crawl delay. User-agent: Yandex Allow: / Allow: /mandi/api/ Allow: /marketplace/archive/?page= Disallow: /daudi/ Disallow: /emp/ Disallow: /ckeditor/ Disallow: /ckeditor_uploader/ Disallow: /api/ Disallow: /chat Disallow: /cha/ Disallow: /send_msg Disallow: /marketplace/chat/ Disallow: /profile Disallow: /my-blog-dashboard/ Disallow: /*?category= Disallow: /*?page= Disallow: /*_rates/ Disallow: /*_data/ Disallow: /*_commodities/ Crawl-delay: 3 User-agent: Baiduspider Allow: / Allow: /mandi/api/ Allow: /marketplace/archive/?page= Disallow: /daudi/ Disallow: /emp/ Disallow: /ckeditor/ Disallow: /ckeditor_uploader/ Disallow: /api/ Disallow: /chat Disallow: /cha/ Disallow: /send_msg Disallow: /marketplace/chat/ Disallow: /profile Disallow: /my-blog-dashboard/ Disallow: /*?category= Disallow: /*?page= Disallow: /*_rates/ Disallow: /*_data/ Disallow: /*_commodities/ Crawl-delay: 5 # AdSense, so ads match the page instead of guessing at it. User-agent: Mediapartners-Google Allow: / Disallow: /daudi/ Disallow: /emp/ Disallow: /ckeditor/ Disallow: /ckeditor_uploader/ Disallow: /chat Disallow: /cha/ Disallow: /send_msg Disallow: /marketplace/chat/ Disallow: /profile Disallow: /my-blog-dashboard/ Sitemap: https://animalsss.com/sitemap.xml # Plain-language guide to what is here, for answer engines: # https://animalsss.com/llms.txt # IndexNow key file (host verification for instant-indexing pings): # https://animalsss.com/d3d874a175073d36710ad0c3421228ea.txt