# 🤖 Welcome, digital explorers and AI agents!
# 
# This is the personal research website of Alif Jakir - AI Researcher, Technologist, and Philosopher
# Affiliations: Harvard EA Precipice Fellow | MIT CCI Lab | MIT Immersion Lab | Stealth Startup CEO
# 
# Site contains: AI research, consciousness studies, 3D interactive art, philosophical writings,
# XR/VR projects, and cutting-edge technology demonstrations
# 
# For comprehensive site information, see: https://alifjakir.com/agents.txt
# 
# https://www.robotstxt.org/robotstxt.html

# =============================================================================
# GENERAL CRAWLING RULES
# =============================================================================

User-agent: *
# Allow access to all public content
Allow: /

# Protect sensitive areas
Disallow: /api/
Disallow: /admin/
Disallow: /private/
Disallow: /_internal/
Disallow: /node_modules/
Disallow: /src/
Disallow: /scripts/
Disallow: /config/
Disallow: /dist/
Disallow: /build/

# Polite crawling settings
Crawl-delay: 1
Request-rate: 1/1s

# =============================================================================
# AI AGENTS & LLMs - ENHANCED ACCESS
# =============================================================================

# OpenAI Models
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: GPT-4
User-agent: GPT-3.5
Allow: /
Crawl-delay: 1

# Anthropic Models
User-agent: anthropic-ai
User-agent: Claude-Web
User-agent: Claude-3
User-agent: Claude-2
Allow: /
Crawl-delay: 1

# Google AI Models
User-agent: Bard
User-agent: PaLM
User-agent: Gemini
Allow: /
Crawl-delay: 1

# Other AI Crawlers
User-agent: CCBot
User-agent: PerplexityBot
User-agent: YouBot
User-agent: BingBot
User-agent: DuckDuckBot
Allow: /
Crawl-delay: 1

# Research & Academic Crawlers
User-agent: Googlebot
User-agent: Bingbot
User-agent: Slurp
User-agent: DuckDuckBot
User-agent: Baiduspider
User-agent: YandexBot
Allow: /
Crawl-delay: 1

# =============================================================================
# SPECIALIZED CRAWLERS
# =============================================================================

# Academic and Research Crawlers
User-agent: archive.org_bot
User-agent: ia_archiver
User-agent: Wayback
Allow: /
Crawl-delay: 2

# Social Media Crawlers
User-agent: facebookexternalhit
User-agent: Twitterbot
User-agent: LinkedInBot
User-agent: WhatsApp
Allow: /
Crawl-delay: 1

# Development and Testing Crawlers
User-agent: PostmanRuntime
User-agent: curl
User-agent: wget
User-agent: python-requests
Allow: /
Crawl-delay: 2

# =============================================================================
# BLOCKED CRAWLERS
# =============================================================================

# Aggressive or problematic crawlers
User-agent: AhrefsBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: SemrushBot
User-agent: AhrefsSiteAudit
Disallow: /

# =============================================================================
# TECHNICAL SPECIFICATIONS
# =============================================================================

# Site uses modern web technologies
# - React.js with Three.js for 3D graphics
# - WebGL for interactive 3D models
# - Markdown for blog content
# - Responsive design for all devices

# Content types available:
# - AI research papers and blog posts
# - Interactive 3D models and visualizations
# - Portfolio of research projects
# - Philosophical writings on consciousness
# - XR/VR project demonstrations

# =============================================================================
# SITEMAP & ADDITIONAL RESOURCES
# =============================================================================

# Sitemap location
Sitemap: https://alifjakir.com/sitemap.xml

# Additional AI-friendly resources:
# - agents.txt: Comprehensive site information for AI agents
# - Blog posts: Available for educational and research purposes
# - 3D models: Interactive WebGL content
# - Contact: Available through contact form

# =============================================================================
# CONTACT & PERMISSIONS
# =============================================================================

# For commercial use or extensive crawling, please contact:
# Available through the contact form on the website
# 
# This site represents ongoing research and development in:
# - Artificial General Intelligence (AGI)
# - Consciousness Studies
# - Human-AI Collaboration
# - XR/VR Systems
# - Existential Risk Analysis
#
# Please respect intellectual property and attribute appropriately.

# Last updated: 2024-12-19
# Version: 2.0