<?php
/**
 * Robots.txt Configuration
 *
 * This file contains all mappings between UI options and robots.txt output.
 *
 * DESIGN PRINCIPLES:
 * - Each step maps to a method that generates output
 * - Mappings are arrays that are easy to modify
 * - Adding new options = adding to the arrays
 * - Step order is explicitly defined
 *
 * TO MODIFY OUTPUT:
 * - Find the relevant section (e.g., STEP 1, STEP 2, etc.)
 * - Modify the bot arrays or output strings
 * - No need to change any logic in RobotsController
 *
 * @package Pagup\BetterRobots
 * @since 3.0.0
 */

namespace Pagup\BetterRobots\Config;

class RobotsConfig
{
    /**
     * Step execution order
     * This defines the exact order in which rules appear in robots.txt
     *
     * To change order: simply reorder this array
     * To add new step: add new entry here and create corresponding method
     */
    const STEP_ORDER = [
        'global_settings',          // Step 1: Global Settings (Sitemaps)
        'search_engine_module',     // Step 2: Search Engine Visibility
        'ai_module',                // Step 3: AI & LLM Governance
        'seo_tools_module',         // Step 4: SEO Tool Protection
        'bad_bots_module',          // Step 5: Bot & Scraper Protection
        'archive_module',           // Step 6: Archive & Wayback Control
        'spam_feeds_module',        // Step 7: Spam & Feed Protection
        'ecommerce_module',         // Step 8: E-commerce Optimization
        'resources_module',         // Step 9: Resources (CSS/JS/Images)
        'social_media_module',      // Step 10: Social Media
        'crawl_traps_module',       // Step 7 backend: Crawl Traps (UI in SpamFirewallModule)
        'ads_module',               // Step 11: Ads & Revenue
        'advanced_settings',        // Step 12: Advanced Settings
    ];

    /**
     * Interpretive Governance signature
     * Reference: https://interpretive-governance.org/
     */
    const SSA_SIGNATURE = "# Generated by Better Robots.txt\n# https://better-robots.com\n# Crawl governance based on the Interpretive Governance framework\n# Reference: https://interpretive-governance.org/\n";

    // =========================================================================
    // GLOBAL BASE RULES
    // =========================================================================

    /**
     * WordPress core paths that should be allowed for all user agents
     * These ensure proper functionality while protecting sensitive areas
     */
    const BASE_ALLOW_PATHS = [
        '/wp-admin/admin-ajax.php',  // Required for AJAX functionality
        '/*/*.css',                   // Allow CSS files
        '/*/*.js',                    // Allow JavaScript files
    ];

    /**
     * WordPress core paths and patterns that should always be blocked
     * These protect sensitive files and prevent unnecessary crawling
     */
    const BASE_DISALLOW_PATHS = [
        '/wp-admin/',                 // WordPress admin area
        '/wp-includes/',              // WordPress core includes
        '/readme.html',               // WordPress readme file
        '/license.txt',               // License file
        '/xmlrpc.php',                // XML-RPC (security concern)
        '/wp-login.php',              // Login page
        '/wp-register.php',           // Registration page
        '*/disclaimer/*',             // Disclaimer pages
        '*?attachment_id=',           // Attachment parameter
        '/*?filtering=',              // Filtering parameter
    ];

    // =========================================================================
    // STEP 1: SEARCH ENGINE VISIBILITY
    // =========================================================================

    /**
     * Map of search engine identifiers to their User-Agent strings
     *
     * To add a new search engine:
     * 1. Add entry here: 'identifier' => 'UserAgentString'
     * 2. Update the visibility level arrays below if needed
     */
    const SEARCH_ENGINE_BOTS = [
        // Major Search Engines
        'google'      => 'Googlebot',
        'bing'        => 'bingbot',
        'yahoo'       => 'Slurp',
        'duckduckgo'  => 'DuckDuckBot',
        // Regional Search Engines
        'baidu'       => 'Baiduspider',
        'yandex'      => 'Yandex',
        'naver'       => 'Yeti',           // Naver's crawler
        'sogou'       => 'Sogou',
        // Privacy-Focused & Eco Search Engines
        'qwant'       => 'Qwantify',
        'ecosia'      => 'Ecosia',         // Uses Bing but has own bot
        'mojeek'      => 'MojeekBot',      // Independent privacy search
        'brave'       => 'Brave',          // Brave Search
        // Other Search Engines
        'seznam'      => 'SeznamBot',      // Czech search engine
        'coccoc'      => 'coccocbot',      // Vietnamese search engine
    ];

    /**
     * Granular search engine bot map used by the custom override UI.
     *
     * These keys must stay aligned with the search engine module in Vue.
     */
    const SEARCH_ENGINE_GRANULAR_BOTS = [
        // Google
        'googlebot' => 'Googlebot',
        'googlebot_image' => 'Googlebot-Image',
        'googlebot_news' => 'Googlebot-News',
        'googlebot_video' => 'Googlebot-Video',
        'mediapartners_google' => 'Mediapartners-Google',
        'adsbot_google' => 'AdsBot-Google',
        'adsbot_google_mobile' => 'AdsBot-Google-Mobile',

        // Bing / Microsoft
        'bingbot' => 'bingbot',
        'bingpreview' => 'BingPreview',

        // Major engines
        'duckduckbot' => 'DuckDuckBot',
        'slurp' => 'Slurp',
        'yandex' => 'Yandex',
        'yandex_images' => 'YandexImages',
        'qwantify' => 'Qwantify',

        // Regional
        'baiduspider' => 'Baiduspider',
        'baiduspider_image' => 'Baiduspider-image',
        'baiduspider_video' => 'Baiduspider-video',
        'baiduspider_2_0' => 'Baiduspider/2.0',
        'sogou_spider' => 'Sogou spider',
        'sogou_web_spider' => 'Sogou web spider',
        'youdaobot' => 'YoudaoBot',
        'youdao' => 'youdao',
        'yodao' => 'yodao',
        'seznambot' => 'SeznamBot',
        'yeti' => 'Yeti',
        'exabot' => 'Exabot',
    ];

    /**
     * Visibility levels and which bots they include
     *
     * To modify which bots appear at each level:
     * Simply edit the arrays below
     */
    const VISIBILITY_LEVELS = [
        // Essential only: Google, Bing, DuckDuckGo
        'minimal_visibility' => [
            'google',
            'bing',
            'duckduckgo'
        ],
        // Standard: Major search engines worldwide
        'recommended_visibility' => [
            'google',
            'bing',
            'yahoo',
            'duckduckgo',
            'baidu',
            'yandex'
        ],
        // Maximum: All search engines including regional & privacy-focused
        'extended_visibility' => [
            'google',
            'bing',
            'yahoo',
            'duckduckgo',
            'baidu',
            'yandex',
            'naver',
            'sogou',
            'qwant',
            'ecosia',
            'mojeek',
            'brave',
            'seznam',
            'coccoc'
        ],
        // 'advanced_custom' uses custom_engines from settings
    ];

    // =========================================================================
    // STEP 2: AI & LLM GOVERNANCE
    // =========================================================================

    /**
     * AI Training Bots - These scrape content to train AI models
     *
     * COMPREHENSIVE LIST - Updated 2024/2025
     * These bots collect data specifically for AI model training
     */
    const AI_TRAINING_BOTS = [
        // OpenAI
        'GPTBot',               // OpenAI's official training crawler
        'OAI-SearchBot',        // OpenAI search feature

        // Google AI
        'Google-Extended',      // Google AI training (Bard/Gemini) - respects robots.txt

        // Anthropic
        'anthropic-ai',         // Anthropic's crawler
        'ClaudeBot',            // Claude's web crawler
        'Claude-Web',           // Claude web access

        // Meta/Facebook
        'FacebookBot',          // Meta AI training
        'Meta-ExternalAgent',   // Meta's external AI agent
        'Meta-ExternalFetcher', // Meta content fetcher

        // ByteDance/TikTok
        'Bytespider',           // ByteDance AI crawler

        // Apple
        'Applebot-Extended',    // Apple AI training (separate from search)

        // Microsoft
        'Bingbot-AI',           // Microsoft AI training variant

        // Other AI Companies
        'cohere-ai',            // Cohere AI training
        'Diffbot',              // AI content extraction service
        'Omgilibot',            // Omgili content aggregator
        'img2dataset',          // Image dataset collection

        // Common Crawl (used by many AI companies)
        'CCBot',                // Common Crawl bot

        // AI Research & Training Crawlers
        'Kangaroo Bot',         // AI training crawler
        'PanguBot',             // Huawei AI training
        'Sidetrade indexer',    // B2B AI company
        'webz.io',              // Data-as-a-service for AI
        'Scrapy',               // Generic scraper often used for AI
        'Timpibot',             // Timpi AI search/training

        // Perplexity (both training and search)
        'PerplexityBot',        // Perplexity AI
    ];

    /**
     * AI Search Engine Bots - These power AI-driven search results
     *
     * block_all_ai_search = block these
     * allow_ai_search = don't block (no rules generated)
     */
    const AI_SEARCH_BOTS = [
        'PerplexityBot',        // Perplexity AI search
        'YouBot',               // You.com AI search
        'ChatGPT-User',         // ChatGPT with browsing capability
        'Amazonbot',            // Amazon Alexa/search integration
        'OAI-SearchBot',        // OpenAI's SearchGPT
        'AI2Bot',               // Allen AI search
        'Applebot',             // Apple search (used in Siri)
        'PhindBot',             // Phind AI developer search
    ];

    // =========================================================================
    // STEP 3: BOT & SCRAPER PROTECTION
    // =========================================================================

    /**
     * Bad bots by protection level
     *
     * BASIC: Common aggressive SEO crawlers
     * ADVANCED: Basic + more SEO tools + data aggregators
     * MAXIMUM: All above + aggressive scrapers + known bad actors
     */
    const BOT_PROTECTION_LEVELS = [
        'basic_protection' => [
            // Most common aggressive SEO crawlers
            'MJ12bot',              // Majestic SEO - very aggressive
            'AhrefsBot',            // Ahrefs - high volume
            'SemrushBot',           // Semrush - frequent crawler
        ],
        'advanced_protection' => [
            // All basic bots
            'MJ12bot',
            'AhrefsBot',
            'SemrushBot',
            // Additional SEO & analysis tools
            'DotBot',               // Moz/OpenSiteExplorer
            'BLEXBot',              // Webmeup backlink crawler
            'DataForSeoBot',        // DataForSEO
            'serpstatbot',          // Serpstat SEO tool
            'Screaming Frog SEO Spider', // Screaming Frog
            // Data aggregators
            'rogerbot',             // Moz
            'LinkpadBot',           // Linkpad backlink checker
            'Exabot',               // Exalead search
        ],
        'maximum_protection' => [
            // All advanced bots
            'MJ12bot',
            'AhrefsBot',
            'SemrushBot',
            'DotBot',
            'BLEXBot',
            'DataForSeoBot',
            'serpstatbot',
            'Screaming Frog SEO Spider',
            'rogerbot',
            'LinkpadBot',
            'Exabot',
            // Aggressive scrapers & crawlers
            'PetalBot',             // Huawei search (aggressive)
            'MegaIndex',            // MegaIndex SEO
            'Seekport',             // Seekport crawler
            'BomboraBot',           // Bombora B2B data
            'ZoominfoBot',          // ZoomInfo B2B scraper
            'Linguee',              // Linguee/DeepL translation scraper
            // Known problematic crawlers
            'SeznamBot',            // Czech search (can be aggressive)
            'YandexBot',            // Yandex (can be aggressive)
            'Mail.RU_Bot',          // Mail.ru crawler
            'Barkrowler',           // SEO tool
            'AspiegelBot',          // Amazon price comparison
            'TurnitinBot',          // Turnitin plagiarism checker
            'BDCbot',               // BigData Corp scraper
            'Buck',                 // Buck web crawler
            'Cliqzbot',             // Cliqz search
            'GrapeshotCrawler',     // Oracle advertising
            'newspaper',            // Python newspaper scraper
            'panscient.com',        // Data scraper
        ],
    ];

    // =========================================================================
    // STEP 4: SEO TOOL PROTECTION
    // =========================================================================

    /**
     * SEO Analysis Tools (Pro tier)
     * block_seo_tools = true blocks these
     */
    const SEO_TOOLS_BASIC = [
        'SemrushBot',               // Semrush
        'DotBot',                   // Moz
    ];

    /**
     * Large-Scale SEO Indexers (Premium tier)
     * block_extra_seo_tools = true blocks these
     */
    const SEO_TOOLS_EXTRA = [
        'AhrefsBot',                // Ahrefs
        'MJ12bot',                  // Majestic
        'BLEXBot',                  // Webmeup
        'DataForSeoBot',            // DataForSEO
        'serpstatbot',              // Serpstat
        'rogerbot',                 // Moz legacy
        'Screaming Frog SEO Spider', // Screaming Frog
    ];

    // =========================================================================
    // STEP 5: SPAM, FEEDS & CRAWL TRAPS
    // =========================================================================

    /**
     * Feed paths to block when block_feeds_spam is enabled
     * Prevents feed scrapers and reduces duplicate content indexing
     */
    const FEED_PATHS = [
        '/feed/',
        '/feed/atom/',
        '/feed/rss/',
        '/feed/rss2/',
        '/feed/rdf/',
        '/comments/feed/',
        '*/feed/',
        '*/trackback/',
        '/trackback/',
        '/xmlrpc.php',              // XML-RPC (security + spam target)
    ];

    /**
     * Author archive paths to block
     * Reduces duplicate content and author enumeration
     */
    const AUTHOR_PATHS = [
        '/author/',
        '/?author=',
    ];

    /**
     * Search and pagination paths to block
     * Prevents indexing of search results and deep pagination
     */
    const SEARCH_PAGINATION_PATHS = [
        '/?s=',
        '/*?s=',
        '/search/',
        '/page/',
        '/*?p=',                    // Page parameter variations
        '/*?paged=',
        '/*?page=',
    ];

    // =========================================================================
    // STEP 6: E-COMMERCE OPTIMIZATION
    // =========================================================================

    /**
     * Basic WooCommerce paths to block
     * Essential private/user pages that shouldn't be indexed
     */
    const ECOMMERCE_BASIC_PATHS = [
        '/cart/',
        '/checkout/',
        '/my-account/',
        '/wishlist/',
        '/compare/',                // Product comparison pages
        '/account/',                // Alternative account path
        '/order-received/',
        '/order-pay/',
        '/view-order/',
    ];

    /**
     * Advanced WooCommerce query patterns to block
     * Filter variations that create duplicate content
     */
    const ECOMMERCE_ADVANCED_PATTERNS = [
        // Add to cart URLs
        '/*?add-to-cart=*',
        '/*?add_to_cart=*',
        '/*?added-to-cart=*',
        // Sorting & ordering
        '/*?orderby=*',
        '/*?order=*',
        // Filtering
        '/*?filter_*',
        '/*?filtering=*',
        '/*?min_price=*',
        '/*?max_price=*',
        '/*?price_range=*',
        // Product attributes
        '/*?pa_*',                  // Product attributes
        '/*?product_cat=*',
        '/*?product_tag=*',
        // Category/shop filters
        '/product-category/*?*',
        '/product-tag/*?*',
        '/shop/*?*',
        '/store/*?*',
        // Session & cart parameters
        '/*?wc-ajax=*',
        '/*?remove_item=*',
        '/*?removed_item=*',
        '/*?undo_item=*',
    ];

    // =========================================================================
    // STEP 7: ARCHIVE & WAYBACK CONTROL
    // =========================================================================

    /**
     * Archive service bots
     * block_archiving = block these
     * allow_archiving = no rules (allow by default)
     */
    const ARCHIVE_BOTS = [
        'ia_archiver',              // Internet Archive main bot
        'archive.org_bot',          // Archive.org
        'Wayback',                  // Wayback Machine variants
        'Wayback Machine',          // Full name variant
        'webrecorder',              // Webrecorder service
        'ArchiveTeam',              // Archive Team projects
        'ArchiveBot',               // Archive Bot
        'SavePageNow',              // Save Page Now service
        'ArchiveBox',               // Self-hosted archiving
    ];

    // =========================================================================
    // STEP 9: RESOURCES & ASSETS
    // =========================================================================

    /**
     * EXTENSIONS: CSS & JS
     * allow_css_js = true (allow these)
     */
    const RESOURCE_CSS_JS = [
        '/*.css$',
        '/*.js$',
    ];

    /**
     * EXTENSIONS: IMAGES
     * allow_images = true (allow these)
     */
    const RESOURCE_IMAGES = [
        '/*.jpg$',
        '/*.jpeg$',
        '/*.png$',
        '/*.gif$',
        '/*.webp$',
        '/*.bmp$',
        '/*.ico$',
        '/*.svg$',
    ];

    // =========================================================================
    // ADDITIONAL BOT CATEGORIES (For future use)
    // =========================================================================

    /**
     * Social Media Crawlers - For potential future feature
     * These crawlers fetch preview data for social sharing
     */
    const SOCIAL_MEDIA_BOTS = [
        'facebookexternalhit',      // Facebook link preview
        'Facebot',                  // Facebook crawler
        'Twitterbot',               // Twitter/X card preview
        'LinkedInBot',              // LinkedIn preview
        'Pinterest',                // Pinterest
        'Slackbot',                 // Slack unfurling
        'Discordbot',               // Discord embeds
        'TelegramBot',              // Telegram preview
        'WhatsApp',                 // WhatsApp preview
    ];

    /**
     * Image Crawlers - For potential future feature
     */
    const IMAGE_BOTS = [
        'Googlebot-Image',          // Google Images
        'bingbot/image',            // Bing Images
        'Yandex/Images',            // Yandex Images
        'PicSearch',                // PicSearch
    ];

    // =========================================================================
    // MASTER PLAN V3: GRANULAR BOT CONTROL
    // Bot User-Agent mappings for individual bot management
    // =========================================================================

    /**
     * MASTER PLAN: Search Engine Bots (Complete List)
     * Maps bot identifiers to exact User-Agent strings
     *
     * NOTE: Some User-Agent strings need client confirmation for exact format
     * (capitalization, version numbers, etc.)
     *
     * Based on Master Plan specification - 25+ search engine bots
     */
    const MASTER_PLAN_SEARCH_BOTS = [
        // Major Search Engines - Google
        'googlebot'                => 'Googlebot',
        'googlebot_image'          => 'Googlebot-Image',
        'googlebot_news'           => 'Googlebot-News',
        'googlebot_video'          => 'Googlebot-Video',
        'mediapartners_google'     => 'Mediapartners-Google',
        'adsbot_google'            => 'AdsBot-Google',
        'adsbot_google_mobile'     => 'AdsBot-Google-Mobile',

        // Major Search Engines - Bing/Microsoft
        'bingbot'                  => 'bingbot',
        'bingpreview'              => 'BingPreview',

        // Other Major Search Engines
        'duckduckbot'              => 'DuckDuckBot',
        'slurp'                    => 'Slurp',              // Yahoo
        'yandex'                   => 'Yandex',
        'yandex_images'            => 'YandexImages',
        'qwantify'                 => 'Qwantify',

        // Regional Search Engines - China
        'baiduspider'              => 'Baiduspider',
        'baiduspider_image'        => 'Baiduspider-image',
        'baiduspider_video'        => 'Baiduspider-video',
        'baiduspider_2_0'          => 'Baiduspider/2.0',
        'sogou_spider'             => 'Sogou spider',
        'sogou_web_spider'         => 'Sogou web spider',
        'youdaobot'                => 'YoudaoBot',
        'youdao'                   => 'youdao',
        'yodao'                    => 'yodao',

        // Regional Search Engines - Other
        'seznambot'                => 'SeznamBot',          // Czech Republic
        'yeti'                     => 'Yeti',               // Naver (Korea)
        'exabot'                   => 'Exabot',             // France
    ];

    /**
     * MASTER PLAN: AI & LLM Bots (Complete List)
     * 23+ AI crawlers for granular control
     *
     * NOTE: Client must confirm exact User-Agent strings for newer bots
     * Some bots may use different capitalization or versioning
     *
     * Based on Master Plan specification
     */
    const MASTER_PLAN_AI_BOTS = [
        // OpenAI
        'gptbot'                   => 'GPTBot',
        'chatgpt_user'             => 'ChatGPT-User',
        'oai_searchbot'            => 'OAI-SearchBot',

        // Anthropic (Claude)
        'claudebot'                => 'ClaudeBot',
        'claude_searchbot'         => 'Claude-SearchBot',    // NOTE: Confirm exact UA
        'claude_user'              => 'Claude-User',          // NOTE: Confirm exact UA

        // Perplexity
        'perplexitybot'            => 'PerplexityBot',
        'perplexity_user'          => 'Perplexity-User',      // NOTE: Confirm exact UA

        // Mistral
        'mistralai_user'           => 'MistralAI-User',       // NOTE: Confirm exact UA

        // Google AI
        'google_extended'          => 'Google-Extended',
        'google_cloudvertexbot'    => 'Google-CloudVertexBot', // NOTE: Confirm exact UA

        // Meta
        'meta_externalagent'       => 'Meta-ExternalAgent',   // NOTE: Confirm capitalization

        // Apple
        'applebot_extended'        => 'Applebot-Extended',

        // Amazon
        'amazonbot'                => 'Amazonbot',

        // Huawei / Asia
        'petalbot'                 => 'PetalBot',
        'pangubot'                 => 'PanguBot',             // NOTE: Confirm if officially documented

        // You.com
        'youbot'                   => 'YouBot',

        // Cohere
        'cohere_ai'                => 'cohere-ai',

        // Open Data / Research
        'ccbot'                    => 'CCBot',                // CommonCrawl
        'ai2bot'                   => 'AI2Bot',               // Allen Institute
        'diffbot'                  => 'Diffbot',
        'timpibot'                 => 'TimpiBot',             // NOTE: Confirm capitalization (Timpibot?)
        'omgili'                   => 'omgili',
    ];

    /**
     * MASTER PLAN: SEO Tool Bots (Complete List)
     * 19+ SEO tool crawlers for competitive intelligence blocking
     *
     * NOTE: Client must provide exact User-Agent strings for all SemrushBot variants
     * Some tools have multiple User-Agent formats
     *
     * Based on Master Plan specification
     */
    const MASTER_PLAN_SEO_BOTS = [
        'ahrefsbot'                => 'AhrefsBot',
        'semrushbot'               => 'SemrushBot',
        'semrushbot_sa'            => 'SemrushBot-SA',        // NOTE: Confirm exact UA
        'semrushbot_ba'            => 'SemrushBot-BA',        // NOTE: Confirm exact UA
        'semrushbot_si'            => 'SemrushBot-SI',        // NOTE: Confirm exact UA
        'semrushbot_swa'           => 'SemrushBot-SWA',       // NOTE: Confirm exact UA
        'semrushbot_ct'            => 'SemrushBot-CT',        // NOTE: Confirm exact UA
        'semrushbot_bm'            => 'SemrushBot-BM',        // NOTE: Confirm exact UA
        'mj12bot'                  => 'MJ12bot',
        'dotbot'                   => 'DotBot',
        'dotbot_1_1'               => 'DotBot/1.1',           // NOTE: Confirm version format
        'domainstatsbot'           => 'DomainStatsBot/1.0',   // NOTE: Confirm exact UA
        'seokicks_robot'           => 'SEOkicks-Robot',
        'sitebulb'                 => 'Sitebulb',             // NOTE: Confirm exact UA
        'screaming_frog'           => 'Screaming Frog SEO Spider',
        'rytebot'                  => 'RyteBot',
        'serpstatbot'              => 'SerpstatBot',          // NOTE: Confirm capitalization
        'oncrawl'                  => 'OnCrawl',
        'deepcrawl'                => 'Deepcrawl',            // NOTE: Confirm capitalization
    ];

    /**
     * MASTER PLAN: Social Media Bots (Complete List)
     * 12+ social media preview/link expansion bots
     *
     * Based on Master Plan specification
     */
    const MASTER_PLAN_SOCIAL_BOTS = [
        // Facebook
        'facebookexternalhit_1_0'  => 'facebookexternalhit/1.0',
        'facebookexternalhit_1_1'  => 'facebookexternalhit/1.1',
        'facebookplatform_1_0'     => 'facebookplatform/1.0',
        'facebot_1_0'              => 'Facebot/1.0',

        // Twitter/X
        'twitterbot'               => 'Twitterbot',

        // LinkedIn
        'linkedinbot'              => 'LinkedInBot',
        'linkedinbot_1_0'          => 'LinkedInBot/1.0',

        // Pinterest
        'pinterest_0_1'            => 'Pinterest/0.1',
        'pinterest_0_2'            => 'Pinterest/0.2',

        // Messaging platforms
        'slackbot_linkexpanding'   => 'Slackbot-LinkExpanding',
        'discordbot'               => 'Discordbot',
    ];

    /**
     * MASTER PLAN: Archive Bots (Complete List)
     * 3 archive service bots
     *
     * Based on Master Plan specification
     */
    const MASTER_PLAN_ARCHIVE_BOTS = [
        'archive_org_bot'          => 'archive.org_bot',
        'archive_it'               => 'Archive-It',           // NOTE: Confirm exact UA
        'special_archiver'         => 'special_archiver',     // NOTE: Confirm if this exists
    ];

    /**
     * MASTER PLAN: Bad Bots Lists
     * AI-curated bot lists (INCOMPLETE - NEEDS CLIENT INPUT)
     *
     * CRITICAL: Master Plan only provides examples, not complete lists
     * Client must provide:
     * 1. BASIC list (10-20 safe universal bots for Free tier)
     * 2. FULL AI-CURATED list (categorized: CLI clients, aggressive scrapers, feed scrapers)
     *
     * Categories from Master Plan:
     * - Scripts & CLI clients (python-requests, curl, wget, libwww-perl, Go-http-client, Apache-HttpClient)
     * - Aggressive scrapers (ZmEu, BUbiNG, Nutch, ZoominfoBot, TurnitinBot, bixo, etc.)
     * - Feed scrapers (FeedlyBot/1.0, Feedspot, Feedbin, etc.)
     *
     * PLACEHOLDER - DO NOT USE IN PRODUCTION UNTIL CLIENT PROVIDES COMPLETE LISTS
     */
    const MASTER_PLAN_BAD_BOTS_BASIC = [
        // Safe, universal blocklist (mostly scripts and CLI tools)
        'python-requests',
        'curl',
        'wget',
        'libwww-perl',
        'Go-http-client',
        'Apache-HttpClient',
        'Scrapy',
        'Java',
    ];

    const MASTER_PLAN_BAD_BOTS_FULL = [
        // Scripts & CLI clients (Basic list included)
        'python-requests',
        'curl',
        'wget',
        'libwww-perl',
        'Go-http-client',
        'Apache-HttpClient',
        'Scrapy',
        'Java',

        // Aggressive Scrapers & Bad Bots (from BAD_BOTS.md)
        'GiftGhostBot',
        'Seznam',
        'PaperLiBot',
        'Genieo',
        'Dataprovider/6.101',
        'DataproviderSiteExplorer',
        'Dataprovider/6.92',
        'Dataprovider Site Explorer',
        'Dazoobot/1.0',
        'Diffbot',
        'Diffbot/0.1',
        'DomainStatsBot/1.0',
        'dubaiindex',
        'eCommerceBot',
        'ExpertSearchSpider',
        'Feedbin',
        'Fetch/2.0a',
        'FFbot/1.0',
        'focusbot/1.1',
        'HuaweiSymantecSpider',
        'HuaweiSymantecSpider/1.0',
        'JobdiggerSpider',
        'LemurWebCrawler',
        'LipperheyLinkExplorer',
        'LSSRocketCrawler/1.0',
        'LYT.SRv1.5',
        'MiaDev/0.0.1',
        'Najdi.si/3.1',
        'BountiiBot',
        'Experibot_v1',
        'bixocrawler',
        'bixocrawler TestCrawler',
        'bixo',
        'bixolabs/1.0',
        'Crawler4j',
        'Crowsnest/0.5',
        'CukBot',
        'DBLBot/1.0',
        'Digg Deeper/v1',
        'discobot/1.0',
        'discobot/1.1',
        'discobot/2.0',
        'discoverybot/2.0',
        'Dlvr.it/1.0',
        'drupact/0.7',
        'Ezooms/1.0',
        'fastbot crawler beta 2.0',
        'fastbot crawler beta 4.0',
        'feedly social',
        'Feedly/1.0',
        'FeedlyBot/1.0',
        'Feedspot',
        'Feedspotbot/1.0',
        'Clickagy Intelligence Bot v2',
        'classbot',
        'CISPA Vulnerability Notification',
        'CirrusExplorer/1.1',
        'Checksem/Nutch-1.10',
        'CatchBot/5.0',
        'CatchBot/3.0',
        'CatchBot/2.0',
        'CatchBot/1.0',
        'CamontSpider/1.0',
        'Buzzbot/1.0',
        'Buzzbot',
        'BusinessSeek.biz_Spider',
        'BUbiNG',
        '008/0.85',
        '008/0.83',
        '008/0.71',
        '^Nail',
        'FyberSpider/1.3',
        'findlinks/1.1.6-beta5',
        'g2reader-bot/1.0',
        'findlinks/1.1.6-beta6',
        'findlinks/2.0',
        'findlinks/2.0.1',
        'findlinks/2.0.2',
        'findlinks/2.0.4',
        'findlinks/2.0.5',
        'findlinks/2.0.9',
        'findlinks/2.1',
        'findlinks/2.1.5',
        'findlinks/2.1.3',
        'findlinks/2.2',
        'findlinks/2.5',
        'findlinks/2.6',
        'findlinks/1.0',
        'findlinks/1.1.3-beta8',
        'findlinks/1.1.3-beta9',
        'findlinks/1.1.4-beta7',
        'findlinks/1.1.6-beta1',
        'findlinks/1.1.6-beta1 Yacy',
        'findlinks/1.1.6-beta2',
        'findlinks/1.1.6-beta3',
        'findlinks/1.1.6-beta4',
        'Crawlera/1.10.2',
        
        // Items from bad_bots_chatgpt that weren't duplicates
        'SiteExplorer',
        'spbot',
        'WBSearchBot',
        'linkdexbot',
        'netEstate NE Crawler',
        'Moreover',
        'sentibot',
        'Aboundexbot',
        'proximic',
        'oBot',
        'meanpathbot',
        'Nutch',
        'TurnitinBot',
        'ZoominfoBot',
        'ZmEu',
        'grapeshot',

        // Note: Archive bots (ia_archiver, etc.) and SEO tools (Screaming Frog, etc.) 
        // are excluded here as they are handled in their own dedicated modules.
    ];

    // =========================================================================
    // HELPER METHODS
    // =========================================================================

    /**
     * Get User-Agent string for a search engine
     *
     * @param string $engine Engine identifier (e.g., 'google')
     * @return string|null User-Agent string or null if not found
     */
    public static function getSearchEngineBot($engine)
    {
        return self::SEARCH_ENGINE_BOTS[$engine] ?? null;
    }

    /**
     * Get a granular search engine bot User-Agent by UI key.
     *
     * @param string $botId Vue bot identifier
     * @return string|null User-Agent string or null if not found
     */
    public static function getGranularSearchEngineBot($botId)
    {
        return self::SEARCH_ENGINE_GRANULAR_BOTS[$botId] ?? null;
    }

    /**
     * Get bots for a visibility level
     *
     * @param string $level Visibility level identifier
     * @return array Array of bot User-Agent strings
     */
    public static function getVisibilityBots($level)
    {
        if (!isset(self::VISIBILITY_LEVELS[$level])) {
            return [];
        }

        $bots = [];
        foreach (self::VISIBILITY_LEVELS[$level] as $engine) {
            if ($bot = self::getSearchEngineBot($engine)) {
                $bots[] = $bot;
            }
        }
        return $bots;
    }

    /**
     * Get bots for a protection level
     *
     * @param string $level Protection level identifier
     * @return array Array of bot User-Agent strings
     */
    public static function getProtectionBots($level)
    {
        return self::BOT_PROTECTION_LEVELS[$level] ?? [];
    }

    /**
     * Generate robots.txt rules for blocking bots
     *
     * @param array $bots Array of bot names
     * @param string $comment Optional comment to add before rules
     * @return string Robots.txt formatted string
     */
    public static function generateDisallowRules($bots, $comment = '')
    {
        if (empty($bots)) {
            return '';
        }

        $output = '';
        if ($comment) {
            $output .= "# {$comment}\n";
        }

        foreach ($bots as $bot) {
            $output .= "User-agent: {$bot}\nDisallow: /\n\n";
        }

        return $output;
    }

    /**
     * Generate robots.txt rules for allowing bots
     *
     * @param array $bots Array of bot names
     * @param string $comment Optional comment to add before rules
     * @return string Robots.txt formatted string
     */
    public static function generateAllowRules($bots, $comment = '')
    {
        if (empty($bots)) {
            return '';
        }

        $output = '';
        if ($comment) {
            $output .= "# {$comment}\n";
        }

        foreach ($bots as $bot) {
            $output .= "User-agent: {$bot}\nAllow: /\n\n";
        }

        return $output;
    }

    /**
     * Generate robots.txt rules for blocking paths
     *
     * @param array $paths Array of paths to block
     * @param string $comment Optional comment
     * @return string Robots.txt formatted string
     */
    public static function generatePathDisallowRules($paths, $comment = '')
    {
        if (empty($paths)) {
            return '';
        }

        $output = '';
        if ($comment) {
            $output .= "# {$comment}\n";
        }

        $output .= "User-agent: *\n";
        foreach ($paths as $path) {
            $output .= "Disallow: {$path}\n";
        }
        $output .= "\n";

        return $output;
    }

    /**
     * Get signature line (or empty if removed)
     *
     * @param bool $removeSignature Whether to remove signature
     * @param string $customText Optional custom footer comment
     * @return string Signature block or empty string
     */
    public static function getSignature($removeSignature = false, $customText = '')
    {
        if ($removeSignature) {
            return '';
        }

        $formattedCustomText = self::formatFooterComment($customText);
        $output = "\n";

        if ($formattedCustomText !== '') {
            $output .= $formattedCustomText . "\n\n";
        }

        return $output;
    }

    /**
     * Format footer text as comment lines.
     *
     * @param string $text
     * @return string
     */
    private static function formatFooterComment($text)
    {
        if (!is_string($text) || trim($text) === '') {
            return '';
        }

        $lines = preg_split('/\r\n|\r|\n/', $text);
        $formatted = [];

        foreach ($lines as $line) {
            $line = trim($line);

            if ($line === '') {
                continue;
            }

            $line = preg_replace('/[^\x20-\x7E]/', '', $line);

            if ($line === '') {
                continue;
            }

            if (strpos($line, '#') !== 0) {
                $line = '# ' . $line;
            }

            $formatted[] = $line;
        }

        return implode("\n", $formatted);
    }

    /**
     * Get all available search engines
     *
     * @return array Associative array of engine_id => bot_name
     */
    public static function getAllSearchEngines()
    {
        return self::SEARCH_ENGINE_BOTS;
    }

    /**
     * Get all AI training bots
     *
     * @return array Array of bot names
     */
    public static function getAITrainingBots()
    {
        return self::AI_TRAINING_BOTS;
    }

    /**
     * Get all AI search bots
     *
     * @return array Array of bot names
     */
    public static function getAISearchBots()
    {
        return self::AI_SEARCH_BOTS;
    }

    /**
     * Generate global base rules for WordPress
     * These rules apply to all user agents and are always included
     *
     * @return string Robots.txt formatted base rules
     */
    public static function generateBaseRules($rules = [])
    {
        $output = "# Global base rules\n";
        $output .= "User-agent: *\n";

        // Add Allow rules
        foreach (self::BASE_ALLOW_PATHS as $path) {
            $output .= "Allow: {$path}\n";
        }

        // --- Granular Disallow Rules ---

        // WordPress Admin (Default: true)
        if ($rules['block_wp_admin'] ?? true) {
            $output .= "Disallow: /wp-admin/\n";
        }

        // WordPress Includes (Default: true)
        if ($rules['block_wp_includes'] ?? true) {
            $output .= "Disallow: /wp-includes/\n";
        }

        // Readme & License (Default: true)
        if ($rules['block_readme'] ?? true) {
            $output .= "Disallow: /readme.html\n";
            $output .= "Disallow: /license.txt\n";
        }

        // XML-RPC (Default: true)
        if ($rules['block_xmlrpc'] ?? true) {
            $output .= "Disallow: /xmlrpc.php\n";
        }

        // Login & Register (Default: true)
        if ($rules['block_login'] ?? true) {
            $output .= "Disallow: /wp-login.php\n";
            $output .= "Disallow: /wp-register.php\n";
        }

        // Disclaimer (Default: true)
        if ($rules['block_disclaimer'] ?? true) {
            $output .= "Disallow: */disclaimer/*\n";
        }

        // Attachments & Filtering (Default: true)
        if ($rules['block_attachments'] ?? true) {
            $output .= "Disallow: *?attachment_id=\n";
            $output .= "Disallow: /*?filtering=\n";
        }

        $output .= "\n";

        return $output;
    }

    // =========================================================================
    // MASTER PLAN V3: HELPER METHODS
    // =========================================================================

    /**
     * Get Master Plan search engine bot User-Agent
     *
     * @param string $botId Bot identifier (e.g., 'googlebot', 'googlebot_image')
     * @return string|null User-Agent string or null if not found
     */
    public static function getMasterPlanSearchBot($botId)
    {
        return self::MASTER_PLAN_SEARCH_BOTS[$botId] ?? null;
    }

    /**
     * Get all Master Plan search engine bots
     *
     * @return array Associative array of bot_id => user_agent
     */
    public static function getAllMasterPlanSearchBots()
    {
        return self::MASTER_PLAN_SEARCH_BOTS;
    }

    /**
     * Get Master Plan AI bot User-Agent
     *
     * @param string $botId Bot identifier (e.g., 'gptbot', 'claudebot')
     * @return string|null User-Agent string or null if not found
     */
    public static function getMasterPlanAIBot($botId)
    {
        return self::MASTER_PLAN_AI_BOTS[$botId] ?? null;
    }

    /**
     * Get all Master Plan AI & LLM bots
     *
     * @return array Associative array of bot_id => user_agent
     */
    public static function getAllMasterPlanAIBots()
    {
        return self::MASTER_PLAN_AI_BOTS;
    }

    /**
     * Get Master Plan SEO tool bot User-Agent
     *
     * @param string $botId Bot identifier (e.g., 'ahrefsbot', 'semrushbot')
     * @return string|null User-Agent string or null if not found
     */
    public static function getMasterPlanSEOBot($botId)
    {
        return self::MASTER_PLAN_SEO_BOTS[$botId] ?? null;
    }

    /**
     * Get all Master Plan SEO tool bots
     *
     * @return array Associative array of bot_id => user_agent
     */
    public static function getAllMasterPlanSEOBots()
    {
        return self::MASTER_PLAN_SEO_BOTS;
    }

    /**
     * Get Master Plan social media bot User-Agent
     *
     * @param string $botId Bot identifier (e.g., 'facebookexternalhit_1_0')
     * @return string|null User-Agent string or null if not found
     */
    public static function getMasterPlanSocialBot($botId)
    {
        return self::MASTER_PLAN_SOCIAL_BOTS[$botId] ?? null;
    }

    /**
     * Get all Master Plan social media bots
     *
     * @return array Associative array of bot_id => user_agent
     */
    public static function getAllMasterPlanSocialBots()
    {
        return self::MASTER_PLAN_SOCIAL_BOTS;
    }

    /**
     * Get Master Plan archive bot User-Agent
     *
     * @param string $botId Bot identifier (e.g., 'archive_org_bot')
     * @return string|null User-Agent string or null if not found
     */
    public static function getMasterPlanArchiveBot($botId)
    {
        return self::MASTER_PLAN_ARCHIVE_BOTS[$botId] ?? null;
    }

    /**
     * Generate Allow rules for specific paths for all user agents
     * Reference: https://developers.google.com/search/docs/crawling-indexing/robots/robots_txt
     *
     * @param array $paths List of paths to allow
     * @param string $comment Optional comment
     * @return string
     */
    public static function generatePathAllowRules(array $paths, string $comment = '')
    {
        if (empty($paths)) {
            return '';
        }

        $output = '';
        if ($comment) {
            $output .= "# {$comment}\n";
        }

        foreach ($paths as $path) {
            $output .= "User-agent: *\nAllow: {$path}\n\n";
        }

        return $output;
    }

    /**
     * Get Master Plan Archive Bots
     */
    public static function getAllMasterPlanArchiveBots()
    {
        return self::MASTER_PLAN_ARCHIVE_BOTS;
    }

    /**
     * Get Master Plan bot by category
     *
     * @param string $category Category: 'search_engines', 'ai_llm', 'seo_tools', 'social_media', 'archive'
     * @return array Associative array of bot_id => user_agent
     */
    public static function getMasterPlanBotsByCategory($category)
    {
        switch ($category) {
            case 'search_engines':
                return self::MASTER_PLAN_SEARCH_BOTS;
            case 'ai_llm':
                return self::MASTER_PLAN_AI_BOTS;
            case 'seo_tools':
                return self::MASTER_PLAN_SEO_BOTS;
            case 'social_media':
                return self::MASTER_PLAN_SOCIAL_BOTS;
            case 'archive':
                return self::MASTER_PLAN_ARCHIVE_BOTS;
            default:
                return [];
        }
    }

    /**
     * Generate robots.txt rules for granular bot control
     * Supports 3-state logic: global / allow / disallow
     *
     * @param array $customBots Associative array of bot_id => state ('global'|'allow'|'disallow')
     * @param string $category Bot category to get default list from
     * @param string $comment Optional comment to add before rules
     * @return string Robots.txt formatted string
     */
    public static function generateGranularBotRules($customBots, $category, $comment = '')
    {
        $allBots = self::getMasterPlanBotsByCategory($category);

        if (empty($allBots)) {
            return '';
        }

        $output = '';
        if ($comment) {
            $output .= "# {$comment}\n";
        }

        foreach ($customBots as $botId => $state) {
            // Skip if state is 'global' (uses User-agent: * rules)
            if ($state === 'global') {
                continue;
            }

            // Get User-Agent string
            $userAgent = $allBots[$botId] ?? null;
            if (!$userAgent) {
                continue;
            }

            // Generate rule based on state
            if ($state === 'allow') {
                $output .= "User-agent: {$userAgent}\nAllow: /\n\n";
            } elseif ($state === 'disallow') {
                $output .= "User-agent: {$userAgent}\nDisallow: /\n\n";
            }
        }

        return $output;
    }
}
