<?php
/**
 * Chat Handler
 *
 * Orchestrates the chat flow between user and Claude AI.
 *
 * @package BoostBot
 * @since 1.0.0
 */

if ( ! defined( 'ABSPATH' ) ) {
    exit;
}

/**
 * Class BoostBot_Chat_Handler
 *
 * Handles incoming chat messages and generates AI responses.
 */
class BoostBot_Chat_Handler {

    /**
     * Per-request RAG trace usage accumulator (populated only when trace logging is on).
     * Declared explicitly to avoid PHP 8.2+ dynamic-property deprecation notices.
     *
     * @var array
     */
    private $trace_usage = array();

    /**
     * Claude client instance.
     *
     * @var BoostBot_Claude_Client
     */
    private $claude_client;

    /**
     * Prompt builder instance.
     *
     * @var BoostBot_Prompt_Builder
     */
    private $prompt_builder;

    /**
     * Conversation model instance.
     *
     * @var BoostBot_Conversation_Model
     */
    private $conversation_model;

    /**
     * Knowledge index instance.
     *
     * @var BoostBot_Knowledge_Index|null
     */
    private $knowledge_index;

    // Trace logs feature archived — see boostbot/_archive/trace-logs/

    /**
     * Week 27.8: Per-turn routing inputs captured from prepare-query (Gemini).
     *
     * Shape:
     * { intents: string[], entities: string[], must_include: string[], rewritten_query: string, original_query: string }
     *
     * @var array
     */
    private $last_routing_inputs = array();

    /**
     * Week 27.8-2: Per-turn tool selection result (Gemini).
     *
     * Shape:
     * { selected_contact_ids:int[], selected_widget_ids:int[], picker_widget_ids:int[] }
     *
     * @var array
     */
    private $last_tool_selection = array();

    /**
     * Maps contact IDs to their parent contact_card widget IDs.
     * Built by get_available_contacts_for_selection() so that prompt builder
     * and token resolution can reference both widget + contact.
     *
     * Shape: [ contact_id => widget_id, ... ]
     *
     * @var array<int, int>
     */
    private $contact_to_widget_map = array();

    /**
     * Resolved bot language for the current request.
     *
     * @var string
     */
    private $bot_lang = 'he';

    /**
     * Constructor.
     *
     * Note: Claude client is initialized lazily to allow exception handling.
     */
    public function __construct() {
        $this->prompt_builder     = new BoostBot_Prompt_Builder();
        $this->conversation_model = new BoostBot_Conversation_Model();
        $this->knowledge_index    = new BoostBot_Knowledge_Index();
        // Claude client is initialized in handle() to catch exceptions properly
    }

    /**
     * Process a chat message.
     *
     * Static entry point for processing messages.
     *
     * @param string $message         The user message.
     * @param string $conversation_id The conversation ID (or null for new).
     * @param string $visitor_id      The visitor ID.
     * @param array  $page_context    Page context data.
     * @return array Response array with message, conversation_id, widgets, suggestions.
     */
    public static function process( string $message, ?string $conversation_id, string $visitor_id, array $page_context = array() ): array {
        $handler = new self();
        return $handler->handle( $message, $conversation_id, $visitor_id, $page_context );
    }

    /**
     * Handle a chat message.
     *
     * @param string $message         The user message.
     * @param string $conversation_id The conversation ID (or null for new).
     * @param string $visitor_id      The visitor ID.
     * @param array  $page_context    Page context data.
     * @return array Response array.
     */
    public function handle( string $message, ?string $conversation_id, string $visitor_id, array $page_context = array() ): array {
        // ===== TIMING DIAGNOSTICS =====
        $timing_start = microtime( true );
        $timing_steps = array();
        $timing_log = function( $step_name ) use ( &$timing_steps, $timing_start ) {
            $now = microtime( true );
            $elapsed_total = round( ( $now - $timing_start ) * 1000, 1 );
            $elapsed_step = 0;
            if ( ! empty( $timing_steps ) ) {
                $last = end( $timing_steps );
                $elapsed_step = round( ( $now - $last['end_time'] ) * 1000, 1 );
            } else {
                $elapsed_step = $elapsed_total;
            }
            $timing_steps[] = array(
                'step'      => $step_name,
                'duration'  => $elapsed_step,
                'cumulative'=> $elapsed_total,
                'end_time'  => $now,
            );
            BoostBot_Logger::info( sprintf( '[TIMING] ⏱️ %s: %dms (total: %dms)', $step_name, $elapsed_step, $elapsed_total ) );
        };
        // ===== END TIMING SETUP =====

        BoostBot_Logger::info( '🔄 Chat Handler: Starting handle()' );

        // Week 27.10: Optional auto-expiration for trace mode.
        $trace_until = absint( BoostBot_Settings::get( 'rag_trace_logs_until', 0 ) );
        if ( $trace_until > 0 && time() > $trace_until ) {
            BoostBot_Settings::set( 'rag_trace_logs', false );
            BoostBot_Settings::set( 'rag_trace_logs_until', 0 );
        }

        $trace_enabled = (bool) BoostBot_Settings::get( 'rag_trace_logs', false );
        $trace_id = $trace_enabled ? ( 'trace:' . wp_generate_uuid4() ) : '';
        if ( $trace_enabled ) {
            BoostBot_Logger::info( '[TRACE ' . $trace_id . '] ====== TRACE START ======' );
            BoostBot_Logger::info( '[TRACE ' . $trace_id . '] visitor_id=' . substr( (string) $visitor_id, 0, 80 ) );
            BoostBot_Logger::info( '[TRACE ' . $trace_id . '] user_message=' . (string) $message );
        }
        
        BoostBot_Logger::debug( 'Processing chat message', array(
            'message_preview'  => substr( $message, 0, 100 ),
            'conversation_id'  => $conversation_id,
            'visitor_id'       => $visitor_id,
        ) );

        try {
            // 0. Initialize AI client (Week 26: backend proxy if configured)
            if ( ! $this->claude_client ) {
                $backend_base_url = (string) BoostBot_Settings::get( 'backend_base_url', '' );
                $backend_base_url = trim( $backend_base_url );

                if ( ! empty( $backend_base_url ) && class_exists( 'BoostBot_Backend_AI_Client' ) ) {
                    BoostBot_Logger::info( '🔧 Initializing Backend AI client...' );
                    BoostBot_Logger::debug( 'Initializing backend AI client' );
                    $this->claude_client = new BoostBot_Backend_AI_Client();
                    BoostBot_Logger::info( '✅ Backend AI client initialized' );
                } else {
                    BoostBot_Logger::info( '🔧 Initializing Claude client...' );
                    BoostBot_Logger::debug( 'Initializing Claude client' );
                    $this->claude_client = new BoostBot_Claude_Client();
                    BoostBot_Logger::info( '✅ Claude client initialized' );
                }
            }
            $timing_log( 'init_client' );

            // 1. Get or create conversation
            BoostBot_Logger::info( '📝 Step 1: Get/create conversation' );
            $conversation = $this->get_or_create_conversation( $conversation_id, $visitor_id );
            BoostBot_Logger::info( '✅ Conversation: ' . $conversation->conversation_id );
            $timing_log( 'get_conversation' );

            // 2. Get conversation history
            BoostBot_Logger::info( '📜 Step 2: Get conversation history' );
            $history = $this->get_conversation_history( $conversation );
            BoostBot_Logger::info( '✅ History messages: ' . count( $history ) );
            $timing_log( 'get_history' );

            // 3. Resolve the active bot for this context (cheap, local - do this early)
            BoostBot_Logger::info( '🤖 Step 3: Resolve bot' );
            $bot = $this->resolve_bot( $page_context );
            BoostBot_Logger::info( '✅ Bot: ' . ( $bot->bot_name ?? 'default' ) );
            $timing_log( 'resolve_bot' );

            // 3.5 PERFORMANCE OPTIMIZATION: First-greeting fast-path
            // For the FIRST message in a conversation that is a simple greeting,
            // skip the unified router and RAG entirely - go straight to LLM.
            $is_first_message = empty( $history ) || count( $history ) === 0;
            $ps_greet          = BoostBot_Prompt_Strings::get( $bot->language ?? 'he' );
            $greeting_patterns = array( $ps_greet['greeting_pattern'] );
            $is_greeting = false;
            $message_trimmed = trim( $message );
            foreach ( $greeting_patterns as $pattern ) {
                if ( preg_match( $pattern, $message_trimmed ) ) {
                    $is_greeting = true;
                    break;
                }
            }
            
            if ( $is_first_message && $is_greeting ) {
                BoostBot_Logger::info( '[PERF] 🚀 FAST-PATH: First greeting detected, skipping unified router + RAG' );
                $timing_log( 'fast_path_check' );
                
                $fast_response = $this->fast_greeting_response( $message, $conversation, $bot, $timing_log, $timing_steps, $timing_start );
                if ( null !== $fast_response ) {
                    return $fast_response;
                }
                // If fast_response is null, fall through to normal flow
                BoostBot_Logger::warning( 'Fast-path returned null, continuing with normal flow' );
            }

            // 4. PERFORMANCE OPTIMIZATION: Call unified router FIRST to determine if RAG is needed.
            // This allows us to skip the expensive search_knowledge call for simple queries.
            BoostBot_Logger::info( '🧭 Step 4: Unified Router (FIRST - decides if RAG needed)' );
            $use_unified_router = (bool) BoostBot_Settings::get( 'use_unified_router', true );
            $unified_router_result = null;
            
            if ( $use_unified_router && class_exists( 'BoostBot_Credits_Backend_Client' ) ) {
                $unified_router_result = $this->call_unified_router( $message, $history, $page_context, $bot, $trace_enabled ? $trace_id : null );
                $timing_log( 'unified_router' );
            }
            
            // Decide routing: use unified router result if available and not fallback, otherwise legacy PHP matching.
            $unified_router_ok = $unified_router_result 
                && is_array( $unified_router_result ) 
                && ! empty( $unified_router_result['routing'] )
                && empty( $unified_router_result['fallback_used'] );
            
            if ( $unified_router_ok ) {
                // Use unified router decisions
                $routing = array(
                    'include_brand_services' => false, // Deprecated field
                    'include_contact_list'   => $unified_router_result['routing']['needs_contacts'] ?? false,
                    'include_actions'        => $unified_router_result['routing']['needs_widgets'] ?? false,
                    'include_page_context'   => $unified_router_result['routing']['needs_page_context'] ?? true,
                    'include_brand_card'     => $unified_router_result['routing']['needs_brand_info'] ?? true,
                    'needs_retrieval'        => $unified_router_result['routing']['needs_retrieval'] ?? true,
                );
                
                // Update routing inputs from unified router
                $this->last_routing_inputs = array(
                    'intents'         => $unified_router_result['intents'] ?? array(),
                    'entities'        => $unified_router_result['entities'] ?? array(),
                    'must_include'    => $unified_router_result['must_include'] ?? array(),
                    'rewritten_query' => $unified_router_result['rewritten_query'] ?? $message,
                    'original_query'  => $message,
                );
                
                // Pre-populate tool selection from unified router
                $this->last_tool_selection = array(
                    'ok'                   => true,
                    'fallback_used'        => false,
                    'selected_contact_ids' => $unified_router_result['selected_contact_ids'] ?? array(),
                    'selected_widget_ids'  => $unified_router_result['selected_widget_ids'] ?? array(),
                    'picker_widget_ids'    => $unified_router_result['picker_widget_ids'] ?? array(),
                );
                
                if ( $trace_enabled ) {
                    BoostBot_Logger::info( '[TRACE ' . $trace_id . '] unified_router=ENABLED' );
                    BoostBot_Logger::info( '[TRACE ' . $trace_id . '] unified_routing=' . wp_json_encode( $unified_router_result['routing'] ?? array(), JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES ) );
                    BoostBot_Logger::info( '[TRACE ' . $trace_id . '] unified_reasoning=' . ( $unified_router_result['reasoning'] ?? '' ) );
                }
            } else {
                // Fall back to legacy PHP keyword matching (always assume needs_retrieval=true for safety)
                $routing = $this->decide_prompt_routing(
                    is_array( $this->last_routing_inputs ) ? $this->last_routing_inputs : array(),
                    $page_context,
                    $bot
                );
                $routing['needs_retrieval'] = true; // Legacy mode: always run RAG
                if ( $trace_enabled ) {
                    BoostBot_Logger::warning( '[TRACE ' . $trace_id . '] unified_router=FALLBACK (legacy PHP routing)' );
                }
            }
            
            if ( $trace_enabled ) {
                $routing_json = wp_json_encode( $routing, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES );
                BoostBot_Logger::info( '[TRACE ' . $trace_id . '] routing=' . ( $routing_json ? $routing_json : '(unavailable)' ) );
            }

            // 5. CONDITIONAL RAG: Only search knowledge base if unified router says it's needed
            $knowledge_context = null;
            $needs_retrieval = isset( $routing['needs_retrieval'] ) ? (bool) $routing['needs_retrieval'] : true;
            
            if ( $needs_retrieval ) {
                BoostBot_Logger::info( '🔍 Step 5: Search knowledge base (needs_retrieval=true)' );
                $knowledge_context = $this->search_knowledge( $message, $history, $trace_enabled ? $trace_id : null );
                $timing_log( 'search_knowledge' );
                if ( $trace_enabled ) {
                    $kc = (string) ( $knowledge_context ?? '' );
                    BoostBot_Logger::info( '[TRACE ' . $trace_id . '] evidence_bundle_chars=' . mb_strlen( $kc ) . ' bytes=' . strlen( $kc ) );
                    BoostBot_Logger::info( "[TRACE {$trace_id}] ====== EVIDENCE BUNDLE START ======\n" . $kc . "\n[TRACE {$trace_id}] ====== EVIDENCE BUNDLE END ======" );
                }
            } else {
                BoostBot_Logger::info( '[PERF] 🚀 Step 5: SKIPPED search_knowledge (needs_retrieval=false, saved ~300ms+)' );
                $timing_log( 'search_knowledge_skipped' );
            }

            // 6. Get page content context (Week 27.8: structured excerpt, capped)
            $page_content = null;
            $needs_page_context = isset( $routing['include_page_context'] ) ? (bool) $routing['include_page_context'] : true;
            if ( $needs_page_context ) {
                BoostBot_Logger::info( '📄 Step 6: Get page content' );
                $page_content = $this->get_page_content( $page_context );
                $timing_log( 'get_page_content' );
            } else {
                BoostBot_Logger::info( '[PERF] 📄 Step 6: SKIPPED page content (not needed)' );
            }

            // 7. ServiceCatalog disabled (removed): do not inject global catalog into prompt.
            $global_context = null;

            // 8. Week 27.6: BrandCard (global + per-bot override)
            $brand_card_text = null;
            $bot_lang        = $bot->language ?? '';
            $this->bot_lang  = $bot_lang ?: 'he';
            $ps              = BoostBot_Prompt_Strings::get( $this->bot_lang );
            $brand_card_ver  = absint( BoostBot_Settings::get_for_language( 'brand_card_global_version', $bot_lang, 1 ) ) ?: 1;
            $brand_card_json = (string) BoostBot_Settings::get_for_language( 'brand_card_global_json', $bot_lang, '' );
            $brand_card_obj  = null;
            if ( '' !== trim( $brand_card_json ) ) {
                $decoded = json_decode( $brand_card_json, true );
                if ( is_array( $decoded ) ) {
                    $brand_card_obj = $decoded;
                }
            }

            // 4.7 Week 27.8-2: Tool selection (Gemini) — skip if already populated by unified router.
            $skip_legacy_tool_selection = isset( $this->last_tool_selection['ok'] ) && $this->last_tool_selection['ok'];
            
            if ( ! $skip_legacy_tool_selection ) {
                // Initialize tool selection if not done by unified router
                $this->last_tool_selection = array(
                    'ok'                 => false,
                    'fallback_used'       => true,
                    'selected_contact_ids' => array(),
                    'selected_widget_ids'  => array(),
                    'picker_widget_ids'    => array(),
                );
            }
            
            $want_contacts = isset( $routing['include_contact_list'] ) ? (bool) $routing['include_contact_list'] : false;
            $want_actions  = isset( $routing['include_actions'] ) ? (bool) $routing['include_actions'] : false;
            if ( ! $skip_legacy_tool_selection && ( $want_contacts || $want_actions ) && class_exists( 'BoostBot_Credits_Backend_Client' ) ) {
                $backend_base_url = (string) BoostBot_Settings::get( 'backend_base_url', '' );
                if ( '' !== trim( $backend_base_url ) ) {
                    $client = new BoostBot_Credits_Backend_Client();
                    $site_id = $client->ensure_site_id();
                    if ( ! is_wp_error( $site_id ) ) {
                        $available_contacts = $want_contacts ? $this->get_available_contacts_for_selection( $bot ) : array();
                        $available_widgets  = $want_actions ? $this->get_available_widgets_for_selection( $bot ) : array();

                        // Avoid calling the backend if there is nothing to select from.
                        if ( ! empty( $available_contacts ) || ! empty( $available_widgets ) ) {
                            $payload = array(
                                'query'              => (string) $message,
                                'intents'            => $this->last_routing_inputs['intents'] ?? array(),
                                'entities'           => $this->last_routing_inputs['entities'] ?? array(),
                                'must_include'       => $this->last_routing_inputs['must_include'] ?? array(),
                                'available_contacts' => $available_contacts,
                                'available_widgets'  => $available_widgets,
                            );

                            $sel = $client->rag_select_tools( (string) $site_id, $payload );
                            if ( ! is_wp_error( $sel ) && is_array( $sel ) ) {
                                // Defensive: Gemini may hallucinate IDs. Filter selections against what we actually provided.
                                $allowed_contact_ids = array();
                                foreach ( (array) $available_contacts as $c ) {
                                    if ( is_array( $c ) && isset( $c['id'] ) ) {
                                        $allowed_contact_ids[] = absint( $c['id'] );
                                    }
                                }
                                $allowed_contact_ids = array_values( array_filter( array_unique( $allowed_contact_ids ) ) );

                                $allowed_widget_ids = array();
                                foreach ( (array) $available_widgets as $w ) {
                                    if ( is_array( $w ) && isset( $w['id'] ) ) {
                                        $allowed_widget_ids[] = absint( $w['id'] );
                                    }
                                }
                                $allowed_widget_ids = array_values( array_filter( array_unique( $allowed_widget_ids ) ) );

                                $raw_selected_contact_ids = isset( $sel['selected_contact_ids'] ) && is_array( $sel['selected_contact_ids'] )
                                    ? array_map( 'absint', $sel['selected_contact_ids'] )
                                    : array();
                                $raw_selected_widget_ids  = isset( $sel['selected_widget_ids'] ) && is_array( $sel['selected_widget_ids'] )
                                    ? array_map( 'absint', $sel['selected_widget_ids'] )
                                    : array();
                                $raw_picker_widget_ids    = isset( $sel['picker_widget_ids'] ) && is_array( $sel['picker_widget_ids'] )
                                    ? array_map( 'absint', $sel['picker_widget_ids'] )
                                    : array();

                                $filtered_contact_ids = ! empty( $allowed_contact_ids )
                                    ? array_values( array_intersect( $raw_selected_contact_ids, $allowed_contact_ids ) )
                                    : array();
                                $filtered_widget_ids  = ! empty( $allowed_widget_ids )
                                    ? array_values( array_intersect( $raw_selected_widget_ids, $allowed_widget_ids ) )
                                    : array();
                                $filtered_picker_ids  = ! empty( $allowed_widget_ids )
                                    ? array_values( array_intersect( $raw_picker_widget_ids, $allowed_widget_ids ) )
                                    : array();

                                $this->last_tool_selection = array(
                                    'ok'                 => true,
                                    'fallback_used'       => isset( $sel['fallback_used'] ) ? (bool) $sel['fallback_used'] : false,
                                    'selected_contact_ids' => $filtered_contact_ids,
                                    'selected_widget_ids'  => $filtered_widget_ids,
                                    'picker_widget_ids'    => $filtered_picker_ids,
                                );
                            }

                            if ( $trace_enabled && ! is_wp_error( $sel ) ) {
                                $sel_json = wp_json_encode( $sel, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES );
                                BoostBot_Logger::info( '[TRACE ' . $trace_id . '] tool_selection=' . ( $sel_json ? $sel_json : '(unavailable)' ) );
                                $usage_json = wp_json_encode( $sel['usage'] ?? null, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES );
                                BoostBot_Logger::info( '[TRACE ' . $trace_id . '] tool_selection_usage=' . ( $usage_json ? $usage_json : '(unavailable)' ) );
                                BoostBot_Logger::info( '[TRACE ' . $trace_id . '] tool_selection_model=' . (string) ( $sel['model'] ?? '' ) );
                                $this->trace_usage['select_tools'] = $this->normalize_usage_snapshot( $sel['usage'] ?? null );
                            }
                        }
                    }
                }
            }

            // Apply per-bot override if configured.
            if ( $bot && isset( $bot->brand_card_mode ) && 'override' === (string) $bot->brand_card_mode ) {
                if ( isset( $bot->brand_card_override ) ) {
                    $override = $bot->brand_card_override;
                    if ( is_string( $override ) && '' !== trim( $override ) ) {
                        $decoded = json_decode( $override, true );
                        if ( is_array( $decoded ) ) {
                            $override = $decoded;
                        }
                    }
                    if ( is_array( $override ) ) {
                        $brand_card_obj = $override;
                        $brand_card_ver = 0; // per-bot override (no global version)
                    }
                }
            }

            // Format BrandCard into a text block for the AI system prompt.
            // Now includes the full brand card — no caps or truncation.
            if ( is_array( $brand_card_obj ) ) {
                $cn = isset( $brand_card_obj['company_name'] ) ? trim( (string) $brand_card_obj['company_name'] ) : '';
                $ol = isset( $brand_card_obj['one_liner'] ) ? trim( (string) $brand_card_obj['one_liner'] ) : '';
                $ab = isset( $brand_card_obj['about_summary'] ) ? trim( (string) $brand_card_obj['about_summary'] ) : '';
                $cta_t = isset( $brand_card_obj['primary_cta_text'] ) ? trim( (string) $brand_card_obj['primary_cta_text'] ) : '';
                $cta_u = isset( $brand_card_obj['primary_cta_url'] ) ? trim( (string) $brand_card_obj['primary_cta_url'] ) : '';
                $phone = isset( $brand_card_obj['phone'] ) ? trim( (string) $brand_card_obj['phone'] ) : '';
                $wa    = isset( $brand_card_obj['whatsapp_number'] ) ? trim( (string) $brand_card_obj['whatsapp_number'] ) : '';

                $lines = array();
                if ( '' !== $cn ) {
                    $lines[] = $ps['brand_company'] . ': ' . $cn;
                }
                if ( '' !== $ol ) {
                    $lines[] = $ps['brand_tagline'] . ': ' . $ol;
                }
                if ( '' !== $ab ) {
                    $lines[] = $ps['brand_about'] . ': ' . $ab;
                }
                if ( '' !== $phone ) {
                    $lines[] = $ps['brand_phone'] . ': ' . $phone;
                }
                if ( '' !== $wa ) {
                    $lines[] = $ps['brand_whatsapp'] . ': ' . $wa;
                }
                if ( '' !== $cta_t || '' !== $cta_u ) {
                    $lines[] = $ps['brand_cta'] . ': ' . ( $cta_t ?: $ps['brand_cta_see_link'] ) . ( $cta_u ? ' — ' . $cta_u : '' );
                }

                // Legacy services (if present and no tables)
                $services = isset( $brand_card_obj['services'] ) && is_array( $brand_card_obj['services'] ) ? $brand_card_obj['services'] : array();
                $tables = isset( $brand_card_obj['tables'] ) && is_array( $brand_card_obj['tables'] ) ? $brand_card_obj['tables'] : array();

                // Format custom tables as structured text for the AI
                if ( ! empty( $tables ) ) {
                    foreach ( $tables as $tbl ) {
                        if ( ! is_array( $tbl ) || empty( $tbl['columns'] ) ) {
                            continue;
                        }
                        $tbl_title = isset( $tbl['title'] ) ? trim( (string) $tbl['title'] ) : $ps['brand_default_table'];
                        $columns   = $tbl['columns'];
                        $rows      = isset( $tbl['rows'] ) && is_array( $tbl['rows'] ) ? $tbl['rows'] : array();

                        $tbl_lines = array();
                        $tbl_lines[] = '--- ' . $tbl_title . ' ---';

                        // Header row
                        $headers = array();
                        foreach ( $columns as $col ) {
                            $headers[] = isset( $col['header'] ) ? trim( (string) $col['header'] ) : '';
                        }
                        $tbl_lines[] = '| ' . implode( ' | ', $headers ) . ' |';

                        // Data rows
                        foreach ( $rows as $row ) {
                            if ( ! is_array( $row ) ) {
                                continue;
                            }
                            $vals = array();
                            foreach ( $columns as $col ) {
                                $cid = $col['id'] ?? '';
                                $val = isset( $row[ $cid ] ) ? trim( (string) $row[ $cid ] ) : '';
                                // For page_select columns, resolve page title
                                if ( ( $col['type'] ?? 'text' ) === 'page_select' && absint( $val ) > 0 ) {
                                    $p = get_post( absint( $val ) );
                                    $val = $p ? get_the_title( $p ) : '#' . $val;
                                }
                                $vals[] = $val;
                            }
                            $tbl_lines[] = '| ' . implode( ' | ', $vals ) . ' |';
                        }

                        $lines[] = implode( "\n", $tbl_lines );
                    }
                } elseif ( ! empty( $services ) ) {
                    // Legacy services fallback
                    $svc_lines = array( $ps['brand_services'] . ':' );
                    foreach ( $services as $s ) {
                        if ( ! is_array( $s ) ) continue;
                        $name = isset( $s['name'] ) ? trim( (string) $s['name'] ) : '';
                        $desc = isset( $s['description'] ) ? trim( (string) $s['description'] ) : '';
                        if ( '' === $name ) continue;
                        $line = '- ' . $name;
                        if ( '' !== $desc ) $line .= ': ' . $desc;
                        $svc_lines[] = $line;
                    }
                    if ( count( $svc_lines ) > 1 ) {
                        $lines[] = implode( "\n", $svc_lines );
                    }
                }

                $brand_card_text = trim( implode( "\n", $lines ) );
            }
            $timing_log( 'tool_selection_and_brand' );

            // 5. Build system prompt with knowledge and bot rules
            BoostBot_Logger::info( '📋 Step 5: Build system prompt' );
            $system_prompt = $this->prompt_builder->build_system_prompt( array(
                'page_content'          => $page_content,
                'knowledge'             => $knowledge_context,
                'global_context'        => $global_context,
                'brand_card'            => $brand_card_text,
                'routing'               => $routing,
                'tool_selection'        => $this->last_tool_selection,
                'bot'                   => $bot,
                'contact_to_widget_map' => $this->contact_to_widget_map,
            ) );
            BoostBot_Logger::info( '✅ System prompt length: ' . strlen( $system_prompt ) );
            if ( $trace_enabled ) {
                BoostBot_Logger::info( '[TRACE ' . $trace_id . '] system_prompt_chars=' . mb_strlen( (string) $system_prompt ) . ' bytes=' . strlen( (string) $system_prompt ) );
                if ( null !== $brand_card_text && '' !== $brand_card_text ) {
                    BoostBot_Logger::info( '[TRACE ' . $trace_id . '] brand_card_version=' . (string) $brand_card_ver );
                    BoostBot_Logger::info( '[TRACE ' . $trace_id . '] brand_card_chars=' . mb_strlen( (string) $brand_card_text ) . ' bytes=' . strlen( (string) $brand_card_text ) );
                } else {
                    BoostBot_Logger::info( '[TRACE ' . $trace_id . '] brand_card_version=(none)' );
                }
            }

            // 5. Build messages array for Claude
            BoostBot_Logger::info( '💬 Step 5b: Build messages array' );
            $messages = $this->build_messages( $history, $message );
            BoostBot_Logger::info( '✅ Messages count: ' . count( $messages ) );
            $timing_log( 'build_prompt_and_messages' );

            // 6. Call AI (Claude BYOK OR Week 26 backend proxy)
            BoostBot_Logger::info( '🚀 Step 6: Calling AI...' );
            // Week 28: output cap (global, with per-bot lower override)
            $global_output_cap = absint( BoostBot_Settings::get( 'bot_output_char_limit_global', 800 ) );
            $bot_cap_raw       = isset( $bot->bot_output_char_limit ) ? absint( $bot->bot_output_char_limit ) : 0;
            $effective_output_cap = $global_output_cap > 0 ? $global_output_cap : 800;
            if ( $bot_cap_raw > 0 ) {
                $effective_output_cap = min( $effective_output_cap, $bot_cap_raw );
            }

            // Week 28: per-turn idempotency key for backend char-metering
            $turn_idempotency_key = 'turn:' . (string) ( $conversation->conversation_id ?? '' ) . ':' . wp_generate_uuid4();

            $response = $this->claude_client->send_message( $messages, array(
                'system'          => $system_prompt,
                'max_tokens'      => 1024,
                'temperature'     => 0.7,
                'bot_id'          => $bot->id ?? null,
                'conversation_id' => $conversation->conversation_id ?? null,
                'idempotency_key' => $turn_idempotency_key,
                'output_char_limit' => $effective_output_cap,
            ) );
            $timing_log( 'ai_generation' );
            BoostBot_Logger::info( '✅ AI returned response' );
            if ( is_array( $response ) ) {
                $p = isset( $response['provider'] ) ? (string) $response['provider'] : '';
                $m = isset( $response['model'] ) ? (string) $response['model'] : '';
                if ( '' !== $p || '' !== $m ) {
                    BoostBot_Logger::info( '🤖 AI model used: ' . ( $p ?: '?' ) . ' / ' . ( $m ?: '?' ) );
                }
                if ( $trace_enabled ) {
                    $usage = isset( $response['usage'] ) ? $response['usage'] : null;
                    $usage_json = wp_json_encode( $usage, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES );
                    BoostBot_Logger::info( '[TRACE ' . $trace_id . '] final_chat_usage=' . ( $usage_json ? $usage_json : '(unavailable)' ) );
                    BoostBot_Logger::info( "[TRACE {$trace_id}] ====== FINAL ANSWER START ======\n" . (string) ( $response['content'] ?? '' ) . "\n[TRACE {$trace_id}] ====== FINAL ANSWER END ======" );

                    // One-line cost summary (human readable).
                    $final_u = $this->normalize_usage_snapshot( $usage );
                    $sum = array(
                        'prepare'  => $this->trace_usage['prepare'] ?? null,
                        'rerank'   => $this->trace_usage['rerank'] ?? null,
                        'compress' => $this->trace_usage['compress'] ?? null,
                        'select'   => $this->trace_usage['select_tools'] ?? null,
                        'final'    => $final_u,
                    );
                    $total_cost = 0.0;
                    $total_tokens = 0;
                    foreach ( $sum as $k2 => $u2 ) {
                        if ( is_array( $u2 ) && isset( $u2['cost'] ) ) {
                            $total_cost += (float) $u2['cost'];
                        }
                        if ( is_array( $u2 ) && isset( $u2['total_tokens'] ) ) {
                            $total_tokens += (int) $u2['total_tokens'];
                        }
                    }
                    BoostBot_Logger::info(
                        '[TRACE ' . $trace_id . '] COST_SUMMARY ' .
                        'prepare=$' . number_format( (float) ( $sum['prepare']['cost'] ?? 0 ), 6 ) . ' ' .
                        'rerank=$' . number_format( (float) ( $sum['rerank']['cost'] ?? 0 ), 6 ) . ' ' .
                        'compress=$' . number_format( (float) ( $sum['compress']['cost'] ?? 0 ), 6 ) . ' ' .
                        'select=$' . number_format( (float) ( $sum['select']['cost'] ?? 0 ), 6 ) . ' ' .
                        'final=$' . number_format( (float) ( $sum['final']['cost'] ?? 0 ), 6 ) . ' ' .
                        'total=$' . number_format( (float) $total_cost, 6 )
                    );

                    // Week 30: Enhanced PROFITABILITY_CHECK log for credit analysis.
                    $input_chars = mb_strlen( (string) $message );
                    $output_chars = mb_strlen( (string) ( $response['content'] ?? '' ) );
                    $billable_chars = $input_chars + $output_chars;
                    $est_credits = $billable_chars > 0 ? max( 1, (int) ceil( $billable_chars / 10000 ) ) : 0;
                    BoostBot_Logger::info(
                        '[TRACE ' . $trace_id . '] PROFITABILITY_CHECK ' .
                        'input_chars=' . $input_chars . ' ' .
                        'output_chars=' . $output_chars . ' ' .
                        'billable_chars=' . $billable_chars . ' ' .
                        'est_credits=' . $est_credits . ' ' .
                        'total_tokens=' . $total_tokens . ' ' .
                        'api_cost_usd=$' . number_format( (float) $total_cost, 6 )
                    );
                    BoostBot_Logger::info( '[TRACE ' . $trace_id . '] ====== TRACE END ======' );
                }
            }

            // 7. Parse response for widgets (pass user message and bot widgets for intent detection)
            $bot_widget_types = $this->get_bot_widget_types( $bot );
            $parsed = $this->parse_response( $response['content'], $message, $bot_widget_types );
            $timing_log( 'parse_response' );

            // 8. Save to conversation
            $this->save_to_conversation( $conversation, $message, $parsed['text'] );
            $timing_log( 'save_conversation' );

            // 9. Check for lead capture from message
            $this->check_lead_capture( $conversation, $message );

            // 9b. Save lead from LEAD_SAVE token (if AI explicitly marked contact data)
            $lead_id_from_token = null;
            if ( ! empty( $parsed['lead_saved'] ) ) {
                $lead_id_from_token = $this->save_lead_from_token( $parsed['lead_saved'], $conversation );
            }

            // 10. Generate suggestions
            $suggestions = $this->generate_suggestions( $parsed['text'] );
            $timing_log( 'generate_suggestions' );

            // ===== TIMING SUMMARY =====
            $total_time = round( ( microtime( true ) - $timing_start ) * 1000, 1 );
            BoostBot_Logger::info( '[TIMING] ===== TIMING SUMMARY =====' );
            BoostBot_Logger::info( '[TIMING] 📊 Total request time: ' . $total_time . 'ms' );
            foreach ( $timing_steps as $step ) {
                $pct = $total_time > 0 ? round( ( $step['duration'] / $total_time ) * 100, 1 ) : 0;
                BoostBot_Logger::info( sprintf( '[TIMING]    %s: %dms (%.1f%%)', $step['step'], $step['duration'], $pct ) );
            }
            BoostBot_Logger::info( '[TIMING] =========================' );
            // ===== END TIMING SUMMARY =====

            BoostBot_Logger::debug( 'Chat response generated', array(
                'response_length' => strlen( $parsed['text'] ),
                'widgets_count'   => count( $parsed['widgets'] ),
                'conversation_id' => $conversation->conversation_id,
                'timing_total_ms' => $total_time,
            ) );

            $result = array(
                'reply'           => $parsed['text'],
                'conversation_id' => $conversation->conversation_id,
                'widgets'         => $parsed['widgets'],
                'suggestions'     => $suggestions,
            );

            // Include lead_saved info if a lead was created/updated from AI token
            if ( $lead_id_from_token ) {
                $result['lead_saved'] = $lead_id_from_token;
            }

            return $result;

        } catch ( Exception $e ) {
            BoostBot_Logger::error( '❌ EXCEPTION in Chat Handler!' );
            BoostBot_Logger::error( '❌ Error: ' . $e->getMessage() );
            BoostBot_Logger::error( '❌ File: ' . $e->getFile() . ' Line: ' . $e->getLine() );
            
            BoostBot_Logger::error( 'Chat handler error', array(
                'error'   => $e->getMessage(),
                'message' => substr( $message, 0, 100 ),
            ) );

            // Provide specific error messages
            $error_message = $e->getMessage();
            $ps_err        = BoostBot_Prompt_Strings::get( $this->bot_lang );
            $user_message  = $ps_err['error_generic'];

            // Check for specific errors
            if ( strpos( $error_message, 'API key' ) !== false || strpos( $error_message, 'api_key' ) !== false ) {
                $user_message = $ps_err['error_no_api_key'];
            } elseif ( strpos( $error_message, 'rate limit' ) !== false || strpos( $error_message, '429' ) !== false ) {
                $user_message = $ps_err['error_rate_limit'];
            } elseif ( strpos( $error_message, 'invalid_api_key' ) !== false || strpos( $error_message, '401' ) !== false ) {
                $user_message = $ps_err['error_invalid_key'];
            } elseif ( strpos( $error_message, 'No credits available' ) !== false || strpos( $error_message, 'no_credits' ) !== false ) {
                // Week 28: backend chat preflight can deny when a message would cross a metering threshold.
                return array(
                    'reply'           => __( "No credits available. Purchase more to continue.", 'boostbot' ),
                    'conversation_id' => $conversation_id,
                    'widgets'         => array(),
                    'suggestions'     => array(),
                    'blocked'         => true,
                    'block_reason'    => 'no_credits',
                );
            }

            return array(
                'reply'           => $user_message,
                'conversation_id' => $conversation_id,
                'widgets'         => array(),
                'suggestions'     => array(),
                'error'           => true,
            );
        }
    }

    /**
     * Get or create a conversation.
     *
     * @param string|null $conversation_id Existing conversation ID.
     * @param string      $visitor_id      Visitor ID.
     * @return object Conversation object.
     */
    private function get_or_create_conversation( ?string $conversation_id, string $visitor_id ) {
        // Try to get existing conversation
        if ( $conversation_id ) {
            $conversation = $this->conversation_model->get_by_conversation_id( $conversation_id );
            if ( $conversation ) {
                return $conversation;
            }
        }

        // Create new conversation
        $new_id = wp_generate_uuid4();

        $this->conversation_model->create( array(
            'conversation_id' => $new_id,
            'visitor_id'      => $visitor_id,
            'session_data'    => wp_json_encode( array() ),
            'started_at'      => current_time( 'mysql' ),
        ) );

        $created = $this->conversation_model->get_by_conversation_id( $new_id );
        if ( $created ) {
            do_action( 'boostbot_conversation_started', $created );
        }
        return $created;
    }

    /**
     * Get conversation history.
     *
     * @param object $conversation Conversation object.
     * @return array Array of previous messages.
     */
    private function get_conversation_history( $conversation ): array {
        $session_data = json_decode( $conversation->session_data ?? '[]', true );

        if ( empty( $session_data ) || ! is_array( $session_data ) ) {
            return array();
        }

        // Tolerate both the nested {"messages":[...]} shape and a legacy flat array.
        $messages = ( isset( $session_data['messages'] ) && is_array( $session_data['messages'] ) )
            ? $session_data['messages']
            : $session_data;

        // Return last 10 messages to keep context manageable
        return array_slice( $messages, -10 );
    }

    /**
     * Build messages array for Claude.
     *
     * @param array  $history     Conversation history.
     * @param string $new_message New user message.
     * @return array Messages array for Claude API.
     */
    private function build_messages( array $history, string $new_message ): array {
        $messages = array();

        // Add history
        foreach ( $history as $msg ) {
            $messages[] = array(
                'role'    => ( $msg['role'] ?? '' ) === 'bot' ? 'assistant' : 'user',
                'content' => $msg['content'] ?? '',
            );
        }

        // Add new message
        $messages[] = array(
            'role'    => 'user',
            'content' => $new_message,
        );

        return $messages;
    }

    /**
     * Parse response for widgets and templates.
     *
     * @param string $content           Response content.
     * @param string $user_message      Original user message (for intent detection).
     * @param array  $bot_widget_types  Widget types assigned to the bot (for intent detection).
     * @return array Parsed response with text and widgets.
     */
    private function parse_response( string $content, string $user_message = '', array $bot_widget_types = array() ): array {
        $widgets = array();
        $text    = $content;

        // First, parse template tags (Week 21 - Pre-built Templates)
        // Templates are converted to HTML inline in the response
        $text = $this->parse_templates( $text );

        // First, check for WIDGET_PICKER token: [WIDGET_PICKER:type1:id1,type2:id2,...]
        if ( preg_match( '/\[WIDGET_PICKER:([^\]]+)\]/i', $content, $picker_match ) ) {
            $picker_data = $picker_match[1];
            $options     = array();

            // Parse each widget option (format: type:id,type:id,...)
            $widget_parts = explode( ',', $picker_data );
            foreach ( $widget_parts as $part ) {
                $part = trim( $part );
                if ( strpos( $part, ':' ) !== false ) {
                    list( $type, $id ) = explode( ':', $part, 2 );
                    $type = strtolower( trim( $type ) );
                    $id   = trim( $id );

                    // Get widget info from database
                    $widget_info = $this->get_widget_info( intval( $id ) );
                    if ( $widget_info ) {
                        $options[] = array(
                            'type'  => $type,
                            'id'    => intval( $id ),
                            'label' => $widget_info->name,
                            'icon'  => $this->get_widget_icon( $type ),
                        );
                    }
                }
            }

            if ( ! empty( $options ) ) {
                $widgets[] = array(
                    'type'    => 'widget_picker',
                    'config'  => array( 'options' => $options ),
                );
            }

            // Remove picker token from text
            $text = str_replace( $picker_match[0], '', $text );
        }

        // Find widget tokens: [WIDGET:TYPE] or [WIDGET:TYPE:DATA] (case insensitive)
        if ( preg_match_all( '/\[WIDGET:([A-Za-z_]+)(?::([^\]]+))?\]/i', $content, $matches, PREG_SET_ORDER ) ) {
            foreach ( $matches as $match ) {
                $widget_type = strtolower( $match[1] );
                $widget_data = $match[2] ?? null;

                // Get widget config from Widget_Manager
                $widget_config = $this->get_widget_config( $widget_type, $widget_data );

                $widgets[] = array(
                    'type'   => $widget_type,
                    'config' => $widget_config,
                    'id'     => $widget_data, // Widget ID for tracking
                );

                // Remove token from text
                $text = str_replace( $match[0], '', $text );
            }
        }

        // Parse LEAD_SAVE tokens: [LEAD_SAVE:name=...|phone=...|summary=...]
        $lead_saved = null;
        if ( preg_match( '/\[LEAD_SAVE:([^\]]+)\]/i', $text, $lead_match ) ) {
            $lead_saved = $this->parse_lead_save_token( $lead_match[1] );
            $text = str_replace( $lead_match[0], '', $text );
        }

        // Auto-detect intent if no widgets found (only if bot has widgets assigned)
        if ( empty( $widgets ) && ! empty( $user_message ) && ! empty( $bot_widget_types ) ) {
            $detected_widget = $this->detect_widget_intent( $user_message, $content, $bot_widget_types );
            if ( $detected_widget ) {
                $widgets[] = $detected_widget;
            }
        }

        // Clean up extra whitespace WITHOUT destroying line breaks: collapse only runs of
        // spaces/tabs and cap 3+ consecutive newlines at 2, so the multi-line package /
        // pricing / list formatting the prompt asks the model to produce is preserved.
        $text = preg_replace( '/[ \t]+/', ' ', $text );
        $text = preg_replace( "/\n{3,}/", "\n\n", $text );
        $text = trim( $text );

        return array(
            'text'       => $text,
            'widgets'    => $widgets,
            'lead_saved' => $lead_saved,
        );
    }

    /**
     * Parse LEAD_SAVE token data from AI response.
     *
     * Token format: [LEAD_SAVE:name=...|phone=...|email=...|summary=...]
     * Required: name + phone. Optional: email, summary.
     *
     * @since 1.9.0
     * @param string $token_data Raw token data (pipe-delimited key=value pairs).
     * @return array|null Parsed lead data or null if invalid.
     */
    private function parse_lead_save_token( string $token_data ): ?array {
        $fields = explode( '|', $token_data );
        $data   = array();

        foreach ( $fields as $field ) {
            $field = trim( $field );
            if ( false !== strpos( $field, '=' ) ) {
                list( $key, $value ) = explode( '=', $field, 2 );
                $key   = strtolower( trim( $key ) );
                $value = trim( $value );
                if ( '' !== $value ) {
                    $data[ $key ] = $value;
                }
            }
        }

        // Must have at least name and phone
        if ( empty( $data['name'] ) || empty( $data['phone'] ) ) {
            BoostBot_Logger::warning( 'LEAD_SAVE token missing required fields', array(
                'raw'    => $token_data,
                'parsed' => $data,
            ) );
            return null;
        }

        // Sanitize
        $lead_info = array(
            'name'    => sanitize_text_field( $data['name'] ),
            'phone'   => sanitize_text_field( $data['phone'] ),
            'email'   => ! empty( $data['email'] ) ? sanitize_email( $data['email'] ) : '',
            'summary' => ! empty( $data['summary'] ) ? sanitize_textarea_field( $data['summary'] ) : '',
        );

        return $lead_info;
    }

    /**
     * Save a lead from a LEAD_SAVE token parsed from the AI response.
     *
     * @since 1.9.0
     * @param array  $lead_info    Parsed lead data (name, phone, email, summary).
     * @param object $conversation Conversation object.
     * @return int|false Lead ID or false on failure.
     */
    private function save_lead_from_token( array $lead_info, $conversation ) {
        $lead_model = new BoostBot_Lead_Model();

        // Check for existing lead in this conversation
        $existing = $lead_model->get_by_conversation_id( $conversation->conversation_id );

        // Also check by phone to avoid duplicates
        if ( ! $existing && ! empty( $lead_info['phone'] ) ) {
            $existing = $lead_model->get_by_phone( $lead_info['phone'] );
        }

        $chatbot_id = $conversation->chatbot_id ?? null;

        if ( $existing ) {
            // Update existing lead with new/better data
            $update_data = array();

            if ( ! empty( $lead_info['name'] ) && ( empty( $existing->name ) || $existing->name === '-' ) ) {
                $update_data['name'] = $lead_info['name'];
            }
            if ( ! empty( $lead_info['phone'] ) && empty( $existing->phone ) ) {
                $update_data['phone'] = $lead_info['phone'];
            }
            if ( ! empty( $lead_info['email'] ) && empty( $existing->email ) ) {
                $update_data['email'] = $lead_info['email'];
            }
            if ( ! empty( $lead_info['summary'] ) ) {
                $update_data['conversation_summary'] = $lead_info['summary'];
            }

            if ( ! empty( $update_data ) ) {
                $lead_model->update( $existing->id, $update_data );
                BoostBot_Logger::info( 'Lead updated from LEAD_SAVE token', array(
                    'lead_id'         => $existing->id,
                    'conversation_id' => $conversation->conversation_id,
                    'updated_fields'  => array_keys( $update_data ),
                ) );
            }

            return $existing->id;
        }

        // Create new lead
        $lead_data = array(
            'visitor_id'           => $conversation->visitor_id,
            'conversation_id'      => $conversation->conversation_id,
            'name'                 => $lead_info['name'],
            'phone'                => $lead_info['phone'],
            'email'                => $lead_info['email'] ?? '',
            'lead_status'          => 'new',
            'source_page'          => '', // Will be filled from context if available
            'conversation_summary' => $lead_info['summary'] ?? '',
            'metadata'             => wp_json_encode( array(
                'source'     => 'ai_conversation',
                'bot_id'     => $chatbot_id,
                'created_by' => 'LEAD_SAVE_token',
            ) ),
        );

        // Calculate lead score
        $lead_data['lead_score'] = $lead_model->calculate_score( $lead_data );

        $lead_id = $lead_model->create( $lead_data );

        if ( $lead_id ) {
            BoostBot_Logger::info( 'Lead created from LEAD_SAVE token', array(
                'lead_id'         => $lead_id,
                'conversation_id' => $conversation->conversation_id,
                'name'            => $lead_info['name'],
                'phone'           => substr( $lead_info['phone'], 0, 4 ) . '****',
            ) );

            do_action( 'boostbot_lead_created', $lead_id, $lead_data, array(
                'source'          => 'ai_conversation',
                'conversation_id' => $conversation->conversation_id,
                'visitor_id'      => $conversation->visitor_id ?? null,
            ) );
        }

        return $lead_id ?: false;
    }

    /**
     * Detect widget intent from user message.
     *
     * @param string $user_message      User message.
     * @param string $bot_response      Bot response.
     * @param array  $bot_widget_types  Widget types assigned to the bot.
     * @return array|null Widget data or null.
     */
    private function detect_widget_intent( string $user_message, string $bot_response, array $bot_widget_types = array() ): ?array {
        $ps = BoostBot_Prompt_Strings::get( $this->bot_lang );
        $message_lower = mb_strtolower( $user_message );
        $response_lower = mb_strtolower( $bot_response );

        // If bot has no widgets assigned, return null
        if ( empty( $bot_widget_types ) ) {
            return null;
        }

        // Lead form intent keywords
        $lead_keywords = $ps['lead_keywords'];

        // Calendar intent keywords
        $calendar_keywords = $ps['calendar_keywords'];

        // Check for lead form intent (bot must have 'form' or 'lead_form' widget assigned)
        if ( in_array( 'form', $bot_widget_types, true ) || in_array( 'lead_form', $bot_widget_types, true ) ) {
            foreach ( $lead_keywords as $keyword ) {
                if ( mb_strpos( $message_lower, $keyword ) !== false ) {
                    // Also check if bot response suggests form
                    $form_response_indicators = $ps['form_response_indicators'];
                    foreach ( $form_response_indicators as $indicator ) {
                        if ( mb_strpos( $response_lower, $indicator ) !== false ) {
                            BoostBot_Logger::debug( 'Auto-detected lead_form intent', array(
                                'keyword' => $keyword,
                                'indicator' => $indicator,
                            ) );
                            return array(
                                'type' => 'lead_form',
                                'data' => $this->get_widget_config( 'lead_form', null ),
                            );
                        }
                    }
                }
            }
        }

        // Check for calendar intent (bot must have calendar widget assigned)
        if ( in_array( 'calendar_meeting', $bot_widget_types, true ) || in_array( 'calendar_call', $bot_widget_types, true ) ) {
            foreach ( $calendar_keywords as $keyword ) {
                if ( mb_strpos( $message_lower, $keyword ) !== false ) {
                    // Prefer calendar_meeting if available, otherwise use calendar_call
                    $calendar_type = in_array( 'calendar_meeting', $bot_widget_types, true ) ? 'calendar_meeting' : 'calendar_call';
                    BoostBot_Logger::debug( 'Auto-detected calendar intent', array(
                        'keyword' => $keyword,
                        'type'    => $calendar_type,
                    ) );
                    return array(
                        'type' => $calendar_type,
                        'data' => $this->get_widget_config( $calendar_type, null ),
                    );
                }
            }
        }

        return null;
    }

    /**
     * Get widget configuration.
     *
     * @param string      $type Widget type.
     * @param string|null $data Additional data from token (widget ID or other data).
     * @return array Widget configuration.
     */
    private function get_widget_config( string $type, ?string $data ): array {
        // Contact card: new format [WIDGET:contact_card:WIDGET_ID:CONTACT_ID]
        // $data will be "WIDGET_ID:CONTACT_ID" (e.g. "42:789").
        if ( 'contact_card' === $type && ! empty( $data ) && strpos( $data, ':' ) !== false ) {
            $parts      = explode( ':', $data, 2 );
            $widget_id  = absint( $parts[0] );
            $contact_id = absint( $parts[1] ?? 0 );

            // Load widget config from DB (settings and contact list selections).
            $db_config = $widget_id > 0 ? $this->get_database_widget_config( $widget_id, $type ) : null;
            $config    = ! empty( $db_config ) ? $db_config : array( 'type' => $type );

            // Resolve the specific contact and inject into config.
            if ( $contact_id > 0 ) {
                $config['contact'] = $this->resolve_contact_for_widget( (string) $contact_id );
            }

            return $config;
        }

        // Check if data is a numeric widget ID - get from database
        if ( ! empty( $data ) && is_numeric( $data ) ) {
            $db_config = $this->get_database_widget_config( intval( $data ), $type );
            if ( ! empty( $db_config ) ) {
                return $db_config;
            }
        }

        // Try to get from Widget_Manager (built-in widgets)
        $widget = BoostBot_Widget_Manager::get_widget( $type );

        if ( $widget ) {
            $config = array(
                'schema' => $widget->get_schema(),
            );

            // Add product data if it's a product widget
            if ( in_array( $type, array( 'product', 'product_card' ), true ) && $data && function_exists( 'wc_get_product' ) ) {
                $product_id = intval( $data );
                $product    = wc_get_product( $product_id );

                if ( $product ) {
                    $config['product'] = array(
                        'id'            => $product->get_id(),
                        'name'          => $product->get_name(),
                        'price'         => $product->get_price(),
                        'regular_price' => $product->get_regular_price(),
                        'image'         => wp_get_attachment_url( $product->get_image_id() ),
                        'description'   => wp_trim_words( $product->get_short_description(), 20 ),
                        'url'           => urldecode( $product->get_permalink() ),
                        'in_stock'      => $product->is_in_stock(),
                        'rating'        => floatval( $product->get_average_rating() ),
                    );
                }
            }

            // Legacy fallback: contact_card with bare contact ID (no widget ID prefix).
            if ( 'contact_card' === $type && $data && is_numeric( $data ) ) {
                $config['contact'] = $this->resolve_contact_for_widget( $data );
            }

            return $config;
        }

        return array( 'type' => $type );
    }

    /**
     * Get widget configuration from database.
     *
     * @param int    $widget_id Widget ID.
     * @param string $type      Widget type.
     * @return array|null Widget configuration or null.
     */
    private function get_database_widget_config( int $widget_id, string $type ): ?array {
        if ( ! class_exists( 'BoostBot_Widget_Model' ) ) {
            return null;
        }

        $widget_model = new BoostBot_Widget_Model();
        $widget       = $widget_model->get_with_fields( $widget_id );

        if ( ! $widget || $widget->type !== $type ) {
            return null;
        }

        $config = array(
            'widget_id' => $widget_id,
            'name'      => $widget->name,
            'type'      => $widget->type,
        );

        // Parse widget config
        $widget_config = $widget->config;
        if ( is_string( $widget_config ) ) {
            $widget_config = json_decode( $widget_config, true );
        }

        // Build config based on widget type
        switch ( $type ) {
            case 'form':
                // Flatten schema into config for frontend compatibility
                $schema = $this->build_form_schema( $widget, $widget_config );
                $config = array_merge( $config, $schema );
                break;

            case 'whatsapp':
                $config = array_merge( $config, $widget_config ?: array() );
                break;

            case 'calendar_call':
            case 'calendar_meeting':
                $config = array_merge( $config, $widget_config ?: array() );
                $config['calendar_type'] = ( $type === 'calendar_call' ) ? 'call' : 'meeting';
                break;

            default:
                $config = array_merge( $config, $widget_config ?: array() );
                break;
        }

        return $config;
    }

    /**
     * Get basic widget info from database.
     *
     * @param int $widget_id Widget ID.
     * @return object|null Widget info or null.
     */
    private function get_widget_info( int $widget_id ): ?object {
        if ( ! class_exists( 'BoostBot_Widget_Model' ) ) {
            return null;
        }

        $widget_model = new BoostBot_Widget_Model();
        return $widget_model->get( $widget_id );
    }

    /**
     * Get icon name for widget type.
     *
     * @param string $type Widget type.
     * @return string Icon name.
     */
    private function get_widget_icon( string $type ): string {
        $icons = array(
            'form'             => 'FileText',
            'whatsapp'         => 'MessageCircle',
            'calendar_call'    => 'Calendar', // backward compat - same as calendar_meeting
            'calendar_meeting' => 'Calendar',
            'coupon'           => 'Ticket',
            'gallery'          => 'Image',
            'youtube'          => 'Youtube',
        );

        return $icons[ $type ] ?? 'Box';
    }

    /**
     * Build form schema from widget and fields.
     *
     * @param object $widget        Widget object.
     * @param array  $widget_config Widget config.
     * @return array Form schema.
     */
    private function build_form_schema( $widget, ?array $widget_config, ?array $initial_values = null ): array {
        $ps     = BoostBot_Prompt_Strings::get( $this->bot_lang );
        $fields = array();

        if ( ! empty( $widget->fields ) && is_array( $widget->fields ) ) {
            foreach ( $widget->fields as $field ) {
                $field_data = array(
                    'name'        => $field->name,
                    'type'        => $field->field_type === 'phone' ? 'tel' : $field->field_type,
                    'label'       => $field->label,
                    'required'    => ! empty( $field->is_required ),
                    'placeholder' => $field->placeholder ?? '',
                );

                // Add options for select type
                if ( $field->field_type === 'select' && ! empty( $field->options ) ) {
                    $options = $field->options;
                    if ( is_string( $options ) ) {
                        $options = json_decode( $options, true );
                    }
                    $field_data['options'] = $options;
                }

                $fields[] = $field_data;
            }
        }

        $schema = array(
            'fields'          => $fields,
            'submit_text'     => $widget_config['submit_text'] ?? $ps['default_form_submit'],
            'success_message' => $widget_config['success_message'] ?? $ps['default_form_success'],
            'animation'       => array(
                'enabled' => true,
                'speed'   => 30,
                'cursor'  => true,
            ),
        );

        // Include initial_values for pre-filling fields (e.g., phone detected by Gemini)
        if ( ! empty( $initial_values ) && is_array( $initial_values ) ) {
            $schema['initial_values'] = $initial_values;
        }

        // Pass through GA4 measurement ID if configured
        if ( ! empty( $widget_config['ga4_measurement_id'] ) ) {
            $schema['ga4_measurement_id'] = sanitize_text_field( $widget_config['ga4_measurement_id'] );
        }

        return $schema;
    }

    /**
     * Save messages to conversation.
     *
     * @param object $conversation  Conversation object.
     * @param string $user_message  User message.
     * @param string $bot_response  Bot response.
     * @return void
     */
    private function save_to_conversation( $conversation, string $user_message, string $bot_response ): void {
        $decoded = json_decode( $conversation->session_data ?? '[]', true );

        if ( ! is_array( $decoded ) ) {
            $decoded = array();
        }

        // All readers expect the nested {"messages":[...]} shape. Tolerate a legacy
        // flat array of messages on read, but always write back the nested shape.
        if ( isset( $decoded['messages'] ) && is_array( $decoded['messages'] ) ) {
            $messages = $decoded['messages'];
        } elseif ( array_key_exists( 'messages', $decoded ) ) {
            $messages = array();
        } else {
            $messages = $decoded; // Legacy flat array.
        }

        // Add user message
        $messages[] = array(
            'role'      => 'user',
            'content'   => $user_message,
            'timestamp' => current_time( 'mysql' ),
        );

        // Add bot response
        $messages[] = array(
            'role'      => 'bot',
            'content'   => $bot_response,
            'timestamp' => current_time( 'mysql' ),
        );

        // Update conversation
        $this->conversation_model->update( $conversation->id, array(
            'session_data'    => wp_json_encode( array( 'messages' => $messages ) ),
            'total_messages'  => count( $messages ),
            'last_message_at' => current_time( 'mysql' ),
        ) );
    }

    /**
     * Check for lead capture in message.
     *
     * Automatically capture emails and phone numbers from messages.
     *
     * @param object $conversation Conversation object.
     * @param string $message      User message.
     * @return void
     */
    /**
     * Check for lead capture from chat message.
     * 
     * Uses ContactExtractor to identify name, phone, and email from free text.
     *
     * @param object $conversation The conversation object.
     * @param string $message      The user message.
     * @return void
     */
    private function check_lead_capture( $conversation, string $message ): void {
        $ps = BoostBot_Prompt_Strings::get( $this->bot_lang );
        // Use ContactExtractor for better extraction
        $extractor = new BoostBot_Contact_Extractor();
        $contact   = $extractor->extract( $message );

        // Only create lead if confidence is high enough
        if ( $contact['confidence'] < $extractor->get_confidence_threshold() ) {
            return;
        }

        // Create or update lead
        $lead_model = new BoostBot_Lead_Model();

        // Check if lead exists for this conversation
        $existing = $lead_model->get_by_conversation_id( $conversation->conversation_id );

        // If no lead for conversation, check by visitor_id
        if ( ! $existing ) {
            $existing = $lead_model->get_by_visitor_id( $conversation->visitor_id );
        }

        $chatbot_id = $conversation->chatbot_id ?? null;

        $lead_data = array(
            'visitor_id'      => $conversation->visitor_id,
            'conversation_id' => $conversation->conversation_id,
            'lead_status'     => 'new',
            'metadata'        => wp_json_encode( array(
                'bot_id' => $chatbot_id,
                'source' => 'free_text',
            ) ),
        );

        // Add extracted data
        if ( $contact['name'] ) {
            $lead_data['name'] = sanitize_text_field( $contact['name'] );
        }

        if ( $contact['email'] ) {
            $lead_data['email'] = sanitize_email( $contact['email'] );
        }

        if ( $contact['phone'] ) {
            $lead_data['phone'] = sanitize_text_field( $contact['phone'] );
        }

        // Add note about extraction
        $lead_data['notes'] = sprintf(
            $ps['lead_extraction_note'],
            $contact['confidence'],
            mb_substr( $contact['raw_text'], 0, 100 )
        );

        if ( $existing ) {
            // Update existing lead with new info (only if fields are empty)
            $update_data = array();

            if ( $contact['name'] && empty( $existing->name ) ) {
                $update_data['name'] = sanitize_text_field( $contact['name'] );
            }

            if ( $contact['phone'] && empty( $existing->phone ) ) {
                $update_data['phone'] = sanitize_text_field( $contact['phone'] );
            }

            if ( $contact['email'] && empty( $existing->email ) ) {
                $update_data['email'] = sanitize_email( $contact['email'] );
            }

            if ( ! empty( $update_data ) ) {
                $lead_model->update( $existing->id, $update_data );

                BoostBot_Logger::info( 'Lead updated from free text', array(
                    'lead_id'         => $existing->id,
                    'conversation_id' => $conversation->conversation_id,
                    'extracted'       => $contact,
                ) );
            }
        } else {
            // Create new lead
            $lead_id = $lead_model->create( $lead_data );

            if ( $lead_id ) {
                BoostBot_Logger::info( 'Lead created from free text', array(
                    'lead_id'         => $lead_id,
                    'conversation_id' => $conversation->conversation_id,
                    'extracted'       => $contact,
                ) );

                do_action( 'boostbot_lead_created', $lead_id, $lead_data, array(
                    'source'          => 'free_text',
                    'conversation_id' => $conversation->conversation_id,
                    'visitor_id'      => $conversation->visitor_id ?? null,
                    'extracted'       => $contact,
                ) );
            }
        }
    }

    /**
     * Generate quick reply suggestions.
     *
     * @param string $response Bot response.
     * @return array Suggestions array.
     */
    private function generate_suggestions( string $response ): array {
        $ps          = BoostBot_Prompt_Strings::get( $this->bot_lang );
        $suggestions = array();

        // Simple keyword-based suggestions
        $response_lower = mb_strtolower( $response );

        foreach ( $ps['trigger_price'] as $kw ) {
            if ( strpos( $response_lower, $kw ) !== false ) { $suggestions[] = $ps['suggest_price_quote']; break; }
        }

        foreach ( $ps['trigger_question'] as $kw ) {
            if ( strpos( $response_lower, $kw ) !== false ) { $suggestions[] = $ps['suggest_question']; break; }
        }

        foreach ( $ps['trigger_details'] as $kw ) {
            if ( strpos( $response_lower, $kw ) !== false ) { $suggestions[] = $ps['suggest_leave_details']; break; }
        }

        foreach ( $ps['trigger_more'] as $kw ) {
            if ( strpos( $response_lower, $kw ) !== false ) { $suggestions[] = $ps['suggest_tell_more']; break; }
        }

        // Limit to 3 suggestions
        return array_slice( $suggestions, 0, 3 );
    }

    /**
     * Get page content for context.
     *
     * Enhanced with full content, technical specs, and price info.
     *
     * @param array $page_context Page context from request.
     * @return array|null Page content array or null.
     */
    private function get_page_content( array $page_context ): ?array {
        if ( empty( $page_context['id'] ) && empty( $page_context['page_id'] ) && empty( $page_context['post_id'] ) ) {
            return null;
        }

        $page_id = (int) ( $page_context['id'] ?? $page_context['page_id'] ?? $page_context['post_id'] ?? 0 );

        if ( ! $page_id ) {
            return null;
        }

        // Get post
        $post = get_post( $page_id );

        if ( ! $post ) {
            return null;
        }

        $url = urldecode( (string) ( $page_context['url'] ?? get_permalink( $page_id ) ) );

        // Week 27.8: Prefer canonical extracted text used for indexing (raw_content) for stable excerpting.
        $raw = '';
        $summary = '';
        try {
            if ( $this->knowledge_index && method_exists( $this->knowledge_index, 'get_by_content_id' ) ) {
                $entry = $this->knowledge_index->get_by_content_id( $page_id, (string) $post->post_type );
                if ( $entry ) {
                    $raw = isset( $entry->raw_content ) ? (string) $entry->raw_content : '';
                    $summary = isset( $entry->summary ) ? (string) $entry->summary : '';
                }
            }
        } catch ( Exception $e ) {
            // best-effort
        }

        if ( '' === trim( $raw ) ) {
            // Fallback: Elementor-aware extractor if available, otherwise post content.
            if ( class_exists( 'BoostBot_Content_Extractor' ) ) {
                $extractor = new BoostBot_Content_Extractor();
                $raw = (string) $extractor->extract( $page_id );
            } else {
                $raw = wp_strip_all_tags( (string) $post->post_content );
            }
        }

        $raw = html_entity_decode( (string) $raw, ENT_QUOTES | ENT_HTML5, 'UTF-8' );
        $raw = preg_replace( '/\s+/u', ' ', $raw );
        $raw = trim( (string) $raw );

        if ( '' === trim( $summary ) ) {
            $summary = wp_trim_words( $raw, 60, '...' );
        }

        // Build a small structured excerpt (hard cap, stable formatting).
        $bullets = array();
        if ( '' !== $raw ) {
            $parts = preg_split( '/[\.!\?]\s+/u', $raw );
            if ( is_array( $parts ) ) {
                foreach ( $parts as $p ) {
                    $p = trim( (string) $p );
                    if ( mb_strlen( $p ) < 25 ) {
                        continue;
                    }
                    $bullets[] = mb_strlen( $p ) > 140 ? ( mb_substr( $p, 0, 140 ) . '…' ) : $p;
                    if ( count( $bullets ) >= 5 ) {
                        break;
                    }
                }
            }
        }

        $excerpt_lines = array();
        $excerpt_lines[] = 'Current page:';
        $excerpt_lines[] = '- Title: ' . (string) $post->post_title;
        $excerpt_lines[] = '- URL: ' . (string) $url;
        if ( ! empty( $bullets ) ) {
            $excerpt_lines[] = '- Key points:';
            foreach ( $bullets as $b ) {
                $excerpt_lines[] = '  - ' . $b;
            }
        } elseif ( '' !== trim( $summary ) ) {
            $excerpt_lines[] = '- Summary: ' . (string) $summary;
        }

        $excerpt = implode( "\n", $excerpt_lines );
        if ( mb_strlen( $excerpt ) > 1600 ) {
            $excerpt = mb_substr( $excerpt, 0, 1600 ) . "\n...[truncated]";
        }

        return array(
            'title'   => (string) $post->post_title,
            'summary' => (string) $summary,
            'url'     => (string) $url,
            'excerpt' => (string) $excerpt,
        );
    }

    /**
     * Week 31: Call the Unified Router (Gemini) for comprehensive routing decisions.
     *
     * Consolidates query understanding, intent detection, and tool selection into one call.
     *
     * @param string      $message         The user message.
     * @param array       $history         Conversation history.
     * @param array       $page_context    Current page context.
     * @param object|null $bot             The active bot.
     * @param string|null $trace_id        Trace ID for logging.
     * @return array|null Unified router result or null on failure.
     */
    private function call_unified_router( string $message, array $history, array $page_context, ?object $bot, ?string $trace_id = null ): ?array {
        $t0 = microtime( true );

        $backend_base_url = (string) BoostBot_Settings::get( 'backend_base_url', '' );
        if ( '' === trim( $backend_base_url ) ) {
            return null;
        }

        $client = new BoostBot_Credits_Backend_Client();
        $site_id = $client->ensure_site_id();
        BoostBot_Logger::info( sprintf( '[TIMING] unified_router.ensure_site_id: %dms', round( ( microtime( true ) - $t0 ) * 1000 ) ) );
        if ( is_wp_error( $site_id ) ) {
            return null;
        }

        // Build context messages (last 4 turns max)
        $context_messages = array();
        $history_slice = array_slice( $history, -8 );
        foreach ( $history_slice as $msg ) {
            if ( ! isset( $msg['role'] ) || ! isset( $msg['content'] ) ) {
                continue;
            }
            $role = ( 'user' === $msg['role'] ) ? 'user' : 'assistant';
            $content = mb_substr( (string) $msg['content'], 0, 800 );
            if ( '' !== trim( $content ) ) {
                $context_messages[] = array(
                    'role'    => $role,
                    'content' => $content,
                );
            }
        }

        // Get available contacts
        $available_contacts = $this->get_available_contacts_for_selection( $bot );

        // Get available widgets
        $available_widgets = $this->get_available_widgets_for_selection( $bot );

        // Get brand name from BrandCard (per-language)
        $brand_name = null;
        $brand_card_json = (string) BoostBot_Settings::get_for_language( 'brand_card_global_json', $bot->language ?? '', '' );
        if ( '' !== trim( $brand_card_json ) ) {
            $decoded = json_decode( $brand_card_json, true );
            if ( is_array( $decoded ) && isset( $decoded['company_name'] ) ) {
                $brand_name = (string) $decoded['company_name'];
            }
        }

        // Get page info
        $page_title = isset( $page_context['post_id'] ) ? get_the_title( absint( $page_context['post_id'] ) ) : null;
        $page_url   = isset( $page_context['url'] ) ? (string) $page_context['url'] : null;

        $payload = array(
            'query'              => $message,
            'language_hint'      => 'he',
            'context_messages'   => $context_messages,
            'available_contacts' => $available_contacts,
            'available_widgets'  => $available_widgets,
            'brand_name'         => $brand_name,
            'page_title'         => $page_title,
            'page_url'           => $page_url,
        );

        $t1 = microtime( true );
        $result = $client->rag_unified_router( (string) $site_id, $payload );
        BoostBot_Logger::info( sprintf( '[TIMING] unified_router.rag_unified_router: %dms', round( ( microtime( true ) - $t1 ) * 1000 ) ) );
        BoostBot_Logger::info( sprintf( '[TIMING] unified_router.total: %dms', round( ( microtime( true ) - $t0 ) * 1000 ) ) );

        if ( is_wp_error( $result ) ) {
            if ( $trace_id ) {
                BoostBot_Logger::error( '[TRACE ' . $trace_id . '] unified_router_error=' . $result->get_error_message() );
            }
            return null;
        }

        if ( ! is_array( $result ) ) {
            return null;
        }

        if ( $trace_id ) {
            $usage_json = wp_json_encode( $result['usage'] ?? null, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES );
            BoostBot_Logger::info( '[TRACE ' . $trace_id . '] unified_router_usage=' . ( $usage_json ?: '(none)' ) );
            BoostBot_Logger::info( '[TRACE ' . $trace_id . '] unified_router_model=' . ( $result['model'] ?? '' ) );
            BoostBot_Logger::info( '[TRACE ' . $trace_id . '] unified_router_intents=' . wp_json_encode( $result['intents'] ?? array() ) );
            
            if ( ! empty( $result['needs_clarification'] ) ) {
                BoostBot_Logger::info( '[TRACE ' . $trace_id . '] unified_router_needs_clarification=true question=' . ( $result['clarification_question'] ?? '' ) );
            }
        }

        return $result;
    }

    /**
     * Week 27.8: Decide which prompt sections to include for this turn.
     * (Legacy fallback when unified router is disabled or unavailable)
     *
     * @param array       $routing_inputs {intents, entities, must_include, rewritten_query, original_query}
     * @param array       $page_context
     * @param object|null $bot
     * @return array
     */
    private function decide_prompt_routing( array $routing_inputs, array $page_context, ?object $bot ): array {
        $ps      = BoostBot_Prompt_Strings::get( $this->bot_lang );
        $intents = isset( $routing_inputs['intents'] ) && is_array( $routing_inputs['intents'] ) ? $routing_inputs['intents'] : array();
        $intents = array_map(
            function( $s ) {
                return is_string( $s ) ? strtolower( trim( $s ) ) : '';
            },
            $intents
        );
        $intents = array_values( array_filter( array_unique( $intents ) ) );

        $msg = isset( $routing_inputs['original_query'] ) ? (string) $routing_inputs['original_query'] : '';
        $msg_l = mb_strtolower( (string) $msg );

        $has_any = function( array $needles ) use ( $intents ): bool {
            foreach ( $needles as $n ) {
                if ( in_array( strtolower( (string) $n ), $intents, true ) ) {
                    return true;
                }
            }
            return false;
        };

        return array(
            // Services list is part of BrandCard; include only for service/pricing style questions.
            'include_brand_services' => $has_any( array( 'pricing', 'price', 'services', 'service', 'offerings', 'packages' ) ),
            // Contacts - include for contact-related intents and language-aware keywords.
            'include_contact_list'   => $has_any( array( 'contact', 'contact_information', 'schedule', 'meeting', 'sales', 'lead', 'representative', 'support' ) )
                || ( function() use ( $msg_l, $ps ) {
                    foreach ( $ps['contact_inclusion_keywords'] as $kw ) {
                        if ( false !== mb_strpos( $msg_l, $kw ) ) return true;
                    }
                    return false;
                } )(),
            // Actions/widgets schema - include for action-related intents and language-aware keywords.
            'include_actions'        => $has_any( array( 
                'widget', 'book', 'calendar', 'lead_capture', 'product', 'products', 
                'woocommerce', 'purchase', 'buy', 'shop', 'contact', 'contact_information', 
                'schedule', 'meeting', 'form', 'video', 'gallery', 'media',
                // Gemini may classify show/display requests as these intents:
                'example', 'template', 'demo', 'display', 'show', 'present', 'request'
            ) )
                || ( function() use ( $msg_l, $ps ) {
                    foreach ( $ps['widget_inclusion_keywords'] as $kw ) {
                        if ( false !== mb_strpos( $msg_l, $kw ) ) return true;
                    }
                    return false;
                } )(),
            // Keep page excerpt always, but it is strictly capped/structured.
            'include_page_context'   => true,
            // Brand card core stays on (stable identity).
            'include_brand_card'     => true,
        );
    }

    /**
     * Build a safe contacts list for Gemini selection (NO private details).
     *
     * Contacts are now sourced from Contact Card widgets attached to the bot,
     * not from the (removed) per-bot contact_list_id setting.
     * Also populates $this->contact_to_widget_map for later use.
     *
     * @param object|null $bot
     * @return array [{id,name,title?,department?,specialties?,description?,meeting_availability?,widget_id?}]
     */
    private function get_available_contacts_for_selection( ?object $bot ): array {
        if ( ! $bot || empty( $bot->id ) ) {
            return array();
        }

        if ( ! class_exists( 'BoostBot_Contact_Model' ) || ! class_exists( 'BoostBot_Widget_Model' ) ) {
            return array();
        }

        $widget_model  = new BoostBot_Widget_Model();
        $widgets       = $widget_model->get_for_chatbot( absint( $bot->id ) );

        if ( empty( $widgets ) || ! is_array( $widgets ) ) {
            return array();
        }

        // Collect contacts from all contact_card widgets.
        $contact_model = new BoostBot_Contact_Model();
        $all_contacts  = array();
        $seen_ids      = array();

        foreach ( $widgets as $w ) {
            $w = (object) $w;
            if ( 'contact_card' !== ( $w->type ?? '' ) ) {
                continue;
            }
            $config          = is_string( $w->config ?? '' ) ? json_decode( $w->config, true ) : ( $w->config ?? array() );
            $contact_list_ids = $config['contact_list_ids'] ?? array();
            if ( empty( $contact_list_ids ) || ! is_array( $contact_list_ids ) ) {
                continue;
            }
            $widget_id = absint( $w->id ?? 0 );
            foreach ( $contact_list_ids as $list_id ) {
                $list_contacts = $contact_model->get_by_list( absint( $list_id ) );
                if ( empty( $list_contacts ) || ! is_array( $list_contacts ) ) {
                    continue;
                }
                foreach ( $list_contacts as $c ) {
                    $c  = (object) $c;
                    $cid = absint( $c->id ?? 0 );
                    if ( $cid <= 0 || in_array( $cid, $seen_ids, true ) ) {
                        continue;
                    }
                    $seen_ids[]            = $cid;
                    $all_contacts[]        = $c;
                    // Map contact -> widget (first widget wins if contact appears in multiple).
                    $this->contact_to_widget_map[ $cid ] = $widget_id;
                }
            }
        }

        if ( empty( $all_contacts ) ) {
            return array();
        }

        // Hard cap to avoid large payloads.
        $all_contacts = array_slice( $all_contacts, 0, 30 );

        $out = array();
        foreach ( $all_contacts as $c ) {
            $c = (object) $c;
            $cid = absint( $c->id ?? 0 );
            $out[] = array(
                'id'           => $cid,
                'name'         => isset( $c->name ) ? (string) $c->name : '',
                'title'        => isset( $c->title ) ? (string) $c->title : '',
                'department'   => isset( $c->department ) ? (string) $c->department : '',
                'specialties'  => isset( $c->specialties ) ? ( is_string( $c->specialties ) ? (string) $c->specialties : wp_json_encode( $c->specialties ) ) : '',
                'description'  => isset( $c->description ) ? (string) $c->description : '',
                'meeting_availability' => $this->normalize_contact_meeting_availability( $c ),
                'widget_id'    => $this->contact_to_widget_map[ $cid ] ?? 0,
            );
        }

        // Remove invalid rows.
        $out = array_filter( $out, function( $row ) {
            return isset( $row['id'] ) && absint( $row['id'] ) > 0 && isset( $row['name'] ) && '' !== trim( (string) $row['name'] );
        } );

        return array_values( $out );
    }

    /**
     * Week 27.8-2: Build a safe widgets list for Gemini selection (minimal metadata).
     *
     * @param object|null $bot
     * @return array [{id,type,name,description?}]
     */
    private function get_available_widgets_for_selection( ?object $bot ): array {
        if ( ! $bot || empty( $bot->id ) || ! class_exists( 'BoostBot_Widget_Model' ) ) {
            return array();
        }

        $widget_model = new BoostBot_Widget_Model();
        $widgets      = $widget_model->get_for_chatbot( absint( $bot->id ) );
        if ( empty( $widgets ) || ! is_array( $widgets ) ) {
            return array();
        }

        // Filter only enabled widgets (both globally active and per-bot enabled).
        // Note: get_for_chatbot() already filters by is_active=1, is_enabled=1,
        // and is_fully_configured, but double-check here for safety.
        $widgets = array_filter( $widgets, function ( $w ) {
            return ! empty( $w->is_active ) && ! empty( $w->is_enabled );
        } );

        $widgets = array_slice( $widgets, 0, 30 );

        $out = array();
        foreach ( $widgets as $w ) {
            $w = (object) $w;
            $item = array(
                'id'          => absint( $w->id ?? 0 ),
                'type'        => isset( $w->type ) ? (string) $w->type : '',
                'name'        => isset( $w->name ) ? (string) $w->name : '',
                'description' => isset( $w->description ) ? (string) $w->description : '',
            );

            // For form widgets, check if the form has a phone field
            if ( 'form' === ( $w->type ?? '' ) && class_exists( 'BoostBot_Form_Field_Model' ) ) {
                $field_model = new BoostBot_Form_Field_Model();
                $fields = $field_model->get_by_widget( absint( $w->id ?? 0 ) );
                $has_phone = false;
                if ( ! empty( $fields ) && is_array( $fields ) ) {
                    foreach ( $fields as $field ) {
                        if ( 'phone' === ( $field->field_type ?? '' ) || 'tel' === ( $field->field_type ?? '' ) ) {
                            $has_phone = true;
                            break;
                        }
                    }
                }
                $item['has_phone_field'] = $has_phone;
            }

            // For contact_card widgets, include their contacts so Gemini knows
            // which contacts belong to which widget.
            if ( 'contact_card' === ( $w->type ?? '' ) && class_exists( 'BoostBot_Contact_Model' ) ) {
                $config           = is_string( $w->config ?? '' ) ? json_decode( $w->config, true ) : ( $w->config ?? array() );
                $contact_list_ids = $config['contact_list_ids'] ?? array();
                $widget_contacts  = array();
                $seen             = array();
                if ( ! empty( $contact_list_ids ) && is_array( $contact_list_ids ) ) {
                    $contact_model = new BoostBot_Contact_Model();
                    foreach ( $contact_list_ids as $list_id ) {
                        $lc = $contact_model->get_by_list( absint( $list_id ) );
                        if ( empty( $lc ) || ! is_array( $lc ) ) {
                            continue;
                        }
                        foreach ( $lc as $c ) {
                            $c   = (object) $c;
                            $cid = absint( $c->id ?? 0 );
                            if ( $cid <= 0 || in_array( $cid, $seen, true ) ) {
                                continue;
                            }
                            $seen[]            = $cid;
                            $widget_contacts[] = array(
                                'id'                   => $cid,
                                'name'                 => isset( $c->name ) ? (string) $c->name : '',
                                'title'                => isset( $c->title ) ? (string) $c->title : '',
                                'department'           => isset( $c->department ) ? (string) $c->department : '',
                                // Important for scheduling/contact-card decisions in Gemini.
                                'meeting_availability' => $this->normalize_contact_meeting_availability( $c ),
                            );
                        }
                    }
                }
                $item['contacts'] = array_slice( $widget_contacts, 0, 30 );
            }

            $out[] = $item;
        }

        $out = array_filter( $out, function( $row ) {
            return isset( $row['id'] ) && absint( $row['id'] ) > 0
                && isset( $row['type'] ) && '' !== trim( (string) $row['type'] )
                && isset( $row['name'] ) && '' !== trim( (string) $row['name'] );
        } );

        return array_values( $out );
    }
    
    /**
     * Get unique widget types assigned to a bot.
     *
     * @param object|null $bot The bot object.
     * @return array Array of widget type strings (e.g., ['form', 'gallery', 'calendar_meeting']).
     */
    private function get_bot_widget_types( ?object $bot ): array {
        if ( ! $bot || empty( $bot->id ) || ! class_exists( 'BoostBot_Widget_Model' ) ) {
            return array();
        }
        
        $widget_types = array();
        try {
            $widget_model = new BoostBot_Widget_Model();
            $assigned_widgets = $widget_model->get_for_chatbot( absint( $bot->id ) );
            if ( ! empty( $assigned_widgets ) ) {
                foreach ( $assigned_widgets as $widget ) {
                    $widget_type = $widget->type ?? 'unknown';
                    if ( ! in_array( $widget_type, $widget_types, true ) ) {
                        $widget_types[] = $widget_type;
                    }
                }
            }
        } catch ( Exception $e ) {
            BoostBot_Logger::error( 'Failed to get bot widget types: ' . $e->getMessage() );
        }
        
        return $widget_types;
    }

    /**
     * Truncate content preserving word boundaries.
     *
     * @param string $content    Content to truncate.
     * @param int    $max_length Maximum length.
     * @return string Truncated content.
     */
    private function truncate_content( string $content, int $max_length ): string {
        if ( mb_strlen( $content ) <= $max_length ) {
            return $content;
        }

        $truncated = mb_substr( $content, 0, $max_length );
        $last_space = mb_strrpos( $truncated, ' ' );

        if ( false !== $last_space && $last_space > $max_length * 0.8 ) {
            $truncated = mb_substr( $truncated, 0, $last_space );
        }

        return $truncated . '...';
    }

    /**
     * Fast-path response for first-message greetings.
     * 
     * Skips the unified router and RAG for simple greetings as the first message,
     * going directly to the LLM for a quick response.
     *
     * @param string   $message      The user message.
     * @param object   $conversation The conversation object.
     * @param object   $bot          The bot object.
     * @param callable $timing_log   Timing logger function.
     * @param array    $timing_steps Timing steps array.
     * @param float    $timing_start Timing start time.
     * @return array|null Response array or null to fall through to normal flow.
     */
    private function fast_greeting_response( string $message, object $conversation, ?object $bot, callable $timing_log, array &$timing_steps, float $timing_start ): ?array {
        try {
            // Get bot's welcome message or use a default greeting personality
            $bot_name = $bot->bot_name ?? 'BoostBot';
            $bot_personality = '';
            if ( $bot && isset( $bot->personality ) && '' !== trim( (string) $bot->personality ) ) {
                $bot_personality = (string) $bot->personality;
            }
            
            // Build a minimal system prompt for greeting
            $system_prompt = "You are {$bot_name}, a friendly AI assistant. ";
            if ( '' !== $bot_personality ) {
                $system_prompt .= "Your personality: {$bot_personality}. ";
            }
            $system_prompt .= "The user is greeting you for the first time. Respond with a warm, friendly greeting and ask how you can help them. Keep your response brief (2-3 sentences max). Respond in the same language as the user.";
            
            // Build simple messages array
            $messages = array(
                array(
                    'role'    => 'user',
                    'content' => $message,
                ),
            );
            
            // Call LLM directly (no RAG context needed for greetings)
            BoostBot_Logger::info( '[PERF] Fast-path: Calling LLM directly for greeting' );
            
            // Initialize AI client if not already done
            if ( ! $this->claude_client ) {
                $backend_base_url = (string) BoostBot_Settings::get( 'backend_base_url', '' );
                if ( ! empty( trim( $backend_base_url ) ) && class_exists( 'BoostBot_Backend_AI_Client' ) ) {
                    $this->claude_client = new BoostBot_Backend_AI_Client();
                } else {
                    $this->claude_client = new BoostBot_Claude_Client();
                }
            }
            
            // Per-turn idempotency key for backend metering
            $turn_idempotency_key = 'turn:' . (string) ( $conversation->conversation_id ?? '' ) . ':' . wp_generate_uuid4();
            
            $response = $this->claude_client->send_message( $messages, array(
                'system'          => $system_prompt,
                'max_tokens'      => 256, // Short response for greetings
                'temperature'     => 0.8, // Slightly more creative for greetings
                'bot_id'          => $bot->id ?? null,
                'conversation_id' => $conversation->conversation_id ?? null,
                'idempotency_key' => $turn_idempotency_key,
                'output_char_limit' => 400, // Greeting shouldn't be long
            ) );
            
            $timing_log( 'fast_path_ai_generation' );
            
            if ( ! is_array( $response ) || ! isset( $response['content'] ) ) {
                BoostBot_Logger::warning( 'Fast-path: LLM response invalid, falling through' );
                return null;
            }
            
            $reply_text = (string) $response['content'];
            
            // Parse for widgets (unlikely in greeting, but maintain consistency)
            $bot_widget_types = $this->get_bot_widget_types( $bot );
            $parsed = $this->parse_response( $reply_text, $message, $bot_widget_types );
            $timing_log( 'fast_path_parse_response' );
            
            // Save to conversation
            $this->save_to_conversation( $conversation, $message, $parsed['text'] );
            $timing_log( 'fast_path_save_conversation' );
            
            // Generate suggestions
            $suggestions = $this->generate_suggestions( $parsed['text'] );
            $timing_log( 'fast_path_generate_suggestions' );
            
            // Log timing summary
            BoostBot_Logger::info( '[TIMING] ===== FAST-PATH TIMING SUMMARY =====' );
            $total_ms = round( ( microtime( true ) - $timing_start ) * 1000 );
            BoostBot_Logger::info( sprintf( '[TIMING] 📊 Fast-path total: %dms', $total_ms ) );
            foreach ( $timing_steps as $step ) {
                BoostBot_Logger::info( sprintf( '[TIMING]    %s: %dms (%.1f%%)', $step['step'], $step['duration'], ( $step['duration'] / max( 1, $total_ms ) ) * 100 ) );
            }
            BoostBot_Logger::info( '[TIMING] =================================' );
            
            BoostBot_Logger::info( '[PERF] 🎉 Fast-path complete! Total: ' . $total_ms . 'ms (vs ~15000ms normal)' );
            
            return array(
                'reply'           => $parsed['text'],
                'conversation_id' => $conversation->conversation_id,
                'widgets'         => $parsed['widgets'] ?? array(),
                'suggestions'     => $suggestions,
                'fast_path'       => true,
            );
            
        } catch ( Exception $e ) {
            BoostBot_Logger::error( 'Fast-path exception: ' . $e->getMessage() );
            return null; // Fall through to normal flow
        }
    }

    /**
     * Resolve the active bot for the current context.
     *
     * @param array $page_context Page context data.
     * @return object|null The bot object or null.
     */
    private function resolve_bot( array $page_context ): ?object {
        try {
            $resolver = new BoostBot_Bot_Resolver();
            
            // Build context from page data
            $post_id = (int) ( $page_context['id'] ?? $page_context['page_id'] ?? $page_context['post_id'] ?? 0 );
            $page_url = $page_context['url'] ?? '';
            
            if ( $post_id || $page_url ) {
                $context = $resolver->build_context_from_request( $post_id, $page_url );
                $bot = $resolver->resolve( $context );
            } else {
                $bot = $resolver->resolve();
            }
            
            if ( $bot ) {
                BoostBot_Logger::debug( 'Resolved bot for chat', array(
                    'bot_id'   => $bot->id,
                    'bot_name' => $bot->bot_name,
                ) );
            }
            
            return $bot;
            
        } catch ( Exception $e ) {
            BoostBot_Logger::error( 'Failed to resolve bot', array(
                'error' => $e->getMessage(),
            ) );
            return null;
        }
    }

    /**
     * Search knowledge base for relevant content.
     *
     * @param string $query User's message/query.
     * @return string|null Formatted knowledge context or null.
     */
    private function search_knowledge( string $query, array $history = array(), ?string $trace_id = null ): ?string {
        $ps = BoostBot_Prompt_Strings::get( $this->bot_lang );
        if ( ! $this->knowledge_index ) {
            return null;
        }

        try {
            $sk_t0 = microtime( true );
            // Week 27.8: initialize routing inputs for this turn (prepare-query outputs).
            $this->last_routing_inputs = array(
                'intents'        => array(),
                'entities'       => array(),
                'must_include'   => array(),
                'rewritten_query'=> (string) $query,
                'original_query' => (string) $query,
            );

            // Check if knowledge base has indexed content
            $stats = $this->knowledge_index->get_stats();

            if ( empty( $stats['total'] ) || $stats['total'] < 1 ) {
                BoostBot_Logger::debug( 'Knowledge base is empty, skipping search' );
                return null;
            }

            // Performance optimization: REMOVED rag_prepare_query call (saves ~2.5s).
            // The unified router (called before search_knowledge in handle()) already provides
            // rewritten_query, intents, entities, and must_include via $this->last_routing_inputs.
            // If unified router hasn't run yet, we fall back to using the original query.
            
            $backend_base_url = (string) BoostBot_Settings::get( 'backend_base_url', '' );
            $site_id = null;
            $client = null;
            if ( ! empty( trim( $backend_base_url ) ) && class_exists( 'BoostBot_Credits_Backend_Client' ) ) {
                $client  = new BoostBot_Credits_Backend_Client();
                $sk_t1 = microtime( true );
                $site_id = $client->ensure_site_id();
                BoostBot_Logger::info( sprintf( '[TIMING] search_knowledge.ensure_site_id: %dms', round( ( microtime( true ) - $sk_t1 ) * 1000 ) ) );
                if ( is_wp_error( $site_id ) ) {
                    $site_id = null;
                }
            }
            
            // Use rewritten_query from unified router if available, otherwise use original query.
            $retrieval_query = isset( $this->last_routing_inputs['rewritten_query'] ) 
                && '' !== trim( (string) $this->last_routing_inputs['rewritten_query'] )
                ? (string) $this->last_routing_inputs['rewritten_query']
                : $query;
            
            BoostBot_Logger::info( '[PERF] Skipped rag_prepare_query (saved ~2.5s), using routing inputs from unified router' );
            
            if ( $retrieval_query !== $query ) {
                BoostBot_Logger::info( '🔎 RAG query rewrite used: ' . mb_substr( $query, 0, 60 ) . ' => ' . mb_substr( $retrieval_query, 0, 60 ) );
            }
            
            if ( $trace_id ) {
                $ri_json = wp_json_encode( $this->last_routing_inputs, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES );
                BoostBot_Logger::info( '[TRACE ' . $trace_id . '] routing_inputs=' . ( $ri_json ? $ri_json : '(unavailable)' ) );
            }

            // Week 27.9: Retrieval hardening + recall guards (guided prefilter + widen-if-empty retry).
            $chunks = array();
            $min_chunks = 2;
            $widen_retry = 0;
            $widen_prefilter_limit = 0;

            $extra_keywords = array();
            $add_kw = function( $v ) use ( &$extra_keywords ) {
                if ( is_string( $v ) ) {
                    $s = trim( $v );
                    if ( '' !== $s ) {
                        $extra_keywords[] = $s;
                    }
                }
            };

            // Base keywords from prepare-query signals.
            $must_include = isset( $this->last_routing_inputs['must_include'] ) && is_array( $this->last_routing_inputs['must_include'] )
                ? $this->last_routing_inputs['must_include']
                : array();
            $entities = isset( $this->last_routing_inputs['entities'] ) && is_array( $this->last_routing_inputs['entities'] )
                ? $this->last_routing_inputs['entities']
                : array();
            foreach ( $must_include as $m ) {
                $add_kw( $m );
            }
            foreach ( $entities as $e2 ) {
                $add_kw( $e2 );
            }

            // Add a few tokens from the rewritten query (language-agnostic best effort).
            $tokens = preg_split( '/\s+/u', trim( (string) $retrieval_query ) );
            if ( is_array( $tokens ) ) {
                $n = 0;
                foreach ( $tokens as $t ) {
                    $t = trim( preg_replace( '/[^\p{L}\p{N}\-_]+/u', '', (string) $t ) );
                    if ( mb_strlen( $t ) < 3 ) {
                        continue;
                    }
                    $add_kw( $t );
                    $n++;
                    if ( $n >= 8 ) {
                        break;
                    }
                }
            }

            // Service names as a keyword booster (BrandCard.services replaces the removed ServiceCatalog).
            $intents = isset( $this->last_routing_inputs['intents'] ) && is_array( $this->last_routing_inputs['intents'] ) ? $this->last_routing_inputs['intents'] : array();
            $intents_l = array_map( function( $s ) { return is_string( $s ) ? strtolower( trim( $s ) ) : ''; }, $intents );
            $intents_l = array_values( array_filter( array_unique( $intents_l ) ) );
            $is_services_intent = in_array( 'services', $intents_l, true ) || in_array( 'service', $intents_l, true ) || in_array( 'pricing', $intents_l, true ) || in_array( 'price', $intents_l, true ) || in_array( 'packages', $intents_l, true );
            $force_urls = array();
            if ( $is_services_intent ) {
                $brand_json = (string) BoostBot_Settings::get_for_language( 'brand_card_global_json', BoostBot_Language::get_primary(), '' );
                $decoded = ( '' !== trim( $brand_json ) ) ? json_decode( $brand_json, true ) : null;
                if ( is_array( $decoded ) && isset( $decoded['services'] ) && is_array( $decoded['services'] ) ) {
                    $svc_n = 0;
                    foreach ( $decoded['services'] as $svc ) {
                        if ( ! is_array( $svc ) ) {
                            continue;
                        }
                        $name = isset( $svc['name'] ) ? trim( (string) $svc['name'] ) : '';
                        $url  = isset( $svc['page_url'] ) ? trim( (string) $svc['page_url'] ) : '';
                        if ( '' !== $name ) {
                            $add_kw( $name );
                        }
                        if ( '' !== $url ) {
                            $force_urls[] = $url;
                            // Add last URL segment as a weak slug hint.
                            $path = wp_parse_url( $url, PHP_URL_PATH );
                            if ( is_string( $path ) && '' !== trim( $path ) ) {
                                $path = trim( $path, '/' );
                                $parts = explode( '/', $path );
                                $last = end( $parts );
                                if ( is_string( $last ) && mb_strlen( $last ) >= 3 ) {
                                    $add_kw( $last );
                                }
                            }
                        }
                        $svc_n++;
                        if ( $svc_n >= 20 ) {
                            break;
                        }
                    }
                }
            }

            $extra_keywords = array_values( array_unique( array_filter( array_map( function( $s ) {
                return is_string( $s ) ? trim( $s ) : '';
            }, $extra_keywords ) ) ) );
            if ( count( $extra_keywords ) > 25 ) {
                $extra_keywords = array_slice( $extra_keywords, 0, 25 );
            }

            // Week 29 (RAPTOR): use summary-tree hints to force-include cohesive leaf chunks.
            $force_chunk_ids = array();
            if ( method_exists( $this->knowledge_index, 'get_raptor_chunk_hints' ) ) {
                $force_chunk_ids = $this->knowledge_index->get_raptor_chunk_hints( $retrieval_query, array(
                    'limit_nodes'   => 8,
                    'max_chunk_ids' => 60,
                ) );
            }

            if ( $trace_id ) {
                $kw_sample = $extra_keywords;
                if ( count( $kw_sample ) > 12 ) {
                    $kw_sample = array_slice( $kw_sample, 0, 12 );
                }
                $u_sample = $force_urls;
                if ( count( $u_sample ) > 8 ) {
                    $u_sample = array_slice( $u_sample, 0, 8 );
                }
                $fc_sample = $force_chunk_ids;
                if ( count( $fc_sample ) > 8 ) {
                    $fc_sample = array_slice( $fc_sample, 0, 8 );
                }
                $retr_meta = array(
                    'extra_keywords_count' => count( $extra_keywords ),
                    'extra_keywords'       => $kw_sample,
                    'force_urls_count'     => count( $force_urls ),
                    'force_urls'           => $u_sample,
                    'force_chunk_ids_count'=> count( (array) $force_chunk_ids ),
                    'force_chunk_ids'      => $fc_sample,
                );
                BoostBot_Logger::info(
                    '[TRACE ' . $trace_id . '] retrieval_hints=' .
                    ( wp_json_encode( $retr_meta, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES ) ?: '(unavailable)' )
                );
            }

            if ( method_exists( $this->knowledge_index, 'semantic_search' ) ) {
                $chunks = $this->knowledge_index->semantic_search( $retrieval_query, array(
                    'limit'           => 20,
                    'prefilter_limit' => 40,
                    'max_chunks'      => 300,
                    'extra_keywords'  => $extra_keywords,
                    'force_chunk_ids' => $force_chunk_ids,
                ) );
            }
            if ( $trace_id ) {
                BoostBot_Logger::info( '[TRACE ' . $trace_id . '] semantic_chunks_found=' . count( (array) $chunks ) );
            }

            // Widen-if-empty retry (one retry only).
            if ( count( (array) $chunks ) < $min_chunks && method_exists( $this->knowledge_index, 'semantic_search' ) ) {
                $widen_retry = 1;
                $widen_prefilter_limit = 80;
                $chunks2 = $this->knowledge_index->semantic_search( $retrieval_query, array(
                    'limit'           => 24,
                    'prefilter_limit' => $widen_prefilter_limit,
                    'max_chunks'      => 450,
                    'extra_keywords'  => $extra_keywords,
                    'force_chunk_ids' => $force_chunk_ids,
                    // Week 27.9: include services URLs explicitly as candidates on widen retry (best-effort).
                    'force_urls'      => is_array( $force_urls ) ? array_slice( array_values( array_unique( array_filter( $force_urls ) ) ), 0, 20 ) : array(),
                ) );
                if ( $trace_id ) {
                    BoostBot_Logger::info( '[TRACE ' . $trace_id . '] widen_retry=1 widen_prefilter_limit=' . (string) $widen_prefilter_limit );
                    BoostBot_Logger::info( '[TRACE ' . $trace_id . '] semantic_chunks_found_after_widen=' . count( (array) $chunks2 ) );
                }
                if ( ! empty( $chunks2 ) ) {
                    $chunks = $chunks2;
                }
            }

            // Recall guard (core): when prepare-query produced a concrete entity/service term,
            // do NOT include unrelated priced pages that don't mention it.
            // This is especially important for pricing questions to avoid wrong prices.
            $hard_terms = array();
            $stop = $ps['price_stop_words'];
            foreach ( array_merge( (array) $must_include, (array) $entities ) as $t ) {
                if ( ! is_string( $t ) ) {
                    continue;
                }
                $s = trim( $t );
                if ( '' === $s ) {
                    continue;
                }
                if ( in_array( mb_strtolower( $s ), $stop, true ) ) {
                    continue;
                }
                if ( mb_strlen( $s ) < 3 ) {
                    continue;
                }
                $hard_terms[] = $s;
            }
            $hard_terms = array_values( array_unique( $hard_terms ) );

            // Week 29: Pre-validate hard terms against indexed content.
            // If a term appears in ZERO chunks, it's likely a language mismatch (e.g., "Facebook" vs Hebrew content)
            // or a concept not in the knowledge base. Remove such terms to avoid filtering out all evidence.
            if ( ! empty( $hard_terms ) && ! empty( $chunks ) ) {
                $validated_hard_terms = array();
                foreach ( $hard_terms as $term ) {
                    $term_found = false;
                    foreach ( (array) $chunks as $c ) {
                        if ( ! is_array( $c ) ) {
                            continue;
                        }
                        $hay = (string) ( $c['title'] ?? '' ) . "\n" . (string) ( $c['chunk_text'] ?? '' );
                        if ( false !== mb_stripos( $hay, $term ) ) {
                            $term_found = true;
                            break;
                        }
                    }
                    if ( $term_found ) {
                        $validated_hard_terms[] = $term;
                    }
                }
                $removed_terms = array_diff( $hard_terms, $validated_hard_terms );
                if ( ! empty( $removed_terms ) && $trace_id ) {
                    BoostBot_Logger::info( '[TRACE ' . $trace_id . '] recall_guard_removed_terms=' . wp_json_encode( array_values( $removed_terms ), JSON_UNESCAPED_UNICODE ) );
                }
                $hard_terms = $validated_hard_terms;
            }

            if ( ! empty( $hard_terms ) && ! empty( $chunks ) ) {
                $before = count( (array) $chunks );
                $chunks = array_values( array_filter( (array) $chunks, function( $c ) use ( $hard_terms ) {
                    if ( ! is_array( $c ) ) {
                        return false;
                    }
                    $hay = (string) ( $c['title'] ?? '' );
                    $hay .= "\n" . (string) ( $c['chunk_text'] ?? '' );
                    foreach ( $hard_terms as $term ) {
                        if ( '' !== $term && false !== mb_stripos( $hay, $term ) ) {
                            return true;
                        }
                    }
                    return false;
                } ) );
                if ( $trace_id ) {
                    $meta = array(
                        'hard_terms'        => array_slice( $hard_terms, 0, 8 ),
                        'chunks_before'     => $before,
                        'chunks_after'      => count( (array) $chunks ),
                        'bypassed_fallback' => ( $is_services_intent && empty( $chunks ) ),
                    );
                    BoostBot_Logger::info(
                        '[TRACE ' . $trace_id . '] recall_guard=' .
                        ( wp_json_encode( $meta, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES ) ?: '(unavailable)' )
                    );
                }
            }

            if ( empty( $chunks ) ) {
                // If we had a hard term but couldn't find any matching evidence, do NOT fall back to legacy
                // summaries (which often contain unrelated priced pages). Return no evidence and let the model
                // ask a clarifying question / fail-soft.
                if ( ! empty( $hard_terms ) && $is_services_intent ) {
                    if ( $trace_id ) {
                        BoostBot_Logger::info( '[TRACE ' . $trace_id . '] fallback=none_hard_terms widen_retry=' . (string) $widen_retry );
                    }
                    return null;
                }

                // Deterministic fallback to legacy summary-based context.
                $context = $this->knowledge_index->get_context_for_chat( $retrieval_query, 3, $this->bot_lang );
                if ( $trace_id ) {
                    BoostBot_Logger::warning( '[TRACE ' . $trace_id . '] fallback=legacy_summary widen_retry=' . (string) $widen_retry );
                }
                return $context ?: null;
            } elseif ( $trace_id ) {
                BoostBot_Logger::info( '[TRACE ' . $trace_id . '] fallback=0 widen_retry=' . (string) $widen_retry );
            }

            // Optional: rerank candidates using backend Gemini helper (cheap), when backend is enabled.
            // Performance optimization: REMOVED rag_rerank call (saves ~2.5s).
            // The semantic search already returns chunks ordered by similarity score.
            // LLM-based reranking adds latency without significant quality improvement.
            $ordered_ids = null;
            BoostBot_Logger::info( '[PERF] Skipped rag_rerank (saved ~2.5s), using semantic search order' );

            // Build evidence bundle (chunk snippets with citations), hard-capped.
            $by_id = array();
            foreach ( $chunks as $c ) {
                $id = isset( $c['chunk_id'] ) ? (string) $c['chunk_id'] : '';
                if ( '' !== $id ) {
                    $by_id[ $id ] = $c;
                }
            }

            $selected = array();
            if ( is_array( $ordered_ids ) ) {
                foreach ( $ordered_ids as $id ) {
                    if ( isset( $by_id[ $id ] ) ) {
                        $selected[] = $by_id[ $id ];
                    }
                    if ( count( $selected ) >= 4 ) {
                        break;
                    }
                }
            }
            if ( empty( $selected ) ) {
                $selected = array_slice( $chunks, 0, 4 );
            }

            // Performance optimization: REMOVED rag_compress call (saves ~2.2s).
            // Evidence is already truncated to 500 chars per item and 2000 chars total below,
            // so paying 2.2s for LLM compression is unnecessary.
            $excerpts_by_id = array();
            BoostBot_Logger::info( '[PERF] Skipped rag_compress (saved ~2.2s), using simple truncation instead' );
            BoostBot_Logger::info( sprintf( '[TIMING] search_knowledge.total: %dms', round( ( microtime( true ) - $sk_t0 ) * 1000 ) ) );

            $context = $ps['rag_evidence_header'];
            $max_total = 2000;
            foreach ( $selected as $item ) {
                $title = isset( $item['title'] ) ? (string) $item['title'] : '';
                $url   = isset( $item['url'] ) ? urldecode( (string) $item['url'] ) : '';
                $text  = isset( $item['chunk_text'] ) ? (string) $item['chunk_text'] : '';
                $cid   = isset( $item['chunk_id'] ) ? (string) $item['chunk_id'] : '';
                if ( '' !== $cid && isset( $excerpts_by_id[ $cid ] ) ) {
                    $text = (string) $excerpts_by_id[ $cid ];
                }
                if ( mb_strlen( $text ) > 500 ) {
                    $text = mb_substr( $text, 0, 500 ) . '...';
                }
                if ( $trace_id ) {
                    $preview = $text;
                    if ( mb_strlen( $preview ) > 280 ) {
                        $preview = mb_substr( $preview, 0, 280 ) . '...';
                    }
                    BoostBot_Logger::info( '[TRACE ' . $trace_id . '] evidence_item chunk_id=' . $cid . ' title=' . $title . ' url=' . $url );
                    BoostBot_Logger::info( '[TRACE ' . $trace_id . '] evidence_text=' . $preview );
                }
                $block = "• {$title}\n\"{$text}\"\n{$ps['rag_link_label']}: {$url}\n\n";
                if ( mb_strlen( $context ) + mb_strlen( $block ) > $max_total ) {
                    break;
                }
                $context .= $block;
            }

            if ( ! empty( $context ) ) {
                BoostBot_Logger::debug( 'Found relevant knowledge', array(
                    'query_preview' => mb_substr( $query, 0, 50 ),
                ) );
            }

            return $context ?: null;

        } catch ( Exception $e ) {
            BoostBot_Logger::error( 'Knowledge search failed', array(
                'error' => $e->getMessage(),
            ) );
            return null;
        }
    }

    /**
     * Normalize backend usage objects into a small numeric snapshot.
     *
     * @param mixed $usage
     * @return array|null {prompt_tokens, completion_tokens, total_tokens, cost}
     */
    private function normalize_usage_snapshot( $usage ): ?array {
        if ( ! is_array( $usage ) ) {
            return null;
        }
        $pt = isset( $usage['prompt_tokens'] ) ? (int) $usage['prompt_tokens'] : 0;
        $ct = isset( $usage['completion_tokens'] ) ? (int) $usage['completion_tokens'] : 0;
        $tt = isset( $usage['total_tokens'] ) ? (int) $usage['total_tokens'] : ( $pt + $ct );
        $cost = isset( $usage['cost'] ) ? (float) $usage['cost'] : 0.0;
        return array(
            'prompt_tokens'     => $pt,
            'completion_tokens' => $ct,
            'total_tokens'      => $tt,
            'cost'              => $cost,
        );
    }

    /**
     * Parse template tags in AI response.
     *
     * Converts template tags like {template:contact-card|{...}} into rendered HTML.
     * Added in Week 21 as part of the Template System.
     *
     * @since 1.1.0
     * @param string $content AI response content.
     * @return string Content with templates rendered.
     */
    private function parse_templates( string $content ): string {
        // Check if Template Parser exists
        if ( ! class_exists( 'BoostBot_Template_Parser' ) ) {
            return $content;
        }

        $parser = new BoostBot_Template_Parser();

        // Only parse if content contains template tags
        if ( ! $parser->has_templates( $content ) ) {
            return $content;
        }

        // Parse and render templates
        $parsed = $parser->parse( $content );

        BoostBot_Logger::debug( 'Parsed templates in response', array(
            'templates_found' => $parser->count_templates( $content ),
        ) );

        return $parsed;
    }

    /**
     * Resolve contact for contact_card widget by numeric contact ID.
     *
     * New format: [WIDGET:contact_card:WIDGET_ID:CONTACT_ID]
     * This function receives only the CONTACT_ID part (numeric).
     *
     * @since 1.2.0
     * @param string $data Contact ID (numeric string).
     * @return array|null Contact data array or null if not found.
     */
    private function resolve_contact_for_widget( string $data ): ?array {
        if ( ! class_exists( 'BoostBot_Contact_Model' ) ) {
            BoostBot_Logger::debug( 'Contact model not available' );
            return null;
        }

        if ( ! is_numeric( $data ) ) {
            BoostBot_Logger::debug( 'Contact resolve: non-numeric data ignored', array( 'data' => $data ) );
            return null;
        }

        $contact_model = new BoostBot_Contact_Model();
        $contact       = $contact_model->get( intval( $data ) );

        if ( ! $contact ) {
            BoostBot_Logger::debug( 'Contact not found', array( 'data' => $data ) );
            return null;
        }

        return array(
            'name'     => $contact->name ?? '',
            'title'    => $contact->title ?? '',
            'phone'    => $contact->phone ?? '',
            'email'    => $contact->email ?? '',
            'whatsapp' => $contact->whatsapp ?? '',
            'location' => $contact->description ?? '',
        );
    }

    /**
     * Normalize contact availability into the canonical meeting_availability field.
     *
     * Supports legacy keys for backward compatibility:
     * - availability
     * - availabilty (historical typo)
     *
     * @param object $contact Contact object.
     * @return string|null JSON string or null when unavailable.
     */
    private function normalize_contact_meeting_availability( $contact ): ?string {
        if ( ! is_object( $contact ) ) {
            return null;
        }

        $raw = null;

        if ( isset( $contact->meeting_availability ) && null !== $contact->meeting_availability ) {
            $raw = $contact->meeting_availability;
        } elseif ( isset( $contact->availability ) && null !== $contact->availability ) {
            $raw = $contact->availability; // Legacy field.
        } elseif ( isset( $contact->availabilty ) && null !== $contact->availabilty ) {
            $raw = $contact->availabilty; // Legacy typo.
        }

        if ( null === $raw ) {
            return null;
        }

        if ( is_string( $raw ) ) {
            $raw = trim( $raw );
            return '' !== $raw ? $raw : null;
        }

        if ( is_array( $raw ) || is_object( $raw ) ) {
            $encoded = wp_json_encode( $raw );
            return is_string( $encoded ) && '' !== $encoded ? $encoded : null;
        }

        return null;
    }
}

