<?php

class TWXChatUtil
{
	// Utill class with static methods
	/* Expects:
			/includes
				chatlog_include.php functions
				knowledge_base_include.php
			/vendor
				Yethee\Tiktoken\EncoderProvider() from autoload

	*/

    public static function getAllChatbotShortcodeAttributes(){
		// All Chatbot Shortcode attributes
        return [
			'type' => ['label' => 'Chatbot Type','type'=>'select','options'=>['avatar','non_avatar'],'default_val'=>'avatar','description'=>__('3D Talking Avatar or Non-Avatar.','twx-chat')],

			'avatar_character' => ['label'=>__('Avatar Character','twx-chat'),'type'=>'select', 'options'=>['','aws:Cristine','aws:Wes'], 'default_val'=>'', 'description'=>__('Select your Avatar\'s Character. NOTE: You may need to download and install the Character if it doesnt exist on your server.','twx-chat'),'parent_attr'=>'type','parent_attr_value'=>'avatar'],
			'avatar_voice_name' => ['label'=>__('Avatar Voice','twx-chat'),'type'=>'select', 'options'=> TWXChatUtil::getAvatarVoiceOptions(), 'default_val'=>'', 'description'=>__('Select the voice you want to use for your Avatar. NOTE: You will need an','twx-chat').' <span class="twx-chat-feature-button" data-feature="aws_cognito_identity_pool_id">AWS Cognito Pool Identity Key</span> '.__('to enable Avatar voices','twx-chat').'. <button id="twxchat_avatar_voice_test_btn" class="button-primary" onclick="onAWSPollyPlayClick()">'.__('Test Play','twx-chat').'</button>','parent_attr'=>'type','parent_attr_value'=>'avatar'],
			'avatar_voice_speed' => ['label'=>__('Avatar Voice Speed (50%-150%)','twx-chat'),'type'=>'number','min'=>50,'max'=>150,'default_val'=>100, 'description'=>__('Avatar Voice speed %. Default 100%','twx-chat'),'parent_attr'=>'type','parent_attr_value'=>'avatar'],
			'voice' => ['label'=>__('Voice','twx-chat'),'type'=>'text', 'description'=>__('The Chatbot\'s voice. Format is \'provider:voice\', eg: \'openai:onyx\'. Use a customized','twx-chat').' <a href="https://platform.openai.com/docs/guides/text-to-speech/quickstart" target="_blank">OpenAI voice</a>. '.__('Google Neural voices (eg: "google:Google italiano" etc) can also be used but the user\'s browser needs to support it otherwise the user\'s default browser voice is used.','twx-chat'),'parent_attr'=>'type','parent_attr_value'=>'non_avatar'],

			'style' => ['label'=>__('Style','twx-chat'),'type'=>'select','options'=>['embedded','floating'], 'default_val'=>'floating','description'=>__('Embedded Chatbots will be automatically maximised to the viewport after load, whereas Floating will be positioned in the lower right corner but can be maximised by the user.','twx-chat')],
			'floating_title' => ['label'=>__('Floating Title','twx-chat'),'type'=>'text', 'description'=>__('Title text visible on Floating chatbot in minimized states. This would typically be short, eg: "Talk to me" or "How can I help?"','twx-chat'),'parent_attr'=>'style','parent_attr_value'=>'floating'],
			'starting_comments' => ['label'=>__('Starting Greeting','twx-chat'),'type'=>'text', 'description'=>__('Initial Chatbot greeting which is displayed and also spoken by Avatar Chatbots, eg: "Ask me anything about XYZ". NOTE: If this is ommitted, the default comments (saved in "Customize" tab) are used.','twx-chat')],
			'continue_comments' => ['label'=>__('Continue Greeting','twx-chat'),'type'=>'text', 'description'=>__('Subsequent Chatbot greeting which is displayed and also spoken by Avatar Chatbots when re-joining a previous conversation, eg: "Welcome back. Let\'s continue our chat".','twx-chat'),'parent_attr'=>'remember_conversation','parent_attr_value'=>'true'],
			'common_questions' => ['label'=>__('Common Questions','twx-chat'),'type'=>'textarea', 'description'=>__('Comma separated list of Common Questions which will appear as buttons in the chatbot. Eg: How many breweries are there?,What is the most popular beer?','twx-chat')],
			'extra_instructions' => ['label'=>__('Extra Instructions to AI Engine','twx-chat'),'type'=>'textarea', 'description'=>__('eg: "Your name is Fred and your goal is to help the user to evaluate this Wordpress site. Remind the user they can see relevant sources in the Conversation Panel at any time."','twx-chat')],
			'llm_model' => ['label'=>__('OpenAI Model','twx-chat'),'type'=>'select', 'options'=> array_keys(TWXChatUtil::getModelCosts(null)), 'default_val'=>'gpt-4o-mini', 'description'=>__('The LLM Model used by OpenAI to make the chatbot queries. NOTE! Non-"mini" models typically cost more to use.','twx-chat'),'parent_attr'=>'kb_or_assistant','parent_attr_value'=>'knowledge_base'],
			'max_output_tokens' => ['label'=>__('Max. Output Tokens','twx-chat'),'type'=>'select', 'options'=> TWXChatOpenAIResponses::getMaxOutputTokenOptions(), 'default_val'=>TWXCHAT_DEFAULT_MAX_OUTPUT_TOKENS, 'description'=>__('Controls how long Chatbot responses can be. Higher values allow more detailed answers, while lower values keep replies short and concise.','twx-chat'),'parent_attr'=>'kb_or_assistant','parent_attr_value'=>'knowledge_base'],

			'kb_or_assistant' => ['label' => __('Knowledge Base or OpenAI Assistant?','twx-chat'),'type'=>'select','options'=>['knowledge_base','openai_assistant'],'default_val'=>'knowledge_base','description'=>__('Select where to get knowledge context from','twx-chat')." 'Knowlege Base' or 'OpenAI Assistant'"],
			'kb_source' => ['label'=>__('Knowledge Base Source Filter','twx-chat'),'type'=>'text', 'description'=>__('During a User query, filter Knowledge Base results to this source, eg: \'mysite.com\' would only match against sources which have \'mysite.com\' in them. You can select multiple sources by separating them with a comma, eg: \'mysite.com,jpg\' would also inlcude JPG images in the Knowledge Base.','twx-chat'),'parent_attr'=>'kb_or_assistant','parent_attr_value'=>'knowledge_base'],
			'min_similarity' => ['label'=>__('Min. Similarity % (1-99)','twx-chat'),'type'=>'number', 'min'=>1, 'max'=>99, 'default_val' => TWXCHAT_DEFAULT_MIN_SIMILARITY, 'description'=>__('Knowledge Base chatbots only. Minimum similarity match score in the Knowledge Base for the Users query. A higher value means more accurate responses but with less Knowledge Base content to work with.','twx-chat'),'parent_attr'=>'kb_or_assistant','parent_attr_value'=>'knowledge_base'],
			'kb_no_match_text' => ['label'=>__('Knowledge Base no match text','twx-chat'),'type'=>'text', 'default_val' => __("I'm sorry. Please be a little more specific, or rephrase your question.",'twx-chat'), 'description'=>__('If no Knowledge Base sources can be matched during a user query, this text will be displayed and spoken if Avatar chatbot.','twx-chat'),'parent_attr'=>'kb_or_assistant','parent_attr_value'=>'knowledge_base'],
			'assistant_id' => ['label'=>__('OpenAI Assistant ID','twx-chat'),'type'=>'text', 'description'=>__("Chatbot will get responses from an OpenAI 'Assistant'. Assistant must be connected to the OpenAI API Key (see 'General' tab).",'twx-chat'),'parent_attr'=>'kb_or_assistant','parent_attr_value'=>'openai_assistant'],
			'flashcards' => ['label'=>__('Show Flashcards','twx-chat'),'type'=>'select','options'=>['true','false'], 'default_val'=>'true', 'description'=>__('Show the relevant Knowledge Base sources that were used to answer a user query as popup Flashcards. Note these do not appear during mobile sessions.','twx-chat'),'parent_attr'=>'type','parent_attr_value'=>'avatar'],
			'allow_feedback' => ['min_plan'=>'pro','label'=>__('Allow Feedback','twx-chat'),'type'=>'select','options'=>['true','false','logged_in_editor'], 'default_val'=>'false', 'description'=>__('Allows User feedback to Chatbot responses to help train the Chatbot over time. Select the "logged_in_editor" option to only allow feedback when a user with edit priviledges is currently logged-in to Wordpress.','twx-chat'),'parent_attr'=>'kb_or_assistant','parent_attr_value'=>'knowledge_base'],
			'remember_conversation' => ['label'=>__('Remember User Conversation?','twx-chat'),'type'=>'select','options'=>['true','logged_in','false'], 'default_val'=>'false', 'description'=>__('User can re-join Chatbot conversation at a later time. Use \'logged_in\' option to allow only WP logged-in users this feature. Chatbot will check for browser cookie or can use \'twxchat_thread_id\' request parameter in URL to store the conversation thread. NOTE: If you use this feature, ensure your website complies with international cookie usage standards such as https://gdpr.eu/ etc.','twx-chat')],
			'stream' => ['label'=>__('Use Streaming','twx-chat'),'type'=>'select','options'=>['true','false'], 'default_val'=>'true', 'description'=>__('Knowledge Base chatbots only. Allow Chatbot to stream responses. Generally faster than without. At this time, OpenAI Assistant Chatbots do not support streaming.','twx-chat')],

			'user_fields' => ['label'=>__('User fields to collect','twx-chat'),'min_plan'=>'plus','type'=>'custom','sanitize_fn'=>'twxchat_sanitizeChatbotUserFields','function'=>'twxchat_getEditChatbotUserFieldsHtml','description'=>__('Add data fields to be collected by the chatbot from the user. These will be captured in the Chat Log and also sent to the "Update Webhook URL" if entered.','twx-chat')],
			'avatar_image_url' => ['label'=>__('Avatar Splash Image URL','twx-chat'),'type'=>'text','sanitize_fn'=>'esc_url_raw','description'=>__('URL to image used in the avatar splash/load page, eg: "https://mywebsite.com/images/splash.jpg"','twx-chat'),'parent_attr'=>'type','parent_attr_value'=>'avatar'],
			'avatar_background_url' => ['label'=>__('Avatar Background Image URL','twx-chat'),'type'=>'text','sanitize_fn'=>'esc_url_raw','description'=>__('URL to background image behind the avatar, eg: "https://mywebsite.com/images/bkg.jpg". Ensure image is hosted on your server or it may not appear.','twx-chat'),'parent_attr'=>'type','parent_attr_value'=>'avatar'],
			'floating_image_url' => ['label'=>__('Floating Image URL','twx-chat'),'type'=>'text','sanitize_fn'=>'esc_url_raw','description'=>__('URL to image used in the floating chatbot. For Avatar chatbots this will appear when User minimizes the chatbot fully.','twx-chat'),'parent_attr'=>'style','parent_attr_value'=>'floating'],
        ];
    }

	public static function loadWordpress() {
		// Load Wordpress Context - used by Stream Handlers etc
		$dir = __DIR__;
		do {
			if (file_exists($dir . '/wp-load.php')) {
				require_once($dir . '/wp-load.php');
				return true;
			}
		} while ($dir = dirname($dir));
		
		$err = 'Could not find wp-load.php';
		exit(esc_html($err));
	}

	public static function getPluginInfo(){
		$plugin_file = dirname(__DIR__) . '/twx-chat.php';
		$plugin_data = get_file_data($plugin_file, [
			'Name'        => 'Plugin Name',
			'Version'     => 'Version',
			'Description' => 'Description',
			'Plan'        => 'Plan',
			'Requires WP' => '5.8',
			'Author'	  => 'Author',
			'Author URI'  => 'Author URI', 
			'Source'	  => 'Source',
		]);
		$plugin_data['plugin_basename'] = plugin_basename($plugin_file);
		$plugin_data['slug'] = 'twx-chat';
		return $plugin_data;
	}

	public static function getPluginAutoUpdateStatus() {

		$pluginInfo = TWXChatUtil::getPluginInfo();
		$plugin_basename = $pluginInfo['plugin_basename'] ?? '';

		// 1. Check if plugin is opted in for auto-updates
		$auto_updates = get_site_option( 'auto_update_plugins', [] );
		$will_auto_update = in_array( $plugin_basename, (array) $auto_updates );

		// 2. Find next scheduled auto-update cron
		$crons = _get_cron_array();
		$next_run = null;

		if ( ! empty( $crons ) ) {
			foreach ( $crons as $timestamp => $hooks ) {
				// Look for either the plugin update check or automatic updater
				if ( isset( $hooks['wp_maybe_auto_update'] ) || isset( $hooks['wp_update_plugins'] ) ) {
					$next_run = $timestamp;
					break; // take the earliest one
				}
			}
		}

		return [
			'will_auto_update' => $will_auto_update,
			'next_scheduled'   => $next_run ? gmdate( 'Y-m-d H:i:s', $next_run + get_option('gmt_offset') * 3600 ) : null,
		];
	}

	public static function getAvatarAssetsRootPath($base = 'basedir'){
		$path = wp_upload_dir()[$base] . '/twx-chat/avatar_assets/';
		return $path;
	}

	public static function getUploadsDir($subDir = '', $trailingSlash = true){
		$dir = wp_upload_dir()['basedir'] . '/twx-chat';
		if($subDir) $dir .= '/' . $subDir;
		if($trailingSlash) $dir .= '/';
		return $dir;
	}

	public static function getUploadsUrl($subDir = '', $fileName = ''){
		$url = wp_upload_dir()['baseurl'] . '/twx-chat';
		if($subDir) $url .= '/' . $subDir;
		if($fileName) $url .= '/'.$fileName;
		return $url;
	}

	public static function getTwxlabsUrl($trailingSlash = true){
		$url = 'https://twxchat.twxlabs.au';
		if($trailingSlash) $url .= '/';
		return $url;
	}

	public function getDatetime($datetime_string = 'now') {
		// I think we can use current_time() instead ??
		$timezone_string = get_option('twx_chat_client_timezone');
		if (!$timezone_string) {
			$timezone_string = get_option('timezone_string');
		}

		try {
			$dt = new DateTime($datetime_string, new DateTimeZone($timezone_string ?: 'UTC'));
			return $dt;
		} catch (Exception $ex) {
			return new DateTime($datetime_string, new DateTimeZone('UTC'));
		}
	}

	public static function escHtml($html){

		$allowed_html = [
			'a' => [
				'href'   => true,
				'title'  => true,
				'target' => true,
				'rel'    => true,
				'download' => true,
			],
		];

		return wp_kses($html, $allowed_html);
	}

	public static function getTextFromPost($post){

		try {
			// Start with post content
			$text = $post->post_content;

			// Apply content filters (shortcodes, embeds, blocks, etc.)
			$text = apply_filters( 'the_content', $text );

			// Strip out all HTML
			$text = wp_strip_all_tags( $text );

			// Decode HTML entities (like &amp;)
			$text = html_entity_decode( $text, ENT_QUOTES | ENT_HTML5, 'UTF-8' );

			// Normalize whitespace
			$text = trim( preg_replace( '/\s+/', ' ', $text ) );

			return $text;

			/*
			$text = $html;

			// Remove scripts and styles first
			$text = preg_replace('#<script(.*?)</script>#is', '', $text);
			$text = preg_replace('#<style(.*?)</style>#is', '', $text);

			// Strip remaining HTML tags
			$text = strip_tags($text);

			// Decode HTML entities (e.g., &amp; → &)
			$text = html_entity_decode($text, ENT_QUOTES | ENT_HTML5, 'UTF-8');

			// Normalize whitespace
			$text = preg_replace('/\s+/', ' ', $text);
			$text = trim($text);
			*/

		} catch (Exception $ex) {
			return '';
		}
		
	}

	public static function getEmailSigHtml(){
		
		return '<p>Kind Regards,<br><strong>The <a href="'.TWXChatUtil::getTwxlabsUrl().'contact-us" target="">TWXLabs Team</a></strong></p>';
	}

	public static function getHttpResponse($url, $args = []){

		$pluginInfo = TWXChatUtil::getPluginInfo();

		$headers = [
			'Plugin-Version' => $pluginInfo['Version'],
			'Plugin-Plan' => $pluginInfo['Plan'],
			'Site-URL' => home_url(),
			'WP-Version' => get_bloginfo('version'),
			'PHP-Version' => PHP_VERSION,
			'User-Agent' => 'Mozilla/5.0 (WordPress Plugin)',
		];

		if(isset($args['headers'])){
			$headers = array_merge($headers, $args['headers']); // Later keys overwrite earlier ones
		}
		$args['headers'] = $headers;

		return wp_remote_get($url, $args);
	}

	public static function getNumAvatarCharactersLoaded() {

		$assets_path = TWXChatUtil::getAvatarAssetsRootPath();
		$num_avatars_loaded = TWXChatUtil::countFoldersInPath($assets_path . 'glTF/characters/adult_female');
		$num_avatars_loaded += TWXChatUtil::countFoldersInPath($assets_path . 'glTF/characters/adult_male');
		return $num_avatars_loaded;
	}

	public static function fetchChatbotAvatars($clientToken) {

		$assets_path = TWXChatUtil::getAvatarAssetsRootPath();
		$num_avatars_loaded = TWXChatUtil::getNumAvatarCharactersLoaded();
		$locale = TWXChatUtil::getSiteLocale();

		$url = TWXChatUtil::getTwxlabsUrl()."api/get_avatars.php?token=$clientToken&ac=$num_avatars_loaded&locale=".rawurlencode($locale); 

		$response = TWXChatUtil::getHttpResponse($url);

		if (is_wp_error($response)) {
			TWXChatUtil::adminLog('TWXChatUtil::fetchChatbotAvatars() failed: ' . $response->get_error_message(),'a',true);
			return [];
		}

		$code = wp_remote_retrieve_response_code($response);
		$body = wp_remote_retrieve_body($response);

		if ($code !== 200 || empty($body)) {
			TWXChatUtil::adminLog("TWXChatUtil::fetchChatbotAvatars() HTTP $code. Body empty or invalid.",'a',true);
			return [];
		}

		$parsed = json_decode($body, true);
		if (!is_array($parsed)) {
			TWXChatUtil::adminLog("TWXChatUtil::fetchChatbotAvatars() JSON decode failed. Body: $body",'a',true);
			return [];
		}

		if (($parsed['status'] ?? '') === 'error') {
			return [];
		}

		$avatars = $parsed['data'] ?? [];

		foreach ($avatars as $key => $avatar) {
			if (!empty($avatar['local_assets_path'])) {
				$avatars[$key]['exists'] = file_exists($assets_path . $avatar['local_assets_path']);
			}
		}

		return $avatars;
	}

	public static function isVersionUpgrade($currVer, $currPlan, $newVer, $newPlan) {

		if($currVer === $newVer){
			if($currPlan === 'free'){
				if($newPlan !== 'free'){
					return true;
				}else{
					return false;
				}
			}else if($currPlan === 'plus'){
				if($newPlan === 'pro'){
					return true;
				}else{
					return false;
				}
			}else{
				// Current is Pro
				return false;
			}
		}else{
			return version_compare($currVer, $newVer,'<');
		}
	}

	public static function isCustomerPlus(){
		return get_option('twx_chat_customer_plan','') === 'plus';
	}

	public static function isCustomerPro(){
		return get_option('twx_chat_customer_plan','') === 'pro';
	}

	public static function customerPlanAccess($plan){
		switch ($plan) {
			case 'plus':
				return TWXChatUtil::isCustomerPlus() || TWXChatUtil::isCustomerPro();
			case 'pro':
				return TWXChatUtil::isCustomerPro();
		}
		return true; // Free
	}

	public static function generateConversationThreadId($short = false){

		// Generate a ThreadID to be used in Chat Log

		// Define the characters to be used in the random string
		$length = 15;
		$characters = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ';
		$charactersLength = strlen($characters);
		$randomString = '';
		
		// Generate the random string
		for ($i = 0; $i < $length; $i++) {
			$randomString .= $characters[wp_rand(0, $charactersLength - 1)];
		}
		
		return ($short ? '' : 'twxchat_thread_').$randomString;
	}

	public static function getParamsForLLM(
		$apiKey, 
		$threadId, 
		$extraInstructions, 
		$chatbotGreeting, 
		$currentUserMsg, 
		$isCommonQuestion, 
		$knowledgeBaseSource, 
		$model, 
		$minSimilarity, 
		$showSources, 
		$chatbotId, 
		$userFieldsWithVals, 
		$numUserFieldsToCollect) 
	{

		// Main function to return paramteres for an LLM query using RAG etc

		// Check if we need to add Bot Greeting to Chatlog
		// We get strange results if we add this. ie: chatbot doesnt seem to gather first user field in user response 1, and also bot finishes engaging after user fields collected
		/*
		$chatLogEntries = TWXChatLog::getChatLogEntriesByThreadId($threadId);
		if(empty($chatLogEntries) && !empty($chatbotGreeting)){
			$atts = TWXChatUtil::getChatbotAttributesById(['id' => $chatbotId]);
			if(isset($atts['user_fields']) && !empty($atts['user_fields'])){
				TWXChatLog::insertChatLog($threadId, ['botResponse' => $chatbotGreeting]);
			}
		}
		*/

		// If not a Common Question button click and if user fields exist and still need to collect more than 1 field (Plus Plan, otherwise values will be empty)
		if(!$isCommonQuestion && $numUserFieldsToCollect > 1 && !empty($userFieldsWithVals)){
			// return message history with instructions geared towards collecting missing user fields
			$newMessages = TWXChatUtil::getMessagesForLLM_CollectUserFields($threadId, $chatbotGreeting, $currentUserMsg, $userFieldsWithVals, $numUserFieldsToCollect);
			return [$newMessages, null, null, null];
		}

		//$lastNPairs = 3;
		$lastNPairs = null;
		$systemPrompt = TWXChatUtil::getLLMSystemPrompt($extraInstructions, $userFieldsWithVals, $numUserFieldsToCollect);
		
		list($prevMessages, $prevSources) = TWXChatLog::getMessagesAndSourcesFromChatLog($threadId, $systemPrompt, $chatbotGreeting, $currentUserMsg, $lastNPairs); // get max last N message pairs and sources
		//TWXChatUtil::log("TWXChatUtil.getParamsForLLM:: prev messages for thread $threadId: ".var_export($prevMessages,true));

		// get Knowledge Base entries
		$kbEntries = [];
		if(!empty($knowledgeBaseSource)){
			$kbEntries = TWXChatKnowledgeBaseUtil::getEntriesBySource($knowledgeBaseSource, false);
		}else{
			$kbEntries = TWXChatKnowledgeBaseUtil::getEntriesNoSource(false);
		}
	
		TWXChatUtil::log('TWXChatUtil::getParamsForLLM() Total knowledge base entries'.(empty($knowledgeBaseSource) ? '' : ' for "'.$knowledgeBaseSource.'"').': '.count($kbEntries));
	
        $options = get_option('twx_chat_options');
        $embeddingModel = $options['openai_embedding_model'] ?? TWXCHAT_DEFAULT_OPENAI_EMBEDDING_MODEL;
	    $tiktokenEncoder = TWXChatUtil::getLocalTiktokenEncoder($embeddingModel);
	
		// determine how many input tokens we have available
		$modelCosts = TWXChatUtil::getModelCosts($model);
	
		$maxTokens = -1;
		if(isset($modelCosts['max_tokens'])){
			$prevMessagesTokens = count(TWXChatUtil::getTokens($tiktokenEncoder, $model, json_encode($prevMessages)));
			$maxTokens = intval(($modelCosts['max_tokens'] - $prevMessagesTokens) * 0.95); // only use 95% of maximum available tokens, can reduce this to be more cost effective but may decrease quality of response
		}
	
		$userMessages = TWXChatUtil::getUserPromptsReverse($prevMessages);
	
		$prevSourceEntries = [];
		if(!empty($prevSources)){
			foreach($prevSources as $id => $prevSource){
				if(isset($kbEntries[$id])) $prevSourceEntries[$id] = $kbEntries[$id];
			}
		}
	
		// get Knowledge Base Entries to use for RAG. If is a follow up q and previous sources exist, use those instead of all entries search
		$similarKnowledgeBaseEntries = [];
		if(TWXChatUtil::isFollowUpQuery($currentUserMsg) && count($prevSourceEntries) > 0 ){
			TWXChatUtil::log("TWXChatUtil::getParamsForLLM() is follow-up query, use previous Knowledge Base entries for RAG");
			$similarKnowledgeBaseEntries = $prevSourceEntries;
		}else{
			$similarKnowledgeBaseEntries = TWXChatKnowledgeBaseUtil::searchSimilarKnowledgeBaseEntries($tiktokenEncoder, $apiKey, $userMessages, $kbEntries, [], $model, $maxTokens, $minSimilarity);
			// If no matching Knowledge Base entries but we have previous Knowledge Base sources, lets use those just in case isFollowUpQuery() didn't work
			if(empty($similarKnowledgeBaseEntries) && count($prevSourceEntries) > 0){
				$similarKnowledgeBaseEntries = $prevSourceEntries;
			}
		}
		TWXChatUtil::log("TWXChatUtil::getParamsForLLM() similarKnowledgeBaseEntries = ".count($similarKnowledgeBaseEntries));
	
		$context = TWXChatKnowledgeBaseUtil::getContextFromEntries($similarKnowledgeBaseEntries, true);
		TWXChatUtil::log("TWXChatUtil.getParamsForLLM:: RAG context length = ".strlen($context));
	
		$newUserPrompt = TWXChatUtil::getLLMUserPrompt($currentUserMsg, $context, $showSources, $userFieldsWithVals);

		// get previous messages and add new modified one
		$newMessages = $prevMessages;
		if(!empty($newMessages)) array_pop($newMessages);  // remove last message as it is the current one and we need to enhance it
		$newMessages[] = ['role' => 'user', 'content' => $newUserPrompt];

		$contentAll = array_column($newMessages,'content');
		$tokens = TWXChatUtil::getTokens($tiktokenEncoder, $model, implode(" ",$contentAll));
		$estimated['tokens'] = count($tokens);
		
		$sources = [];
		foreach($similarKnowledgeBaseEntries as $id => $entry){
			$newEntry = [
				'source_id' => $id, 
				'source' => $entry['source'], 
				'similarity_score' => $entry['similarity_score'] ?? 0, 
				'content_length_chars' => strlen($entry['orig_text']),
				'last_modified' => $entry['created_at'],
			];
			$title = $entry['title'] ?? '';
			if(!empty($title)) $newEntry['title'] = $title;
			$sources[$id] = $newEntry;
		}

		return [$newMessages, $similarKnowledgeBaseEntries, $estimated, $sources];
	}

	public static function getMessagesForLLM_CollectUserFields($threadId, $chatbotGreeting, $currentUserMsg, $userFieldsWithVals, $numUserFieldsToCollect){

		// Return Messages to be used in LLM query in order to collect missing user fields
		// Assumes $userFieldsWithVals not empty and still more user fields to collect

		$lastNPairs = null; //count($userFieldsWithVals);

		$systemPrompt = TWXChatUtil::getLLMSystemPrompt(null, $userFieldsWithVals, $numUserFieldsToCollect);

		// Below is a chatlog_include function
		list($prevMessages, $prevSources) = TWXChatLog::getMessagesAndSourcesFromChatLog($threadId, $systemPrompt, $chatbotGreeting, $currentUserMsg, $lastNPairs);

		$newUserPrompt = TWXChatUtil::getLLMUserPrompt($currentUserMsg, null, null, $userFieldsWithVals);

		$newMessages = $prevMessages;
		if(!empty($newMessages)) array_pop($newMessages);  // remove last message as it is the current one and we need to enhance it
		$newMessages[] = ['role' => 'user', 'content' => $newUserPrompt];

		return $newMessages;
	}

	public static function getLLMSystemPrompt($extraInstructions, $userFieldsWithVals, $numUserFieldsToCollect = 0){

		// Construct a System Prompt based on extra instructions and user fields (if they exist)
		/*
		$userFieldsString = '';
		if(!empty($userFieldsWithVals)){
			for($i=0; $i<count($userFieldsWithVals); $i++){
				$field = $userFieldsWithVals[$i];
				$userFieldsString .= ($i+1) . '. ' . $field['name'];
				if(isset($field['description']) && !empty($field['description'])) {
					$userFieldsString .= ' ('.$field['description'] .')';
				}
				$userFieldsString .= "\n";
			}
		}
		*/	

		$prompt = '';

		if(!empty($extraInstructions)) {
			$prompt .= $extraInstructions . "\n\n";
		}else{
			$prompt .= "You are a helpful assistant.\n\n";
		}

		if(!empty($userFieldsWithVals)){

			$jsonUserFields = json_encode($userFieldsWithVals, JSON_PRETTY_PRINT);
			//$prompt .= "Another objective is to collect the following fields from the user and use that information as context for the above instructions. Politely keep asking for missing fields.\n\n"
			$prompt .= "Another objective is to collect the following fields, one at a time, from the user and use that information as context for the above instructions. Do not repeat or confirm the values of the fields provided by the user. Politely keep asking for missing fields.\n\n"
				. "User Fields:\n"
				. $jsonUserFields . "\n\n"
				. "As user field values are detected or changed in user messages, you need to capture these, and possibly other data, in a JSON object at the end of each response starting with a § character, for example:\n"
				. "§{\"user_fields\":{\"user_name\":\"John\",\"user_email\":\"john@example.com\"},\"source_ids\":[\"id1\",\"id34\",\"etc\"],\"other_data\":\"etc\"}\n\n"
				. "User prompts may contain additional instructions.\n";
		}

		return $prompt;
	}

	public static function getLLMUserPrompt($currentUserMsg, $context, $showSources, $userFieldsWithVals){

		// Return modified user prompt. User fields may exist and if they do we can assume they have been fully collected, but could be changed

		$prompt = '';

		if(!empty($context) && empty($userFieldsWithVals)){
			$prompt = "Based on the sources below, answer the following question:\n$currentUserMsg\n\n" 
                ."Do not make statements like 'according to the sources' or 'based on the sources' etc.\n\n";
			//TODO
			if($showSources && $showSources === 'avatar_flashcards'){
				/*
				$prompt .= "You are an avatar chatbot with SSML capabilities. Feel free to insert SSML within your response. For example if you want the avatar to look at the Flashcards (which contain sources) you could insert something like:\n"
				. ' For more information click on the sources <mark name=\'{"feature":"PointOfInterestFeature","method":"setTargetByName","args":["Flashcards"]}\'/> here.' . "\n\n";
				*/
			}
			$prompt .= "At the end of your answer, return the source_ids used to generate the response as a JSON string on a new line, starting with a § character exactly like this:\n"
				.'§{"source_ids":["id1","id4","id25","etc"]}'
				."\nThe source_ids should be in relevance order from highest to lowest.\n\n";

			$prompt .= "Sources:\n$context\nEnd of Sources\n\n";
		}

		$userFieldsAnswerTemplate = TWXChatUtil::getUserFieldsLLMAnswerTemplate($userFieldsWithVals);

		if(!empty($context) && !empty($userFieldsWithVals)){
			//$prompt = 'Current User Fields: '.json_encode($userFieldsWithVals)."\n\n";
			$prompt = "Based on the sources below and the current user fields, answer the following question:\n$currentUserMsg\n\n" 
                ."Do not make statements like 'according to the sources' or 'based on the sources' etc.\n\n";
			//TODO
			if($showSources && $showSources === 'avatar_flashcards'){
				/*
				$prompt .= "You are an avatar chatbot with SSML capabilities. Feel free to insert SSML within your response. For example if you want the avatar to look at the Flashcards (which contain sources) you could insert something like:\n"
				. ' For more information click on the sources <mark name=\'{"feature":"PointOfInterestFeature","method":"setTargetByName","args":["Flashcards"]}\'/> here.' . "\n\n";
				*/
			}
			$prompt .= "Remember, at the end of your answer, return the source_ids used to generate the response, and any changes to user fields, as a JSON object on a new line, starting with a § character exactly like this:\n"
				.'§{"source_ids":["id1","id4","id25","etc"],'.$userFieldsAnswerTemplate.'}'
				."\nThe source_ids should be in relevance order from highest to lowest.\n\n";

			$prompt .= "Sources:\n$context\nEnd of Sources\n\n";
		}

		if(empty($context) && !empty($userFieldsWithVals)){
			//$prompt = 'Current User Fields: '.json_encode($userFieldsWithVals)."\n\n";
			$prompt = "Based on the current user fields, answer the following question:\n$currentUserMsg\n\n" 
                ."Do not make statements like 'according to the fields' or 'based on the fields' etc.\n"
				. "Remember to include user field values at the end in a JSON object.\n\n";
			/*
			$prompt .= "Remember, at the end of your answer, return any changes to user fields as a JSON string on a new line, starting with a § character exactly like this:\n"
				.'§{'.$userFieldsAnswerTemplate.'}';
			*/
		}

		// Added for Multilingual Support
		$prompt .= "Also, at the end of your answer, within the JSON data, return the ISO 639-1 international language code used in the current user query. For example:\n"
			.'§{"source_ids":["id1","id4","id25","etc"],"user_query_language_code":"en"}'
			."\n\n";
		
		return rtrim($prompt, "\n");
	}

	public static function getUserFieldsLLMAnswerTemplate($userFieldsWithVals){

		if(empty($userFieldsWithVals)) return '';
		$answerTemplate = [];
		foreach($userFieldsWithVals as $field){
			$fieldName = $field['name'] ?? '';
			if(empty($fieldName)) continue;
			$fieldValue = $field['value'] ?? '';
			$answerTemplate['user_fields'][$fieldName] = (!empty($fieldValue) ? $fieldValue : 'value');
		}

		return substr(json_encode($answerTemplate), 1, -1);
	}

	public static function getUserPromptsReverse($messages){

		if(empty($messages)) return [];
		//if(count($messages)==1) return [];
		
		$ret = [];
	
		for($i=count($messages)-1; $i>=0; $i--){
			$msg = $messages[$i];
			if($msg['role'] == 'user'){
				$ret[] = TWXChatUtil::appendQuestionMark($msg['content']);
			}
		}
	
		return $ret;
	}

	public static function getLastBotMessageContent($messages) {
		if(empty($messages)) return null;
		for ($i = count($messages) - 1; $i >= 0; $i--) {
			if ($messages[$i]['role'] === 'assistant') {
				return $messages[$i]['content'];
			}
		}
		return null; // none found
	}
	
	public static function splitDataFromBotResponse($botResponse){
		if(empty($botResponse)) return ['',''];
		//$botResponse = mb_convert_encoding($botResponse, 'UTF-8', 'UTF-8');
		$lastDelimPos = strrpos($botResponse, '§');

		if ($lastDelimPos !== false) {
			$before = trim(substr($botResponse, 0, $lastDelimPos));
			$after = trim(substr($botResponse, $lastDelimPos + 2)); // for some reason +1 is leading with a strange char �, +2 works ok
		} else {
			$before = $botResponse;
			$after = '';
		}
		return [$before, $after];
	}

	public static function splitStringByMaxTokens($text, $maxTokens, $model, $tiktokenEncoder = null) {

		try{

			if(empty($tiktokenEncoder)) $tiktokenEncoder = TWXChatUtil::getLocalTiktokenEncoder($model); //  new Yethee\Tiktoken\EncoderProvider();  // from vendor autoload

			$sentences = preg_split('/(?<=[.?!])\s+|\r?\n/', $text, -1, PREG_SPLIT_NO_EMPTY);
			$chunks = [];
			$currentChunk = "";

			foreach ($sentences as $sentence) {
				$candidate = $currentChunk === "" ? $sentence : ($currentChunk . " " . $sentence);

				// Count tokens of candidate
				$tokens = TWXChatUtil::getTokens($tiktokenEncoder, $model, $candidate);
				//TWXChatUtil::adminLog("splitStringByMaxTokens:: tokens: ".var_export($tokens,true));
				$tokenCount = count($tokens);

				if ($tokenCount > $maxTokens) {
					// Finalize current chunk
					if ($currentChunk !== "") {
						$chunks[] = $currentChunk;
					}
					// Start new chunk with the long sentence
					$currentChunk = $sentence;
				} else {
					// Safe to append
					$currentChunk = $candidate;
				}
			}

			// Push last chunk
			if ($currentChunk !== "") {
				$chunks[] = $currentChunk;
			}

			return $chunks;

		} catch(Exception $ex) {
			throw new Exception($ex->getMessage());
		}

	}

	public static function getSiteLanguageCode() {
		$locale = get_locale(); // e.g., en_US
		if (empty($locale)) {
			$locale = get_bloginfo('language'); // e.g., en-US
		}

		// Normalize both formats
		$locale = str_replace('-', '_', $locale);

		// Return two-char language code
		return substr($locale, 0, 2);
	}

	public static function getSiteLocale(){
		return function_exists('determine_locale') ? determine_locale() : get_locale();
	}

	public static function getAWSPollyVoices() {

		$pollyVoices = [
			["language" => "Arabic (Gulf)", "code" => "ar-AE", "name" => "Hala", "gender" => "Female", "type" => "Neural"],
			["language" => "Arabic (Gulf)", "code" => "ar-AE", "name" => "Zayd", "gender" => "Male", "type" => "Neural"],
			["language" => "Belgian Dutch (Flemish)", "code" => "nl-BE", "name" => "Lisa", "gender" => "Female", "type" => "Neural"],
			["language" => "Catalan", "code" => "ca-ES", "name" => "Arlet", "gender" => "Female", "type" => "Neural"],
			["language" => "Czech", "code" => "cs-CZ", "name" => "Jitka", "gender" => "Female", "type" => "Neural"],
			["language" => "Chinese (Cantonese)", "code" => "yue-CN", "name" => "Hiujin", "gender" => "Female", "type" => "Neural"],
			["language" => "Chinese (Mandarin)", "code" => "cmn-CN", "name" => "Zhiyu", "gender" => "Female", "type" => "Neural"],
			["language" => "Danish", "code" => "da-DK", "name" => "Sofie", "gender" => "Female", "type" => "Neural"],
			["language" => "Dutch", "code" => "nl-NL", "name" => "Laura", "gender" => "Female", "type" => "Neural"],
			["language" => "English (Australian)", "code" => "en-AU", "name" => "Olivia", "gender" => "Female", "type" => "Neural"],
			["language" => "English (British)", "code" => "en-GB", "name" => "Amy", "gender" => "Female", "type" => "Neural"],
			["language" => "English (British)", "code" => "en-GB", "name" => "Emma", "gender" => "Female", "type" => "Neural"],
			["language" => "English (British)", "code" => "en-GB", "name" => "Brian", "gender" => "Male", "type" => "Neural"],
			["language" => "English (British)", "code" => "en-GB", "name" => "Arthur", "gender" => "Male", "type" => "Neural"],
			["language" => "English (Indian)", "code" => "en-IN", "name" => "Kajal", "gender" => "Female", "type" => "Neural"],
			["language" => "English (Irish)", "code" => "en-IE", "name" => "Niamh", "gender" => "Female", "type" => "Neural"],
			["language" => "English (New Zealand)", "code" => "en-NZ", "name" => "Aria", "gender" => "Female", "type" => "Neural"],
			["language" => "English (Singaporean)", "code" => "en-SG", "name" => "Jasmine", "gender" => "Female", "type" => "Neural"],
			["language" => "English (South African)", "code" => "en-ZA", "name" => "Ayanda", "gender" => "Female", "type" => "Neural"],
			["language" => "English (US)", "code" => "en-US", "name" => "Danielle", "gender" => "Female", "type" => "Neural"],
			["language" => "English (US)", "code" => "en-US", "name" => "Gregory", "gender" => "Male", "type" => "Neural"],
			["language" => "English (US)", "code" => "en-US", "name" => "Ivy", "gender" => "Female (child)", "type" => "Neural"],
			["language" => "English (US)", "code" => "en-US", "name" => "Joanna", "gender" => "Female", "type" => "Neural"],
			["language" => "English (US)", "code" => "en-US", "name" => "Kendra", "gender" => "Female", "type" => "Neural"],
			["language" => "English (US)", "code" => "en-US", "name" => "Kimberly", "gender" => "Female", "type" => "Neural"],
			["language" => "English (US)", "code" => "en-US", "name" => "Salli", "gender" => "Female", "type" => "Neural"],
			["language" => "English (US)", "code" => "en-US", "name" => "Joey", "gender" => "Male", "type" => "Neural"],
			["language" => "English (US)", "code" => "en-US", "name" => "Justin", "gender" => "Male (child)", "type" => "Neural"],
			["language" => "English (US)", "code" => "en-US", "name" => "Kevin", "gender" => "Male (child)", "type" => "Neural"],
			["language" => "English (US)", "code" => "en-US", "name" => "Matthew", "gender" => "Male", "type" => "Neural"],
			["language" => "English (US)", "code" => "en-US", "name" => "Ruth", "gender" => "Female", "type" => "Neural"],
			["language" => "English (US)", "code" => "en-US", "name" => "Stephen", "gender" => "Male", "type" => "Neural"],
			["language" => "Finnish", "code" => "fi-FI", "name" => "Suvi", "gender" => "Female", "type" => "Neural"],
			["language" => "French (Belgian)", "code" => "fr-BE", "name" => "Isabelle", "gender" => "Female", "type" => "Neural"],
			["language" => "French (Canadian)", "code" => "fr-CA", "name" => "Gabrielle", "gender" => "Female", "type" => "Neural"],
			["language" => "French (Canadian)", "code" => "fr-CA", "name" => "Liam", "gender" => "Male", "type" => "Neural"],
			["language" => "French", "code" => "fr-FR", "name" => "Lea", "gender" => "Female", "type" => "Neural"],
			["language" => "French", "code" => "fr-FR", "name" => "Remi", "gender" => "Male", "type" => "Neural"],
			["language" => "German", "code" => "de-DE", "name" => "Vicki", "gender" => "Female", "type" => "Neural"],
			["language" => "German", "code" => "de-DE", "name" => "Daniel", "gender" => "Male", "type" => "Neural"],
			["language" => "German (Austrian)", "code" => "de-AT", "name" => "Hannah", "gender" => "Female", "type" => "Neural"],
			["language" => "German (Swiss)", "code" => "de-CH", "name" => "Sabrina", "gender" => "Female", "type" => "Neural"],
			["language" => "Hindi", "code" => "hi-IN", "name" => "Kajal", "gender" => "Female", "type" => "Neural"],
			["language" => "Italian", "code" => "it-IT", "name" => "Bianca", "gender" => "Female", "type" => "Neural"],
			["language" => "Italian", "code" => "it-IT", "name" => "Adriano", "gender" => "Male", "type" => "Neural"],
			["language" => "Japanese", "code" => "ja-JP", "name" => "Takumi", "gender" => "Male", "type" => "Neural"],
			["language" => "Japanese", "code" => "ja-JP", "name" => "Kazuha", "gender" => "Female", "type" => "Neural"],
			["language" => "Japanese", "code" => "ja-JP", "name" => "Tomoko", "gender" => "Female", "type" => "Neural"],
			["language" => "Korean", "code" => "ko-KR", "name" => "Seoyeon", "gender" => "Female", "type" => "Neural"],
			["language" => "Korean", "code" => "ko-KR", "name" => "Jihye", "gender" => "Female", "type" => "Neural"],
			["language" => "Norwegian", "code" => "nb-NO", "name" => "Ida", "gender" => "Female", "type" => "Neural"],
			["language" => "Polish", "code" => "pl-PL", "name" => "Ola", "gender" => "Female", "type" => "Neural"],
			["language" => "Portuguese (Brazilian)", "code" => "pt-BR", "name" => "Camila", "gender" => "Female", "type" => "Neural"],
			["language" => "Portuguese (Brazilian)", "code" => "pt-BR", "name" => "Vitoria", "gender" => "Female", "type" => "Neural"],
			["language" => "Portuguese (Brazilian)", "code" => "pt-BR", "name" => "Thiago", "gender" => "Male", "type" => "Neural"],
			["language" => "Portuguese (European)", "code" => "pt-PT", "name" => "Ines", "gender" => "Female", "type" => "Neural"],
			["language" => "Spanish (Spain)", "code" => "es-ES", "name" => "Lucia", "gender" => "Female", "type" => "Neural"],
			["language" => "Spanish (Spain)", "code" => "es-ES", "name" => "Sergio", "gender" => "Male", "type" => "Neural"],
			["language" => "Spanish (Mexican)", "code" => "es-MX", "name" => "Mia", "gender" => "Female", "type" => "Neural"],
			["language" => "Spanish (Mexican)", "code" => "es-MX", "name" => "Andres", "gender" => "Male", "type" => "Neural"],
			["language" => "Spanish (US)", "code" => "es-US", "name" => "Lupe", "gender" => "Female", "type" => "Neural"],
			["language" => "Spanish (US)", "code" => "es-US", "name" => "Pedro", "gender" => "Male", "type" => "Neural"],
			["language" => "Swedish", "code" => "sv-SE", "name" => "Elin", "gender" => "Female", "type" => "Neural"],
			["language" => "Turkish", "code" => "tr-TR", "name" => "Burcu", "gender" => "Female", "type" => "Neural"],
		];

		return $pollyVoices;
	}

	public static function getAvatarVoiceOptions(){

		$pollyVoices = TWXChatUtil::getAWSPollyVoices();

		$options = [];
		foreach($pollyVoices as $voice){
			$opt = [];
			$opt['value'] = "aws:{$voice['name']}";
			$opt['display'] = "{$voice['language']} {$voice['name']} ({$voice['gender']})";
			$options[] = $opt;
		}

		return $options;
	}

	public static function getModelCosts_OLD($model = null, $llmOnly = false) {
		
		$models = [
			'gpt-4o-mini' => ['in'=>0.0000005,'out'=>0.0000006, 'max_tokens' => 128000],
			'gpt-4o-mini-2024-07-18' => ['in'=>0.0000005,'out'=>0.0000006, 'max_tokens' => 128000],
			'gpt-3.5-turbo' => ['in'=>0.0000005,'out'=>0.0000015, 'max_tokens' => 16385],
			'gpt-3.5-turbo-0125' => ['in'=>0.0000005,'out'=>0.0000015, 'max_tokens' => 16385],
			'gpt-3.5-turbo-1106' => ['in'=>0.0000005,'out'=>0.0000015, 'max_tokens' => 16385],
			'gpt-4o' => ['in'=>0.000005,'out'=>0.000015, 'max_tokens' => 128000],
			'gpt-4-turbo' => ['in'=>0.000005,'out'=>0.000015, 'max_tokens' => 128000],
			'text-embedding-ada-002' => ['in'=>0.00000002,'out'=>0.00000013, 'max_tokens' => 8192],
		];

		if(!$llmOnly) $models['embedding'] = ['in'=>0.00000002,'out'=>0.00000013];
		
		if(empty($model)) return $models;
		
		if(isset($models[$model])) return $models[$model];
		
		return null;
	}

	public static function getModelCosts($model = null) {

		$models = [
			// --- LLMs ---
			'gpt-4o-mini' => [
				'in' => 0.0000005,   // $0.0005 per 1K tokens
				'out' => 0.0000006,  // $0.0006 per 1K tokens
				'max_tokens' => 128000
			],
			'gpt-4o-mini-2024-07-18' => [
				'in' => 0.0000005,
				'out' => 0.0000006,
				'max_tokens' => 128000
			],
			'gpt-3.5-turbo' => [
				'in' => 0.0000005,   // $0.0005 / 1K input
				'out' => 0.0000015,  // $0.0015 / 1K output
				'max_tokens' => 16385
			],
			'gpt-3.5-turbo-0125' => [
				'in' => 0.0000005,
				'out' => 0.0000015,
				'max_tokens' => 16385
			],
			'gpt-4o' => [
				'in' => 0.000005,    // $0.005 / 1K input
				'out' => 0.000015,   // $0.015 / 1K output
				'max_tokens' => 128000
			],
			'gpt-4-turbo' => [
				'in' => 0.000005,
				'out' => 0.000015,
				'max_tokens' => 128000
			],
		];

		if(empty($model)){
			return $models;
		}

		return $models[$model] ?? null;
	}

	public static function getEmbeddingModelCosts($model = null){

		$models = [
			'text-embedding-ada-002' => [
				'in' => 0.0000001,   // $0.0001 / 1K tokens (corrected)
				'max_tokens' => 8192,
				'dims' => 1536,
				//'similarity_multiplier' => 1.0, // ada-002 is the baseline
			],
			'text-embedding-3-small' => [
				'in' => 0.00000002,  // $0.00002 / 1K tokens
				'max_tokens' => 8192,
				'dims' => 1536,
				//'similarity_multiplier' => 1.35, // to get to ada-002 comparable similarity
			],
			'text-embedding-3-large' => [
				'in' => 0.00000013,  // $0.00013 / 1K tokens
				'max_tokens' => 8192,
				'dims' => 3072,
				//'similarity_multiplier' => 1.15, // to get to ada-002 comparable similarity
			],
		];

		if(empty($model)){
			return $models;
		}

		return $models[$model] ?? null;
	}

	public static function getEmbedding($apiKey, $text, $embeddingModel) {

		if(empty($apiKey) || empty($text) || empty($embeddingModel)) return [];

		$url = 'https://api.openai.com/v1/embeddings';
		
		$data = [
			'input' => $text,
			'model' => $embeddingModel,
		];
		
		$response = self::getHttpOpenAI_WP($apiKey, $url, $data, 'POST');

		if(isset($response['error'])){
			throw new Exception(esc_html($response['error']['message']));
		}

		if(isset($response['data'][0]['embedding'])) return $response['data'][0]['embedding'];
		
		return [];
	}	
	
	public static function cosineSimilarity($embedding1, $embedding2)
	{
		// Magic function that compares embeddings

		if(empty($embedding1) || empty($embedding2)) return 0;
		$dotProduct = 0.0;
		$normA = 0.0;
		$normB = 0.0;
		for ($i = 0; $i < count($embedding1); $i++) {
			$dotProduct += $embedding1[$i] * $embedding2[$i];
			$normA += pow($embedding1[$i], 2);
			$normB += pow($embedding2[$i], 2);
		}
		if(empty($normA) || empty($normB)) return 0;
		return $dotProduct / (sqrt($normA) * sqrt($normB));
	}	
	
	public static function getLocalTiktokenEncoder($model){

		try {
			$loader = new TWXChatLocalVocabLoader(
				WP_PLUGIN_DIR . '/twx-chat/assets/tiktoken'
			);

			$provider = new Yethee\Tiktoken\EncoderProvider();
			$provider->setVocabLoader($loader);

			$encoder = $provider->getForModel($model);
			return $encoder;

		} catch (Exception $ex) {
			TWXChatUtil::adminLog("getLocalTiktokenEncoder:: model: $model, error: " . $ex->getMessage());
		}

		return null;
	}

	public static function getTokens($tiktokenEncoder, $model, $text)
	{
		try {
			if($tiktokenEncoder === null){
				$tiktokenEncoder = TWXChatUtil::getLocalTiktokenEncoder($model);
			}

			return $tiktokenEncoder->encode($text);

		} catch (Exception $ex) {
			TWXChatUtil::adminLog("getTokens:: model: $model, error: " . $ex->getMessage());
		}

		return [];
	}

	public static function isFollowUpQuery($text) {
		$text = strtolower(trim($text));
	
		// List of vague pronouns or references
		$contextualWords = ['they', 'them', 'it', 'those', 'these', 'that', 'he', 'she', 'their'];
	
		// Check for presence of any of the above words anywhere in the sentence
		foreach ($contextualWords as $word) {
			if (preg_match('/\b' . preg_quote($word, '/') . '\b/', $text)) {
				return true;
			}
		}
	
		// Optional: Short question detection (often contextual)
		/*
		if (str_word_count($text) < 4) {
			return true;
		}
		*/
	
		return false;
	}

	public static function isValidJson($string) {
		json_decode($string);
		return json_last_error() === JSON_ERROR_NONE;
	}

	public static function sanitizeJS($input) {
		// Remove any <script> tags (opening or closing)
		return preg_replace('#<\s*/?\s*script[^>]*>#i', '', $input);
	}	

	public static function sanitizeCSS($input) {
    	// Remove <style> tags if present (they're unnecessary in inline CSS)
    	return preg_replace('#<\s*/?\s*style[^>]*>#i', '', $input);		
	}	

	/**
	 * Sanitize and validate uploaded file array.
	 *
	 * @param array $file The $_FILES['your_input'] array.
	 * @param int   $max_size Maximum allowed file size in bytes (default 5MB).
	 * @param array $allowed_types Optional array of allowed MIME types/extensions.
	 *
	 * @return array|WP_Error Sanitized file array or WP_Error on failure.
	 */
	public static function sanitizeFileUpload( $file, $max_size = 5242880, $allowed_types = [] ) {
		if ( empty( $file ) || ! isset( $file['name'] ) || ! isset( $file['tmp_name'] ) ) {
			return new WP_Error( 'no_file', __( 'No file uploaded.', 'twx-chat' ) );
		}

		// Sanitize filename
		$file['name'] = sanitize_file_name( wp_unslash( $file['name'] ) );

		// Validate file size
		if ( $file['size'] > $max_size ) {
			return new WP_Error( 'file_too_large', __( 'Uploaded file exceeds size limit.', 'twx-chat' ) );
		}

		// Validate file type and extension
		$filetype = wp_check_filetype_and_ext( $file['tmp_name'], $file['name'] );

		if ( ! $filetype['ext'] || ! $filetype['type'] ) {
			return new WP_Error( 'invalid_file_type', __( 'Invalid file type.', 'twx-chat' ) );
		}

		// If allowed_types specified, enforce
		if ( ! empty( $allowed_types ) && ! in_array( $filetype['type'], $allowed_types, true ) ) {
			return new WP_Error( 'disallowed_file_type', __( 'File type not allowed.', 'twx-chat' ) );
		}

		// Replace detected type
		$file['type'] = $filetype['type'];

		return $file;
	}

	public static function obscureApiKey($key, $prefix = ''){
		if(empty($key)) return '';
		$key = trim($key);
		if(empty($key)) return '';
		if(strlen($key)<6) return '******';
		return $prefix.'**********'.substr($key, -6);
	}

	public static function urlExists($url) {
		if (!filter_var($url, FILTER_VALIDATE_URL)) {
			return false;
		}

		$response = TWXChatUtil::getHttpResponse($url, [
			'method'      => 'GET',
			'timeout'     => 10,
			'redirection' => 5,
			'limit_response_size' => 2048, // Optional: limit response size to save memory
		]);

		if (is_wp_error($response)) {
			TWXChatUtil::log("TWXUtil::urlExists(): error for $url - " . $response->get_error_message(), 'a', true);
			return false;
		}

		$status_code = wp_remote_retrieve_response_code($response);
		return ($status_code >= 200 && $status_code < 400);
	}

	public static function synthesizeSpeech($apiKey, $text, $outputFile, $voice = 'onyx') {
		
		// Endpoint URL
		$url = 'https://api.openai.com/v1/audio/speech';

		$data = [
			'model' => 'tts-1',
			'input' => $text,
			'voice' => $voice,
		];
		
		$response = self::getHttpOpenAI_WP($apiKey, $url, $data, 'POST', false);

		if(self::isValidJson($response)){
			
			$response = json_decode($response,true);
			if(isset($response['error'])){
				return $response['error'];
			}else{
				return $response;
			}
			
		}else{
			
			file_put_contents($outputFile, $response);
			return file_exists($outputFile);
		}

	}

	public static function getHttpOpenAI_WP($apiKey, $url, $data = [], $method = 'POST', $decodeResponse = true) {

		// OpenAI HTTP Request Similar to goCURL but can be used within Wordpress environments
		
		$headers = [
			'Authorization' => 'Bearer ' . $apiKey,
			'Content-Type' => 'application/json',
			'OpenAI-Beta' => 'assistants=v2',
		];

		$args = [
			'headers' => $headers,
			'timeout' => 30,
		];

		if ($method === 'POST' || $method === 'PATCH') {
			$args['body'] = json_encode($data);
			$response = wp_remote_post($url, $args);
		} else if ($method === 'GET') {
			if (!empty($data)) {
				$url = add_query_arg($data, $url);
			}
			$response = wp_remote_get($url, $args);
		} else {
			return false; // unsupported method
		}

		if (is_wp_error($response)) {
			return false;
		}

		$body = wp_remote_retrieve_body($response);

		return $decodeResponse ? json_decode($body, true) : $body;
	}
	
    public static function parseAnswer($in){
		
		$out = $in;
		
		$out = preg_replace('/【.*?】/', '', $out); // remove annoations/sources for the moment
		
		$out = str_ireplace('***','',$out);
		
		$out = str_ireplace('**','',$out);
		
		$out = str_ireplace('###','',$out);
		
		$out = str_ireplace('##','',$out);

		return $out;
	}
	
	public static function validateAnswerWithKnowledgeBaseEntries($answer, &$entries){
		
		// attempt to reduce AI Hallucinations by analysing AI Answer (Response) with the Knowledge Base entries used in the input prompt
		$ret = $answer;
		
		// First Check Hyperlinks
		$ret = self::updateHyperlinks($ret, $entries);
		
		return $ret;
	}
	
    public static function getPluginUpdateTransient($transient){

        if (empty($transient->checked)) return $transient;

        $updates = TWXChatUtil::checkForPluginUpdates();
        if(!isset($updates['update_is_newer']) || $updates['update_is_newer'] === false || !isset($updates['download_url'])){
            return $transient; // No update available
        }

        TWXChatUtil::adminLog('TWXChatUtil::getPluginUpdateTransient() updates: '.var_export($updates,true));

        $pluginInfo = TWXChatUtil::getPluginInfo();
        $plugin_basename = $pluginInfo['plugin_basename'];

        // Inject your plugin update data
        $transient->response[$plugin_basename] = (object) array(
            'slug' => $pluginInfo['slug'],
            'plugin' => $plugin_basename,
            'new_version' => $updates['update_version'],
            'package' => $updates['download_url'], // This is where WP gets the .zip
            'url' => TWXChatUtil::getTwxlabsUrl(), // Optional - plugin page
        );

        return $transient;
    }

	public static function checkForPluginUpdates(){

		$options = get_option('twx_chat_options'); 
		$customerKey = $options['twx_customer_key'] ?? '';

		// Check TWXLabs server for new updates
		$url = TWXChatUtil::getTwxlabsUrl()."api/check_plugin_update.php?token=".urlencode($customerKey);
		$response = TWXChatUtil::getHttpResponse($url);

		$ret = [];

		if (is_wp_error($response)) {
			$ret['error'] = var_export($response,true);
			return $ret;
		}
		
		$serverData = json_decode(wp_remote_retrieve_body($response), true);

		$pluginInfo = TWXChatUtil::getPluginInfo();
		$currVer = $pluginInfo['Version'] ?? '0.0.0';
		$currPlan = $pluginInfo['Plan'] ?? 'free';
		
		if(isset($serverData['version']) && isset($serverData['plan'])){
			$ret['update_version'] = $serverData['version'];
			$ret['update_plan'] = $serverData['plan'];
			$ret['update_is_newer'] = TWXChatUtil::isVersionUpgrade($currVer, $currPlan, $serverData['version'], $serverData['plan']);
			if(isset($serverData['download_url'])) $ret['download_url'] = $serverData['download_url'];
		}

		return $ret;
	}

	public static function sendEmail( $to, $subject, $html, $headers = [], $attachments = [] ) {
		
		// Send HTML Email
		$errors = [];

		if(empty(get_option( 'admin_email',''))){
			$errors[] = 'Admin email not configured — aborting email send';
			return [false, $errors];
		}

		add_filter( 'wp_mail_content_type', function() { return 'text/html'; } );

		// Default headers.
		$default_headers = [
			'From: ' . get_bloginfo( 'name' ) . ' <' . get_option( 'admin_email' ) . '>'
		];

		$headers = array_merge( $default_headers, (array) $headers );

		// Check if SMTP configured.
		if ( ! TWXChatUtil::isSMTPConfigured() ) {
			//TWXChatUtil::adminLog( 'SMTP not configured — skipping email send.' );
			remove_filter( 'wp_mail_content_type', 'wp_mail_content_type' );
			$errors[] = 'SMTP not configured — aborting email send';
			return [false, $errors];
		}

		// Send email.
		$sent = wp_mail( $to, $subject, $html, $headers, $attachments );

		// Remove filter after sending.
		remove_filter( 'wp_mail_content_type', 'wp_mail_content_type' );

		return [$sent, $errors];
	}

	public static function isSMTPConfigured() {

		$mailer = new PHPMailer\PHPMailer\PHPMailer( true );
		do_action_ref_array( 'phpmailer_init', [ &$mailer ] );

		return (
			$mailer->Mailer === 'smtp' &&
			!empty( $mailer->Host ) &&
			!empty( $mailer->Port ) &&
			!empty( $mailer->Username )
		);
	}

	public static function clearScheduledCronJob($hook){

		wp_clear_scheduled_hook($hook);
		wp_cron(); // forces hooks out of memory - apparently
		$time = wp_next_scheduled($hook);
		if($time) {
			wp_unschedule_event($time, $hook);
			wp_cron();
		}
	}

	//// PRIVATES /////

    private static function updateHyperlinks($html, &$entries, $removeLinkIfNotFound = true) {
        // Define a regex pattern to match <a> tags and capture href
        $pattern = '/<a\s+([^>]*?)href=["\']([^"\']+)["\']([^>]*?)>(.*?)<\/a>/i';
		
        // Callback function to process matches
        $callback = function($matches) use (&$entries, $removeLinkIfNotFound) {
			
            $beforeHref = $matches[1];
            $href = $matches[2];
            $afterHref = $matches[3];
            $text = $matches[4];

			$newHref = '';
            
			foreach($entries as $entry){
				$markdown = $entry['orig_text'];
				$source = $entry['source'];
				$pos = stripos($markdown,$text);
				
				//TWXChatUtil::log('checking in source: '.$source);

				if($pos !== false){
					//TWXChatUtil::log('found mention of: '.$text);
					
					$newHref = self::findMarkdownLinksWithText($markdown, $text);
					if(!empty($newHref)){
						break;
					}
				}
			}
			
			if(!empty($newHref)){
				return '<a ' . $beforeHref . 'href="' . htmlspecialchars($newHref) . '"' . $afterHref . '>' . htmlspecialchars($text) . '</a>';
			}

			if($removeLinkIfNotFound) return $text;

			return '<a ' . $beforeHref . 'href="' . htmlspecialchars($href) . '"' . $afterHref . '>' . htmlspecialchars($text) . '</a>';
        };

        // Perform the regex replacement
        $updatedHtml = preg_replace_callback($pattern, $callback, $html);

        return $updatedHtml;
    }

	public static function findMarkdownLinksWithText($markdown, $searchText) {
		// Define a regex pattern to match markdown links and capture the text and URL
		$pattern = '/\[(.*?)\]\((.*?)\)/i';
		
		// Initialize an array to hold matching markdown links
		$matches = [];
		
		// Perform the regex match
		if (preg_match_all($pattern, $markdown, $found)) {
			foreach ($found[1] as $index => $text) {
				// Check if the search text matches the hyperlink text
				if (stripos(trim($text), $searchText) !== false) {
					// Store the entire markdown link portion
					return $found[2][$index];
					//$matches[] = $found[0][$index];
				}
			}
		}
		
		return '';
	}
	
	public static function extractMarkdownImages($markdown) {
		$images = [];

		// Match ![alt text](image-url)
		preg_match_all('/!\[(.*?)\]\((.*?)\)/', $markdown, $matches, PREG_SET_ORDER);

		foreach ($matches as $match) {
			$images[] = [
				'alt' => $match[1],
				'url' => $match[2],
			];
		}

		return $images;
	}

	public static function extractBlurbFromMarkdown($markdown, $maxLength = 150) {
		// Remove image tags: ![alt](url)
		$markdown = preg_replace('/!\[.*?\]\(.*?\)/', '', $markdown);

		// Remove links but keep link text: [text](url) → text
		$markdown = preg_replace('/\[(.*?)\]\(.*?\)/', '$1', $markdown);

		// Strip other Markdown syntax (*, _, `, >, etc.)
		$markdown = wp_strip_all_tags($markdown); // basic HTML tags (if any)
		$markdown = preg_replace('/[_*`>#-]/', '', $markdown); // common MD chars
		$markdown = trim($markdown);

		// Split into lines/paragraphs
		$lines = preg_split('/\r\n|\r|\n/', $markdown);
		
		foreach ($lines as $line) {
			$line = trim($line);
			if (!empty($line)) {
				// Truncate to maxLength if needed
				if (strlen($line) > $maxLength) {
					$line = substr($line, 0, $maxLength);
					$line = preg_replace('/\s+\S*$/', '', $line); // Avoid cutting mid-word
					$line .= '...';
				}
				return $line;
			}
		}

		return '';
	}

	public static function parseInlineSourcesOLD($text){

		$uniqueIds = [];

		// Match any word or phrase immediately followed by §[...]
		$pattern = '/(\b.+?)§\[(.*?)\]/';
	
		$processedText = preg_replace_callback($pattern, function ($matches) use (&$uniqueIds) {
			$label = trim($matches[1]);      // Text before the §
			$rawIds = $matches[2];           // Contents inside brackets
	
			// Attempt to parse the array
			$ids = json_decode("[$rawIds]", true);
			if (!is_array($ids)) return $matches[0]; // Leave unchanged if parsing fails
	
			// Collect unique source IDs
			foreach ($ids as $id) {
				if (!in_array($id, $uniqueIds)) {
					$uniqueIds[] = $id;
				}
			}
	
			$dataAttr = htmlspecialchars(implode(',', $ids), ENT_QUOTES);
			return '<span class="inline-source" data-source-ids="' . $dataAttr . '">' . $label . '</span>';
		}, $text);
	
		return [
			'parsed' => $processedText,
			'source_ids' => $uniqueIds
		];		
	}

	public static function stripInLineSources($text){
		return rtrim(strstr($text, '§', true));
	}

	public static function parseInlineSources($text) {

		/*
		in: Here is an inline source§["1766"], followed by two inline sources§["1234","1766"]. I hope that helps!
		out: Here is an inline <span class="inline-source" data-source-ids="1766">source</span>, followed by two inline <span class="inline-source" data-source-ids="1234,1766">sources</span>. I hope that helps!

		Also returns an array of unique source ids: eg: ["1766","1234"];
		*/

		$uniqueIds = [];

		// Match a word (including punctuation) immediately before §[...] — avoids greediness
		$pattern = '/(\S+?)§\[(.*?)\]/';
	
		$processedText = preg_replace_callback($pattern, function ($matches) use (&$uniqueIds) {
			//TWXChatUtil::log('parseInlineSources. matches: '.var_export($matches,true));
			$label = $matches[1];
			$rawIds = $matches[2];
	
			// Decode source IDs (comma-separated)
			$ids = json_decode("[$rawIds]", true);
			//TWXChatUtil::log('parseInlineSources. ids: '.var_export($ids,true));
			if (!is_array($ids)) return $matches[0]; // Return original if decoding fails
	
			// Add to unique IDs
			foreach ($ids as $id) {
				if (!in_array($id, $uniqueIds)) {
					$uniqueIds[] = $id;
				}
			}
			$dataAttr = htmlspecialchars(implode(',', $ids), ENT_QUOTES);
			$span = '<span class="inline-source" data-source-ids="' . $dataAttr . '">' . $label . '</span>'; 

			return $span; //'<span class="inline-source" data-source-ids="' . $dataAttr . '">' . $label . '</span>';
		}, $text);
	
		return [$processedText, $uniqueIds];
	}	

	public static function extractSourceIdsFromText($text) {
		$sourceIds = [];
	
		// Match all spans with class "inline-source" and extract data-source-id
		$pattern = '/<span[^>]*class="[^"]*inline-source[^"]*"[^>]*data-source-id="([^"]+)"[^>]*>/i';
	
		if (preg_match_all($pattern, $text, $matches)) {
			foreach ($matches[1] as $idString) {
				// If multiple IDs are comma-separated, split them
				$ids = array_map('trim', explode(',', $idString));
				$sourceIds = array_merge($sourceIds, $ids);
			}
		}
	
		// Return unique IDs
		return array_values(array_unique($sourceIds));
	}
	

	public static function parseLinks($text) {
		
		/* eg:
			Seeker Brewing - [Seeker Brewing Website](https://mysite.net.au/seeker) and ABC Ales [ABC Brewing](https://mysite.net.au/abc) etc could be (lots) of text with [brackets] etc

			to

			Seeker Brewing - <a href="https://mysite.net.au/seeker" target="_new">Seeker Brewing Website</a> and ABC Ales <a href="https://mysite.net.au/abc" target="_new">ABC Brewing</a> etc

			parts:

				Seeker Brewing - [Seeker Brewing Website
				s://mysite.net.au/seeker) and ABC Ales [ABC Brewing
				s://mysite.net.au/abc) etc could be (lots) of text with [brackets] etc
		*/
		
		$ret = '';
		
		$delim = '](http';
		$split = explode($delim, $text);
		$linkText = '';
		$linkUrl = '';
		$leftLinkSurround = '';  // [
		$rightLinkSurround = '';  // ]
		
		for($i=0;$i<count($split);$i++){
			
			$part = $split[$i];
			
			if($i==0){
				// start part
				$linkText = '';
				$sb_pos = strrpos($part,'[');
				if($sb_pos !== false){
					$ret.= substr($part,0,$sb_pos);
					$ret.= $leftLinkSurround.'<a target="_new" href="';
					$linkText = substr($part,$sb_pos+1);
				}else{
					$ret.= $part;
				}
			} elseif($i == count($split)-1) {
				// end part
				if(!empty($linkText)){
					$rb_pos = strpos($part,')');
					if($rb_pos !== false){
						$linkUrl = 'http'.substr($part,0,$rb_pos);
						$ret.= $linkUrl.'">'.$linkText.'</a>'.$rightLinkSurround;
						$ret.= substr($part,$rb_pos+1);
					}else{
						$ret.= $part;
					}
				}else{
					$ret.= $part;
				}
			} else {
				// middle part(s)
				if(!empty($linkText)){
					$rb_pos = strpos($part,')');
					if($rb_pos !== false){
						$linkUrl = 'http'.substr($part,0,$rb_pos);
						$ret.= $linkUrl.'">'.$linkText.'</a>'.$rightLinkSurround;
						$sb_pos = strrpos($part,'[');
						if($sb_pos !== false){
							$ret.= substr($part,$rb_pos+1,($sb_pos-$rb_pos-1)); 
							$ret.= $leftLinkSurround.'<a target="_new" href="';
							$linkText = substr($part,$sb_pos+1);
						}else{
							$ret.= substr($part,$rb_pos+1);
							$linkText = '';
						}
					}else{
						$ret.= $part;
					}
				}else{
					$ret.= $part;
				}
					
			}
			
		}
		
		return $ret;
		
	}

	public static function getChatbotAttributesFromPost($post) {
		if (!isset($post->post_content)) {
			return [];
		}

		$shortcode_tag = 'twxchat';
		$pattern = get_shortcode_regex([$shortcode_tag]);
		$results = [];

		if (preg_match_all('/' . $pattern . '/s', $post->post_content, $matches, PREG_SET_ORDER)) {
			foreach ($matches as $shortcode) {
				$atts = shortcode_parse_atts($shortcode[3]);
				if (is_array($atts)) {
					// Merge with existing attributes (later ones override)
					$results = array_merge($results, $atts);
				}
			}
		}
				
		return $results;
	}

	public static function getChatbots($atts_filters = []){

		global $wpdb;
		$table_name = $wpdb->prefix . 'twxchat_chatbots';

		$limit = TWXChatUtil::customerPlanAccess('plus') ? null : 2;
		$rows = [];
		if($limit){
			$rows = $wpdb->get_results(
				$wpdb->prepare("SELECT * FROM $table_name ORDER BY id DESC LIMIT %d", $limit)
			);
			// reverse to get ascending order (lowest ID first)
			$rows = array_reverse($rows);
			if(count($rows) >= $limit) $limitReached = true;
		}else{
			$rows = $wpdb->get_results("SELECT * FROM $table_name ORDER BY id DESC");
		}

		$chatbots = [];
		foreach ($rows as $row) {

			$atts = json_decode($row->settings ?? '{}', true);

			$exclude = false;
			foreach($atts_filters as $att_name => $att_filter){

				if(!isset($atts[$att_name])){
					$exclude = true;
					break;
				}

				if(is_array($att_filter)){
					// OR logic
					$or_success = false;
					foreach($att_filter as $filter_val){
						if($atts[$att_name] === $filter_val){
							$or_success = true;
							break;
						}
					}
					if(!$or_success){
						$exclude = true;
						break;
					}
				}else{
					if($atts[$att_name] !== $att_filter){
						$exclude = true;
						break;
					}
				}
			}

			if($exclude) continue;

			$chatbot = [];
			$chatbot['id'] = $row->id;
			$chatbot['name'] = $row->name;
			$chatbot = array_merge($chatbot, $atts);

			$chatbots[] = $chatbot;
		}

		return $chatbots;
	}

	public static function getChatbotIdFromShortcode($string) {
		if (preg_match('/\[twxchat\s+id="(\d+)"\]/', $string, $m)) {
			return (int) $m[1];
		}
		return null;
	}

	public static function getChatbotAttributesById($atts, $mergeWithAll = false) {

		/*
		$atts needs to contain 'id' but may also contain override atts, eg: twx_debug etc 
		*/

		if(!isset($atts['id'])) return [];
		$id = $atts['id'];

		global $wpdb;
		$table_name = $wpdb->prefix . 'twxchat_chatbots';

		$row = null;
		if ($id) {
			$row = $wpdb->get_row($wpdb->prepare("SELECT * FROM $table_name WHERE id = %d", $id));
			if(!$row) return [];
			$chatbot_atts = json_decode($row->settings, true);
			$atts = array_merge($chatbot_atts, $atts); // $atts takes priority
		}

		if($mergeWithAll){
			$atts = TWXChatUtil::mergeChatbotAttributesWithAll($atts);
		}

		return $atts;
	}

	public static function mergeChatbotAttributesWithAll($atts){

		$allAtts = TWXChatUtil::getAllChatbotShortcodeAttributes();
		foreach($allAtts as $attName => $att){
			if(!isset($atts[$attName])){
				$atts[$attName] = isset($att['default_val']) ? $att['default_val'] : '';
			}
		}

		return $atts;
	}

	public static function isValidAvatarShortcodeAtts($atts){
		return isset($atts['avatar_character']) && !empty($atts['avatar_character']) && isset($atts['avatar_voice']) && !empty($atts['avatar_voice']);
	}

	public static function findPostsWithChatbotId($id) {
		global $wpdb;

    	$search = '[twxchat id="' . (int)$id;
    	$like = '%' . $wpdb->esc_like($search) . '%';

		// Search posts and pages where the post_content contains the shortcode
		//$like = '%' . $wpdb->esc_like('[' . $shortcode) . '%';

		$results = $wpdb->get_results(
			$wpdb->prepare("
				SELECT ID, post_title, post_type 
				FROM {$wpdb->posts}
				WHERE post_status IN ('publish','private') 
				AND post_type IN ('post', 'page')
				AND post_content LIKE %s
			", $like)
		);

		return $results;
	}

	public static function getFiles($apiKey, $purpose = 'assistants') {

		// Get all OpenAI files related to the API Key
		$url = "https://api.openai.com/v1/files";
		
		$params = [];
		if(!empty($purpose)) $params['purpose'] = $purpose;
		
		$response = self::getHttpOpenAI_WP($apiKey, $url, $params, 'GET');

		return $response['data'];	
	}

	public static function countFoldersInPath($path) {
		if (!is_dir($path)) {
			return 0; // not a valid directory
		}

		$count = 0;
		foreach (scandir($path) as $item) {
			if ($item === '.' || $item === '..') {
				continue;
			}

			if (is_dir($path . DIRECTORY_SEPARATOR . $item)) {
				$count++;
			}
		}

		return $count;
	}

	
	public static function deleteFile($apiKey, $fileId){
		
		// Delete a file from OpenAI storage by its File ID (must be owned by the apiKey)
		$url = "https://api.openai.com/v1/files/$fileId";
		$response = TWXChatUtil::getHttpOpenAI_WP($apiKey, $url, null, 'DELETE');
		
		return $response;  // Deletion Status
	}

	public static function getAssistants($apiKey) {

		// Retrieves all Assistants connected to ApiKey
		$url = "https://api.openai.com/v1/assistants";
		
		$response = TWXChatUtil::getHttpOpenAI_WP($apiKey, $url, null, 'GET');
		if(isset($response['error'])){
			$ret['error']['message'] = $response['error']['message'];
			return $ret;
		}

		return $response['data'];
	}

	public static function deleteUnusedFiles($apiKey){
		
		// Deletes any Files in OpenAI Project associated with apiKey that are not in use (by Assistants/Vector Stores etc)

		TWXChatUtil::adminLog("TWXChatUtil::deleteUnusedFiles() start");
		
		$assistants = TWXChatUtil::getAssistants($apiKey);
		if(isset($assistants['error'])){
			TWXChatUtil::adminLog("TWXChatUtil::deleteUnusedFiles() error deleting unused spider files: ".$assistants['error']['message']);
			return false;
		}

		$inUseFileIds = [];
		$count = 0;
		
		foreach($assistants as $asst){
			if(isset($asst['tool_resources']['file_search']['vector_store_ids'])){
				$asstId = $asst['id'];
				$vsIds = $asst['tool_resources']['file_search']['vector_store_ids'];
				if(empty($vsIds)) continue;
				$assistObj = new TWXChatOpenAIAssistant($apiKey, $asstId);
				foreach($vsIds as $vsId){
					$vsFiles = $assistObj->getVectorStoreFiles($vsId);
					if(isset($vsFiles['error'])) continue;
					foreach($vsFiles as $vsFile){
						$inUseFileIds[$vsFile['id']] = true;
					}
				}
			}
		}

		$files = TWXChatUtil::getFiles($apiKey);

		if(is_array($files)){
			foreach($files as $file){
				$fileId = $file['id'];
				$fileString = $file['filename']." ($fileId)";

				//TWXChatUtil::log("deleteUnusedFiles:: check file $fileString");

				if(isset($inUseFileIds[$fileId])) {
					//TWXChatUtil::log("deleteUnusedFiles:: $fileString aready in use, skip");
					continue;  // dont delete in use file
				}
				
				//TWXChatUtil::log("deleteUnusedFiles:: delete unused file $fileString");
				
				$response = TWXChatUtil::deleteFile($apiKey, $fileId);
				if(isset($response['error']['message'])){
					TWXChatUtil::adminLog($response['error']['message']);
				}else{
					$count++;
				}
			}
		}

		return $count;
	}

	public static function appendQuestionMark($string) {
		// append ? to end of string if not already there - can be the difference between Knowledge Base entry match or not for a user query
		if (substr($string, -1) !== '?') {
			$string .= '?';
		}
		return $string;
	}

	public static function generateRandomFilename( $extension = '', $length = 10 ) {
		// Generate a unique string using wp_generate_password
		$filename = strtolower( wp_generate_password( $length, false, false ) );

		// Sanitize for safe use in file systems
		$filename = sanitize_file_name( $filename );

		// Append extension if given
		if ( $extension ) {
			$extension = ltrim( $extension, '.' ); // remove leading dot if present
			$filename .= '.' . $extension;
		}

		return $filename;
	}


	public static function makeDir($dir_path, $chmod = 0755) {
		if ( ! function_exists('WP_Filesystem') ) {
			require_once ABSPATH . 'wp-admin/includes/file.php';
		}

		global $wp_filesystem;

		if ( ! $wp_filesystem ) {
			WP_Filesystem();
		}

		$dir_path = untrailingslashit($dir_path);

		if ( $wp_filesystem->is_dir($dir_path) ) {
			return true;
		}

		$parent_dir = dirname($dir_path);
		if ( ! $wp_filesystem->is_dir($parent_dir) ) {
			// Recursively create parent directory
			self::makeDir($parent_dir, $chmod);
		}

		return $wp_filesystem->mkdir($dir_path, $chmod);
	}

	public static function removeDir( $dir_path, $recursive = false ) {
		global $wp_filesystem;

		if ( empty( $wp_filesystem ) ) {
			require_once ABSPATH . 'wp-admin/includes/file.php';
			WP_Filesystem();
		}

		// Check if the directory exists
		if ( $wp_filesystem->is_dir( $dir_path ) ) {
			return $wp_filesystem->rmdir( $dir_path, $recursive );
		}

		return false; // Directory did not exist or could not be removed
	}

	public static function touch($file_path) {
		global $wp_filesystem;

		if ( empty( $wp_filesystem ) ) {
			require_once ABSPATH . 'wp-admin/includes/file.php';
			WP_Filesystem();
		}

		// Create file if it doesn't exist
		if ( ! $wp_filesystem->exists( $file_path ) ) {
			$wp_filesystem->put_contents( $file_path, '', FS_CHMOD_FILE );
			return true;
		}

		// Update modified time by re-writing contents
		$contents = $wp_filesystem->get_contents( $file_path );
		return $wp_filesystem->put_contents( $file_path, $contents, FS_CHMOD_FILE );
	}

	public static function fileWrite( $file_path, $content, $append = false, $max_lines = null) {
		global $wp_filesystem;

		if ( empty( $wp_filesystem ) ) {
			require_once ABSPATH . 'wp-admin/includes/file.php';
			WP_Filesystem();
		}

		// If appending, read the current content
		if ( $append && $wp_filesystem->exists( $file_path ) ) {
			$existing = $wp_filesystem->get_contents( $file_path );
			$content = $existing . $content;
		}

		// If max_lines is set, trim to the last X lines
		if ( $max_lines !== null && is_numeric( $max_lines ) && $max_lines > 0 ) {
			$lines = explode( "\n", $content );
			if ( count( $lines ) > $max_lines ) {
				$lines = array_slice( $lines, -$max_lines ); // Keep last X lines
			}
			$content = implode( "\n", $lines );
		}

		return $wp_filesystem->put_contents( $file_path, $content, FS_CHMOD_FILE );
	}

	public static function recursiveCopy($src, $dst) {
		$dir = opendir($src);
		TWXChatUtil::makeDir($dst);

		while(false !== ($file = readdir($dir))) {
			if (($file !== '.') && ($file !== '..')) {
				$srcPath = $src . '/' . $file;
				$dstPath = $dst . '/' . $file;

				if (is_dir($srcPath)) {
					TWXChatUtil::recursiveCopy($srcPath, $dstPath);
				} else {
					copy($srcPath, $dstPath);
				}
			}
		}
		closedir($dir);
	}

	public static function recursiveDelete($dir_path){
		// Make sure WP_Filesystem is loaded
		if ( ! function_exists('WP_Filesystem') ) {
			require_once ABSPATH . 'wp-admin/includes/file.php';
		}

		global $wp_filesystem;

		// Initialize the filesystem if not already done
		if ( ! $wp_filesystem ) {
			WP_Filesystem();
		}

		// Normalize and validate path
		$dir_path = trailingslashit($dir_path);

		if ( ! $wp_filesystem->is_dir($dir_path) ) {
			return false; // Not a directory
		}

		// Attempt to delete directory recursively
		return $wp_filesystem->delete($dir_path, true, 'd'); // recursive, type = 'd' for directory
	}

	public static function updateOption($key,$val){
		// Needs access to WP context
		if(empty($key)) return false;
		try {
			$options = get_option('twx_chat_options', []);
			$options[$key] = $val;
			update_option('twx_chat_options', $options);
			return true;
		}catch(Exception $ex){
			TWXChatUtil::adminLog("TWXChatUtil::updateOption() Error updating Option '$key' to '$val'. ".$ex->getMessage(),'a',true);
			return false;
		}
	}

	public static function log($message, $mode = 'a', $isError = false) {
		
		// Main Log
		try {
			$dir = TWXChatUtil::getUploadsDir('logs');
			if(!file_exists($dir)) TWXChatUtil::makeDir($dir);
			$logfile = $dir.'twx_chat.log';
			$logEntry = current_time('mysql') . " - " . $message . "\n";
			$append = ($mode === 'a');
			TWXChatUtil::fileWrite($logfile, $logEntry, $append, 1000);

			if($isError){
				if ( defined( 'WP_DEBUG' ) && WP_DEBUG && defined( 'WP_DEBUG_LOG' ) && WP_DEBUG_LOG ) {
					error_log('[TWX-Chat Plugin] '. current_time('mysql') . " - " . $message);
				}
			}
		
		} catch(Exception $ex) {
			// Do nothing
		}
	}	

	public static function getChatLogText(){

		$file_path = TWXChatUtil::getUploadsDir('logs').'twx_chat.log';

		global $wp_filesystem;
		if ( empty( $wp_filesystem ) ) {
			require_once ABSPATH . 'wp-admin/includes/file.php';
			WP_Filesystem();
		}

		if(!$wp_filesystem->exists($file_path)) return '';

		return $wp_filesystem->get_contents($file_path);
	}

	public static function adminLog($message, $mode = 'a', $isError = false) {
		
		// Main Log
		try {
			$dir = TWXChatUtil::getUploadsDir('logs');
			if(!file_exists($dir)) TWXChatUtil::makeDir($dir);
			$logfile = $dir.'twx_chat_admin.log';
			$logEntry = current_time('mysql') . " - " . $message . "\n";
			$append = ($mode === 'a');
			TWXChatUtil::fileWrite($logfile, $logEntry, $append, 1000);

			if($isError){
				if ( defined( 'WP_DEBUG' ) && WP_DEBUG && defined( 'WP_DEBUG_LOG' ) && WP_DEBUG_LOG ) {
					error_log('[TWX-Chat Plugin] '. current_time('mysql') . " - " . $message);
				}
			}
		
		} catch(Exception $ex) {
			// Do nothing
		}
	}

	public static function getAdminLogText(){

		$file_path = TWXChatUtil::getUploadsDir('logs').'twx_chat_admin.log';

		global $wp_filesystem;
		if ( empty( $wp_filesystem ) ) {
			require_once ABSPATH . 'wp-admin/includes/file.php';
			WP_Filesystem();
		}

		if(!$wp_filesystem->exists($file_path)) return '';

		return $wp_filesystem->get_contents($file_path);
	}

	public static function consoleLog($message) {

		try {
			$dir = TWXChatUtil::getUploadsDir('logs');
			if(!file_exists($dir)) TWXChatUtil::makeDir($dir);
			$date = current_time('Y-m-d'); // e.g., 2025-04-16
			$logFile = $dir."{$date}-console.log";
		
			$logLine = $message . PHP_EOL;
		
			// Open the file and lock it - Need locking thus can't use WP Filesystem methods
			$fp = fopen($logFile, 'a');
			if ($fp) {
				// Exclusive lock (waits until it gets the lock)
				if (flock($fp, LOCK_EX)) {
					fwrite($fp, $logLine);
					fflush($fp); // ensure it's written
					flock($fp, LOCK_UN); // release lock
				}
				fclose($fp);
			} else {
				TWXChatUtil::log("TWXChatUtil::consoleLog() Failed to open log file: $logFile");
				return false;
			}
		
		} catch(Exception $ex) {
			// Do nothing
			return false;
		}
		return true;
	}	

	public static function getWpUserLogin(){

		if ( is_user_logged_in() ) {
			$user = wp_get_current_user();
			return $user->user_login;
		} else {
			return 'visitor';
		}
	}

	public static function getCurrentUserRole() {
		if ( ! is_user_logged_in() ) {
			return 'visitor';
		}

		$user = wp_get_current_user();
		if ( empty( $user->roles ) || ! is_array( $user->roles ) ) {
			return 'visitor';
		}

		// Define priority list from highest to lowest
		$role_priority = [
			'administrator',
			'editor',
			'author',
			'contributor',
			'subscriber',
		];

		// Find the highest matching role
		foreach ( $role_priority as $role ) {
			if ( in_array( $role, $user->roles, true ) ) {
				return $role;
			}
		}

		// If user has custom roles not in the list, just return the first
		return $user->roles[0];
	}

}

class TWXChatLocalVocabLoader implements Yethee\Tiktoken\Vocab\VocabLoader
{
    private string $basePath;

    public function __construct(string $basePath)
    {
        $this->basePath = rtrim($basePath, '/');
    }

    public function load(string $uri, ?string $checksum = null): Yethee\Tiktoken\Vocab\Vocab
    {
		//TWXChatUtil::adminLog("TWXChatLocalVocabLoader::load");

		// extract filename from URI
        $filename = basename($uri);

        $path = $this->basePath . '/' . $filename;

        if (!file_exists($path)) {
            throw new Exception("Local vocab not found: " . $path);
        }

		//TWXChatUtil::adminLog("TWXChatLocalVocabLoader::load path: $path");
        return Yethee\Tiktoken\Vocab\Vocab::fromFile($path);
    }

    /**
     * Returns raw file contents instead of downloading from the internet
     */
    public function loadFile(string $uri, ?string $checksum = null): string
    {
		//TWXChatUtil::adminLog("TWXChatLocalVocabLoader::loadFile uri: $uri");
        $filename = basename($uri);
        $path = $this->basePath . '/' . $filename;

        if (!file_exists($path)) {
            throw new \RuntimeException("Local vocab not found: $path");
        }

        $contents = file_get_contents($path);
        if ($contents === false) {
            throw new \RuntimeException("Failed to read vocab file: $path");
        }

        return $contents;
    }

}