<?php
/* Website Spider/Crawler Class 
Expects
	Wordpress context
	TWXChatUtil
	TWXChatKnowledgeBaseUtil
	TWXChatPro (optional)
	Vendor autoload: HtmlConverter
*/

class TWXChatSpider
{
	//public const RESULTS_FILENAME = 'twxchat-sitecrawl.md';

    // Class properties (module variables)
    private $_visitedUrls = [];
	private $_topDomain = '';

    // Constructor method
    public function __construct()
    {
        // Assign values to properties
    }

    // Destructor method
    public function __destruct()
    {
        // Log or perform actions needed at object destruction 
    }

	public static function runSiteCrawl($schedule) {

		// run site crawler now

		TWXChatSpider::log('TWXChatSpider start', 'w');
		TWXChatSpider::clearProgress();
		TWXChatSpider::appendProgress('Crawler Started...');

		try {
			
			$options = get_option('twx_chat_options');
			$openaiApiKey = $options['openai_apikey'] ?? '';
			$crawlPublicPosts = ($options['site_crawler_public_posts'] ?? '0') === '1';
			$crawlPrivatePosts = ($options['site_crawler_private_posts'] ?? '0') === '1';
			$crawlSiteMedia = ($options['site_crawler_onsite_media'] ?? '0') === '1';
			$excludeSources = ($options['site_crawler_exclude_sources'] ?? '0') === '1';
			$excludeSourcesList = array_map('trim', explode(',', ($options['site_crawler_exclude_sources_list'] ?? '')));
			$crawlSitemap = ($options['site_crawler_sitemap'] ?? '0') === '1';
			$sitemapUrl = $options['site_crawler_start_url'] ?? '';
			$updateAssistant = ($options['site_crawler_update_assistant'] ?? '0') === '1';
			$assistantId = $options['site_crawler_assistant_id'] ?? '';
			$crawlGoogleDrive = ($options['site_crawler_google_drive'] ?? '0') === '1';
			$gdriveFolderId = $options['site_crawler_gdrive_folder_id'] ?? '';
			$googleApiKey = $options['google_apikey'] ?? '';
			$createLlmsTxt = ($options['site_crawler_create_llms_txt'] ?? '0') === '1';
			$crawlWoocommerceProducts = ($options['site_crawler_woocommerce_products'] ?? '0') === '1';

			TWXChatSpider::log('TWXChatSpider params retrieved');

			$embeddingModel = $options['openai_embedding_model'] ?? TWXCHAT_DEFAULT_OPENAI_EMBEDDING_MODEL;
			TWXChatSpider::log("Embedding Model: $embeddingModel");

			update_option('twx_chat_site_crawler_running', true);

			// Delete completed file if exists. This file can be monitored by ajax calls to detect crawl completed
			$spiderCompletedFile = TWXChatUtil::getUploadsDir('spider_results').'twx_spider_completed.txt';
			if(file_exists($spiderCompletedFile)) wp_delete_file($spiderCompletedFile);

			TWXChatSpider::appendProgress("Schedule: $schedule");
			
			$completeAfterScriptDone = true;
			$spiderResultsDir = TWXChatUtil::getUploadsDir('spider_results');
			$bigMarkdownFileName = 'twxchat-'.TWXChatUtil::generateRandomFilename('md');
			$bigMarkdownFilePath = $spiderResultsDir.$bigMarkdownFileName;
			
			// Remove all previous .md files
			if (is_dir($spiderResultsDir)) {
				$mdfiles = glob($spiderResultsDir . '*.md');
				if ($mdfiles !== false) {
					foreach ($mdfiles as $file) {
						if (is_file($file)) {
							wp_delete_file($file);
						}
					}
				}
			}

			$llmsTxtFilePath = '';
			if($createLlmsTxt){
				$llmsTxtFilePath = TWXChatUtil::getUploadsDir('spider_results').'llms.txt';
				if(file_exists($llmsTxtFilePath)) wp_delete_file($llmsTxtFilePath);
			}

			// remove KB entries that dont exist anymore on site
			$num_kb_entries_deleted = TWXChatSpider::deleteNonExistingSources();
			if($num_kb_entries_deleted > 0) TWXChatSpider::appendProgress($num_kb_entries_deleted.' '.__("old sources deleted from Knowledge Base",'twx-chat'));

			if($crawlPublicPosts || $crawlPrivatePosts){
				list($num_items, $num_items_excluded) = TWXChatSpider::crawlPosts($openaiApiKey, $crawlPublicPosts, $crawlPrivatePosts, $excludeSources, $excludeSourcesList, $bigMarkdownFilePath, $llmsTxtFilePath);
				TWXChatSpider::appendProgress($num_items.' '.__("changed onsite posts/pages refreshed in Knowledge Base",'twx-chat'));
				if($num_items_excluded > 0) TWXChatSpider::appendProgress($num_items_excluded.' '.__("sources excluded from crawl",'twx-chat'));
			}

			if($crawlSiteMedia){
				$num_items = TWXChatSpider::crawlSiteMedia($openaiApiKey, $bigMarkdownFilePath);
				TWXChatSpider::appendProgress($num_items.' '.__("onsite media items with descriptions/captions refreshed in Knowledge Base",'twx-chat'));
			}

			if($crawlGoogleDrive && !empty($gdriveFolderId) && !empty($googleApiKey)){
				if(TWXChatUtil::customerPlanAccess('pro') && class_exists('TWXChatPro')){
					$num_items = TWXChatPro::crawlGoogleFolder($googleApiKey, $gdriveFolderId, $updateAssistant, $assistantId, 'TWXChatSpider::appendProgress', $bigMarkdownFilePath);
					$assistant = ($updateAssistant && !empty($assistantId)) ? ' '.__("and Updated to OpenAI Assistant ID",'twx-chat').' '.$assistantId : '';
					TWXChatSpider::appendProgress($num_items.' '.__("new/changed files in Google Drive Folder",'twx-chat').$gdriveFolderId.' '.__("loaded/refreshed in Knowledge Base",'twx-chat').$assistant);
				}else{
					TWXChatSpider::log(__('Unauthorized access to Google Drive Folder Crawl feature','twx-chat'));
				}
			}

			if($crawlWoocommerceProducts){
				if(TWXChatUtil::customerPlanAccess('pro') && class_exists('TWXChatPro')){
					$num_items = TWXChatPro::crawlWooCommerceProducts($bigMarkdownFilePath);
					TWXChatSpider::appendProgress($num_items.' '.__("new/changed products in WooCommerce loaded/refreshed in Knowledge Base",'twx-chat'));
				}else{
					TWXChatSpider::log(__('Unauthorized access to WooCommerce Product Crawl feature','twx-chat'));
				}
			}

			if($crawlSitemap && !empty($sitemapUrl)){

				if(!TWXChatUtil::urlExists($sitemapUrl)){
					TWXChatSpider::appendProgress($sitemapUrl.' '.__("does not exist. Aborting sitemap crawl",'twx-chat'));
				}else{
					if(TWXChatUtil::customerPlanAccess('pro') && class_exists('TWXChatPro')){
						list($started, $numUrls, $numBatches, $message) = TWXChatSpider::crawlSitemapInBackground($sitemapUrl, $updateAssistant, $assistantId);
						if(!$started){
							TWXChatSpider::appendProgress($message);
						}else{
							// Leave twx_chat_site_crawler_running option as true until batches complete
							$completeAfterScriptDone = false;
						}
					}else{
						TWXChatSpider::log(__('Unauthorized access to Sitemap Crawler feature','twx-chat'));
					}
				}
			}

			if($completeAfterScriptDone){
				$bigMarkdownFileUrl = TWXChatUtil::getUploadsUrl('spider_results', $bigMarkdownFileName);
				TWXChatSpider::appendProgress(__('Site Crawler Markdown File Generated','twx-chat') . '. <a href="'.$bigMarkdownFileUrl.'" download="'.$bigMarkdownFileName.'">' . __('Download here','twx-chat') . '</a>');

				if ($createLlmsTxt && file_exists($llmsTxtFilePath)) {
					// Copy to root of website as /llms.txt (overwrite)
					$targetPath = rtrim(ABSPATH, DIRECTORY_SEPARATOR) . DIRECTORY_SEPARATOR . 'llms.txt';
					if (!@copy($llmsTxtFilePath, $targetPath)) {
						TWXChatSpider::log("Failed to copy llms.txt to site root: $targetPath");
					} else {
						TWXChatSpider::appendProgress(
							'LLMS.txt '.__('File Generated','twx-chat').' <a href="/llms.txt" target="_blank">View here</a>.'
						);
					}
				}

				TWXChatSpider::appendProgress(__("Completed Crawl",'twx-chat'));
				update_option('twx_chat_site_crawler_running', false);
			}


		}catch(Exception $ex){
			
			TWXChatSpider::log($ex->getMessage());
			TWXChatSpider::appendProgress(__('Error occurred','twx-chat').': '.$ex->getMessage());
			update_option('twx_chat_site_crawler_running', false);

		}
	}

	public static function deleteNonExistingSources() {

		$num_deleted = 0;

		$args = [
			'post_type'      => ['post', 'page'],
			'posts_per_page' => -1,
		];

		$query = new WP_Query($args);
		$posts = $query->posts;

		$currentSources = [];
		foreach ($posts as $index => $post) {
			setup_postdata($post);
			$post_id    = $post->ID;
			$source     = rtrim(get_permalink($post_id), '/');
			$currentSources[$source] = true;
		}

		//TWXChatUtil::adminLog("deleteNonExistingSources:: currentWPSources: ".var_export($currentSources,true));

		$base_url = get_site_url();
		$kbEntries = TWXChatKnowledgeBaseUtil::getFilteredEntries(['source' => $base_url]);

		foreach($kbEntries as $index => $entry){
			$kbSource = rtrim(($entry['source'] ?? ''), '/');
			if(empty($kbSource) || isset($currentSources[$kbSource])) continue;

			// Exclude external URLs - they may have been crawled with Sitemap Crawler feature
			if(strpos($kbSource, $base_url) !== 0) continue;

			// Exclude WC Products
			if(strpos($kbSource, $base_url.'/product/') === 0) continue;

			// Exclude uploads
			if(strpos($kbSource, $base_url.'/wp-content/uploads/') === 0) continue;

			// Check if url actually exists
			$response = wp_remote_head( $kbSource );
			if ( is_wp_error( $response ) ) {
				// Doesnt exist, delete it
				$delete_success = TWXChatKnowledgeBaseUtil::deleteEntryBySource($entry['source']); // use orig source in case it has a trailing slash
				if($delete_success){
					TWXChatSpider::appendProgress("Deleted old KB source: $kbSource");
					$num_deleted++;
				}else{
					TWXChatSpider::appendProgress("Couldn't delete old KB source: $kbSource");
				}
			}
			
		}

		return $num_deleted;

	}

	public static function crawlPosts(
		$openaiApiKey,
		$include_public,
		$include_private,
		$exclude_sources,
		$exclude_sources_list, // string array
		$big_markdown_file_path = '',
		$llms_txt_file_path = ''
	) {

		// Crawl onsite Posts/Pages
		if (!$include_public && !$include_private) return 0;

		$hcParams = ['strip_tags' => true];
		try {
			$converter = new League\HTMLToMarkdown\HtmlConverter($hcParams); // from vendor autoload
		} catch (Exception $ex) {
			TWXChatSpider::log("twx_chat_crawlPosts:: Error creating HtmlConverter: " . $ex->getMessage());
			TWXChatSpider::appendProgress("An error occurred during Page/Post crawl: Error creating HtmlConverter. Please contact TWX-Chat support at ".TWXChatUtil::getTwxlabsUrl().'contact-us');
			return [0, 0];
		}

        $options = get_option('twx_chat_options');
        $embeddingModel = $options['openai_embedding_model'] ?? TWXCHAT_DEFAULT_OPENAI_EMBEDDING_MODEL;
	    $tiktokenEncoder = TWXChatUtil::getLocalTiktokenEncoder($embeddingModel);

		$num_loaded = 0;
		$num_excluded = 0;
		$allow_password_protected = false;
		$post_statuses = [];
		if ($include_public) $post_statuses[] = 'publish';
		if ($include_private) {
			$post_statuses[] = 'private';
			$allow_password_protected = true;
		}

		$args = [
			'post_type'      => ['post', 'page'],
			'post_status'    => $post_statuses,
			'posts_per_page' => -1,
		];

		if(!$allow_password_protected) $args['has_password'] = false; 

		$query = new WP_Query($args);
		$posts = $query->posts;
		$front_page_id = (int) get_option( 'page_on_front' );
		$llms_entries = []; // Collect for LLMS.txt
		$website_description = '';
		$num_posts = count($posts);
		$checkpoint = floor($num_posts / 10);

		TWXChatSpider::appendProgress("Inspecting $num_posts pages/posts");

        $options = get_option('twx_chat_options');
        $embeddingModel = $options['openai_embedding_model'] ?? TWXCHAT_DEFAULT_OPENAI_EMBEDDING_MODEL;

		foreach ($posts as $index => $post) {
			setup_postdata($post);

			// Get basic fields
			$post_id    = $post->ID;
			$title      = get_the_title($post_id);
			$html       = apply_filters('the_content', $post->post_content); // Rendered HTML content
			$summary    = get_the_excerpt($post_id);
			$date       = get_the_date('c', $post_id); // ISO 8601 format
			$modified   = get_the_modified_date('c', $post_id);
			$source     = get_permalink($post_id);
			
			if($exclude_sources){
				$exclude = false;
				foreach($exclude_sources_list as $exclude_source){
					if (str_contains($source, $exclude_source)) {
						$exclude = true;
						break;
					}
				}
				if($exclude){
					$num_excluded++;
					continue;
				}
			}

			// Get SEO metadata
			$meta_description = get_post_meta($post_id, '_yoast_wpseo_metadesc', true);
			if (empty($meta_description)) $meta_description = $summary;
			$meta_keywords = get_post_meta($post_id, 'keywords', true);

			if ($front_page_id && $post->ID === $front_page_id ){
				// Is Home Page
				$website_description = $meta_description;
			}

			if (empty($html)) continue;

			$markdown = TWXChatSpider::getMarkdown($converter, $html);

			$text = (!empty($meta_description) ? "description: $meta_description\n" : '');
			$text .= (!empty($meta_keywords) ? "keywords: $meta_keywords\n" : '');
			$text .= $markdown;

			try {
				// if new or changed content
				if (TWXChatKnowledgeBaseUtil::hasOrigTextChanged($source, $text) || TWXChatKnowledgeBaseUtil::hasEmbeddingModelChanged($source, $embeddingModel)) {
					TWXChatSpider::log("load post/page into Knowledge Base: $source");
					$inserted = TWXChatKnowledgeBaseUtil::storeEntry($openaiApiKey, $source, $title, $text, true, '', $tiktokenEncoder);
					$num_loaded++;
				} else {
					TWXChatSpider::log("no change to source: $source");
				}

				if (!empty($big_markdown_file_path)) {
					TWXChatUtil::fileWrite(
						$big_markdown_file_path,
						"source: $source\ntitle: $title\n" . $text . "\n\n",
						true
					);
				}

				if ($post->post_status === 'publish' && !empty($llms_txt_file_path)) {

					// Add to LLMS entries
					$llms_content = TWXChatUtil::getTextFromPost($post);
					$entry = "## $title\n\n";
					$entry .= "- url: $source\n";
					if(!empty($meta_description) && $llms_content !== $meta_description){
						$entry .= "- description: $meta_description\n";
					}
					$entry .= "- content text:\n" . $llms_content . "\n\n";

					$llms_entries[] = $entry;
				}

			} catch (Exception $ex) {
				TWXChatSpider::log("twx_chat_crawlPosts:: Error " . $ex->getMessage());
				TWXChatSpider::appendProgress("Error crawling $source: ".$ex->getMessage() . ". Skipping...");
			}

			// --- Progress check ---
			// Because array keys may not be sequential, use a counter
			$current = $index + 1;
			if ($checkpoint > 0 && ($current % $checkpoint === 0 || $current === $num_posts)) {
				$percent = floor(($current / $num_posts) * 100);
				TWXChatSpider::appendProgress("Page/Post Crawl Progress: {$percent}% complete");
			}
			
		}

		// Check if dynamic front page and if so, include
		if(empty($front_page_id)){
			$response = wp_remote_get(home_url('/'));

			if (!is_wp_error($response)) {
				$html = wp_remote_retrieve_body($response);

				if (!empty($html)) {
					$markdown = TWXChatSpider::getMarkdown($converter, $html);
					$source = home_url('/');
					$title = 'Homepage';
					$text = $markdown;

					if(empty($website_description)){
						$metatags = TWXChatSpider::extractMetaTags($html);
						$website_description = $metatags['description'] ?? TWXChatSpider::extractTitle($html);
					}

					if (TWXChatKnowledgeBaseUtil::hasOrigTextChanged($source, $text) || TWXChatKnowledgeBaseUtil::hasEmbeddingModelChanged($source, $embeddingModel)) {
						try{
							TWXChatKnowledgeBaseUtil::storeEntry($openaiApiKey, $source, $title, $text, true, '', $tiktokenEncoder);
							TWXChatSpider::log("Crawled and stored rendered homepage: $source");
							$num_loaded++;
						}catch(Exception $ex){
							TWXChatSpider::appendProgress("Error crawling $source: ".$ex->getMessage() . ". Skipping...");
						}
					} else {
						TWXChatSpider::log("No changes detected in rendered homepage.");
					}

					// Add homepage to LLMS.txt
					$entry = "## $title\n\n";
					$entry .= "- url: $source\n";
					$entry .= "- description: $website_description\n";
					$entry .= "\ncontent markdown: \n" . $markdown . "\n\n";

					$llms_entries[] = $entry;
					
				} else {
					TWXChatSpider::log("Empty HTML body returned from homepage.");
				}
			} else {
				TWXChatSpider::log("Error fetching homepage: " . $response->get_error_message());
			}
		}

		wp_reset_postdata();

		// === Write LLMS.txt file ===
		if (!empty($llms_txt_file_path)) {
			$website_name = get_bloginfo('name') ?? '';
			$llms_txt = "# $website_name\n\n> $website_description\n\n" . implode("\n", $llms_entries);
			TWXChatUtil::fileWrite($llms_txt_file_path, $llms_txt, false); // overwrite
		}

		return [$num_loaded, $num_excluded];
	}

	public static function crawlSiteMedia($openaiApiKey, $big_markdown_file_path = '') {

		$num_loaded = 0;

		$args = [
			'post_type'      => 'attachment',
			'post_status'    => 'inherit',
			'posts_per_page' => -1, // load all media
			'post_mime_type' => '', // empty = all types (images, pdf, audio, etc.)
		];

        $options = get_option('twx_chat_options');
        $embeddingModel = $options['openai_embedding_model'] ?? TWXCHAT_DEFAULT_OPENAI_EMBEDDING_MODEL;
	    $tiktokenEncoder = TWXChatUtil::getLocalTiktokenEncoder($embeddingModel);

		$media_query = new WP_Query($args);
		if ($media_query->have_posts()) {
			while ($media_query->have_posts()) {
				$media_query->the_post();

				$id   = get_the_ID();
				$mime = get_post_mime_type($id);
				$url  = wp_get_attachment_url($id);
				$title = get_the_title();

				// Collect meta
				$keywords = isset(wp_get_attachment_metadata($id)['image_meta']['keywords']) && !empty(wp_get_attachment_metadata($id)['image_meta']['keywords'])
					? json_encode(wp_get_attachment_metadata($id)['image_meta']['keywords'])
					: null;

				$meta = [
					'id'           => $id,
					'title'        => $title,
					'caption'      => wp_get_attachment_caption($id),
					'description'  => get_the_content(),
					'alt_text'     => get_post_meta($id, '_wp_attachment_image_alt', true),
					'mime_type'    => $mime,
					'file_url'     => $url,
					'meta_data'    => wp_get_attachment_metadata($id),
					'keywords'     => $keywords,
					'uploaded_date'=> get_the_date(),
				];

				$source = $meta['file_url'] ?? "media_{$id}";
				$text   = "";

				// --- PDF handling ---
				if ($mime === 'application/pdf') {
					if (empty($meta['description']) && empty($meta['caption'])) continue;

					try {
						if (class_exists(\Smalot\PdfParser\Parser::class)) {
							$parser = new \Smalot\PdfParser\Parser();

							// Parse the PDF
							$pdf = $parser->parseFile(get_attached_file($id));
							$pages = $pdf->getPages();
							$pages_loaded = 0;
							// Loop through each page and get the text
							foreach ($pages as $index => $page) {

								$page_text = (!empty($meta['caption']) ? "caption: {$meta['caption']}\n" : "")
									. (!empty($meta['description']) ? "description: {$meta['description']}\n" : "") 
									. trim($page->getText());

								// https://sqrl.eeny.com.au/wp-content/uploads/2025/09/asx-tradeaccept-user-manual.pdf#page=23

								$page_title = "{$title}_page".($index+1);
								$page_source = "{$source}#page=".($index+1); 
								try {
									if(TWXChatKnowledgeBaseUtil::hasOrigTextChanged($page_source, $page_text) || TWXChatKnowledgeBaseUtil::hasEmbeddingModelChanged($page_source, $embeddingModel)){
										TWXChatKnowledgeBaseUtil::storeEntry($openaiApiKey, $page_source, $page_title, $page_text, true, '', $tiktokenEncoder);
										TWXChatSpider::log("refresh/load PDF page into Knowledge Base: $page_source");
										$pages_loaded++;
									}else{
										TWXChatSpider::log("no change to PDF source: $page_source");
									}
									if(!empty($big_markdown_file_path)) TWXChatUtil::fileWrite($big_markdown_file_path, "source: $page_source\ntitle: $page_title\n".$page_text."\n\n", true);

								} catch (Exception $ex) {
									TWXChatSpider::log("error parsing PDF $page_source: ".$ex->getMessage());
									TWXChatSpider::appendProgress("Error parsing PDF $page_source: ".$ex->getMessage() . ". Skipping...");
								}
							}
							if($pages_loaded > 0) $num_loaded++;

						} else {
							TWXChatSpider::log("PDF Parser not available for $source");
							TWXChatSpider::appendProgress("PDF Parser not available for $source. Skipping...");
						}
					} catch (Exception $ex) {
						TWXChatSpider::log("error parsing PDF $source: ".$ex->getMessage());
						TWXChatSpider::appendProgress("Error parsing PDF $source: ".$ex->getMessage() . ". Skipping...");
						continue; // skip this file
					}
				} 
				// --- Images & others ---
				else {
					// skip items with no useful text
					if (empty($meta['description']) && empty($meta['caption']) && empty($meta['alt_text'])) continue;

					$text = (!empty($meta['caption']) ? "caption: {$meta['caption']}\n" : "")
						. (!empty($meta['description']) ? "description: {$meta['description']}\n" : "") 
						. (!empty($meta['alt_text']) ? "alt_text: {$meta['alt_text']}" : "");
				}

				try {
					if (!empty(trim($text))) {
						if (TWXChatKnowledgeBaseUtil::hasOrigTextChanged($source, $text) || TWXChatKnowledgeBaseUtil::hasEmbeddingModelChanged($source, $embeddingModel)) {
							TWXChatKnowledgeBaseUtil::storeEntry($openaiApiKey, $source, $title, $text, true, '', $tiktokenEncoder);
							TWXChatSpider::log("parsed $mime file into Knowledge Base: $source");
							$num_loaded++;
						} else {
							TWXChatSpider::log("no change to source: $source");
						}

						if (!empty($big_markdown_file_path)) {
							TWXChatUtil::fileWrite(
								$big_markdown_file_path,
								"source: $source\ntitle: $title\n".$text."\n\n",
								true
							);
						}
					}
				} catch (Exception $ex) {
					TWXChatSpider::log("error processing $source: ".$ex->getMessage());
					TWXChatSpider::appendProgress("Error crawling $source: ".$ex->getMessage() . ". Skipping...");
				}
			}
			wp_reset_postdata();
		}

		return $num_loaded;
	}

	public static function crawlSitemapInBackground($sitemapUrl, $updateAssistant, $assistantId){

		//$path = plugin_dir_path(__FILE__) . 'spider_batch.php';
		$path = plugin_dir_path(dirname(__FILE__)) . 'spider_batch.php';
		if(!file_exists($path)){
			return [false, 0, 0, __('Background processor file not available','twx-chat')];
		}

		$batchesOf = 35;
		$spider = new TWXChatSpider();
		$token = get_option('twx_chat_server_token','');
		
		TWXChatSpider::appendProgress('Crawl Sitemap: '.$sitemapUrl);
		$urlsToCrawl = TWXChatPro::getSitemapUrlsToCrawl($sitemapUrl, $spider);
		if(empty($urlsToCrawl)){
			return [false, 0, 0, __('No valid URLs in sitemap to crawl','twx-chat')];
		}
		TWXChatSpider::appendProgress('Found '.count($urlsToCrawl).' unique URLs');

		$urlBatches = array_chunk($urlsToCrawl, $batchesOf);
		foreach ($urlBatches as $index => $urls) {

			$batchNo = ($index+1);
			$params = [ 
				'assistantId' => ($updateAssistant && $assistantId) ? $assistantId : '',
				'batchNo' => $batchNo,
				'numBatches' => count($urlBatches),
				'urls' => $urls,
				'token' => $token,
				'nocache' => substr(md5(uniqid(wp_rand(), true)), 0, 8),
			];
			
			$url = plugins_url('spider_batch.php', dirname(__FILE__));

			// Start Batch Crawl in Background
			TWXChatSpider::log("twx_chat_executeBackground batch no $batchNo @ $url");
			TWXChatPro::executeBackground($url, $params);
		}

		return [true, count($urlsToCrawl), count($urlBatches), __('Go!','twx-chat')];
	}


	public function getUrlsToCrawl($url){
		
		// inspect a url's contents and extract links (urls) within it
		// url could be a sitemap for instance
		
		try {
			$html = $this->fetchUrlContent($url);
			if($html === false){
				TWXChatSpider::log("Unable to get content from url: $url");
				return [];
			}

			if($this->isSitemapContent($html)){
				$links = $this->getSitemapLinks($html);
				if(!empty($links) && $this->isSitemapUrl($links[0])) {
					// Sitemap contains links to other Sitemaps
					TWXChatSpider::log("Sitemap $url contains links to other sitemaps. Traversing.");
					$nonSiteMapLinks = [];
					foreach($links as $link){
						$nonSiteMapLinks = array_merge($nonSiteMapLinks, $this->getUrlsToCrawl($link));
					}
					$links = $nonSiteMapLinks;
				}
			}else{
				$links = $this->extractLinksFromHtml($html, $url);
			}
			
			return $links;
			
		}catch(Exception $ex){
			TWXChatSpider::log("Error getting urls to crawl: " . $ex->getMessage());
			return [];
		}
		
	}
	
	public function crawlUrlsNonRecursive($urls, $batchNo, $createFiles, $progressCallback){
		
		$this->resetVars();
		
		// Crawl array of Urls, non recursivley
		$preserveLinks = true;
		$allowOffsite = true; // as we are looping through a list of user defined urls
		$i = 0;
		foreach ($urls as $url) {
			if($this->crawlUrl($url, 0, 0, $allowOffsite, $batchNo, $preserveLinks, $createFiles)) {
				$i++;
				$progressCallback($batchNo, count($urls), $i);
			}
		}
		
	}
	
	public function isSitemapUrl($url) {
		
		$html = $this->fetchUrlContent($url);
		if($html === false){
			throw new Exception("Unable to get content from url: $url");
		}
		
		return $this->isSitemapContent($html);
	}
	
	public function startCrawl($topDomain, $startUrl, $maxDepth, $allowOffsite = false, $createFiles = true, $mustMatch = null){
		
		// Begin recursive crawl
		
		$this->resetVars();
		
		// causes issues if not a dir eg: sitemap.xml
		//$startUrl = rtrim($startUrl, '/') . '/'; // ensure last char is a /
		
		if(empty($topDomain)){
			$this->_topDomain = wp_parse_url($startUrl)['host'] ?? '';
		}else{
			$this->_topDomain = $topDomain;
		}
		
		try {
			if($createFiles) {
				$dir = TWXChatUtil::getUploadsDir('spider_results',false);
				$this->removeAllSubdirsAndFiles($dir, ['twxchatspider_progress.txt']);
			} 
		} catch (Exception $e) {
			TWXChatSpider::log("Error removing previous spider_results: " . $e->getMessage());
			return false;
		}

		$batchNo = 0;
		$preserveLinks = true;
		$ret = $this->crawlUrl($startUrl, 0, $maxDepth, $allowOffsite, $batchNo, $preserveLinks, $createFiles, $mustMatch);
		
		TWXChatSpider::log('crawl done');
		
		return $ret;
	}
	
	public function removeAllSubdirsAndFiles($dir, $exceptFiles = [])
	{
		if (!is_dir($dir)) {
			throw new InvalidArgumentException($dir . ' must be a directory');
		}

		$dir = rtrim($dir, '/\\');

		$items = @scandir($dir);
		if ($items === false) {
			throw new RuntimeException("Unable to read directory: $dir");
		}

		// Best effort init WP_Filesystem (works on many hosts; if it fails we fall back to PHP)
		$fs = null;
		if (!function_exists('WP_Filesystem')) {
			require_once ABSPATH . 'wp-admin/includes/file.php';
		}
		if (function_exists('WP_Filesystem') && WP_Filesystem()) {
			global $wp_filesystem;
			$fs = $wp_filesystem;
		}

		// helper: delete a directory tree without exec()
		$deleteTree = function (string $path) use (&$deleteTree, $fs) {
			$path = rtrim($path, '/\\');

			if (!is_dir($path)) {
				return;
			}

			// Do not follow symlinks (safety)
			if (is_link($path)) {
				// treat symlink as a file-like item
				if (!@unlink($path)) {
					throw new RuntimeException("Failed to delete symlink: $path");
				}
				return;
			}

			// Prefer WP_Filesystem if available
			if ($fs && is_object($fs)) {
				if ($fs->delete($path, true)) { // true = recursive
					return;
				}
				// fall through to PHP recursion if it fails
			}

			$children = @scandir($path);
			if ($children === false) {
				throw new RuntimeException("Unable to read directory: $path");
			}

			foreach ($children as $child) {
				if ($child === '.' || $child === '..') {
					continue;
				}

				$childPath = $path . DIRECTORY_SEPARATOR . $child;

				if (is_dir($childPath) && !is_link($childPath)) {
					$deleteTree($childPath);
				} else {
					if (!@unlink($childPath)) {
						// Try WP helper as a fallback (honors WP hooks)
						if (!wp_delete_file($childPath)) {
							throw new RuntimeException("Failed to delete file: $childPath");
						}
					}
				}
			}

			if (!@rmdir($path)) {
				throw new RuntimeException("Failed to remove directory: $path");
			}
		};

		foreach ($items as $item) {
			if ($item === '.' || $item === '..') {
				continue;
			}

			// Exclude matching filenames at this directory level
			if (in_array($item, $exceptFiles, true)) {
				continue;
			}

			$path = $dir . DIRECTORY_SEPARATOR . $item;

			if (is_dir($path)) {
				$deleteTree($path);
			} else {
				// Remove files (and symlinks) unless excluded
				if (is_file($path) || is_link($path)) {
					if (!wp_delete_file($path)) {
						// wp_delete_file returns the filename on success, false on failure
						if (!@unlink($path)) {
							throw new RuntimeException("Failed to delete file: $path");
						}
					}
				}
			}
		}
	}

	public function removeAllFilesInDir($dir)
	{
		if (!is_dir($dir)) {
			throw new InvalidArgumentException(esc_html($dir) . ' must be a directory');
		}

		// Get all files in the directory
		$files = glob($dir . '/*');

		// Iterate over the files
		foreach ($files as $file) {
			// If it's a file, delete it
			if (is_file($file)) {
				wp_delete_file($file);
			}
		}
		
	}

	public function getVisitedUrls()
	{
		return $this->_visitedUrls;
	}
	
	public function zipDirectory($sourceDir, $zipFilePath) {
		
		TWXChatSpider::log("Zipping spider results in $sourceDir to $zipFilePath");

		if (!extension_loaded('zip') || !file_exists($sourceDir)) {
			return false;
		}

		$zip = new ZipArchive();
		if (!$zip->open($zipFilePath, ZipArchive::CREATE)) {
			return false;
		}

		$sourceDir = realpath($sourceDir);

		$iterator = new RecursiveDirectoryIterator($sourceDir, RecursiveDirectoryIterator::SKIP_DOTS);
		$files = new RecursiveIteratorIterator($iterator, RecursiveIteratorIterator::SELF_FIRST);

		foreach ($files as $file) {
			$filePath = realpath($file);
			TWXChatSpider::log("Zipping $filePath...");
			if (is_dir($filePath)) {
				$zip->addEmptyDir(str_replace($sourceDir . DIRECTORY_SEPARATOR, '', $filePath . DIRECTORY_SEPARATOR));
			} elseif (is_file($filePath)) {
				$zip->addFile($filePath, str_replace($sourceDir . DIRECTORY_SEPARATOR, '', $filePath));
			}
		}

		return $zip->close();
	}
	
	public static function extractMetaTags($html) {
		
		$metaTags = [];
		
		$dom = new DOMDocument();
		libxml_use_internal_errors(true); // Suppress HTML parsing errors
		$dom->loadHTML($html);
		libxml_clear_errors();
		
		// Get all meta tags
		$metaElements = $dom->getElementsByTagName('meta');
		
		$names = ['keywords','description']; //, 'og:image'];

		// Loop through meta tags and extract their attributes
		foreach ($metaElements as $metaElement) {
			$nameAttribute = $metaElement->getAttribute('name');
			if (in_array($nameAttribute, $names)) {
				$metaTags[$nameAttribute] = $metaElement->getAttribute('content');
			}
			if(!isset($metaTags[$nameAttribute])){
				$propertyAttribute = $metaElement->getAttribute('property');
				if( in_array($propertyAttribute, $names)){
					$metaTags[$propertyAttribute] = $metaElement->getAttribute('content');
				}
			}
		}
		
		return $metaTags;
	}

	public static function extractTitle($html) {
		
		$title = '';
		
		$dom = new DOMDocument();
		libxml_use_internal_errors(true); // Suppress HTML parsing errors
		$dom->loadHTML($html);
		libxml_clear_errors();
		
		$titleElements = $dom->getElementsByTagName('title');
		if(!empty($titleElements)) {
			$title = $titleElements->item(0)->nodeValue;
		}
		
		return $title;
	}
	
	public static function getMarkdown(&$converter, $html) {

		// $converter is an instance of League\HTMLToMarkdown\HtmlConverter() with $hcParams = ['strip_tags' => true];

		if (empty($html)) return '';

		// Ensure UTF-8 encoding
		$html = mb_convert_encoding($html, 'UTF-8', 'UTF-8');

		// Normalize only ASCII punctuation to HTML entities
		$replace = [
			"‘" => "&#8216;",
			"’" => "&#8217;",
			"“" => "&#8220;",
			"”" => "&#8221;",
			"—" => "&#8212;",
			"…" => "&#8230;",
			"–" => "&#8211;"
		];
		$html = str_replace(array_keys($replace), array_values($replace), $html);

		// Strip down HTML
		$html = self::stripDownHtml($html);

		// Convert to Markdown
		$markdown = $converter->convert($html);

		// Remove empty links (not images)
		$markdown = preg_replace('/(?<!\!)\[\s*\]\((.*?)\)/', '', $markdown);

		// Remove leading spaces on lines
		$markdown = preg_replace('/^\s+/m', '', $markdown);

		// Clean up javascript placeholders
		$markdown = str_replace(["javascript:void(0);", "javascript:void(0)"], "", $markdown);

		// Normalize line breaks
		$markdown = preg_replace('/[ \t]*\r?\n/', "\n", $markdown);
		$markdown = preg_replace('/(\r?\n){2,}/', "\n", $markdown);

		// Remove empty list items
		$markdown = preg_replace('/^-$/m', '', $markdown);
		$markdown = preg_replace('/(\r?\n){2,}/', "\n", $markdown);

		// Final UTF-8 normalization
		$markdown = mb_convert_encoding($markdown, 'UTF-8', 'UTF-8');

		// Remove all 4-byte UTF-8 characters (emojis, rare symbols)
		$markdown = preg_replace('/[\x{10000}-\x{10FFFF}]/u', '', $markdown);		

		return $markdown;
	}


	public static function getMarkdown_OLD(&$converter, $html){
		
		// $converter is an instance of League\HTMLToMarkdown\HtmlConverter() with $hcParams = ['strip_tags' => true];

		// Needed for Non-Latin Multi Lingual chars
		$html = mb_convert_encoding($html, 'UTF-8', 'UTF-8');

		// apostrophe and other char normalization to html codes, needs to happen at top, otherwise render issues in KB browser
		$html = str_replace("‘", "&#8216;", $html);
		$html = str_replace("’", "&#8217;", $html); 
		$html = str_replace("“", "&#8220;", $html);
		$html = str_replace("”", "&#8221;", $html);
		$html = str_replace("—", "&#8212;", $html);
		$html = str_replace("…", "&#8230;", $html);
		$html = str_replace("–", "&#8211;", $html);

		$html = self::stripDownHtml($html);
	
		$markdown = $converter->convert($html);

		$markdown = preg_replace('/(?<!\!)\[\s*\]\((.*?)\)/', '', $markdown); // Only remove links that have truly empty link text (not images)

		$markdown = preg_replace('/^\s+/m', '', $markdown); // remove leading spaces
		
		$markdown = str_replace("javascript:void(0);", "", $markdown);
		$markdown = str_replace("javascript:void(0)", "", $markdown);

		$markdown = str_replace("   \n", "\n", $markdown);
		$markdown = str_replace("  \n", "\n", $markdown);
		$markdown = str_replace(" \n", "\n", $markdown);
		$markdown = preg_replace('/(\r?\n){2,}/', "\n", $markdown); // turn multiple line breaks into single

		$markdown = preg_replace('/^-$/m', '', $markdown); // replace empty line items (-) with single line break
		$markdown = preg_replace('/(\r?\n){2,}/', "\n", $markdown); // turn multiple line breaks into single (again)

		return $markdown;
	}
	
	public static function removeUniformColumns(array $array) {
		
		if (empty($array)) {
			return $array;
		}

		// Extract column names
		$columns = array_keys(reset($array));
		$columnsToRemove = [];

		// Check each column
		foreach ($columns as $column) {
			$firstValue = null;
			$allSame = true;

			foreach ($array as $row) {
				if ($firstValue === null) {
					$firstValue = $row[$column];
				} elseif (isset($row[$column]) && $row[$column] !== $firstValue) {
					$allSame = false;
					break;
				}
			}

			if ($allSame) {
				$columnsToRemove[] = $column;
			}
		}

		// Remove uniform columns while preserving keys
		foreach ($array as $key => &$row) {
			foreach ($columnsToRemove as $column) {
				unset($row[$column]);
			}
		}

		return $array;
	}
	
	public static function getCommonHeadersAndFooters($markdowns) {
		
		$headers = [];
		$footers = [];

		// Extract all potential headers and footers
		foreach ($markdowns as $markdown) {
			$lines = explode("\n", $markdown);
			$headers[] = self::extractHeader($lines);
			$footers[] = self::extractFooter($lines);
		}
		
		// Identify the common headers and footers
		$commonHeaderLines = self::findCommonLines($headers);
		$commonFooterLines = self::findCommonLines($footers);
		if (!empty($commonFooterLines)) $commonFooterLines = array_reverse($commonFooterLines);
		//if(!empty($commonFooter)) krsort($commonFooter); // need to revert to original sort order for footer lines

		/*
		return [
			'header' => implode("\n", $commonHeader),
			'footer' => implode("\n", $commonFooter)
		];
		*/
		return [
			'headerLines' => $commonHeaderLines,
			'footerLines' => $commonFooterLines,
		];
	}

	///////////////////// PRIVATES ////////////////////////

	private function resetVars(){
		
		$this->_visitedUrls = [];
		$this->_topDomain = '';
	}

	private function crawlUrl($url, $currentDepth, $maxDepth = 2, $allowOffsite = false, $batchNo = 1, $preserveLinks = true, $createFiles = true, $mustMatch = null){
		
		// Main function. Recursive crawl a url and its url links until maxDepth reached
		// If $createFiles is false, no files are generated but the visited links will still increment

		TWXChatSpider::log("crawl batch $batchNo, depth $currentDepth maxDepth $maxDepth : $url");
		
		// Check if URL has already been visited to avoid duplicates
		if (isset($this->_visitedUrls[$url])) {
			TWXChatSpider::log('already visited. skipping');
			return true;
		}
		
		if(!$allowOffsite){
			if(wp_parse_url($url)['host'] != $this->_topDomain) {
				TWXChatSpider::log('url offsite. skipping');
				return true;
			}
		}

		try {
			// Fetch HTML content from URL
			$html = $this->fetchUrlContent($url);
			if ($html === false) {
				throw new Exception("Failed to fetch HTML content for URL: {$url}");
			}
			
			$isSiteMap = false;
			if($this->isSitemapContent($html)){
				$isSiteMap = true;
				TWXChatSpider::log("is sitemap. not saving but will get links to crawl");
			}

			if(!$isSiteMap && $mustMatch !== null){
				if(strpos($url, $mustMatch) === false){
					TWXChatSpider::log("url does not match '$mustMatch'. skipping");
					return true;
				}
			}

			if(!$isSiteMap){
				if($createFiles){
					$filepath = TWXChatUtil::getUploadsDir('spider_results') . wp_parse_url($url, PHP_URL_HOST) . wp_parse_url($url, PHP_URL_PATH) . '.html';
					$this->saveContentToFile($html, $filepath);
					$this->_visitedUrls[$url] = $filepath;
				}else{
					$this->_visitedUrls[$url] = null;
				}
			}

			// Recursive crawl if current depth is less than max depth
			if ($currentDepth < $maxDepth) {
				$links = [];
				if($isSiteMap){
					$links = $this->getSitemapLinks($html, $mustMatch);
				}else{
					$links = $this->extractLinksFromHtml($html, $url, $mustMatch);
				}
				TWXChatSpider::log(count($links)." links found in page. traversing.");
				
				foreach ($links as $link) {
					$this->crawlUrl($link, ($currentDepth + 1), $maxDepth, $allowOffsite, $batchNo, $preserveLinks, $createFiles, $mustMatch);
				}
			}
		} catch (Exception $e) {
			TWXChatSpider::log("Error crawling URL {$url}: " . $e->getMessage());
			return false;
		}
		
		//TWXChatSpider::log('crawl finished for '.$url);
		
		return true;
	}
	

	private static function extractHeader(array $lines) {
		// Extract first few lines, assuming headers are usually at the start
		return array_slice($lines, 0, min(100, count($lines)));
	}

	private static function extractFooter(array $lines) {
		krsort($lines); // turn upside down to allow common lines check from bottom-up
		return array_slice($lines, 0, min(100, count($lines)));
	}

	private static function findCommonLines(array $lineArrays, int $minOccurrences = 2): array {
		$lineCount = [];

		foreach ($lineArrays as $lines) {
			$seen = [];
			foreach ($lines as $line) {
				$line = trim($line);
				if ($line === '') continue;
				// Avoid double counting within the same document
				if (!isset($seen[$line])) {
					$lineCount[$line] = ($lineCount[$line] ?? 0) + 1;
					$seen[$line] = true;
				}
			}
		}

		// Keep lines appearing at least $minOccurrences times
		$commonLines = [];
		foreach ($lineCount as $line => $count) {
			if ($count >= $minOccurrences) {
				$commonLines[] = $line;
			}
		}

		return $commonLines;
	}
	
	private static function findShortestArrayIndex($arrayOfArrays) {
		
		$minLength = PHP_INT_MAX;
		$shortestArrayIndex = null;

		foreach ($arrayOfArrays as $key => $subArray) {
			if (is_array($subArray)) {
				$currentLength = count($subArray);
				if ($currentLength < $minLength) {
					$minLength = $currentLength;
					$shortestArrayIndex = $key;
				}
			}
		}

		return $shortestArrayIndex;
	}

	/*
	private function fetchUrlContent($url, &$retMsg)
	{
		if(!ini_get('allow_url_fopen')){
			return $this->fetchUrlContentWP($url);
		}
		
		// Suppress errors for failed fetches
		$html = @file_get_contents($url);
		
		// Check if the fetch was successful
		if ($html === false) {
			// Log the error
			$error = error_get_last();
			$retMsg = "Error fetching URL $url. Error was: ".$error['message'];
			TWXChatSpider::log($retMsg);
			return false;
		}
		
		return $html;
	}
	*/
	
	private function fetchUrlContent($url) {
		// Use WordPress HTTP API
		$response = wp_remote_get($url, [
			'timeout' => 30,
			'redirection' => 5, // max redirects
			'sslverify' => true,
		]);

		if (is_wp_error($response)) {
			// Log the error
			$error_message = $response->get_error_message();
			TWXChatSpider::log("fetchUrlContent() error for url {$url}: {$error_message}");
			return false;
		}

		$http_status = wp_remote_retrieve_response_code($response);

		if ($http_status != 200) {
			TWXChatSpider::log("HTTP error: {$http_status} - Failed to fetch URL: {$url}");
			return false;
		}

		$body = wp_remote_retrieve_body($response);

		return $body;
	}

	/*
	private function fetchUrlContentCURL($url)
	{
		// Initialize cURL session - Using CURL instead of wp_remote_get() as this Class may be used in other Non-WP Projects
		$ch = curl_init();
		
		// Set cURL options
		curl_setopt($ch, CURLOPT_URL, $url);
		curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
		curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
		curl_setopt($ch, CURLOPT_TIMEOUT, 30);

		// Execute cURL request
		$html = curl_exec($ch);
		
		// Check for cURL errors
		if (curl_errno($ch)) {
			$error_msg = curl_error($ch);
			TWXChatSpider::log('fetchUrlContentCURL error for url '.$url.': '. $error_msg);
			curl_close($ch);
			return false;
		}
		
		// Get HTTP status code
		$http_status = curl_getinfo($ch, CURLINFO_HTTP_CODE);
		
		// Close cURL session
		curl_close($ch);
		
		// Check if the request was successful
		if ($http_status != 200) {
			TWXChatSpider::log('HTTP error: ' . $http_status . ' - Failed to fetch URL: ' . $url);
			return false;
		}
		
		return $html;
	}
	*/
	
	private function parseHtmlContent($html)
	{
		$dom = new DOMDocument();
		libxml_use_internal_errors(true); // Suppress HTML parsing errors
		$dom->loadHTML($html);
		libxml_clear_errors();

		$xpath = new DOMXPath($dom);

		// Remove script, style, and other non-visible elements
		foreach ($xpath->query('//script | //style | //noscript | //iframe | //object | //embed | //svg | //img') as $element) {
			$element->parentNode->removeChild($element);
		}

		// Extract only visible content
		/*
		$mainContent = '';
		$body = $dom->getElementsByTagName('body')->item(0);
		if ($body) {
			foreach ($body->childNodes as $node) {
				if ($node->nodeType == XML_ELEMENT_NODE) {
					$mainContent .= $dom->saveHTML($node);
				} elseif ($node->nodeType == XML_TEXT_NODE) {
					$mainContent .= htmlspecialchars($node->nodeValue, ENT_QUOTES | ENT_HTML5, 'UTF-8');
				}
			}
		}
		*/

		$mainContent = '';
		foreach ($dom->getElementsByTagName('body')->item(0)->childNodes as $node) {
			$mainContent .= $dom->saveHTML($node);
		}

		return $mainContent;
	}
	
	private static function stripDownHtml($html) {
		if (empty($html)) return '';

		// Force UTF-8 encoding (for Multi-Lingual)
		//$html = mb_convert_encoding($html, 'HTML-ENTITIES', 'UTF-8'); // Deprecated
		$html = mb_encode_numericentity($html, [0x80, 0x10FFFF, 0, ~0], 'UTF-8');

		$dom = new DOMDocument();
		libxml_use_internal_errors(true);

		// Wrap content in a <div> to prevent truncation and enforce UTF-8
    	$htmlUtf8 = '<?xml encoding="UTF-8"><div>' . $html . '</div>';
    	$dom->loadHTML($htmlUtf8, LIBXML_HTML_NOIMPLIED | LIBXML_HTML_NODEFDTD);

		libxml_clear_errors();
		$xpath = new DOMXPath($dom);

		// Remove non-visible or script/style elements
		foreach ($xpath->query('//script | //style | //noscript | //iframe | //object | //embed | //svg') as $element) {
			$element->parentNode->removeChild($element);
		}

		// Extract content from the wrapper <div>
		$container = $dom->getElementsByTagName('div')->item(0);
		if ($container) {
			$innerHtml = '';
			foreach ($container->childNodes as $child) {
				// Use saveXML for fragments to preserve UTF-8 properly
				$innerHtml .= $dom->saveXML($child);
			}

			// Decode HTML entities back to UTF-8 for Markdown
			return html_entity_decode($innerHtml, ENT_QUOTES | ENT_HTML5, 'UTF-8');
		}

		return '';		
	}

	private static function stripDownHtml_OLD($html){
		
		if(empty($html)) return '';

		$dom = new DOMDocument();
		libxml_use_internal_errors(true); // Suppress HTML parsing errors
		$dom->loadHTML($html);
		libxml_clear_errors();
		
		$xpath = new DOMXPath($dom);

		// Remove script, style, and other non-visible elements
		//foreach ($xpath->query('//script | //style | //noscript | //iframe | //object | //embed | //svg | //img') as $element) {
		foreach ($xpath->query('//script | //style | //noscript | //iframe | //object | //embed | //svg') as $element) {
			$element->parentNode->removeChild($element);
		}
		
		$body = $dom->getElementsByTagName('body')->item(0);
		if ($body) {
			$innerHtml = '';
			foreach ($body->childNodes as $child) {
				$innerHtml .= $dom->saveHTML($child);
			}
			return $innerHtml;
		}

		return '';
	}
	
	private function hasAllSameValues($arr) {
		if (empty($arr)) {
			return true; // Consider an empty array as having all the same values
		}

		$firstValue = $arr[0];
		foreach ($arr as $value) {
			if ($value !== $firstValue) {
				return false;
			}
		}

		return true;
	}
	
	private function getTextContent($html, $preserveLinks){
		
		// returns only Text contents of given html
		$dom = new DOMDocument();
		libxml_use_internal_errors(true); // Suppress HTML parsing errors
		$dom->loadHTML($html);
		libxml_clear_errors();
		
		try{
			$xpath = new DOMXPath($dom);

			// Remove script, style, and other non-visible elements
			foreach ($xpath->query('//script | //style | //noscript | //iframe | //object | //embed | //svg | //img') as $element) {
				$element->parentNode->removeChild($element);
			}
			
			$mainContent = '';
			$body = $dom->getElementsByTagName('body')->item(0);

			if ($body) {
				$mainContent = $this->getTextFromNode($body, $preserveLinks);
			}else{
				TWXChatSpider::log("could not retrieve body of html");
			}
		
			// remove whitespace
			$mainContent = trim(preg_replace('/\s+/', ' ', $mainContent));
			
			return $mainContent;
			
		}catch(Exception $ex){
			return null;
		}
	}
	
	private function getTextFromNode($node, $preserveLinks = false) {
		
		// Recursive get inner text of a node
		
		$textContent = '';
		foreach ($node->childNodes as $child) {
			if ($child->nodeType == XML_ELEMENT_NODE) {
				if ($child->nodeName == 'br') {
					$textContent .= ' ';
				} elseif ($child->nodeName == 'a' && $preserveLinks) {
					// Recursively process the anchor's child nodes
					$innerText = trim($this->getTextFromNode($child, $preserveLinks));
					if(!$this->nonContent($innerText)){
					//if(!empty($innerText)){
						// Append the opening anchor tag with its href attribute
						$href = $child->getAttribute('href');
						$textContent .= ' <a href="' . htmlspecialchars($href, ENT_QUOTES) . '">';
						$textContent .= $innerText;
						// Append the closing anchor tag
						$textContent .= '</a> ';
					}
				} elseif ($child->nodeName == 'li') {
					// Keep li elements otherwise openai can seperate/merge text unexpectedly
					$innerText = trim($this->getTextFromNode($child, $preserveLinks));
					if(!$this->nonContent($innerText)) $textContent .= '<li>'.$innerText.'</li>';
					//if(!empty($innerText)) $textContent .= '<li>'.$innerText.'</li>';
				} else {
					$textContent .= ' ' . $this->getTextFromNode($child, $preserveLinks);
				}
			} elseif ($child->nodeType == XML_TEXT_NODE) {
				$textContent .= ' ' . $child->nodeValue;
			}
		}
		
		return $textContent;
	}
	
	private function nonContent($text){

		if(empty(trim($text))) return true;
		
		$non = ['Home','|'];
		if(in_array(trim($text), $non)) return true;
		return false;
	}

	// Function to save content to file
	private function saveContentToFile($content, $filename)
	{
		$directory = dirname($filename);
		if (!is_dir($directory)) {
			TWXChatUtil::makeDir($directory);
		}
		file_put_contents($filename, $content);
	}

	
	// Function to extract links from HTML content
	private function extractLinksFromHtml($html, $baseUrl, $mustMatch = null)
	{
		$links = [];

		try{
			$dom = new DOMDocument();
			libxml_use_internal_errors(true); // Suppress HTML parsing errors
			$dom->loadHTML($html);
			libxml_clear_errors();

			$baseHost = wp_parse_url($baseUrl, PHP_URL_HOST);
			$baseScheme = wp_parse_url($baseUrl, PHP_URL_SCHEME);

			$anchorTags = $dom->getElementsByTagName('a');
			foreach ($anchorTags as $tag) {
				$href = $tag->getAttribute('href');
				if ($href && strpos($href, '#') !== 0) { // Exclude anchor links
					$absoluteUrl = $this->resolveUrl($baseScheme, $baseHost, $baseUrl, $href);
					if ($absoluteUrl) {
						if($mustMatch !== null && strpos($absoluteUrl, $mustMatch) === false) continue;
						$links[] = $absoluteUrl;
					}
				}
			}
		}catch(Exception $ex){
			TWXChatSpider::log($ex->getMessage());
		}

		return $links;
	}

	private function resolveUrl($baseScheme, $baseHost, $baseUrl, $href)
	{
		if (strpos($href, 'http') === 0) {
			return $href; // Absolute URL
		} elseif (strpos($href, '/') === 0) {
			return "$baseScheme://$baseHost$href"; // Root-relative URL
		} else {
			$basePath = rtrim(wp_parse_url($baseUrl, PHP_URL_PATH), '/') . '/';
			return "$baseScheme://$baseHost/$basePath$href"; // Relative URL
		}
	}

	private function removeAllSubdirs($dir)
	{
		if (!is_dir($dir)) {
			throw new InvalidArgumentException(esc_html($dir) . ' must be a directory');
		}

		$items = scandir($dir);

		foreach ($items as $item) {
			if ($item === '.' || $item === '..') {
				continue;
			}

			$path = $dir . DIRECTORY_SEPARATOR . $item;

			if (is_dir($path)) {
				// Unix-based systems
				if (strtoupper(substr(PHP_OS, 0, 3)) !== 'WIN') {
					exec("rm -rf " . escapeshellarg($path));
				} else {
					// Windows systems
					exec("rmdir /S /Q " . escapeshellarg($path));
				}
			}
		}
	}

	private function isSitemapContent($content) {
		// Check if the content is XML
		libxml_use_internal_errors(true);
		$xml = simplexml_load_string($content);
        libxml_clear_errors();

		if ($xml === false) {
			return false; // Not valid XML
		}

		// Check for common sitemap attributes and elements
		$rootName = $xml->getName();
		$namespaces = $xml->getNamespaces(true);

		// Sitemap namespaces to look for
		$sitemapNamespaces = [
			'http://www.sitemaps.org/schemas/sitemap/0.9',
			'http://www.google.com/schemas/sitemap/0.9'
		];

		// Check if the root element is 'urlset' or 'sitemapindex' and if namespaces match
		if($rootName == 'urlset' || $rootName == 'sitemapindex') return true;
		if(isset($namespaces['']) && in_array($namespaces[''], $sitemapNamespaces)) return true;

		return false;
	}

	private function display_xml_error($error, $xml) {
		$return = $xml[$error->line - 1] . "\n";
		$return .= str_repeat('-', $error->column) . "^\n";

		switch ($error->level) {
			case LIBXML_ERR_WARNING:
				$return .= "Warning $error->code: ";
				break;
			case LIBXML_ERR_ERROR:
				$return .= "Error $error->code: ";
				break;
			case LIBXML_ERR_FATAL:
				$return .= "Fatal Error $error->code: ";
				break;
		}

		$return .= trim($error->message) .
				   "\n  Line: $error->line" .
				   "\n  Column: $error->column";

		if ($error->file) {
			$return .= "\n  File: $error->file";
		}

		return "$return\n\n--------------------------------------------\n\n";
	}
	
	private function getSitemapLinks($content, $mustMatch = null) {
		// Check if the content is XML
		libxml_use_internal_errors(true);
		$xml = simplexml_load_string($content);
		libxml_clear_errors();

		// Initialize an empty array to store the links
		$links = [];

		// Check if the XML is valid
		if ($xml !== false) {
			$rootName = $xml->getName();

			// If the root element is 'urlset', extract links from 'url' elements
			if ($rootName == 'urlset') {
				foreach ($xml->url as $url) {
					$link = (string) $url->loc;
					if($mustMatch !== null && strpos($link, $mustMatch) === false) continue;
					$links[] = $link;
				}
			}
			// If the root element is 'sitemapindex', extract links from 'sitemap' elements
			elseif ($rootName == 'sitemapindex') {
				foreach ($xml->sitemap as $sitemap) {
					$links[] = (string) $sitemap->loc;
				}
			}
		}

		return $links;
	}

	public static function log($message, $mode = 'a') {
		try {
			$dir = TWXChatUtil::getUploadsDir('logs');
			TWXChatUtil::makeDir($dir);
			$logfile = $dir.'twx_spider.log';
			$logEntry = current_time('mysql') . " - " . $message . "\n";
			$append = ($mode === 'a');
			TWXChatUtil::fileWrite($logfile, $logEntry, $append);

		} catch (Exception $ex) {
			// Silently fail as before
		}
	}

	public static function getSpiderLogText(){

		$file_path = TWXChatUtil::getUploadsDir('logs').'twx_spider.log';

		global $wp_filesystem;
		if ( empty( $wp_filesystem ) ) {
			require_once ABSPATH . 'wp-admin/includes/file.php';
			WP_Filesystem();
		}

		if(!$wp_filesystem->exists($file_path)) return '';

		return $wp_filesystem->get_contents($file_path);
	}


	public static function clearProgress() {

		// Clear Spider Progress Log
		try{
			$dir = TWXChatUtil::getUploadsDir('spider_results');
			$logFile = $dir.'twxchatspider_progress.txt';
			if(file_exists($logFile)) wp_delete_file($logFile);
		}catch(Exception $ex){
			// Do nothing
		}
	}	


	public static function appendProgress($message, $andAppendToLog = true) {

		try {
			$dir = TWXChatUtil::getUploadsDir('spider_results');
			TWXChatUtil::makeDir($dir);
			$logfile = $dir.'twxchatspider_progress.txt';
			$logEntry = current_time('mysql') . " - " . $message . "\n";
			TWXChatUtil::fileWrite($logfile, $logEntry, true);

			if($andAppendToLog) TWXChatSpider::log("[progress] $message");

		} catch (Exception $ex) {
			// Silently fail as before
		}
	}	

}

