<?php
/**
 * External Sources Handler class for managing external URL content.
 *
 * This class handles the crawling, processing, and management of external website content
 * for the Ask Any plugin, similar to the External Sources feature.
 *
 * @package Askany
 * @since 1.3.1
 */

namespace Askany;

/**
 * External Sources Handler class
 *
 * @package Askany
 * @since 1.3.1
 */
class ExternalSourcesHandler {
	/**
	 * Database instance for managing external sources data.
	 *
	 * @var Database
	 */
	private $database;

	/**
	 * External sources table name.
	 *
	 * @var string
	 */
	private $external_sources_table;

	/**
	 * Maximum crawl depth.
	 *
	 * @var int
	 */
	private $max_depth = 2;

	/**
	 * Maximum pages to crawl per source.
	 *
	 * @var int
	 */
	private $max_pages = 15;

	/**
	 * Request timeout in seconds.
	 *
	 * @var int
	 */
	private $request_timeout = 15;

	/**
	 * Processing batch size.
	 *
	 * @var int
	 */
	private $batch_size = 5;

	/**
	 * Constructor for ExternalSourcesHandler.
	 *
	 * @param Database $database Database instance.
	 */
	public function __construct( $database ) {
		$this->database = $database;
		global $wpdb;
		$this->external_sources_table = $wpdb->prefix . 'askany_external_sources';
	}

	/**
	 * Initialize the External Sources Handler.
	 *
	 * @return void
	 */
	public function init() {
		// Create external sources table
		$this->create_external_sources_table();
		
		// Run database migrations
		$this->run_database_migrations();

		// Register REST API endpoints
		add_action( 'rest_api_init', array( $this, 'register_rest_routes' ) );

		// Register background processing action
		add_action( 'askany_process_external_url', array( $this, 'process_external_url_background' ), 10, 2 );
	}

	/**
	 * Create the external sources table.
	 *
	 * @return void
	 */
	private function create_external_sources_table() {
		global $wpdb;
		require_once ABSPATH . 'wp-admin/includes/upgrade.php';

		$charset_collate = $wpdb->get_charset_collate();

		$sql = "CREATE TABLE IF NOT EXISTS {$this->external_sources_table} (
			id bigint(20) NOT NULL AUTO_INCREMENT,
			url varchar(500) NOT NULL,
			title varchar(255) DEFAULT '',
			status varchar(20) NOT NULL DEFAULT 'pending',
			follow_links tinyint(1) NOT NULL DEFAULT 0,
			include_paths text DEFAULT '',
			exclude_paths text DEFAULT '',
			include_selectors text DEFAULT '',
			exclude_selectors text DEFAULT '',
			crawled_urls longtext DEFAULT '',
			processing_progress text DEFAULT '',
			last_crawled datetime NULL,
			error_message text DEFAULT '',
			created_at datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
			updated_at datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
			PRIMARY KEY (id),
			KEY url (url(191)),
			KEY status (status),
			KEY last_crawled (last_crawled)
		) $charset_collate;";

		dbDelta( $sql );
	}
	
	/**
	 * Run database migrations for external sources table.
	 * 
	 * This method checks if migrations are needed and applies them.
	 * Uses WordPress options to track which migrations have been run.
	 *
	 * @return void
	 */
	private function run_database_migrations() {
		global $wpdb;
		
		// Get current migration version
		$current_version = get_option( 'askany_external_sources_db_version', '1.0.0' );
		$target_version = '1.1.0'; // Version with processing_progress column
		
		// Check if migration is needed
		if ( version_compare( $current_version, $target_version, '<' ) ) {
			// Check if processing_progress column exists
			$column_exists = $wpdb->get_results( 
				$wpdb->prepare(
					"SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS 
					WHERE TABLE_SCHEMA = %s 
					AND TABLE_NAME = %s 
					AND COLUMN_NAME = 'processing_progress'",
					DB_NAME,
					$this->external_sources_table
				)
			);
			
			// Add processing_progress column if it doesn't exist
			if ( empty( $column_exists ) ) {
				$wpdb->query(
					"ALTER TABLE {$this->external_sources_table} 
					ADD COLUMN processing_progress text DEFAULT '' 
					AFTER crawled_urls"
				);
				
				// Log the migration
				// error_log( 'AskAny: Added processing_progress column to external sources table' );
			}
			
			// Update migration version
			update_option( 'askany_external_sources_db_version', $target_version );
		}
	}

	/**
	 * Register REST API routes for external sources.
	 *
	 * @return void
	 */
	public function register_rest_routes() {
		// Add external source
		register_rest_route( 'askany/v1', '/external-sources', array(
			'methods' => 'POST',
			'callback' => array( $this, 'add_external_source' ),
			'permission_callback' => function() {
				return current_user_can( 'manage_options' );
			},
			'args' => array(
				'url' => array(
					'required' => true,
					'type' => 'string',
					'validate_callback' => function( $param ) {
						return filter_var( $param, FILTER_VALIDATE_URL );
					}
				),
				'follow_links' => array(
					'required' => false,
					'type' => 'boolean',
					'default' => false
				),
				'include_paths' => array(
					'required' => false,
					'type' => 'string',
					'default' => ''
				),
				'exclude_paths' => array(
					'required' => false,
					'type' => 'string',
					'default' => ''
				),
				'include_selectors' => array(
					'required' => false,
					'type' => 'string',
					'default' => ''
				),
				'exclude_selectors' => array(
					'required' => false,
					'type' => 'string',
					'default' => ''
				)
			)
		) );

		// Get external sources
		register_rest_route( 'askany/v1', '/external-sources', array(
			'methods' => 'GET',
			'callback' => array( $this, 'get_external_sources' ),
			'permission_callback' => function() {
				return current_user_can( 'manage_options' );
			}
		) );

		// Delete external source
		register_rest_route( 'askany/v1', '/external-sources/(?P<id>\d+)', array(
			'methods' => 'DELETE',
			'callback' => array( $this, 'delete_external_source' ),
			'permission_callback' => function() {
				return current_user_can( 'manage_options' );
			}
		) );

		// Get crawled URLs for a source
		register_rest_route( 'askany/v1', '/external-sources/(?P<id>\d+)/urls', array(
			'methods' => 'GET',
			'callback' => array( $this, 'get_crawled_urls' ),
			'permission_callback' => function() {
				return current_user_can( 'manage_options' );
			}
		) );

		// Reprocess external source
		register_rest_route( 'askany/v1', '/external-sources/(?P<id>\d+)/reprocess', array(
			'methods' => 'POST',
			'callback' => array( $this, 'reprocess_external_source' ),
			'permission_callback' => function() {
				return current_user_can( 'manage_options' );
			}
		) );

		// Check processing status
		register_rest_route( 'askany/v1', '/external-sources/status', array(
			'methods' => 'GET',
			'callback' => array( $this, 'check_processing_status' ),
			'permission_callback' => function() {
				return current_user_can( 'manage_options' );
			}
		) );
	}

	/**
	 * Add a new external source.
	 *
	 * @param \WP_REST_Request $request The REST request object.
	 * @return \WP_REST_Response|\WP_Error The response object or error.
	 */
	public function add_external_source( $request ) {
		global $wpdb;

		$url = esc_url_raw( $request->get_param( 'url' ) );
		$follow_links = (bool) $request->get_param( 'follow_links' );
		$include_paths = sanitize_text_field( $request->get_param( 'include_paths' ) );
		$exclude_paths = sanitize_text_field( $request->get_param( 'exclude_paths' ) );
		$include_selectors = sanitize_text_field( $request->get_param( 'include_selectors' ) );
		$exclude_selectors = sanitize_text_field( $request->get_param( 'exclude_selectors' ) );

		// Check Pro limits
		$plugin = \Askany\Plugin::get_instance();
		$is_pro = $plugin->is_pro_active();
		$free_limit = 2;
		
		if ( ! $is_pro ) {
			// Count current external sources
			$sources_count = $wpdb->get_var( "SELECT COUNT(*) FROM {$this->external_sources_table}" );
			
			if ( $sources_count >= $free_limit ) {
				return new \WP_Error( 
					'pro_limit_reached', 
					sprintf(
						__( 'Free plan allows up to %d external sources. Upgrade to Pro for unlimited external sources.', 'askany' ),
						$free_limit
					),
					array( 'status' => 403, 'is_pro_limit' => true )
				);
			}
		}

		// Check if URL already exists
		$existing = $wpdb->get_var( $wpdb->prepare(
			"SELECT id FROM {$this->external_sources_table} WHERE url = %s",
			$url
		) );

		if ( $existing ) {
			return new \WP_Error( 'url_exists', __( 'This URL has already been added.', 'askany' ), array( 'status' => 400 ) );
		}

		// Insert new external source
		$result = $wpdb->insert(
			$this->external_sources_table,
			array(
				'url' => $url,
				'status' => 'pending',
				'follow_links' => $follow_links,
				'include_paths' => $include_paths,
				'exclude_paths' => $exclude_paths,
				'include_selectors' => $include_selectors,
				'exclude_selectors' => $exclude_selectors,
				'created_at' => current_time( 'mysql' ),
				'updated_at' => current_time( 'mysql' )
			),
			array( '%s', '%s', '%d', '%s', '%s', '%s', '%s', '%s', '%s' )
		);

		if ( $result === false ) {
			return new \WP_Error( 'insert_failed', __( 'Failed to add external source.', 'askany' ), array( 'status' => 500 ) );
		}

		$source_id = $wpdb->insert_id;

		// Schedule background processing
		wp_schedule_single_event( time() + 5, 'askany_process_external_url', array( $source_id, $url ) );

		return new \WP_REST_Response( array(
			'success' => true,
			'message' => __( 'External source added successfully. Processing will begin shortly.', 'askany' ),
			'source_id' => $source_id
		), 201 );
	}

	/**
	 * Get all external sources.
	 *
	 * @param \WP_REST_Request $request The REST request object.
	 * @return \WP_REST_Response The response object.
	 */
	public function get_external_sources( $request ) {
		global $wpdb;

		$sources = $wpdb->get_results(
			"SELECT * FROM {$this->external_sources_table} ORDER BY created_at DESC"
		);

		// Process crawled URLs for display
		foreach ( $sources as $source ) {
			$source->crawled_urls_array = ! empty( $source->crawled_urls ) ? json_decode( $source->crawled_urls, true ) : array();
			$source->crawled_count = is_array( $source->crawled_urls_array ) ? count( $source->crawled_urls_array ) : 0;
		}

		return new \WP_REST_Response( array(
			'success' => true,
			'sources' => $sources
		) );
	}

	/**
	 * Delete an external source.
	 *
	 * @param \WP_REST_Request $request The REST request object.
	 * @return \WP_REST_Response|\WP_Error The response object or error.
	 */
	public function delete_external_source( $request ) {
		global $wpdb;

		$source_id = (int) $request->get_param( 'id' );

		// Get source details first
		$source = $wpdb->get_row( $wpdb->prepare(
			"SELECT * FROM {$this->external_sources_table} WHERE id = %d",
			$source_id
		) );

		if ( ! $source ) {
			return new \WP_Error( 'source_not_found', __( 'External source not found.', 'askany' ), array( 'status' => 404 ) );
		}

		// Delete related content from content index
		$crawled_urls = ! empty( $source->crawled_urls ) ? json_decode( $source->crawled_urls, true ) : array();
		if ( is_array( $crawled_urls ) ) {
			foreach ( $crawled_urls as $crawled_url ) {
				// Delete from content index using URL as identifier
				$wpdb->delete(
					$wpdb->prefix . 'askany_content_index',
					array( 'url' => $crawled_url, 'object_type' => 'external_url' ),
					array( '%s', '%s' )
				);
			}
		}

		// Delete the external source
		$result = $wpdb->delete(
			$this->external_sources_table,
			array( 'id' => $source_id ),
			array( '%d' )
		);

		if ( $result === false ) {
			return new \WP_Error( 'delete_failed', __( 'Failed to delete external source.', 'askany' ), array( 'status' => 500 ) );
		}

		return new \WP_REST_Response( array(
			'success' => true,
			'message' => __( 'External source deleted successfully.', 'askany' )
		) );
	}

	/**
	 * Get crawled URLs for a specific source.
	 *
	 * @param \WP_REST_Request $request The REST request object.
	 * @return \WP_REST_Response|\WP_Error The response object or error.
	 */
	public function get_crawled_urls( $request ) {
		global $wpdb;

		$source_id = (int) $request->get_param( 'id' );

		$source = $wpdb->get_row( $wpdb->prepare(
			"SELECT crawled_urls FROM {$this->external_sources_table} WHERE id = %d",
			$source_id
		) );

		if ( ! $source ) {
			return new \WP_Error( 'source_not_found', __( 'External source not found.', 'askany' ), array( 'status' => 404 ) );
		}

		$crawled_urls = ! empty( $source->crawled_urls ) ? json_decode( $source->crawled_urls, true ) : array();

		return new \WP_REST_Response( array(
			'success' => true,
			'urls' => is_array( $crawled_urls ) ? $crawled_urls : array()
		) );
	}

	/**
	 * Reprocess an external source.
	 *
	 * @param \WP_REST_Request $request The REST request object.
	 * @return \WP_REST_Response|\WP_Error The response object or error.
	 */
	public function reprocess_external_source( $request ) {
		global $wpdb;

		$source_id = (int) $request->get_param( 'id' );

		$source = $wpdb->get_row( $wpdb->prepare(
			"SELECT * FROM {$this->external_sources_table} WHERE id = %d",
			$source_id
		) );

		if ( ! $source ) {
			return new \WP_Error( 'source_not_found', __( 'External source not found.', 'askany' ), array( 'status' => 404 ) );
		}

		// Update status to pending
		$wpdb->update(
			$this->external_sources_table,
			array( 'status' => 'pending', 'error_message' => '', 'updated_at' => current_time( 'mysql' ) ),
			array( 'id' => $source_id ),
			array( '%s', '%s', '%s' ),
			array( '%d' )
		);

		// Schedule background processing
		wp_schedule_single_event( time() + 5, 'askany_process_external_url', array( $source_id, $source->url ) );

		return new \WP_REST_Response( array(
			'success' => true,
			'message' => __( 'External source reprocessing started.', 'askany' )
		) );
	}

	/**
	 * Check processing status of external sources.
	 *
	 * @param \WP_REST_Request $request The REST request object.
	 * @return \WP_REST_Response The response object.
	 */
	public function check_processing_status( $request ) {
		global $wpdb;

		// Get sources that are pending or processing
		$processing_sources = $wpdb->get_results(
			"SELECT id, url, status FROM {$this->external_sources_table} 
			WHERE status IN ('pending', 'processing')"
		);
		
		// Get total sources count
		$total_sources = $wpdb->get_var(
			"SELECT COUNT(*) FROM {$this->external_sources_table}"
		);

		return new \WP_REST_Response( array(
			'success' => true,
			'processing_sources' => $processing_sources,
			'total_sources' => (int) $total_sources
		) );
	}

	/**
	 * Process external URL in background.
	 *
	 * @param int    $source_id The source ID.
	 * @param string $url       The URL to process.
	 * @return void
	 */
	public function process_external_url_background( $source_id, $url ) {
		global $wpdb;

		// Update status to processing with initial progress
		$wpdb->update(
			$this->external_sources_table,
			array( 
				'status' => 'processing',
				'processing_progress' => json_encode(array(
					'current' => 0,
					'total' => 1,
					'message' => 'Starting crawl...',
					'current_url' => $url
				)),
				'updated_at' => current_time( 'mysql' )
			),
			array( 'id' => $source_id ),
			array( '%s', '%s', '%s' ),
			array( '%d' )
		);

		try {
			// Get source configuration
			$source = $wpdb->get_row( $wpdb->prepare(
				"SELECT * FROM {$this->external_sources_table} WHERE id = %d",
				$source_id
			) );

			if ( ! $source ) {
				throw new \Exception( 'Source not found' );
			}

			// Process the URL with progress tracking
			$result = $this->crawl_and_index_url( $source, $source_id );

			if ( $result['success'] ) {
				// Update status to completed
				$wpdb->update(
					$this->external_sources_table,
					array(
						'status' => 'completed',
						'title' => $result['title'],
						'crawled_urls' => json_encode( $result['crawled_urls'] ),
						'last_crawled' => current_time( 'mysql' ),
						'error_message' => '',
						'processing_progress' => json_encode(array(
							'current' => count($result['crawled_urls']),
							'total' => count($result['crawled_urls']),
							'message' => 'Completed successfully',
							'current_url' => ''
						)),
						'updated_at' => current_time( 'mysql' )
					),
					array( 'id' => $source_id ),
					array( '%s', '%s', '%s', '%s', '%s', '%s', '%s' ),
					array( '%d' )
				);
			} else {
				// Update status to failed
				$wpdb->update(
					$this->external_sources_table,
					array(
						'status' => 'failed',
						'error_message' => $result['error'],
						'processing_progress' => '',
						'updated_at' => current_time( 'mysql' )
					),
					array( 'id' => $source_id ),
					array( '%s', '%s', '%s', '%s' ),
					array( '%d' )
				);
			}
		} catch ( \Exception $e ) {
			// Update status to failed
			$wpdb->update(
				$this->external_sources_table,
				array(
					'status' => 'failed',
					'error_message' => $e->getMessage(),
					'processing_progress' => '',
					'updated_at' => current_time( 'mysql' )
				),
				array( 'id' => $source_id ),
				array( '%s', '%s', '%s', '%s' ),
				array( '%d' )
			);
		}
	}

	/**
	 * Crawl and index a URL.
	 *
	 * @param object $source The source configuration.
	 * @param int $source_id The source ID for progress tracking.
	 * @return array The result array with success status and data.
	 */
	private function crawl_and_index_url( $source, $source_id = null ) {
		global $wpdb;
		$crawled_urls = array();
		$title = '';

		try {
			// Update progress: Fetching main URL
			if ( $source_id ) {
				$this->update_processing_progress( $source_id, 0, 1, 'Fetching main URL...', $source->url );
			}
			
			// Fetch main URL content
			$main_content = $this->fetch_url_content( $source->url );
			
			if ( ! $main_content ) {
				return array(
					'success' => false,
					'error' => 'Failed to fetch main URL content'
				);
			}

			$title = $main_content['title'];
			$crawled_urls[] = $source->url;

			// Update progress: Indexing main URL
			if ( $source_id ) {
				$this->update_processing_progress( $source_id, 1, 1, 'Indexing main URL...', $source->url );
			}
			
			// Index main URL content
			$this->database->index_content(
				md5( $source->url ), // Use URL hash as object_id
				'external_url',
				$main_content['content'],
				$main_content['title'],
				$source->url
			);

			// If follow_links is enabled, crawl linked pages
			if ( $source->follow_links ) {
				// Update progress: Discovering links
				if ( $source_id ) {
					$this->update_processing_progress( $source_id, 1, 1, 'Discovering linked pages...', $source->url );
				}
				
				$linked_urls = $this->discover_linked_urls( $source->url, $main_content['html'], $source );
				$total_urls = min( count( $linked_urls ), $this->max_pages - 1 );
				$current_index = 1; // Already processed main URL
				
				foreach ( array_slice( $linked_urls, 0, $this->max_pages - 1 ) as $linked_url ) {
					// Update progress: Processing linked URL
					if ( $source_id ) {
						$this->update_processing_progress( 
							$source_id, 
							$current_index, 
							$total_urls + 1, 
							sprintf( 'Processing page %d of %d...', $current_index + 1, $total_urls + 1 ),
							$linked_url 
						);
					}
					
					$linked_content = $this->fetch_url_content( $linked_url );
					
					if ( $linked_content ) {
						$crawled_urls[] = $linked_url;
						
						// Index linked URL content
						$this->database->index_content(
							md5( $linked_url ), // Use URL hash as object_id
							'external_url',
							$linked_content['content'],
							$linked_content['title'],
							$linked_url
						);
					}

					$current_index++;
					
					// Small delay to be respectful
					usleep( 500000 ); // 0.5 seconds
				}
			}

			return array(
				'success' => true,
				'title' => $title,
				'crawled_urls' => $crawled_urls
			);

		} catch ( \Exception $e ) {
			return array(
				'success' => false,
				'error' => $e->getMessage()
			);
		}
	}
	
	/**
	 * Update processing progress for a source.
	 *
	 * @param int $source_id The source ID.
	 * @param int $current Current progress count.
	 * @param int $total Total items to process.
	 * @param string $message Progress message.
	 * @param string $current_url Currently processing URL.
	 * @return void
	 */
	private function update_processing_progress( $source_id, $current, $total, $message, $current_url = '' ) {
		global $wpdb;
		
		$progress_data = array(
			'current' => $current,
			'total' => $total,
			'percentage' => $total > 0 ? round( ( $current / $total ) * 100 ) : 0,
			'message' => $message,
			'current_url' => $current_url,
			'timestamp' => current_time( 'mysql' )
		);
		
		$wpdb->update(
			$this->external_sources_table,
			array( 
				'processing_progress' => json_encode( $progress_data ),
				'updated_at' => current_time( 'mysql' )
			),
			array( 'id' => $source_id ),
			array( '%s', '%s' ),
			array( '%d' )
		);
	}

	/**
	 * Fetch content from a URL.
	 *
	 * @param string $url The URL to fetch.
	 * @return array|false The content array or false on failure.
	 */
	private function fetch_url_content( $url ) {
		$response = wp_remote_get( $url, array(
			'timeout' => $this->request_timeout,
			'user-agent' => 'AskAny Bot (+' . home_url() . ')',
			'headers' => array(
				'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
			)
		) );

		if ( is_wp_error( $response ) ) {
			return false;
		}

		$body = wp_remote_retrieve_body( $response );
		$status_code = wp_remote_retrieve_response_code( $response );

		if ( $status_code !== 200 || empty( $body ) ) {
			return false;
		}

		// Parse HTML content
		$dom = new \DOMDocument();
		@$dom->loadHTML( $body );

		// Extract title
		$title_nodes = $dom->getElementsByTagName( 'title' );
		$title = $title_nodes->length > 0 ? $title_nodes->item( 0 )->textContent : '';

		// Extract main content (remove scripts, styles, etc.)
		$this->remove_unwanted_elements( $dom );
		$content = $this->extract_text_content( $dom );

		return array(
			'title' => trim( $title ),
			'content' => trim( $content ),
			'html' => $body
		);
	}

	/**
	 * Discover linked URLs from HTML content.
	 *
	 * @param string $base_url The base URL.
	 * @param string $html     The HTML content.
	 * @param object $source   The source configuration.
	 * @return array Array of discovered URLs.
	 */
	private function discover_linked_urls( $base_url, $html, $source ) {
		$dom = new \DOMDocument();
		@$dom->loadHTML( $html );

		$links = $dom->getElementsByTagName( 'a' );
		$discovered_urls = array();
		$base_domain = parse_url( $base_url, PHP_URL_HOST );

		foreach ( $links as $link ) {
			$href = $link->getAttribute( 'href' );
			
			if ( empty( $href ) || $href === '#' ) {
				continue;
			}

			// Convert relative URLs to absolute
			$absolute_url = $this->make_absolute_url( $href, $base_url );
			
			if ( ! $absolute_url ) {
				continue;
			}

			// Only include URLs from the same domain
			$link_domain = parse_url( $absolute_url, PHP_URL_HOST );
			if ( $link_domain !== $base_domain ) {
				continue;
			}

			// Apply include/exclude path filters
			if ( ! $this->should_include_url( $absolute_url, $source ) ) {
				continue;
			}

			$discovered_urls[] = $absolute_url;
		}

		return array_unique( $discovered_urls );
	}

	/**
	 * Make a URL absolute.
	 *
	 * @param string $url      The URL to make absolute.
	 * @param string $base_url The base URL.
	 * @return string|false The absolute URL or false on failure.
	 */
	private function make_absolute_url( $url, $base_url ) {
		// Already absolute
		if ( parse_url( $url, PHP_URL_SCHEME ) ) {
			return $url;
		}

		$base_parts = parse_url( $base_url );
		
		if ( ! $base_parts ) {
			return false;
		}

		// Protocol-relative URL
		if ( substr( $url, 0, 2 ) === '//' ) {
			return $base_parts['scheme'] . ':' . $url;
		}

		// Absolute path
		if ( substr( $url, 0, 1 ) === '/' ) {
			return $base_parts['scheme'] . '://' . $base_parts['host'] . 
				   ( isset( $base_parts['port'] ) ? ':' . $base_parts['port'] : '' ) . $url;
		}

		// Relative path
		$base_path = isset( $base_parts['path'] ) ? dirname( $base_parts['path'] ) : '';
		if ( $base_path === '.' ) {
			$base_path = '';
		}

		return $base_parts['scheme'] . '://' . $base_parts['host'] . 
			   ( isset( $base_parts['port'] ) ? ':' . $base_parts['port'] : '' ) . 
			   rtrim( $base_path, '/' ) . '/' . $url;
	}

	/**
	 * Check if a URL should be included based on include/exclude filters.
	 *
	 * @param string $url    The URL to check.
	 * @param object $source The source configuration.
	 * @return bool Whether the URL should be included.
	 */
	private function should_include_url( $url, $source ) {
		$path = parse_url( $url, PHP_URL_PATH );
		
		if ( ! $path ) {
			$path = '/';
		}

		// Check include paths
		if ( ! empty( $source->include_paths ) ) {
			$include_paths = array_filter( array_map( 'trim', explode( ',', $source->include_paths ) ) );
			$included = false;
			
			foreach ( $include_paths as $include_path ) {
				if ( strpos( $path, $include_path ) !== false ) {
					$included = true;
					break;
				}
			}
			
			if ( ! $included ) {
				return false;
			}
		}

		// Check exclude paths
		if ( ! empty( $source->exclude_paths ) ) {
			$exclude_paths = array_filter( array_map( 'trim', explode( ',', $source->exclude_paths ) ) );
			
			foreach ( $exclude_paths as $exclude_path ) {
				if ( strpos( $path, $exclude_path ) !== false ) {
					return false;
				}
			}
		}

		return true;
	}

	/**
	 * Remove unwanted elements from DOM.
	 *
	 * @param \DOMDocument $dom The DOM document.
	 * @return void
	 */
	private function remove_unwanted_elements( $dom ) {
		$unwanted_tags = array( 'script', 'style', 'nav', 'header', 'footer', 'aside', 'noscript' );
		
		foreach ( $unwanted_tags as $tag ) {
			$elements = $dom->getElementsByTagName( $tag );
			$to_remove = array();
			
			foreach ( $elements as $element ) {
				$to_remove[] = $element;
			}
			
			foreach ( $to_remove as $element ) {
				if ( $element->parentNode ) {
					$element->parentNode->removeChild( $element );
				}
			}
		}
	}

	/**
	 * Extract text content from DOM.
	 *
	 * @param \DOMDocument $dom The DOM document.
	 * @return string The extracted text content.
	 */
	private function extract_text_content( $dom ) {
		$body = $dom->getElementsByTagName( 'body' )->item( 0 );
		
		if ( ! $body ) {
			return '';
		}

		$text = $body->textContent;
		
		// Clean up whitespace
		$text = preg_replace( '/\s+/', ' ', $text );
		$text = trim( $text );

		return $text;
	}

	/**
	 * Get external sources statistics.
	 *
	 * @return array Statistics array.
	 */
	public function get_stats() {
		global $wpdb;

		return array(
			'total_sources' => (int) $wpdb->get_var( "SELECT COUNT(*) FROM {$this->external_sources_table}" ),
			'completed_sources' => (int) $wpdb->get_var( "SELECT COUNT(*) FROM {$this->external_sources_table} WHERE status = 'completed'" ),
			'processing_sources' => (int) $wpdb->get_var( "SELECT COUNT(*) FROM {$this->external_sources_table} WHERE status = 'processing'" ),
			'failed_sources' => (int) $wpdb->get_var( "SELECT COUNT(*) FROM {$this->external_sources_table} WHERE status = 'failed'" ),
			'total_crawled_urls' => (int) $wpdb->get_var( "SELECT COUNT(*) FROM {$wpdb->prefix}askany_content_index WHERE object_type = 'external_url'" ),
			'last_crawled' => $wpdb->get_var( "SELECT MAX(last_crawled) FROM {$this->external_sources_table}" )
		);
	}
}