<?php

/**
 * WordPress content crawler service.
 *
 * Crawls WordPress posts/pages and extracts content for the knowledge base.
 *
 * @package Inqyra
 */
// If this file is called directly, abort.
if ( !defined( 'WPINC' ) ) {
    die;
}
/**
 * Crawler service class.
 */
class Inqyra_Crawler {
    /**
     * Default chunk size in characters.
     * Smaller chunks (800-1200) work better for semantic search.
     */
    const CHUNK_SIZE = 1000;

    /**
     * Default chunk overlap in characters.
     * ~20% overlap ensures context continuity between chunks.
     */
    const CHUNK_OVERLAP = 200;

    /**
     * Current crawl run ID.
     *
     * @var int
     */
    private $run_id;

    /**
     * Current agent being crawled.
     *
     * @var object
     */
    private $agent;

    /**
     * Crawl statistics.
     *
     * @var array
     */
    private $stats = [
        'posts_processed'     => 0,
        'chunks_created'      => 0,
        'chunks_updated'      => 0,
        'chunks_skipped'      => 0,
        'documents_processed' => 0,
        'errors'              => [],
    ];

    /**
     * Debug logs collected during crawl.
     *
     * @var array
     */
    private $debug_logs = [];

    /**
     * Cached full page HTML from last fetch (avoids double requests).
     *
     * @var string
     */
    private $last_fetched_html = '';

    /**
     * URL of last fetched HTML.
     *
     * @var string
     */
    private $last_fetched_url = '';

    /**
     * Crawl an agent's configured content.
     *
     * @param int $agent_id The agent ID to crawl for.
     * @param int $run_id   Optional existing run ID to use.
     * @return array Crawl statistics.
     */
    public function crawl_agent( $agent_id, $run_id = null ) {
        global $wpdb;
        // Get agent configuration
        // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching -- Agent config needed fresh for crawl.
        $this->agent = $wpdb->get_row( $wpdb->prepare( 'SELECT * FROM %i WHERE id = %d', $wpdb->prefix . 'inqyra_agents', $agent_id ) );
        if ( !$this->agent ) {
            return [
                'error' => 'Agent not found',
            ];
        }
        // Parse crawl configuration
        // Default to common content tags - p, headings, list items, and divs
        $crawl_parts = ( json_decode( $this->agent->crawl_parts, true ) ?: [
            'p',
            'h1',
            'h2',
            'h3',
            'h4',
            'li',
            'td',
            'blockquote',
            'figcaption'
        ] );
        $post_types = array('page');
        // Use existing run ID or create new one
        $this->run_id = ( $run_id ?: $this->start_crawl_run( $agent_id ) );
        try {
            // Delete all existing crawl-generated chunks for this agent to prevent duplicates
            // (e.g. after upgrading from free to premium tier with different crawl scope).
            // This removes website chunks AND linked document chunks (source_id IS NULL),
            // but preserves manually uploaded document chunks (source_id IS NOT NULL).
            // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching -- Bulk delete before re-crawl to prevent duplicates.
            $wpdb->query( $wpdb->prepare( 'DELETE FROM %i
                 WHERE agent_id = %d AND (content_type = \'website\' OR (content_type = \'document\' AND source_id IS NULL))', $wpdb->prefix . 'inqyra_knowledge_chunks', $agent_id ) );
            // Get all published posts of configured types
            $posts = get_posts( [
                'post_type'   => $post_types,
                'post_status' => 'publish',
                'numberposts' => -1,
                'orderby'     => 'modified',
                'order'       => 'DESC',
            ] );
            $total_posts = count( $posts );
            $this->update_crawl_run( [
                'total_posts' => $total_posts,
                'status'      => 'running',
            ] );
            // Process each post
            foreach ( $posts as $post ) {
                try {
                    $this->process_post( $post, $crawl_parts );
                    $this->stats['posts_processed']++;
                    $this->update_crawl_run( [
                        'processed_posts' => $this->stats['posts_processed'],
                    ] );
                } catch ( Exception $e ) {
                    $this->stats['errors'][] = "Post {$post->ID}: " . $e->getMessage();
                    $this->log_error( "Error processing post {$post->ID}: " . $e->getMessage() );
                }
            }
            // Clean up orphaned chunks (posts that were deleted or unpublished)
            $this->cleanup_orphaned_chunks( $agent_id, $post_types );
            // Complete the crawl run
            $this->complete_crawl_run();
        } catch ( Exception $e ) {
            $this->fail_crawl_run( $e->getMessage() );
            return [
                'error'      => $e->getMessage(),
                'debug_logs' => $this->debug_logs,
            ];
        }
        $this->stats['debug_logs'] = $this->debug_logs;
        return $this->stats;
    }

    /**
     * Re-crawl specific posts (incremental update).
     *
     * Unlike crawl_agent(), this does NOT delete all chunks first.
     * It only updates chunks for the given post IDs, using the
     * content hash check to skip unchanged content.
     *
     * @param int   $agent_id The agent ID.
     * @param int[] $post_ids Array of post IDs to re-crawl.
     * @return array Crawl statistics.
     */
    public function crawl_posts( $agent_id, array $post_ids ) {
        global $wpdb;
        // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching -- Agent config needed fresh for crawl.
        $this->agent = $wpdb->get_row( $wpdb->prepare( 'SELECT * FROM %i WHERE id = %d', $wpdb->prefix . 'inqyra_agents', $agent_id ) );
        if ( !$this->agent ) {
            return [
                'error' => 'Agent not found',
            ];
        }
        $crawl_parts = ( json_decode( $this->agent->crawl_parts, true ) ?: [
            'p',
            'h1',
            'h2',
            'h3',
            'h4',
            'li',
            'td',
            'blockquote',
            'figcaption'
        ] );
        foreach ( $post_ids as $post_id ) {
            $post = get_post( $post_id );
            if ( !$post || $post->post_status !== 'publish' ) {
                // Post was unpublished or deleted — remove its chunks
                // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching -- Delete chunks for unpublished/deleted post.
                $wpdb->query( $wpdb->prepare(
                    'DELETE FROM %i
                     WHERE agent_id = %d AND source_id = %d AND content_type = \'website\'',
                    $wpdb->prefix . 'inqyra_knowledge_chunks',
                    $agent_id,
                    $post_id
                ) );
                continue;
            }
            try {
                $this->process_post( $post, $crawl_parts );
                $this->stats['posts_processed']++;
            } catch ( Exception $e ) {
                $this->stats['errors'][] = "Post {$post_id}: " . $e->getMessage();
            }
        }
        $this->stats['debug_logs'] = $this->debug_logs;
        return $this->stats;
    }

    /**
     * Process a single post.
     *
     * @param WP_Post $post       The post to process.
     * @param array   $crawl_parts HTML tags to extract.
     */
    private function process_post( $post, $crawl_parts ) {
        global $wpdb;
        // Skip attachment posts (media library items)
        if ( $post->post_type === 'attachment' ) {
            $this->stats['chunks_skipped']++;
            return;
        }
        // Extract content from post
        $extracted = $this->extract_content( $post, $crawl_parts );
        $title = $extracted['title'];
        $content = $extracted['content'];
        if ( empty( trim( $content ) ) ) {
            $this->stats['chunks_skipped']++;
            return;
        }
        // Skip if content looks like WordPress attachment metadata
        if ( $this->is_attachment_metadata( $content ) ) {
            $this->log_error( "Skipping post {$post->ID}: content appears to be attachment metadata" );
            $this->stats['chunks_skipped']++;
            return;
        }
        // Normalize source_type to valid enum value
        $source_type = $this->normalize_source_type( $post->post_type );
        $permalink = get_permalink( $post );
        // Calculate content hash (include title so changes are detected)
        $content_hash = md5( $title . $content );
        // First, clean up any orphaned chunks with wrong source_type (by URL)
        // This fixes duplicates from before the source_type normalization
        // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching -- Check for orphaned chunks with wrong source_type.
        $orphaned = $wpdb->get_var( $wpdb->prepare(
            'SELECT COUNT(*) FROM %i
             WHERE agent_id = %d AND source_url = %s AND source_type != %s',
            $wpdb->prefix . 'inqyra_knowledge_chunks',
            $this->agent->id,
            $permalink,
            $source_type
        ) );
        if ( $orphaned > 0 ) {
            // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching -- Clean up orphaned chunks.
            $wpdb->query( $wpdb->prepare(
                'DELETE FROM %i
                 WHERE agent_id = %d AND source_url = %s AND source_type != %s',
                $wpdb->prefix . 'inqyra_knowledge_chunks',
                $this->agent->id,
                $permalink,
                $source_type
            ) );
            $this->log_error( "Cleaned up {$orphaned} orphaned chunks for post {$post->ID}" );
        }
        // Check if content has changed (use source_url for more reliable matching)
        // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching -- Check content hash to skip unchanged posts.
        $existing_hash = $wpdb->get_var( $wpdb->prepare(
            'SELECT content_hash FROM %i
             WHERE agent_id = %d AND source_url = %s AND chunk_index = 0
             LIMIT 1',
            $wpdb->prefix . 'inqyra_knowledge_chunks',
            $this->agent->id,
            $permalink
        ) );
        if ( $existing_hash === $content_hash ) {
            $this->stats['chunks_skipped']++;
            // Content unchanged, but still check for linked documents
            // Get existing chunk IDs for linking
            // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching -- Fetch existing chunk IDs for document linking.
            $existing_chunk_ids = $wpdb->get_col( $wpdb->prepare(
                'SELECT id FROM %i
                 WHERE agent_id = %d AND source_url = %s AND content_type = \'website\'
                 ORDER BY chunk_index ASC',
                $wpdb->prefix . 'inqyra_knowledge_chunks',
                $this->agent->id,
                $permalink
            ) );
            if ( !empty( $existing_chunk_ids ) ) {
                $this->process_linked_documents( $post, $permalink, $existing_chunk_ids );
            }
            return;
        }
        // Use a transaction to ensure chunks are not lost if insertion fails mid-way
        $chunks = $this->chunk_text( $content );
        $url_path = $this->extract_url_path( $permalink );
        $website_chunk_ids = [];
        // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching -- Transaction start for atomic chunk replacement.
        $wpdb->query( 'START TRANSACTION' );
        try {
            // Delete all existing chunks for this URL
            // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching -- Delete old chunks inside transaction.
            $wpdb->query( $wpdb->prepare(
                'DELETE FROM %i
                 WHERE agent_id = %d AND source_url = %s',
                $wpdb->prefix . 'inqyra_knowledge_chunks',
                $this->agent->id,
                $permalink
            ) );
            // Create new chunks
            foreach ( $chunks as $index => $chunk_content ) {
                // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery -- Insert chunk inside transaction.
                $result = $wpdb->insert( $wpdb->prefix . 'inqyra_knowledge_chunks', [
                    'agent_id'        => $this->agent->id,
                    'source_type'     => $source_type,
                    'source_id'       => $post->ID,
                    'source_url'      => $permalink,
                    'url_path'        => $url_path,
                    'title'           => $title,
                    'content_type'    => 'website',
                    'chunk_index'     => $index,
                    'content'         => $chunk_content,
                    'content_hash'    => $content_hash,
                    'needs_embedding' => 1,
                    'created_at'      => current_time( 'mysql' ),
                    'updated_at'      => current_time( 'mysql' ),
                ], [
                    '%d',
                    '%s',
                    '%d',
                    '%s',
                    '%s',
                    '%s',
                    '%s',
                    '%d',
                    '%s',
                    '%s',
                    '%d',
                    '%s',
                    '%s'
                ] );
                if ( $result === false ) {
                    throw new Exception(esc_html( "Failed to insert chunk {$index}: " . $wpdb->last_error ));
                }
                $website_chunk_ids[] = $wpdb->insert_id;
                $this->stats['chunks_created']++;
            }
            // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching -- Transaction commit.
            $wpdb->query( 'COMMIT' );
        } catch ( Exception $e ) {
            // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching -- Transaction rollback on failure.
            $wpdb->query( 'ROLLBACK' );
            $this->log_error( "Transaction failed for post {$post->ID}: " . $e->getMessage() );
            return;
        }
        // Process linked documents (outside transaction - these are independent)
        if ( !empty( $website_chunk_ids ) ) {
            $this->process_linked_documents( $post, $permalink, $website_chunk_ids );
        }
    }

    /**
     * Process documents linked from a page.
     *
     * @param WP_Post $post              The post being crawled.
     * @param string  $permalink         The post permalink.
     * @param array   $website_chunk_ids The chunk IDs created for this post.
     */
    private function process_linked_documents( $post, $permalink, $website_chunk_ids ) {
        // Document processing is premium-only
        if ( !class_exists( 'Inqyra_Document_Processor' ) ) {
            return;
        }
        // Initialize document processor
        $doc_processor = new Inqyra_Document_Processor();
        $doc_links = [];
        // 1. Extract document links from HTML (use cached HTML from fetch_frontend_html if available)
        $html = ( $this->last_fetched_url === $permalink && !empty( $this->last_fetched_html ) ? $this->last_fetched_html : $this->fetch_full_page_html( $permalink ) );
        if ( !empty( $html ) ) {
            $html_links = $doc_processor->extract_document_links( $html, $permalink );
            $doc_links = array_merge( $doc_links, $html_links );
            $this->log_error( "Found " . count( $html_links ) . " document links in HTML on {$permalink}" );
        }
        // 2. Extract document links from ACF fields
        $acf_links = $this->extract_acf_document_links( $post->ID, $permalink );
        if ( !empty( $acf_links ) ) {
            $doc_links = array_merge( $doc_links, $acf_links );
            $this->log_error( "Found " . count( $acf_links ) . " document links in ACF fields on {$permalink}" );
        }
        // Remove duplicates by URL
        $seen_urls = [];
        $unique_links = [];
        foreach ( $doc_links as $link ) {
            if ( !isset( $seen_urls[$link['url']] ) ) {
                $seen_urls[$link['url']] = true;
                $unique_links[] = $link;
            }
        }
        $doc_links = $unique_links;
        $this->log_error( "Total unique document links: " . count( $doc_links ) );
        if ( empty( $doc_links ) ) {
            return;
        }
        foreach ( $doc_links as $doc_link ) {
            try {
                $this->log_error( "Processing linked document: {$doc_link['url']}" );
                // Process the document
                $result = $doc_processor->process_url( $this->agent->id, $doc_link['url'], $doc_link['text'] );
                if ( isset( $result['error'] ) ) {
                    $this->log_error( "Document processing failed: {$result['error']}" );
                    continue;
                }
                // Create links between website chunks and document chunks
                if ( !empty( $result['chunk_ids'] ) ) {
                    $doc_processor->create_chunk_links(
                        $website_chunk_ids,
                        $result['chunk_ids'],
                        $doc_link['url'],
                        $doc_link['text']
                    );
                    if ( !isset( $result['unchanged'] ) || !$result['unchanged'] ) {
                        $this->stats['documents_processed'] = ($this->stats['documents_processed'] ?? 0) + 1;
                    }
                }
            } catch ( Exception $e ) {
                $this->log_error( "Error processing document {$doc_link['url']}: " . $e->getMessage() );
            }
            // Collect document processor logs
            $doc_logs = Inqyra_Document_Processor::get_and_clear_logs();
            foreach ( $doc_logs as $log ) {
                $this->debug_logs[] = $log;
            }
        }
    }

    /**
     * Extract content from a post.
     *
     * @param WP_Post $post       The post to extract from.
     * @param array   $tags       HTML tags to extract.
     * @return array Array with 'title' and 'content' keys.
     */
    private function extract_content( $post, $tags ) {
        // Primary method: Fetch the front-end rendered HTML
        // This captures all content regardless of how it's stored (ACF, page builders, etc.)
        $html = $this->fetch_frontend_html( $post );
        // Fallback: if fetch failed, use standard post content + ACF
        if ( empty( $html ) ) {
            // phpcs:ignore WordPress.NamingConventions.PrefixAllGlobals.NonPrefixedHooknameFound -- Applying core WordPress filter, not defining a custom hook.
            $html = apply_filters( 'the_content', $post->post_content );
            // Also try to get content from ACF fields if available
            $acf_content = $this->extract_acf_content( $post->ID );
            if ( !empty( $acf_content ) ) {
                $html .= "\n\n" . $acf_content;
            }
        }
        // Remove scripts and styles
        $html = preg_replace( '/<script\\b[^>]*>(.*?)<\\/script>/is', '', $html );
        $html = preg_replace( '/<style\\b[^>]*>(.*?)<\\/style>/is', '', $html );
        // Parse HTML
        $dom = new DOMDocument();
        libxml_use_internal_errors( true );
        $dom->loadHTML( '<?xml encoding="utf-8"?>' . $html, LIBXML_HTML_NOIMPLIED | LIBXML_HTML_NODEFDTD );
        libxml_clear_errors();
        $content_parts = [];
        // Store title separately (not in content)
        $title = $post->post_title;
        // Also extract meta description if available (add early)
        $meta_description = get_post_meta( $post->ID, '_yoast_wpseo_metadesc', true );
        if ( empty( $meta_description ) ) {
            $meta_description = get_post_meta( $post->ID, '_aioseop_description', true );
        }
        if ( !empty( $meta_description ) ) {
            $content_parts[] = $meta_description;
        }
        // Extract content from specified tags
        $found_content = false;
        $title_normalized = strtolower( trim( $title ) );
        foreach ( $tags as $tag ) {
            $elements = $dom->getElementsByTagName( $tag );
            foreach ( $elements as $element ) {
                $text = $this->get_text_content( $element );
                $text = trim( $text );
                // Headings carry semantic meaning even when short (e.g. "About us", "Contact")
                $is_heading = in_array( $tag, [
                    'h1',
                    'h2',
                    'h3',
                    'h4',
                    'h5',
                    'h6'
                ] );
                $min_length = ( $is_heading ? 2 : 10 );
                if ( !empty( $text ) && strlen( $text ) > $min_length ) {
                    // Skip H1 that matches the page title (already stored separately)
                    if ( $tag === 'h1' && strtolower( $text ) === $title_normalized ) {
                        continue;
                    }
                    $found_content = true;
                    // Add tag context for headings
                    if ( $is_heading ) {
                        $content_parts[] = "[{$tag}] " . $text;
                    } else {
                        $content_parts[] = $text;
                    }
                }
            }
        }
        // Fallback: if no content found from specified tags, extract all text
        if ( !$found_content && !empty( $html ) ) {
            $fallback_text = $this->extract_all_text( $dom );
            if ( !empty( $fallback_text ) && strlen( $fallback_text ) > 20 ) {
                $content_parts[] = $fallback_text;
                $found_content = true;
            }
        }
        // Second fallback: use raw post content stripped of HTML
        if ( !$found_content && !empty( $post->post_content ) ) {
            $raw_text = wp_strip_all_tags( $post->post_content );
            $raw_text = preg_replace( '/\\s+/', ' ', $raw_text );
            $raw_text = trim( $raw_text );
            if ( !empty( $raw_text ) && strlen( $raw_text ) > 20 ) {
                $content_parts[] = $raw_text;
            }
        }
        // Also try to get content from post excerpt (add at beginning of content)
        if ( !empty( $post->post_excerpt ) ) {
            $excerpt = trim( $post->post_excerpt );
            if ( !empty( $excerpt ) && strlen( $excerpt ) > 10 ) {
                // Add at the beginning of content parts
                array_unshift( $content_parts, $excerpt );
            }
        }
        // Filter out content parts that are just URLs or image metadata
        $content_parts = array_filter( $content_parts, function ( $part ) {
            // Keep heading markers but check for URL-only content
            if ( preg_match( '/^\\[h[1-6]\\]/', $part ) ) {
                $content_after_marker = preg_replace( '/^\\[[^\\]]+\\]\\s*/', '', $part );
                if ( $this->is_mostly_urls( $content_after_marker ) ) {
                    return false;
                }
                return true;
            }
            // Skip parts that are mostly URLs or image paths
            if ( $this->is_mostly_urls( $part ) ) {
                return false;
            }
            return true;
        } );
        // Deduplicate content parts to avoid repeated paragraphs
        // Use normalized version for comparison but keep original
        $seen = [];
        $seen_words = [];
        // Track all words we've seen for condensed duplicate detection
        $unique_parts = [];
        foreach ( $content_parts as $part ) {
            // Normalize for comparison: lowercase, collapse whitespace
            $normalized = strtolower( preg_replace( '/\\s+/', ' ', trim( $part ) ) );
            // Skip very short parts that might be common phrases
            if ( strlen( $normalized ) < 30 ) {
                $unique_parts[] = $part;
                // Still track words from short parts
                $words = preg_split( '/\\s+/', $normalized );
                foreach ( $words as $word ) {
                    if ( strlen( $word ) > 3 ) {
                        $seen_words[$word] = true;
                    }
                }
                continue;
            }
            // Check for exact or near-duplicate
            $is_duplicate = false;
            foreach ( $seen as $seen_text ) {
                // Exact match
                if ( $normalized === $seen_text ) {
                    $is_duplicate = true;
                    break;
                }
                // Check if one is substring of the other (covers overlap cases)
                if ( strlen( $normalized ) > 50 && strlen( $seen_text ) > 50 ) {
                    if ( strpos( $seen_text, $normalized ) !== false || strpos( $normalized, $seen_text ) !== false ) {
                        $is_duplicate = true;
                        break;
                    }
                }
            }
            // Check for condensed duplicates: if 80%+ of words already seen, skip it
            // This catches cases like "A B C D" when we already have "A", "B", "C", "D" separately
            if ( !$is_duplicate && !empty( $seen_words ) ) {
                $words = preg_split( '/\\s+/', $normalized );
                $words = array_filter( $words, function ( $w ) {
                    return strlen( $w ) > 3;
                } );
                if ( count( $words ) >= 3 ) {
                    $overlap_count = 0;
                    foreach ( $words as $word ) {
                        if ( isset( $seen_words[$word] ) ) {
                            $overlap_count++;
                        }
                    }
                    // If 80%+ of significant words already seen, it's likely a condensed duplicate
                    if ( $overlap_count / count( $words ) >= 0.8 ) {
                        $is_duplicate = true;
                    }
                }
            }
            if ( !$is_duplicate ) {
                $seen[] = $normalized;
                // Track words for condensed duplicate detection
                $words = preg_split( '/\\s+/', $normalized );
                foreach ( $words as $word ) {
                    if ( strlen( $word ) > 3 ) {
                        $seen_words[$word] = true;
                    }
                }
                $unique_parts[] = $part;
            }
        }
        return [
            'title'   => $title,
            'content' => implode( "\n\n", $unique_parts ),
        ];
    }

    /**
     * Check if a text string is mostly URLs or file paths.
     *
     * @param string $text The text to check.
     * @return bool True if text is mostly URLs.
     */
    private function is_mostly_urls( $text ) {
        // Count URLs
        $url_count = preg_match_all( '/https?:\\/\\/[^\\s]+/i', $text );
        // Remove URLs and see what's left
        $stripped = preg_replace( '/https?:\\/\\/[^\\s]+/i', '', $text );
        // Remove common image extensions that might appear as fragments
        $stripped = preg_replace( '/\\b[\\w\\-]+\\.(jpg|jpeg|png|gif|webp|svg|pdf)\\b/i', '', $stripped );
        // Remove dimension patterns like 150x150, 1024x768
        $stripped = preg_replace( '/\\b\\d+x\\d+\\b/', '', $stripped );
        $stripped = preg_replace( '/\\s+/', ' ', $stripped );
        $stripped = trim( $stripped );
        // If we have URLs and very little meaningful text left
        if ( $url_count >= 2 && strlen( $stripped ) < 50 ) {
            return true;
        }
        // If URLs significantly outnumber words
        $word_count = str_word_count( $stripped );
        if ( $url_count >= 3 && $url_count > $word_count ) {
            return true;
        }
        return false;
    }

    /**
     * Last fetch timestamp for rate limiting.
     *
     * @var float
     */
    private $last_fetch_time = 0;

    /**
     * Apply rate limiting delay between HTTP requests.
     */
    private function apply_rate_limit() {
        $now = microtime( true );
        $elapsed = $now - $this->last_fetch_time;
        // Wait at least 300ms between requests to avoid 429 errors
        if ( $elapsed < 0.3 && $this->last_fetch_time > 0 ) {
            usleep( (int) ((0.3 - $elapsed) * 1000000) );
        }
        $this->last_fetch_time = microtime( true );
    }

    /**
     * Fetch the front-end rendered HTML of a post.
     *
     * This method retrieves the actual HTML that visitors see,
     * capturing all content regardless of storage method (ACF, page builders, etc.)
     *
     * @param WP_Post $post The post to fetch.
     * @return string The HTML content, or empty string on failure.
     */
    private function fetch_frontend_html( $post ) {
        $url = get_permalink( $post );
        if ( empty( $url ) ) {
            return '';
        }
        // SSRF protection: validate URL is safe to fetch
        if ( !$this->is_safe_url( $url ) ) {
            $this->log_error( "URL blocked by SSRF protection: {$url}" );
            return '';
        }
        // Apply rate limiting
        $this->apply_rate_limit();
        // Use wp_remote_get to fetch the page (with retry on 429)
        $max_retries = 2;
        $response = null;
        for ($attempt = 0; $attempt <= $max_retries; $attempt++) {
            $response = wp_remote_get( $url, [
                'timeout'    => 30,
                'sslverify'  => $this->should_verify_ssl(),
                'user-agent' => 'Inqyra Crawler/1.0 (WordPress Plugin)',
                'headers'    => [
                    'Accept' => 'text/html',
                ],
            ] );
            if ( is_wp_error( $response ) ) {
                $this->log_error( "Failed to fetch {$url}: " . $response->get_error_message() );
                return '';
            }
            $status_code = wp_remote_retrieve_response_code( $response );
            // Retry on 429 with exponential backoff
            if ( $status_code === 429 && $attempt < $max_retries ) {
                $wait_time = pow( 2, $attempt + 1 );
                // 2s, 4s
                $this->log_error( "HTTP 429 on {$url}, waiting {$wait_time}s before retry..." );
                sleep( $wait_time );
                continue;
            }
            break;
        }
        $status_code = wp_remote_retrieve_response_code( $response );
        if ( $status_code !== 200 ) {
            $this->log_error( "HTTP {$status_code} when fetching {$url}" );
            return '';
        }
        $html = wp_remote_retrieve_body( $response );
        // Cache the full HTML for document link extraction later
        $this->last_fetched_html = $html;
        $this->last_fetched_url = $url;
        // Extract just the main content area if possible
        // Try to find the <main> tag or common content wrappers
        $main_content = $this->extract_main_content( $html );
        return ( $main_content ?: $html );
    }

    /**
     * Fetch the full page HTML (including header, footer, sidebar).
     *
     * Used for document link extraction where links may be anywhere on the page.
     *
     * @param string $url The URL to fetch.
     * @return string The full HTML content, or empty string on failure.
     */
    private function fetch_full_page_html( $url ) {
        if ( empty( $url ) ) {
            return '';
        }
        // SSRF protection: validate URL is safe to fetch
        if ( !$this->is_safe_url( $url ) ) {
            $this->log_error( "URL blocked by SSRF protection: {$url}" );
            return '';
        }
        // Apply rate limiting
        $this->apply_rate_limit();
        // Use wp_remote_get to fetch the page
        $response = wp_remote_get( $url, [
            'timeout'    => 30,
            'sslverify'  => $this->should_verify_ssl(),
            'user-agent' => 'Inqyra Crawler/1.0 (WordPress Plugin)',
            'headers'    => [
                'Accept' => 'text/html',
            ],
        ] );
        if ( is_wp_error( $response ) ) {
            $this->log_error( "Failed to fetch {$url}: " . $response->get_error_message() );
            return '';
        }
        $status_code = wp_remote_retrieve_response_code( $response );
        if ( $status_code !== 200 ) {
            $this->log_error( "HTTP {$status_code} when fetching {$url}" );
            return '';
        }
        // Return the FULL HTML without extracting main content
        return wp_remote_retrieve_body( $response );
    }

    /**
     * Extract the main content area from full page HTML.
     *
     * Tries to identify and extract only the main content,
     * excluding header, footer, sidebar, navigation, etc.
     *
     * @param string $html Full page HTML.
     * @return string Main content HTML, or empty string if not found.
     */
    private function extract_main_content( $html ) {
        // Parse the HTML
        $dom = new DOMDocument();
        libxml_use_internal_errors( true );
        $dom->loadHTML( '<?xml encoding="utf-8"?>' . $html, LIBXML_HTML_NOIMPLIED | LIBXML_HTML_NODEFDTD );
        libxml_clear_errors();
        $xpath = new DOMXPath($dom);
        // Try various selectors for main content areas (in order of preference)
        $selectors = [
            '//main',
            '//article',
            '//*[contains(@class, "entry-content")]',
            '//*[contains(@class, "post-content")]',
            '//*[contains(@class, "page-content")]',
            '//*[contains(@class, "content-area")]',
            '//*[@id="content"]',
            '//*[@role="main"]'
        ];
        foreach ( $selectors as $selector ) {
            $nodes = $xpath->query( $selector );
            if ( $nodes && $nodes->length > 0 ) {
                // Find the largest/main content node (by text length)
                // This avoids including multiple article elements (e.g., related articles)
                $best_node = null;
                $best_length = 0;
                foreach ( $nodes as $node ) {
                    $text_length = strlen( trim( wp_strip_all_tags( $dom->saveHTML( $node ) ) ) );
                    if ( $text_length > $best_length ) {
                        $best_length = $text_length;
                        $best_node = $node;
                    }
                }
                if ( $best_node && $best_length > 50 ) {
                    return $dom->saveHTML( $best_node );
                }
            }
        }
        // If no main content area found, try to remove common non-content elements
        $remove_selectors = [
            '//header',
            '//footer',
            '//nav',
            '//*[contains(@class, "sidebar")]',
            '//*[contains(@class, "menu")]',
            '//*[contains(@class, "navigation")]',
            '//*[@id="wpadminbar"]',
            '//*[contains(@class, "cookie")]',
            '//script',
            '//style',
            '//noscript'
        ];
        foreach ( $remove_selectors as $selector ) {
            $nodes = $xpath->query( $selector );
            if ( $nodes ) {
                foreach ( $nodes as $node ) {
                    if ( $node->parentNode ) {
                        $node->parentNode->removeChild( $node );
                    }
                }
            }
        }
        // Return the body content
        $body = $dom->getElementsByTagName( 'body' )->item( 0 );
        if ( $body ) {
            return $dom->saveHTML( $body );
        }
        return '';
    }

    /**
     * Extract all text content from DOM, ignoring structure.
     *
     * @param DOMDocument $dom The DOM document.
     * @return string All text content.
     */
    private function extract_all_text( $dom ) {
        $body = $dom->getElementsByTagName( 'body' )->item( 0 );
        if ( !$body ) {
            // If no body, get text from the entire document
            $text = $dom->textContent;
        } else {
            $text = $this->get_text_content( $body );
        }
        // Clean up whitespace
        $text = preg_replace( '/\\s+/', ' ', $text );
        $text = trim( $text );
        return $text;
    }

    /**
     * Extract document links from ACF fields.
     *
     * @param int    $post_id  The post ID.
     * @param string $base_url The base URL for resolving relative links.
     * @return array Array of document links with url and text.
     */
    private function extract_acf_document_links( $post_id, $base_url ) {
        // Check if ACF is active
        if ( !function_exists( 'get_fields' ) ) {
            return [];
        }
        $fields = get_fields( $post_id );
        if ( empty( $fields ) || !is_array( $fields ) ) {
            return [];
        }
        $doc_links = [];
        $this->extract_acf_document_links_recursive( $fields, $doc_links, $base_url );
        return $doc_links;
    }

    /**
     * Recursively extract document URLs from ACF fields.
     *
     * @param mixed  $data      The field data.
     * @param array  &$doc_links Array to collect document links.
     * @param string $base_url  The base URL for resolving relative links.
     * @param int    $depth     Current recursion depth.
     */
    private function extract_acf_document_links_recursive(
        $data,
        &$doc_links,
        $base_url,
        $depth = 0
    ) {
        // Prevent infinite recursion
        if ( $depth > 10 ) {
            return;
        }
        if ( is_string( $data ) ) {
            // Check if it's a document URL
            if ( $this->is_document_url_string( $data ) ) {
                $full_url = $this->resolve_relative_url( $data, $base_url );
                if ( $full_url && $this->is_same_domain_url( $full_url, $base_url ) ) {
                    $doc_links[] = [
                        'url'  => $full_url,
                        'text' => basename( wp_parse_url( $full_url, PHP_URL_PATH ) ),
                    ];
                    $this->log_error( "Found ACF document URL: {$full_url}" );
                }
            }
        } elseif ( is_array( $data ) ) {
            // Check for ACF file field format: array with 'url' key
            if ( isset( $data['url'] ) && is_string( $data['url'] ) ) {
                $url = $data['url'];
                if ( $this->is_document_url_string( $url ) ) {
                    $full_url = $this->resolve_relative_url( $url, $base_url );
                    if ( $full_url && $this->is_same_domain_url( $full_url, $base_url ) ) {
                        // Use title or filename as link text
                        $link_text = $data['title'] ?? $data['filename'] ?? basename( wp_parse_url( $full_url, PHP_URL_PATH ) );
                        $doc_links[] = [
                            'url'  => $full_url,
                            'text' => $link_text,
                        ];
                        $this->log_error( "Found ACF file field document: {$full_url} ({$link_text})" );
                    }
                }
            }
            // Check for ACF link field format: array with 'url' key
            if ( isset( $data['link'] ) && is_string( $data['link'] ) ) {
                $url = $data['link'];
                if ( $this->is_document_url_string( $url ) ) {
                    $full_url = $this->resolve_relative_url( $url, $base_url );
                    if ( $full_url && $this->is_same_domain_url( $full_url, $base_url ) ) {
                        $link_text = $data['title'] ?? basename( wp_parse_url( $full_url, PHP_URL_PATH ) );
                        $doc_links[] = [
                            'url'  => $full_url,
                            'text' => $link_text,
                        ];
                        $this->log_error( "Found ACF link field document: {$full_url}" );
                    }
                }
            }
            // Recursively process all array values
            foreach ( $data as $key => $value ) {
                // Skip keys that are clearly not document-related
                if ( is_string( $key ) && in_array( strtolower( $key ), [
                    'id',
                    'ID',
                    'width',
                    'height',
                    'sizes',
                    'mime_type'
                ] ) ) {
                    continue;
                }
                $this->extract_acf_document_links_recursive(
                    $value,
                    $doc_links,
                    $base_url,
                    $depth + 1
                );
            }
        }
    }

    /**
     * Check if a string looks like a document URL.
     *
     * @param string $str The string to check.
     * @return bool True if it looks like a document URL.
     */
    private function is_document_url_string( $str ) {
        if ( empty( $str ) || !is_string( $str ) ) {
            return false;
        }
        // Check for document extensions
        $extensions = ['pdf', 'docx', 'doc'];
        foreach ( $extensions as $ext ) {
            if ( preg_match( '/\\.' . preg_quote( $ext, '/' ) . '(\\?|$)/i', $str ) ) {
                return true;
            }
        }
        return false;
    }

    /**
     * Resolve a relative URL to absolute.
     *
     * @param string $url      The URL (possibly relative).
     * @param string $base_url The base URL.
     * @return string|null The resolved URL or null.
     */
    private function resolve_relative_url( $url, $base_url ) {
        // Already absolute
        if ( preg_match( '/^https?:\\/\\//i', $url ) ) {
            return $url;
        }
        // Protocol-relative
        if ( strpos( $url, '//' ) === 0 ) {
            $scheme = ( wp_parse_url( $base_url, PHP_URL_SCHEME ) ?: 'https' );
            return $scheme . ':' . $url;
        }
        // Parse base URL
        $base = wp_parse_url( $base_url );
        if ( empty( $base['scheme'] ) || empty( $base['host'] ) ) {
            return null;
        }
        $root = $base['scheme'] . '://' . $base['host'];
        if ( !empty( $base['port'] ) ) {
            $root .= ':' . $base['port'];
        }
        // Absolute path
        if ( strpos( $url, '/' ) === 0 ) {
            return $root . $url;
        }
        // Relative path
        $path = ( !empty( $base['path'] ) ? $base['path'] : '/' );
        $path = dirname( $path );
        if ( $path === '\\' || $path === '.' ) {
            $path = '/';
        }
        return $root . rtrim( $path, '/' ) . '/' . $url;
    }

    /**
     * Check if two URLs are on the same domain.
     *
     * @param string $url1 First URL.
     * @param string $url2 Second URL.
     * @return bool True if same domain.
     */
    private function is_same_domain_url( $url1, $url2 ) {
        $host1 = wp_parse_url( $url1, PHP_URL_HOST );
        $host2 = wp_parse_url( $url2, PHP_URL_HOST );
        return strtolower( $host1 ) === strtolower( $host2 );
    }

    /**
     * Extract content from ACF (Advanced Custom Fields) if available.
     *
     * @param int $post_id The post ID.
     * @return string Extracted ACF content.
     */
    private function extract_acf_content( $post_id ) {
        // Check if ACF is active
        if ( !function_exists( 'get_fields' ) ) {
            return '';
        }
        $fields = get_fields( $post_id );
        if ( empty( $fields ) || !is_array( $fields ) ) {
            return '';
        }
        $content_parts = [];
        $this->extract_acf_fields_recursive( $fields, $content_parts );
        return implode( "\n\n", $content_parts );
    }

    /**
     * Recursively extract text content from ACF fields.
     *
     * @param mixed $data          The field data (can be array, string, etc.).
     * @param array &$content_parts Array to collect content parts.
     * @param int   $depth         Current recursion depth.
     */
    private function extract_acf_fields_recursive( $data, &$content_parts, $depth = 0 ) {
        // Prevent infinite recursion
        if ( $depth > 10 ) {
            return;
        }
        if ( is_string( $data ) ) {
            // Clean HTML and extract text
            $text = wp_strip_all_tags( $data );
            $text = html_entity_decode( $text, ENT_QUOTES, 'UTF-8' );
            $text = preg_replace( '/\\s+/', ' ', $text );
            $text = trim( $text );
            // Only add if meaningful content (more than just a few characters)
            if ( !empty( $text ) && strlen( $text ) > 20 ) {
                $content_parts[] = $text;
            }
        } elseif ( is_array( $data ) ) {
            foreach ( $data as $key => $value ) {
                // Skip certain field types that don't contain useful text
                if ( is_string( $key ) ) {
                    $skip_keys = [
                        'image',
                        'images',
                        'file',
                        'files',
                        'url',
                        'link',
                        'video',
                        'gallery',
                        'id',
                        'ID',
                        'post_id',
                        'attachment_id',
                        'media',
                        'thumbnail',
                        'icon',
                        'logo',
                        'background',
                        'background_image',
                        'featured_image',
                        'avatar',
                        'photo',
                        'picture',
                        'banner',
                        'header_image',
                        'cover',
                        'poster',
                        'src',
                        'href',
                        'alt',
                        'width',
                        'height',
                        'mime_type',
                        'filesize',
                        'sizes',
                        'filename',
                        'caption',
                        'description',
                        'title'
                    ];
                    if ( in_array( strtolower( $key ), array_map( 'strtolower', $skip_keys ) ) ) {
                        continue;
                    }
                    // Also skip keys containing common image/file patterns
                    if ( preg_match( '/(image|file|url|link|media|thumb|icon|logo|photo|picture|avatar|banner|poster|src|href)_?/i', $key ) ) {
                        continue;
                    }
                }
                // Skip values that look like URLs or file paths
                if ( is_string( $value ) ) {
                    if ( preg_match( '/^https?:\\/\\//i', $value ) || preg_match( '/^\\/.*\\.(jpg|jpeg|png|gif|webp|svg|pdf|doc|docx|mp4|webm)$/i', $value ) ) {
                        continue;
                    }
                    // Skip WordPress upload paths
                    if ( preg_match( '/wp-content\\/uploads\\//i', $value ) ) {
                        continue;
                    }
                }
                // Recursively process the value
                $this->extract_acf_fields_recursive( $value, $content_parts, $depth + 1 );
            }
        }
    }

    /**
     * Get text content from a DOM element, handling nested elements.
     *
     * @param DOMElement $element The element to extract text from.
     * @return string The text content.
     */
    private function get_text_content( $element ) {
        $text = '';
        foreach ( $element->childNodes as $child ) {
            if ( $child->nodeType === XML_TEXT_NODE ) {
                $text .= $child->textContent;
            } elseif ( $child->nodeType === XML_ELEMENT_NODE ) {
                // Skip certain elements
                if ( in_array( $child->nodeName, ['script', 'style', 'noscript'] ) ) {
                    continue;
                }
                $text .= ' ' . $this->get_text_content( $child ) . ' ';
            }
        }
        // Normalize whitespace
        $text = preg_replace( '/\\s+/', ' ', $text );
        return trim( $text );
    }

    /**
     * Split text into chunks with overlap.
     *
     * @param string $text      The text to chunk.
     * @param int    $chunk_size Maximum chunk size.
     * @param int    $overlap   Overlap between chunks.
     * @return array Array of chunk strings.
     */
    public function chunk_text( $text, $chunk_size = null, $overlap = null ) {
        $chunk_size = $chunk_size ?? self::CHUNK_SIZE;
        $overlap = $overlap ?? self::CHUNK_OVERLAP;
        // Split into paragraphs
        $paragraphs = array_filter( array_map( 'trim', preg_split( '/\\n\\n+/', $text ) ), function ( $p ) {
            return strlen( $p ) > 0;
        } );
        if ( empty( $paragraphs ) ) {
            return [];
        }
        $chunks = [];
        $current_chunk = [];
        $current_length = 0;
        foreach ( $paragraphs as $paragraph ) {
            $para_length = strlen( $paragraph );
            // If single paragraph exceeds chunk size, split it
            if ( $para_length > $chunk_size ) {
                // First, save current chunk if not empty
                if ( !empty( $current_chunk ) ) {
                    $chunks[] = implode( "\n\n", $current_chunk );
                    $current_chunk = [];
                    $current_length = 0;
                }
                // Split long paragraph by sentences
                $sentences = $this->split_into_sentences( $paragraph );
                $sentence_chunk = [];
                $sentence_length = 0;
                foreach ( $sentences as $sentence ) {
                    if ( $sentence_length + strlen( $sentence ) > $chunk_size && !empty( $sentence_chunk ) ) {
                        $chunks[] = implode( ' ', $sentence_chunk );
                        // Keep last sentence for overlap
                        $sentence_chunk = [end( $sentence_chunk )];
                        $sentence_length = strlen( end( $sentence_chunk ) );
                    }
                    $sentence_chunk[] = $sentence;
                    $sentence_length += strlen( $sentence );
                }
                if ( !empty( $sentence_chunk ) ) {
                    $current_chunk = [implode( ' ', $sentence_chunk )];
                    $current_length = strlen( $current_chunk[0] );
                }
                continue;
            }
            // Check if adding this paragraph exceeds chunk size
            if ( $current_length + $para_length > $chunk_size && !empty( $current_chunk ) ) {
                // Save current chunk
                $chunks[] = implode( "\n\n", $current_chunk );
                // Create overlap: keep last paragraphs up to overlap size
                $overlap_buffer = [];
                $overlap_length = 0;
                for ($i = count( $current_chunk ) - 1; $i >= 0; $i--) {
                    $para_len = strlen( $current_chunk[$i] );
                    if ( $overlap_length + $para_len > $overlap ) {
                        // If this is the first (last) paragraph and it exceeds overlap,
                        // still include a truncated version to ensure some overlap
                        if ( empty( $overlap_buffer ) ) {
                            $truncated = mb_substr( $current_chunk[$i], -$overlap );
                            $overlap_buffer = [$truncated];
                            $overlap_length = strlen( $truncated );
                        }
                        break;
                    }
                    array_unshift( $overlap_buffer, $current_chunk[$i] );
                    $overlap_length += $para_len;
                }
                $current_chunk = $overlap_buffer;
                $current_length = $overlap_length;
            }
            $current_chunk[] = $paragraph;
            $current_length += $para_length;
        }
        // Don't forget the last chunk
        if ( !empty( $current_chunk ) ) {
            $chunks[] = implode( "\n\n", $current_chunk );
        }
        return $chunks;
    }

    /**
     * Split text into sentences.
     *
     * @param string $text The text to split.
     * @return array Array of sentences.
     */
    private function split_into_sentences( $text ) {
        // Split on sentence-ending punctuation followed by space
        $sentences = preg_split(
            '/(?<=[.!?])\\s+/',
            $text,
            -1,
            PREG_SPLIT_NO_EMPTY
        );
        return array_filter( $sentences, function ( $s ) {
            return strlen( trim( $s ) ) > 0;
        } );
    }

    /**
     * Extract URL path for path-based search.
     *
     * @param string $url The full URL.
     * @return string Extracted path words.
     */
    private function extract_url_path( $url ) {
        $path = wp_parse_url( $url, PHP_URL_PATH );
        if ( empty( $path ) ) {
            return '';
        }
        // Remove leading/trailing slashes
        $path = trim( $path, '/' );
        // Remove file extensions
        $path = preg_replace( '/\\.(html?|php|aspx?)$/i', '', $path );
        // Replace separators with spaces
        $path = preg_replace( '/[\\/\\-_]/', ' ', $path );
        // Normalize whitespace
        $path = preg_replace( '/\\s+/', ' ', $path );
        return strtolower( trim( $path ) );
    }

    /**
     * Clean up chunks for deleted/unpublished posts.
     *
     * @param int   $agent_id   The agent ID.
     * @param array $post_types Post types being crawled.
     */
    private function cleanup_orphaned_chunks( $agent_id, $post_types ) {
        global $wpdb;
        // Get all source IDs we have chunks for
        // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching -- Fetch all source IDs to check for orphans.
        $chunk_sources = $wpdb->get_results( $wpdb->prepare( 'SELECT DISTINCT source_id, source_type
             FROM %i
             WHERE agent_id = %d AND content_type = \'website\' AND source_id IS NOT NULL', $wpdb->prefix . 'inqyra_knowledge_chunks', $agent_id ) );
        foreach ( $chunk_sources as $source ) {
            // Check if post still exists and is published
            $post = get_post( $source->source_id );
            if ( !$post || $post->post_status !== 'publish' || !in_array( $post->post_type, $post_types ) ) {
                // Delete orphaned chunks
                // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching -- Delete orphaned chunks for removed/unpublished posts.
                $wpdb->delete( $wpdb->prefix . 'inqyra_knowledge_chunks', [
                    'agent_id'    => $agent_id,
                    'source_id'   => $source->source_id,
                    'source_type' => $source->source_type,
                ], ['%d', '%d', '%s'] );
            }
        }
    }

    /**
     * Start a new crawl run.
     *
     * @param int $agent_id The agent ID.
     * @return int The run ID.
     */
    private function start_crawl_run( $agent_id ) {
        global $wpdb;
        // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery -- Insert new crawl run record.
        $wpdb->insert( $wpdb->prefix . 'inqyra_crawl_runs', [
            'agent_id'   => $agent_id,
            'status'     => 'pending',
            'started_at' => current_time( 'mysql' ),
            'created_at' => current_time( 'mysql' ),
        ], [
            '%d',
            '%s',
            '%s',
            '%s'
        ] );
        return $wpdb->insert_id;
    }

    /**
     * Update the current crawl run.
     *
     * @param array $data Data to update.
     */
    private function update_crawl_run( $data ) {
        global $wpdb;
        if ( !$this->run_id ) {
            return;
        }
        // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching -- Update crawl run progress.
        $wpdb->update( $wpdb->prefix . 'inqyra_crawl_runs', $data, [
            'id' => $this->run_id,
        ] );
    }

    /**
     * Mark the crawl run as complete.
     */
    private function complete_crawl_run() {
        global $wpdb;
        // Count all chunks for this agent (website + linked documents) for accurate reporting
        // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching -- Count total chunks after crawl completes.
        $total_chunks = (int) $wpdb->get_var( $wpdb->prepare( 'SELECT COUNT(*) FROM %i WHERE agent_id = %d', $wpdb->prefix . 'inqyra_knowledge_chunks', $this->agent->id ) );
        $this->update_crawl_run( [
            'status'         => 'completed',
            'chunks_created' => $total_chunks,
            'completed_at'   => current_time( 'mysql' ),
        ] );
        // Update stats to reflect the total
        $this->stats['chunks_created'] = $total_chunks;
    }

    /**
     * Mark the crawl run as failed.
     *
     * @param string $error_message The error message.
     */
    private function fail_crawl_run( $error_message ) {
        $this->update_crawl_run( [
            'status'        => 'failed',
            'error_message' => $error_message,
            'completed_at'  => current_time( 'mysql' ),
        ] );
    }

    /**
     * Log an error/debug message.
     *
     * @param string $message The message.
     */
    private function log_error( $message ) {
        // Always collect in debug_logs for response
        $this->debug_logs[] = '[' . gmdate( 'H:i:s' ) . '] ' . $message;
        // Also log to error_log if WP_DEBUG is enabled
        if ( defined( 'WP_DEBUG' ) && WP_DEBUG ) {
            // phpcs:ignore WordPress.PHP.DevelopmentFunctions.error_log_error_log -- Gated behind WP_DEBUG.
            error_log( '[Inqyra Crawler] ' . $message );
        }
    }

    /**
     * Get collected debug logs.
     *
     * @return array Debug log messages.
     */
    public function get_debug_logs() {
        return $this->debug_logs;
    }

    /**
     * Get crawl run by ID.
     *
     * @param int $run_id The run ID.
     * @return object|null The crawl run.
     */
    public static function get_run( $run_id ) {
        global $wpdb;
        // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching -- Crawl run lookup by primary key.
        return $wpdb->get_row( $wpdb->prepare( 'SELECT * FROM %i WHERE id = %d', $wpdb->prefix . 'inqyra_crawl_runs', $run_id ) );
    }

    /**
     * Get latest crawl run for an agent.
     *
     * @param int $agent_id The agent ID.
     * @return object|null The crawl run.
     */
    public static function get_latest_run( $agent_id ) {
        global $wpdb;
        // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching -- Latest crawl run lookup for agent.
        return $wpdb->get_row( $wpdb->prepare( 'SELECT * FROM %i
             WHERE agent_id = %d
             ORDER BY created_at DESC
             LIMIT 1', $wpdb->prefix . 'inqyra_crawl_runs', $agent_id ) );
    }

    /**
     * Normalize WordPress post type to valid database enum value.
     *
     * The source_type column is an enum: 'post', 'page', 'cpt', 'document', 'manual'
     * Custom post types (like 'team', 'werk') need to be mapped to 'cpt'.
     *
     * @param string $post_type The WordPress post type.
     * @return string Normalized source type for database.
     */
    private function normalize_source_type( $post_type ) {
        $valid_types = [
            'post',
            'page',
            'document',
            'manual'
        ];
        if ( in_array( $post_type, $valid_types, true ) ) {
            return $post_type;
        }
        // Map all custom post types to 'cpt'
        return 'cpt';
    }

    /**
     * Check if content looks like WordPress attachment metadata.
     *
     * Attachment metadata typically contains:
     * - Image title/filename
     * - Multiple URLs with WordPress thumbnail sizes (150x150, 300x..., 768x..., 1024x..., etc.)
     * - wp-content/uploads paths
     *
     * @param string $content The content to check.
     * @return bool True if content appears to be attachment metadata.
     */
    private function is_attachment_metadata( $content ) {
        // Remove the title line for analysis
        $lines = explode( "\n", $content );
        $content_without_title = implode( "\n", array_slice( $lines, 1 ) );
        // Count image URLs with WordPress thumbnail patterns
        $thumbnail_patterns = [
            '/wp-content\\/uploads\\/.*-\\d+x\\d+\\.(jpg|jpeg|png|gif|webp)/i',
            // -150x150.jpg style
            '/wp-content\\/uploads\\/.*-scaled\\.(jpg|jpeg|png|gif|webp)/i',
            // -scaled.jpg
            '/wp-includes\\/images\\/media\\//i',
        ];
        $url_count = 0;
        foreach ( $thumbnail_patterns as $pattern ) {
            $url_count += preg_match_all( $pattern, $content_without_title );
        }
        // If we have 3+ thumbnail URLs and very little other content, it's probably attachment metadata
        if ( $url_count >= 3 ) {
            // Strip all URLs and see what's left
            $stripped = preg_replace( '/https?:\\/\\/[^\\s]+/i', '', $content_without_title );
            $stripped = preg_replace( '/\\.(jpg|jpeg|png|gif|webp|pdf|doc|docx)/i', '', $stripped );
            $stripped = preg_replace( '/\\s+/', ' ', $stripped );
            $stripped = trim( $stripped );
            // If remaining content is very short or just contains filename-like strings
            if ( strlen( $stripped ) < 100 ) {
                return true;
            }
            // Check if remaining content is mostly image filename patterns
            $remaining_words = str_word_count( $stripped );
            if ( $remaining_words < 20 && $url_count > $remaining_words ) {
                return true;
            }
        }
        return false;
    }

    /**
     * Validate URL is safe to fetch (SSRF protection).
     *
     * Prevents fetching internal/private IP addresses and localhost.
     *
     * @param string $url URL to validate.
     * @return bool True if URL is safe to fetch.
     */
    private function is_safe_url( $url ) {
        $parsed = wp_parse_url( $url );
        if ( !$parsed || empty( $parsed['host'] ) ) {
            return false;
        }
        $host = strtolower( $parsed['host'] );
        // Block localhost variations
        $blocked_hosts = [
            'localhost',
            '127.0.0.1',
            '::1',
            '0.0.0.0'
        ];
        if ( in_array( $host, $blocked_hosts, true ) ) {
            return false;
        }
        // Block .local domains
        if ( preg_match( '/\\.local$/', $host ) ) {
            return false;
        }
        // Resolve hostname to IP for checking
        $ip = gethostbyname( $host );
        // If resolution failed, host equals ip — block to prevent DNS rebinding
        if ( $ip === $host && !filter_var( $host, FILTER_VALIDATE_IP ) ) {
            return false;
        }
        // Check if IP is in private/reserved ranges
        if ( !filter_var( $ip, FILTER_VALIDATE_IP, FILTER_FLAG_NO_PRIV_RANGE | FILTER_FLAG_NO_RES_RANGE ) ) {
            return false;
        }
        return true;
    }

    /**
     * Determine if SSL should be verified based on environment.
     *
     * @return bool True if SSL should be verified.
     */
    private function should_verify_ssl() {
        // Explicit opt-out for development with self-signed certs
        if ( defined( 'INQYRA_SKIP_SSL_VERIFY' ) && INQYRA_SKIP_SSL_VERIFY ) {
            return false;
        }
        // Only disable SSL for explicitly declared local/development environments
        if ( defined( 'WP_ENVIRONMENT_TYPE' ) && in_array( WP_ENVIRONMENT_TYPE, ['local', 'development'], true ) ) {
            return false;
        }
        // Default to verifying SSL (including when WP_DEBUG is true)
        return true;
    }

}
