<?php
/**
 * Background Indexer
 *
 * Handles background processing of content indexing using WordPress cron.
 *
 * @package BoostBot
 * @since 1.0.0
 */

if (!defined('ABSPATH')) {
    exit;
}

/**
 * Class BoostBot_Background_Indexer
 *
 * Processes content indexing in the background using WordPress cron.
 */
class BoostBot_Background_Indexer
{
    /**
     * Cron hook name.
     *
     * @var string
     */
    public const CRON_HOOK = 'boostbot_process_index_batch';

    /**
     * Recovery cron hook name.
     *
     * @var string
     */
    public const RECOVERY_HOOK = 'boostbot_index_recover_stuck';

    /**
     * Option name for loopback self-test result.
     *
     * @var string
     */
    private const OPTION_LOOPBACK_WORKS = 'boostbot_index_loopback_works';

    /**
     * Option name for loopback auth token.
     *
     * @var string
     */
    private const OPTION_LOOPBACK_TOKEN = 'boostbot_index_loopback_token';

    /**
     * Option name for pause flag.
     *
     * @var string
     */
    private const OPTION_PAUSED = 'boostbot_index_paused';

    /**
     * Option name for last processing attempt timestamp (unix seconds).
     *
     * Used for best-effort rate limiting between items.
     *
     * @var string
     */
    private const OPTION_LAST_RUN = 'boostbot_index_last_run';

    /**
     * Option name for loopback consecutive failure count (circuit breaker).
     *
     * @var string
     */
    private const OPTION_LOOPBACK_FAILURES = 'boostbot_index_loopback_failures';

    /**
     * Option name for loopback last failure timestamp (circuit breaker).
     *
     * @var string
     */
    private const OPTION_LOOPBACK_LAST_FAIL = 'boostbot_index_loopback_last_fail';

    /**
     * Consecutive loopback failures before entering cooldown.
     *
     * @var int
     */
    private const LOOPBACK_FAILURE_THRESHOLD = 3;

    /**
     * Seconds to wait before retesting loopback after hitting the failure threshold.
     *
     * @var int
     */
    private const LOOPBACK_COOLDOWN_SECONDS = 600; // 10 minutes

    /**
     * Default batch size for processing (basic mode).
     *
     * Deep indexing can be much heavier (extraction + chunking + embeddings),
     * so we dynamically reduce it per job mode to avoid OOM on shared hosts.
     *
     * @var int
     */
    private const BATCH_SIZE = 5;

    /**
     * Maximum number of items to process in a single batch call.
     *
     * Each batch runs in a single PHP process. We loop through multiple items
     * within one call to avoid the massive overhead of one-item-per-cron-event,
     * while still checking memory/time between each item.
     *
     * @var int
     */
    private const MAX_ITEMS_PER_BATCH = 10;

    /**
     * Number of items to pre-fetch for batch summarization.
     *
     * Items are dequeued, their content is extracted, and all summaries are
     * requested in a single API call (parallel Gemini on the backend).
     * This dramatically reduces the per-item latency for AI summarization.
     *
     * @var int
     */
    private const BATCH_SUMMARIZE_SIZE = 5;

    /**
     * Maximum seconds a single batch call is allowed to run.
     *
     * We leave a safety buffer before max_execution_time kicks in.
     * If an item takes longer than expected, the batch stops and
     * schedules the next one.
     *
     * @var int
     */
    private const BATCH_TIME_LIMIT_SECONDS = 90;

    /**
     * Initializes the background indexer.
     *
     * @since 1.0.0
     */
    public static function init(): void
    {
        add_action( self::CRON_HOOK, array( __CLASS__, 'process_batch' ) );
        add_action( self::RECOVERY_HOOK, array( __CLASS__, 'run_recovery' ) );
    }

    /**
     * Dispatches a job for background processing.
     *
     * @since 1.0.0
     * @param string $job_id The job ID to process.
     * @return bool True if dispatched successfully.
     */
    public static function dispatch(string $job_id): bool
    {
        BoostBot_Logger::info( 'Dispatching background indexer', array( 'job_id' => $job_id ) );

        // Get job details
        $job = BoostBot_Index_Job::get( $job_id );
        if ( ! $job ) {
            BoostBot_Logger::error( 'Job not found for dispatch', array( 'job_id' => $job_id ) );
            return false;
        }

        // Reset pause flag, loopback circuit breaker, and set a fresh loopback token.
        delete_option( self::OPTION_PAUSED );
        update_option( self::OPTION_LOOPBACK_TOKEN, wp_generate_password( 32, false, false ) );
        self::reset_loopback_failures();
        update_option( self::OPTION_LOOPBACK_WORKS, true, false );

        $scanner = new BoostBot_Site_Scanner();
        $queue   = new BoostBot_Index_Queue();

        // Fresh run: clear any previous queue rows (single-queue model per site for now).
        $queue->clear();

        // Pre-scan sizes (SQL-only) and pick safe extraction levels per item.
        $sizes = $scanner->get_content_sizes( (array) $job->content_types );

        // Filter by selected_ids (if provided).
        $selected = array();
        if ( ! empty( $job->selected_ids ) && is_array( $job->selected_ids ) ) {
            foreach ( $job->selected_ids as $sid ) {
                $selected[(int) $sid] = true;
            }
        }

        // Load excluded IDs from settings.
        $excluded = array();
        $excluded_raw = get_option( 'boostbot_index_excluded_ids', '[]' );
        $excluded_arr = json_decode( (string) $excluded_raw, true );
        if ( is_array( $excluded_arr ) ) {
            foreach ( $excluded_arr as $eid ) {
                $excluded[ (int) $eid ] = true;
            }
        }

        $items_to_enqueue = array();
        foreach ( $sizes as $row ) {
            $id   = (int) ( $row['id'] ?? 0 );
            $type = (string) ( $row['type'] ?? '' );
            if ( ! $id || '' === $type ) {
                continue;
            }
            if ( ! empty( $selected ) && ! isset( $selected[ $id ] ) ) {
                continue;
            }
            // Skip items that are in the exclusion list.
            if ( isset( $excluded[ $id ] ) ) {
                continue;
            }

            $content_size = (int) ( $row['content_size'] ?? 0 );

            // =====================================================
            // SIZE-BASED PRIORITY ASSIGNMENT
            // Smaller items get higher priority to ensure progress.
            // Larger items are processed last to avoid early stuck states.
            // Priority scale: 10 (highest) to 1 (lowest)
            // =====================================================
            $priority = 5; // Default priority
            if ( $content_size < 10000 ) {
                $priority = 10; // < 10KB: highest priority
            } elseif ( $content_size < 50000 ) {
                $priority = 8;  // 10-50KB: high priority
            } elseif ( $content_size < 100000 ) {
                $priority = 6;  // 50-100KB: normal-high priority
            } elseif ( $content_size < self::LARGE_CONTENT_THRESHOLD ) {
                $priority = 4;  // 100-150KB: normal-low priority
            } elseif ( $content_size < 300000 ) {
                $priority = 2;  // 150-300KB: low priority
            } else {
                $priority = 1;  // 300KB+: lowest priority (processed last)
            }

            $items_to_enqueue[] = array(
                'content_id'       => $id,
                'content_type'     => $type,
                'priority'         => $priority,
                'content_size'     => $content_size,
                'extraction_level' => (int) ( $row['recommended_level'] ?? 5 ),
                'max_attempts'     => 3,
            );
        }

        if ( empty( $items_to_enqueue ) ) {
            BoostBot_Logger::warning( 'No items to index', array( 'job_id' => $job_id ) );
            BoostBot_Index_Job::update_status( $job_id, BoostBot_Index_Job::STATUS_COMPLETED );
            return false;
        }

        $res = $queue->enqueue_many( $items_to_enqueue );

        BoostBot_Index_Job::update_progress( $job_id, 0, count( $items_to_enqueue ) );
        BoostBot_Index_Job::update_status( $job_id, BoostBot_Index_Job::STATUS_RUNNING );

        BoostBot_Logger::info( 'Index queue populated', array(
            'job_id'    => $job_id,
            'inserted'  => (int) ( $res['inserted'] ?? 0 ),
            'errors'    => (int) ( $res['errors'] ?? 0 ),
            'total'     => count( $items_to_enqueue ),
        ) );

        // Trigger first batch immediately (loopback preferred).
        self::schedule_next_batch();

        return true;
    }

    /**
     * Stuck item timeout in minutes for inline recovery.
     * Shorter than the cron-based recovery for faster response.
     *
     * @var int
     */
    private const STUCK_TIMEOUT_MINUTES = 2;

    /**
     * Maximum content size (bytes) for full processing.
     * Larger items get auto-downgraded extraction level.
     *
     * @var int
     */
    private const LARGE_CONTENT_THRESHOLD = 150000; // 150KB

    /**
     * Processes a batch of items.
     *
     * @since 1.0.0
     */
    public static function process_batch(): void
    {
        // If paused, do nothing.
        if ( get_option( self::OPTION_PAUSED, false ) ) {
            BoostBot_Logger::debug( 'Indexing paused; skipping batch.' );
            return;
        }

        // =====================================================
        // INLINE STUCK RECOVERY - Run before each batch
        // This ensures stuck items are recovered immediately,
        // not waiting for the hourly cron.
        // =====================================================
        $recovered = self::recover_stuck_items( self::STUCK_TIMEOUT_MINUTES );
        if ( $recovered > 0 ) {
            BoostBot_Logger::info( 'Inline recovery: reset stuck items', array( 'count' => $recovered ) );
        }

        // =====================================================
        // MEMORY SAFETY GATE
        // Check current memory usage before starting heavy work.
        // If we're already using > 70% of the limit, skip this cycle
        // and let the next cron/kick pick it up when memory is fresh.
        // This prevents OOM crashes on shared hosting (128-256MB).
        // =====================================================
        $current_limit = ini_get( 'memory_limit' );
        $limit_bytes   = wp_convert_hr_to_bytes( $current_limit );

        // Don't try to raise the limit — work within what the host provides.
        // Shared hosts often ignore ini_set('memory_limit') anyway.
        // Only raise if we're below 256MB (very constrained environments).
        if ( $limit_bytes > 0 && $limit_bytes < 268435456 ) { // < 256MB
            // phpcs:ignore Squiz.PHP.DiscouragedFunctions.Discouraged -- background indexing needs adequate memory
            @ini_set( 'memory_limit', '256M' );
            $limit_bytes = wp_convert_hr_to_bytes( ini_get( 'memory_limit' ) );
        }

        // phpcs:ignore Squiz.PHP.DiscouragedFunctions.Discouraged -- background indexing may exceed default 30s
        @set_time_limit( 120 );

        $min_free = 50 * 1024 * 1024; // 50MB minimum free memory per item.

        if ( $limit_bytes > 0 ) {
            $used_bytes = function_exists( 'memory_get_usage' ) ? memory_get_usage( true ) : 0;
            $available  = $limit_bytes - $used_bytes;
            if ( $available < $min_free ) {
                BoostBot_Logger::warning( 'Memory safety gate: not enough free memory, deferring batch', array(
                    'limit_mb'     => round( $limit_bytes / 1048576 ),
                    'used_mb'      => round( $used_bytes / 1048576 ),
                    'available_mb' => round( $available / 1048576 ),
                    'required_mb'  => round( $min_free / 1048576 ),
                ) );
                self::schedule_next_batch();
                return;
            }
        }

        $job = BoostBot_Index_Job::get();
        if ( ! $job ) {
            BoostBot_Logger::debug( 'No active indexing job.' );
            return;
        }

        $job_id = (string) $job->id;

        // Check cancelled/completed
        if ( $job->status === BoostBot_Index_Job::STATUS_CANCELLED ) {
            BoostBot_Logger::info( 'Job cancelled; stopping indexer.', array( 'job_id' => $job_id ) );
            wp_clear_scheduled_hook( self::CRON_HOOK );
            return;
        }

        $queue = new BoostBot_Index_Queue();
        $stats = $queue->get_stats();

        // If no pending/processing remains, job is complete.
        $pending    = (int) ( $stats['pending'] ?? 0 );
        $processing = (int) ( $stats['processing'] ?? 0 );
        $done       = (int) ( $stats['completed'] ?? 0 ) + (int) ( $stats['failed'] ?? 0 ) + (int) ( $stats['skipped'] ?? 0 );

        // Keep job total stable; if missing, derive from queue total.
        $total = isset( $job->total ) && (int) $job->total > 0 ? (int) $job->total : (int) ( $stats['total'] ?? 0 );

        // Update progress best-effort (processed == done)
        BoostBot_Index_Job::update_progress( $job_id, $done, max( 0, $total ) );

        if ( $pending <= 0 && $processing <= 0 ) {
            BoostBot_Logger::info( 'Background indexing completed', array(
                'job_id'    => $job_id,
                'processed' => $done,
                'total'     => $total,
            ) );

            BoostBot_Index_Job::update_status( $job_id, BoostBot_Index_Job::STATUS_COMPLETED );
            update_option( 'boostbot_last_full_index', current_time( 'mysql' ) );
            wp_clear_scheduled_hook( self::CRON_HOOK );
            return;
        }

        $mode = isset( $job->index_mode ) ? (string) $job->index_mode : 'basic';
        $batch_start   = microtime( true );
        $items_done    = 0;
        $rate_limit    = max( 0, (int) BoostBot_Settings::get( 'indexing_rate_limit', 5 ) );
        $max_items     = self::MAX_ITEMS_PER_BATCH;
        $batch_wall    = self::BATCH_TIME_LIMIT_SECONDS;

        // =====================================================
        // PRE-SCHEDULE SAFETY NET
        // Schedule the next batch BEFORE processing. If the PHP
        // process is killed (OOM, host timeout, max_execution_time),
        // this pre-scheduled event ensures the cron chain continues.
        // After the batch completes normally, we reschedule with
        // proper timing.
        // =====================================================
        wp_clear_scheduled_hook( self::CRON_HOOK );
        wp_schedule_single_event( time() + 60, self::CRON_HOOK );

        // =====================================================
        // PHASE 1: PRE-WARM SUMMARY CACHE (BATCH SUMMARIZATION)
        //
        // Dequeue up to BATCH_SUMMARIZE_SIZE items, extract their
        // content, send all for AI summarization in a single API
        // call (backend runs them in parallel with Promise.allSettled).
        // Cache each result as a transient so that the normal
        // process_content() call finds it and skips the API.
        //
        // This turns N sequential ~6s API calls into one ~6s call.
        // =====================================================
        $pre_dequeued = array(); // Items dequeued for batch summarization.

        if ( 'backend' === (string) BoostBot_Settings::get( 'indexing_llm_provider', 'backend' )
             && class_exists( 'BoostBot_Indexing_Backend_Client' )
             && class_exists( 'BoostBot_Content_Processor' )
        ) {
            $batch_sz = self::BATCH_SUMMARIZE_SIZE;
            $summarize_items = array();

            for ( $i = 0; $i < $batch_sz; $i++ ) {
                // Time guard.
                $elapsed = microtime( true ) - $batch_start;
                if ( $elapsed >= $batch_wall * 0.3 ) {
                    break; // Leave >=70% of time budget for actual processing.
                }

                // Memory guard.
                if ( $limit_bytes > 0 ) {
                    $used_now = function_exists( 'memory_get_usage' ) ? memory_get_usage( true ) : 0;
                    if ( ( $limit_bytes - $used_now ) < $min_free ) {
                        break;
                    }
                }

                $item = $queue->dequeue();
                if ( ! $item ) {
                    break;
                }

                $cid   = (int) ( $item['content_id'] ?? 0 );
                $ctype = (string) ( $item['content_type'] ?? '' );
                $clvl  = (int) ( $item['extraction_level'] ?? 5 );
                $csz   = (int) ( $item['content_size'] ?? 0 );
                $catts = (int) ( $item['attempts'] ?? 1 );

                // Apply extraction level adjustments (same logic as main loop).
                if ( $csz > self::LARGE_CONTENT_THRESHOLD ) {
                    if ( $csz > 500000 ) {
                        $clvl = min( $clvl, 1 );
                    } elseif ( $csz > 300000 ) {
                        $clvl = min( $clvl, 2 );
                    } else {
                        $clvl = min( $clvl, 3 );
                    }
                }
                if ( $catts >= 3 && $clvl > 1 ) {
                    $clvl = 1;
                }

                $item['_adjusted_level'] = $clvl;
                $pre_dequeued[] = $item;

                // Extract content for summarization (local, fast).
                $tmp_processor = new BoostBot_Content_Processor();
                $extracted     = $tmp_processor->extract_at_level( $cid, $clvl );
                unset( $tmp_processor );

                if ( is_array( $extracted ) && ! empty( $extracted['content'] ) ) {
                    $content_for_ai = (string) $extracted['content'];
                    if ( mb_strlen( $content_for_ai ) > 30000 ) {
                        $content_for_ai = mb_substr( $content_for_ai, 0, 30000 );
                    }

                    $summarize_items[] = array(
                        'id'      => (string) $cid,
                        'title'   => (string) ( $extracted['title'] ?? '' ),
                        'content' => $content_for_ai,
                        'type'    => (string) ( $extracted['type'] ?? 'page' ),
                    );
                }
                unset( $extracted );
            }

            // Send batch summarization request if we have items.
            if ( ! empty( $summarize_items ) ) {
                $backend_client = new BoostBot_Indexing_Backend_Client();
                $batch_results  = $backend_client->summarize_batch( $summarize_items );

                if ( ! is_wp_error( $batch_results ) && is_array( $batch_results ) ) {
                    $cached_count = 0;
                    foreach ( $batch_results as $rid => $rdata ) {
                        if ( ! is_array( $rdata ) ) {
                            continue;
                        }
                        // Find the matching item to compute the correct cache key.
                        foreach ( $summarize_items as $si ) {
                            if ( $si['id'] === (string) $rid ) {
                                $cache_key = 'boostbot_summary_' . md5( $si['content'] . $si['title'] );
                                set_transient( $cache_key, $rdata, WEEK_IN_SECONDS );
                                $cached_count++;
                                break;
                            }
                        }
                    }
                    BoostBot_Logger::info( 'Batch summarization completed', array(
                        'requested' => count( $summarize_items ),
                        'cached'    => $cached_count,
                        'elapsed_s' => round( microtime( true ) - $batch_start, 2 ),
                    ) );
                } else {
                    $err_msg = is_wp_error( $batch_results ) ? $batch_results->get_error_message() : 'unknown';
                    BoostBot_Logger::warning( 'Batch summarization failed; items will fall back to single calls', array(
                        'error' => $err_msg,
                        'count' => count( $summarize_items ),
                    ) );
                }
                unset( $backend_client, $batch_results, $summarize_items );
            }

            // Force GC after content extraction phase.
            if ( function_exists( 'gc_collect_cycles' ) ) {
                gc_collect_cycles();
            }
        }

        // =====================================================
        // PHASE 2: PROCESS ITEMS (pre-dequeued + any additional)
        //
        // First process the items we already dequeued for batch
        // summarization, then continue dequeuing more if time/
        // memory allows.
        // =====================================================
        $all_items_to_process = $pre_dequeued; // Start with batch-summarized items.
        unset( $pre_dequeued );

        // Process all items (pre-dequeued first, then dequeue more).
        $item_index = 0;
        while ( $items_done < $max_items ) {
            // Time guard: stop if we've used most of our wall-clock budget.
            $elapsed = microtime( true ) - $batch_start;
            if ( $elapsed >= $batch_wall ) {
                BoostBot_Logger::debug( 'Batch time limit reached', array(
                    'items_done' => $items_done,
                    'elapsed'    => round( $elapsed, 1 ),
                ) );
                break;
            }

            // Memory guard: stop if we're running low.
            if ( $limit_bytes > 0 ) {
                $used_now  = function_exists( 'memory_get_usage' ) ? memory_get_usage( true ) : 0;
                $free_now  = $limit_bytes - $used_now;
                if ( $free_now < $min_free ) {
                    BoostBot_Logger::debug( 'Batch memory limit reached', array(
                        'items_done'   => $items_done,
                        'free_mb'      => round( $free_now / 1048576, 1 ),
                        'required_mb'  => round( $min_free / 1048576 ),
                    ) );
                    break;
                }
            }

            // Check if job was cancelled mid-batch.
            if ( $items_done > 0 && $items_done % 5 === 0 ) {
                $job = BoostBot_Index_Job::get();
                if ( ! $job || $job->status === BoostBot_Index_Job::STATUS_CANCELLED ) {
                    BoostBot_Logger::info( 'Job cancelled mid-batch', array( 'items_done' => $items_done ) );
                    wp_clear_scheduled_hook( self::CRON_HOOK );
                    return;
                }
            }

            // Get next item: from pre-dequeued list or fresh dequeue.
            if ( $item_index < count( $all_items_to_process ) ) {
                $item = $all_items_to_process[ $item_index ];
                $item_index++;
            } else {
                $item = $queue->dequeue();
                if ( ! $item ) {
                    break; // No more items to process.
                }
            }

            // Record claim time for rate limiting (best-effort).
            update_option( self::OPTION_LAST_RUN, time(), false );

            $content_id       = (int) ( $item['content_id'] ?? 0 );
            $content_type     = (string) ( $item['content_type'] ?? '' );
            $queue_id         = (int) ( $item['id'] ?? 0 );
            $extraction_level = isset( $item['_adjusted_level'] )
                ? (int) $item['_adjusted_level']
                : (int) ( $item['extraction_level'] ?? 5 );
            $content_size     = (int) ( $item['content_size'] ?? 0 );
            $attempts         = (int) ( $item['attempts'] ?? 1 );

            // Apply extraction level adjustments for freshly dequeued items
            // (pre-dequeued items already have _adjusted_level set).
            if ( ! isset( $item['_adjusted_level'] ) ) {
                $original_level = $extraction_level;
                if ( $content_size > self::LARGE_CONTENT_THRESHOLD ) {
                    if ( $content_size > 500000 ) {
                        $extraction_level = min( $extraction_level, 1 );
                    } elseif ( $content_size > 300000 ) {
                        $extraction_level = min( $extraction_level, 2 );
                    } else {
                        $extraction_level = min( $extraction_level, 3 );
                    }

                    if ( $extraction_level !== $original_level ) {
                        BoostBot_Logger::info( 'Large content: auto-reduced extraction level', array(
                            'content_id'     => $content_id,
                            'content_size'   => $content_size,
                            'original_level' => $original_level,
                            'new_level'      => $extraction_level,
                        ) );
                    }
                }

                if ( $attempts >= 3 && $extraction_level > 1 ) {
                    $extraction_level = 1;
                    BoostBot_Logger::info( 'Last attempt: forcing minimal extraction level', array(
                        'content_id' => $content_id,
                        'attempts'   => $attempts,
                    ) );
                }
            }

            BoostBot_Logger::debug( 'Processing queued item', array(
                'job_id'            => $job_id,
                'queue_id'          => $queue_id,
                'content_id'        => $content_id,
                'content_type'      => $content_type,
                'content_size'      => $content_size,
                'index_mode'        => $mode,
                'extraction_level'  => $extraction_level,
                'attempts'          => $attempts,
                'batch_item'        => $items_done + 1,
            ) );

            $index = new BoostBot_Knowledge_Index();
            $t0    = microtime( true );

            try {
                $index->index_content(
                    $content_id,
                    $content_type,
                    array(
                        'mode'             => $mode,
                        'job_id'           => $job_id,
                        'queue_id'         => $queue_id,
                        'extraction_level' => $extraction_level,
                    )
                );

                $dt = microtime( true ) - $t0;
                $queue->complete( $queue_id, array(
                    'memory_peak'     => function_exists( 'memory_get_peak_usage' ) ? (int) memory_get_peak_usage( true ) : null,
                    'processing_time' => (float) $dt,
                ) );

                if ( $content_size > self::LARGE_CONTENT_THRESHOLD ) {
                    BoostBot_Logger::info( 'Large content processed successfully', array(
                        'content_id'      => $content_id,
                        'content_size'    => $content_size,
                        'processing_time' => round( $dt, 2 ),
                        'extraction_level'=> $extraction_level,
                    ) );
                }
            } catch ( Exception $e ) {
                $next_level = max( 1, $extraction_level - 1 );
                $queue->fail( $queue_id, $e->getMessage(), $next_level );
                BoostBot_Index_Job::add_error( $job_id, array(
                    'content_id'   => $content_id,
                    'content_type' => $content_type,
                    'message'      => $e->getMessage(),
                ) );
            } catch ( Error $e ) {
                $next_level = max( 1, $extraction_level - 1 );
                $queue->fail( $queue_id, 'Fatal: ' . $e->getMessage(), $next_level );
                BoostBot_Index_Job::add_error( $job_id, array(
                    'content_id'   => $content_id,
                    'content_type' => $content_type,
                    'message'      => 'Fatal: ' . $e->getMessage(),
                ) );
            }

            // Free the index object and force garbage collection between items.
            unset( $index );
            if ( function_exists( 'gc_collect_cycles' ) ) {
                gc_collect_cycles();
            }

            $items_done++;

            // Optional micro-delay between items when rate_limit > 0 and batch is fast.
            // Only sleep within the batch if the rate limit is > 1s (user explicitly slowed it).
            if ( $rate_limit > 1 && $items_done < $max_items ) {
                $item_elapsed = microtime( true ) - $t0;
                $sleep_s = max( 0, $rate_limit - $item_elapsed );
                if ( $sleep_s > 0 && $sleep_s < 5 ) {
                    usleep( (int) ( $sleep_s * 1000000 ) );
                }
            }
        }

        // Reliability: items pre-dequeued in PHASE 1 for batch summarization are marked
        // 'processing'. If PHASE 2 broke early (time/memory guard) or there were more
        // pre-dequeued items than max_items, some were never processed and would be left
        // stranded in 'processing' until the watchdog downgrades them. Re-queue those back
        // to 'pending' (started_at reset, attempts decremented) so the next batch picks
        // them up cleanly. The status guard avoids clobbering rows a concurrent run touched.
        if ( isset( $all_items_to_process ) && $item_index < count( $all_items_to_process ) ) {
            global $wpdb;
            $queue_table  = $wpdb->prefix . 'boostbot_index_queue';
            $requeued     = 0;
            for ( $ri = $item_index; $ri < count( $all_items_to_process ); $ri++ ) {
                $rq_id = (int) ( $all_items_to_process[ $ri ]['id'] ?? 0 );
                if ( $rq_id <= 0 ) {
                    continue;
                }
                // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching, WordPress.DB.PreparedSQL.InterpolatedNotPrepared, PluginCheck.Security.DirectDB.UnescapedDBParameter -- Table name is $wpdb->prefix plus an internal constant; the id is bound via prepare().
                $updated = $wpdb->query(
                    // phpcs:ignore WordPress.DB.PreparedSQL.NotPrepared, WordPress.DB.PreparedSQL.InterpolatedNotPrepared
                    $wpdb->prepare(
                        // phpcs:ignore WordPress.DB.PreparedSQL.InterpolatedNotPrepared
                        "UPDATE `{$queue_table}` SET status = 'pending', started_at = NULL, attempts = GREATEST(attempts - 1, 0) WHERE id = %d AND status = 'processing'",
                        $rq_id
                    )
                );
                if ( $updated ) {
                    $requeued++;
                }
            }
            if ( $requeued > 0 ) {
                BoostBot_Logger::info( 'Re-queued unprocessed pre-dequeued items', array(
                    'job_id'   => $job_id,
                    'requeued' => $requeued,
                ) );
            }
        }

        // Update progress after the batch loop.
        $stats = $queue->get_stats();
        $done  = (int) ( $stats['completed'] ?? 0 ) + (int) ( $stats['failed'] ?? 0 ) + (int) ( $stats['skipped'] ?? 0 );
        $total = isset( $job->total ) && (int) $job->total > 0 ? (int) $job->total : (int) ( $stats['total'] ?? 0 );
        BoostBot_Index_Job::update_progress( $job_id, $done, max( 0, $total ) );

        // Check if we're now done.
        $pending    = (int) ( $stats['pending'] ?? 0 );
        $processing = (int) ( $stats['processing'] ?? 0 );

        if ( $pending <= 0 && $processing <= 0 ) {
            BoostBot_Logger::info( 'Background indexing completed', array(
                'job_id'    => $job_id,
                'processed' => $done,
                'total'     => $total,
            ) );

            BoostBot_Index_Job::update_status( $job_id, BoostBot_Index_Job::STATUS_COMPLETED );
            update_option( 'boostbot_last_full_index', current_time( 'mysql' ) );
            wp_clear_scheduled_hook( self::CRON_HOOK );
            return;
        }

        $batch_elapsed = microtime( true ) - $batch_start;

        // Log batch summary.
        if ( $limit_bytes > 0 ) {
            $post_mem = function_exists( 'memory_get_usage' ) ? memory_get_usage( true ) : 0;
            $peak_mem = function_exists( 'memory_get_peak_usage' ) ? memory_get_peak_usage( true ) : 0;
            BoostBot_Logger::info( 'Batch completed', array(
                'items_done'   => $items_done,
                'elapsed_s'    => round( $batch_elapsed, 2 ),
                'current_mb'   => round( $post_mem / 1048576, 1 ),
                'peak_mb'      => round( $peak_mem / 1048576, 1 ),
                'limit_mb'     => round( $limit_bytes / 1048576 ),
                'remaining'    => $pending,
            ) );
        }

        // Continue processing.
        self::schedule_next_batch();
    }

    /**
     * Schedules the next batch processing.
     *
     * Uses HTTP loopback for memory isolation - each batch runs in a fresh PHP process.
     * ALWAYS schedules a WP-Cron fallback event to guarantee progress even when
     * loopback is blocked and admin UI is not on the KB page.
     *
     * @since 1.0.0
     */
    public static function schedule_next_batch(): void
    {
        // Clear any existing scheduled event
        wp_clear_scheduled_hook(self::CRON_HOOK);

        // Best-effort scheduler controls.
        // - Rate limit between items
        // - Optional scheduled window
        // - Optional load gating (Linux only)
        $gate = self::get_scheduler_gate();

        if ( ! (bool) ( $gate['ok'] ?? false ) ) {
            $delay = (int) ( $gate['delay'] ?? 60 );
            wp_schedule_single_event( time() + max( 5, $delay ), self::CRON_HOOK );
            BoostBot_Logger::debug( 'Next batch delayed by scheduler gate', array(
                'delay'  => $delay,
                'reason' => (string) ( $gate['reason'] ?? 'unknown' ),
            ) );
            self::ensure_cron_fires();
            return;
        }

        $delay = (int) ( $gate['delay'] ?? 0 );
        if ( $delay > 0 ) {
            wp_schedule_single_event( time() + $delay, self::CRON_HOOK );
            BoostBot_Logger::debug( 'Next batch scheduled via WP-Cron (rate limit)', array( 'delay' => $delay ) );
            self::ensure_cron_fires();
            return;
        }

        // Circuit-breaker: decide whether to attempt loopback.
        $loopback_allowed = self::is_loopback_allowed();

        $loopback_success = false;
        if ( $loopback_allowed ) {
            $loopback_success = self::trigger_http_loopback();
        }

        // ALWAYS schedule a WP-Cron fallback, regardless of loopback result.
        // This ensures indexing continues even when:
        // - HTTP loopback is silently dropped by host firewall
        // - Admin is not on the Knowledge Base page
        // - DISABLE_WP_CRON is false but cron is triggered by external traffic
        // Cron delay is kept short so the batch fires quickly. The multi-item
        // batch loop already handles rate limiting internally.
        $cron_delay = 5; // Minimal delay — batch loop handles rate limiting.
        if ( $loopback_success ) {
            // Loopback fired — use a slightly longer cron delay as a safety net.
            $cron_delay = 10;
            BoostBot_Logger::debug( 'Next batch triggered via HTTP loopback + WP-Cron safety net', array(
                'cron_delay' => $cron_delay,
            ) );
        } else {
            BoostBot_Logger::debug( 'Next batch scheduled via WP-Cron (loopback ' . ( $loopback_allowed ? 'failed' : 'cooldown' ) . ')', array(
                'cron_delay' => $cron_delay,
            ) );
        }

        wp_schedule_single_event( time() + $cron_delay, self::CRON_HOOK );

        // Proactively trigger WP-Cron spawn to ensure the event actually fires.
        self::ensure_cron_fires();
    }

    /**
     * Ensure WP-Cron fires by calling spawn_cron() if needed.
     *
     * WP-Cron relies on incoming HTTP requests (pseudo-cron). On low-traffic
     * sites, scheduled events can go minutes or hours without firing. This
     * method calls spawn_cron() to trigger an immediate cron runner, ensuring
     * the next batch fires promptly regardless of site traffic.
     *
     * @since 1.16.0
     */
    private static function ensure_cron_fires(): void {
        if ( function_exists( 'spawn_cron' ) ) {
            spawn_cron();
        }
    }

    /**
     * Scheduler gate: check load, scheduled window, and rate limit.
     *
     * @since 1.13.0
     * @return array{ok:bool,delay:int,reason?:string}
     */
    private static function get_scheduler_gate(): array {
        // Guard: paused.
        if ( (bool) get_option( self::OPTION_PAUSED, false ) ) {
            return array( 'ok' => false, 'delay' => 60, 'reason' => 'paused' );
        }

        $mode = (string) BoostBot_Settings::get( 'indexing_mode', 'normal' );

        // Scheduled window check (only when mode == scheduled).
        if ( 'scheduled' === $mode ) {
            $start = (int) BoostBot_Settings::get( 'indexing_schedule_start', 2 );
            $end   = (int) BoostBot_Settings::get( 'indexing_schedule_end', 6 );

            $start = max( 0, min( 23, $start ) );
            $end   = max( 0, min( 23, $end ) );

            $hour = (int) current_time( 'G' );
            $in_window = false;
            if ( $start === $end ) {
                // Degenerate case: treat as always allowed.
                $in_window = true;
            } elseif ( $start < $end ) {
                $in_window = ( $hour >= $start && $hour < $end );
            } else {
                // Window wraps midnight.
                $in_window = ( $hour >= $start || $hour < $end );
            }

            if ( ! $in_window ) {
                // Try again in ~15 minutes (best-effort).
                return array( 'ok' => false, 'delay' => 900, 'reason' => 'outside_schedule_window' );
            }
        }

        // Load check (Linux only; sys_getloadavg not available on Windows).
        $max_load = (float) BoostBot_Settings::get( 'indexing_max_load', 2.0 );
        if ( $max_load > 0 && function_exists( 'sys_getloadavg' ) ) {
            $load = sys_getloadavg();
            $one  = is_array( $load ) && isset( $load[0] ) ? (float) $load[0] : 0.0;
            if ( $one > $max_load ) {
                return array( 'ok' => false, 'delay' => 300, 'reason' => 'high_load' );
            }
        }

        // Rate limiting between batches is no longer needed here because
        // the multi-item batch loop (process_batch) handles rate limiting
        // internally with usleep() between items. Between-batch delays
        // only slowed down indexing without benefit.
        return array( 'ok' => true, 'delay' => 0 );
    }

    /**
     * Trigger next batch via HTTP loopback for memory isolation.
     *
     * Uses a circuit-breaker pattern: after LOOPBACK_FAILURE_THRESHOLD consecutive
     * failures, loopback is temporarily disabled for LOOPBACK_COOLDOWN_SECONDS,
     * then automatically retested. This prevents permanent degradation from
     * transient DNS/firewall glitches.
     *
     * @since 1.8.0
     * @return bool True if loopback was triggered successfully.
     */
    private static function trigger_http_loopback(): bool {
        $url = rest_url( 'boostbot/v1/index/process-next' );

        $token = (string) get_option( self::OPTION_LOOPBACK_TOKEN, '' );
        if ( '' === $token ) {
            return false;
        }

        // Use a short blocking request so we can actually verify the connection.
        // With blocking:false, wp_remote_post always returns non-error, making
        // the circuit breaker useless. A 2-second timeout is enough to confirm
        // the server received the request without waiting for processing.
        $response = wp_remote_post( $url, array(
            'timeout'   => 2,
            'blocking'  => true,
            'sslverify' => false,
            'headers'   => array(
                'X-BoostBot-Loopback' => $token,
            ),
        ) );

        if ( is_wp_error( $response ) ) {
            self::record_loopback_failure( $response->get_error_message() );
            return false;
        }

        $code = (int) wp_remote_retrieve_response_code( $response );
        if ( $code >= 400 ) {
            self::record_loopback_failure( 'HTTP ' . $code );
            return false;
        }

        // Confirmed reachable — reset the circuit breaker.
        self::reset_loopback_failures();
        return true;
    }

    /**
     * Check if loopback is currently allowed by the circuit breaker.
     *
     * Loopback is allowed when:
     * - Consecutive failures are below the threshold, OR
     * - The cooldown period has elapsed (time to retest).
     *
     * The activation-time self-test result is used as the initial seed,
     * but is no longer a permanent kill switch.
     *
     * @since 1.15.0
     * @return bool True if loopback should be attempted.
     */
    private static function is_loopback_allowed(): bool {
        // Respect the activation self-test as a starting hint only on the very
        // first run (before any circuit-breaker data exists).
        $failures  = (int) get_option( self::OPTION_LOOPBACK_FAILURES, 0 );
        $last_fail = (int) get_option( self::OPTION_LOOPBACK_LAST_FAIL, 0 );

        // If no failures recorded yet, fall back to the self-test flag.
        if ( 0 === $failures && 0 === $last_fail ) {
            return (bool) get_option( self::OPTION_LOOPBACK_WORKS, true );
        }

        // Below threshold — always allowed.
        if ( $failures < self::LOOPBACK_FAILURE_THRESHOLD ) {
            return true;
        }

        // Above threshold — check cooldown.
        $elapsed = time() - $last_fail;
        if ( $elapsed >= self::LOOPBACK_COOLDOWN_SECONDS ) {
            // Cooldown expired — allow one retest attempt.
            BoostBot_Logger::info( 'Loopback circuit breaker: cooldown expired, retesting', array(
                'failures'    => $failures,
                'cooldown_s'  => self::LOOPBACK_COOLDOWN_SECONDS,
                'elapsed_s'   => $elapsed,
            ) );
            return true;
        }

        return false;
    }

    /**
     * Record a loopback failure (circuit breaker).
     *
     * @since 1.15.0
     * @param string $error Error message for logging.
     */
    private static function record_loopback_failure( string $error ): void {
        $failures = (int) get_option( self::OPTION_LOOPBACK_FAILURES, 0 ) + 1;
        update_option( self::OPTION_LOOPBACK_FAILURES, $failures, false );
        update_option( self::OPTION_LOOPBACK_LAST_FAIL, time(), false );

        BoostBot_Logger::warning( 'HTTP loopback failed (circuit breaker)', array(
            'error'                => $error,
            'consecutive_failures' => $failures,
            'threshold'            => self::LOOPBACK_FAILURE_THRESHOLD,
            'cooldown_s'           => self::LOOPBACK_COOLDOWN_SECONDS,
        ) );
    }

    /**
     * Reset the loopback circuit breaker on success.
     *
     * @since 1.15.0
     */
    private static function reset_loopback_failures(): void {
        $prev = (int) get_option( self::OPTION_LOOPBACK_FAILURES, 0 );
        if ( $prev > 0 ) {
            update_option( self::OPTION_LOOPBACK_FAILURES, 0, false );
            delete_option( self::OPTION_LOOPBACK_LAST_FAIL );
            update_option( self::OPTION_LOOPBACK_WORKS, true, false );
            BoostBot_Logger::info( 'Loopback circuit breaker: recovered after failures', array(
                'previous_failures' => $prev,
            ) );
        }
    }

    /**
     * Self-test: check if HTTP loopback works on this host.
     *
     * Called on plugin activation to determine if the host supports loopback.
     * Seeds the circuit breaker with an initial state.
     *
     * @since 1.14.1
     * @return bool True if loopback is reachable.
     */
    public static function test_loopback(): bool {
        $url = rest_url( 'boostbot/v1/index/status' );

        $response = wp_remote_get( $url, array(
            'timeout'   => 5,
            'sslverify' => false,
        ) );

        $works = ! is_wp_error( $response ) && wp_remote_retrieve_response_code( $response ) < 500;

        update_option( self::OPTION_LOOPBACK_WORKS, $works, false );

        // Seed circuit breaker state.
        if ( $works ) {
            self::reset_loopback_failures();
        } else {
            self::record_loopback_failure(
                is_wp_error( $response ) ? $response->get_error_message() : 'HTTP ' . wp_remote_retrieve_response_code( $response )
            );
        }

        BoostBot_Logger::info( 'Loopback self-test result', array(
            'works' => $works,
            'url'   => $url,
            'error' => is_wp_error( $response ) ? $response->get_error_message() : null,
            'code'  => ! is_wp_error( $response ) ? wp_remote_retrieve_response_code( $response ) : null,
        ) );

        return $works;
    }

    /**
     * Stops the background indexer.
     *
     * @since 1.0.0
     * @return bool True on success.
     */
    public static function stop(): bool
    {
        $job = BoostBot_Index_Job::get();
        if ( $job ) {
            BoostBot_Index_Job::cancel( (string) $job->id );
        }

        // Pause future processing.
        update_option( self::OPTION_PAUSED, true );

        wp_clear_scheduled_hook( self::CRON_HOOK );

        BoostBot_Logger::info( 'Background indexer stopped' );

        return true;
    }

    /**
     * Gets the current queue status.
     *
     * @since 1.0.0
     * @return array|null Queue status or null.
     */
    public static function get_queue_status(): ?array
    {
        $job = BoostBot_Index_Job::get();
        if ( ! $job ) {
            return null;
        }

        $queue = new BoostBot_Index_Queue();
        $stats = $queue->get_stats();

        return array(
            'job_id'    => (string) $job->id,
            'stats'     => $stats,
        );
    }

    /**
     * Checks if the indexer is currently processing.
     *
     * @since 1.0.0
     * @return bool True if processing.
     */
    public static function is_processing(): bool
    {
        $job = BoostBot_Index_Job::get();
        if ( ! $job || $job->status !== BoostBot_Index_Job::STATUS_RUNNING ) {
            return false;
        }
        $queue = new BoostBot_Index_Queue();
        $stats = $queue->get_stats();
        return ( (int) ( $stats['pending'] ?? 0 ) > 0 ) || ( (int) ( $stats['processing'] ?? 0 ) > 0 );
    }

    /**
     * Recovery: detect and reset stuck processing items.
     *
     * Enhanced logic:
     * - Items at extraction_level 1 that are on their last attempt get skipped
     *   (they've exhausted all recovery options)
     * - Other items get their extraction level reduced and retried
     *
     * @since 1.13.0
     * @param int $timeout_minutes
     * @return int Number of items reset or skipped
     */
    public static function recover_stuck_items( int $timeout_minutes = 5 ): int {
        $queue = new BoostBot_Index_Queue();
        $stuck = $queue->get_stuck_items( $timeout_minutes );
        $reset = 0;

        foreach ( $stuck as $row ) {
            $id           = (int) ( $row['id'] ?? 0 );
            $level        = (int) ( $row['extraction_level'] ?? 5 );
            $attempts     = (int) ( $row['attempts'] ?? 0 );
            $max_attempts = (int) ( $row['max_attempts'] ?? 3 );
            $content_id   = (int) ( $row['content_id'] ?? 0 );
            $content_size = (int) ( $row['content_size'] ?? 0 );

            if ( ! $id ) {
                continue;
            }

            // If at minimum extraction level AND on last attempt, skip this item entirely
            // It has exhausted all recovery options and is likely problematic content
            if ( $level <= 1 && $attempts >= $max_attempts ) {
                $reason = sprintf(
                    'Auto-skipped: exhausted all recovery options (size: %d bytes, attempts: %d)',
                    $content_size,
                    $attempts
                );
                if ( $queue->skip( $id, $reason ) ) {
                    BoostBot_Logger::warning( 'Auto-skipped permanently stuck item', array(
                        'queue_id'    => $id,
                        'content_id'  => $content_id,
                        'content_size'=> $content_size,
                        'attempts'    => $attempts,
                    ) );
                    $reset++;
                }
                continue;
            }

            // Normal recovery: reduce extraction level and retry
            $next_level = max( 1, $level - 1 );
            if ( $queue->fail( $id, 'Stuck processing timeout; retrying', $next_level ) ) {
                $reset++;
            }
        }

        return $reset;
    }

    /**
     * Cron handler for recovery hook (runs every 5 minutes).
     *
     * Three-stage recovery:
     * 1. Reset stuck processing items.
     * 2. If stalled, re-establish the batch cron chain (schedule_next_batch).
     * 3. If the batch cron chain is still missing after scheduling, process
     *    one batch inline as a last resort.
     *
     * @since 1.13.0
     * @return void
     */
    public static function run_recovery(): void {
        $n = self::recover_stuck_items( 5 );
        if ( $n > 0 ) {
            BoostBot_Logger::warning( 'Recovered stuck indexing items', array( 'count' => $n ) );
        }

        if ( ! self::is_stalled() ) {
            return;
        }

        // Stage 1: try to re-establish the cron chain (preferred, non-blocking).
        BoostBot_Logger::info( 'Recovery detected stalled indexing — re-establishing cron chain' );
        self::schedule_next_batch();

        // Stage 2: verify the cron event was actually created.
        // On some hosts wp_schedule_single_event silently fails.
        $next = wp_next_scheduled( self::CRON_HOOK );
        if ( $next && $next <= time() + 120 ) {
            // Cron chain re-established — it will fire soon.
            return;
        }

        // Stage 3: cron chain couldn't be re-established. Process one batch inline.
        BoostBot_Logger::warning( 'Recovery: cron chain could not be re-established, processing batch inline' );
        self::process_batch();
    }

    /**
     * Check if indexing is stalled.
     *
     * Stalled = there's a running job with pending items, nothing is processing,
     * and either no WP-Cron event is scheduled or the scheduled event is overdue
     * (meaning WP-Cron itself isn't firing).
     *
     * @since 1.14.1
     * @return bool True if indexing appears stalled.
     */
    public static function is_stalled(): bool {
        if ( (bool) get_option( self::OPTION_PAUSED, false ) ) {
            return false;
        }

        $job = BoostBot_Index_Job::get();
        if ( ! $job || $job->status !== BoostBot_Index_Job::STATUS_RUNNING ) {
            return false;
        }

        $queue = new BoostBot_Index_Queue();
        $stats = $queue->get_stats();
        $pending    = (int) ( $stats['pending'] ?? 0 );
        $processing = (int) ( $stats['processing'] ?? 0 );

        if ( $pending <= 0 ) {
            return false;
        }

        if ( $processing > 0 ) {
            return false;
        }

        // Check if a cron event is already scheduled AND not overdue.
        $next_cron = wp_next_scheduled( self::CRON_HOOK );
        if ( $next_cron ) {
            // If the cron event is in the near future, not stalled.
            if ( $next_cron > time() ) {
                return false;
            }
            // If it's overdue by less than 2 minutes, give WP-Cron a chance to fire.
            if ( ( time() - $next_cron ) < 120 ) {
                return false;
            }
            // Overdue by 2+ minutes — WP-Cron isn't working, treat as stalled.
        }

        return true;
    }

    /**
     * Get queue health metrics for monitoring.
     *
     * @since 1.14.0
     * @return array Health metrics
     */
    public static function get_queue_health(): array {
        $queue = new BoostBot_Index_Queue();
        $stats = $queue->get_stats();
        $stuck = $queue->get_stuck_items( self::STUCK_TIMEOUT_MINUTES );

        $total      = (int) ( $stats['total'] ?? 0 );
        $completed  = (int) ( $stats['completed'] ?? 0 );
        $failed     = (int) ( $stats['failed'] ?? 0 );
        $skipped    = (int) ( $stats['skipped'] ?? 0 );
        $pending    = (int) ( $stats['pending'] ?? 0 );
        $processing = (int) ( $stats['processing'] ?? 0 );

        $progress_pct = $total > 0 ? round( ( $completed + $failed + $skipped ) / $total * 100, 1 ) : 0;
        $success_pct  = ( $completed + $failed + $skipped ) > 0
            ? round( $completed / ( $completed + $failed + $skipped ) * 100, 1 )
            : 100;

        return array(
            'stats'          => $stats,
            'stuck_count'    => count( $stuck ),
            'progress_pct'   => $progress_pct,
            'success_pct'    => $success_pct,
            'is_healthy'     => count( $stuck ) === 0 && $failed < ( $total * 0.1 ), // < 10% failure rate
            'large_content_threshold' => self::LARGE_CONTENT_THRESHOLD,
            'stuck_timeout_minutes'   => self::STUCK_TIMEOUT_MINUTES,
        );
    }
}

