<?php
/**
 * Data collection and analysis class with content scraping
 *
 * @package ContentPerformanceAnalyzer
 * @since 1.0.0
 */

// Prevent direct access
if (!defined('ABSPATH')) {
    exit;
}


class CPAP_Data_Collector {

    private $content_scraper;

    /**
     * API errors captured during the current collection run.
     *
     * @var string[]
     */
    private $api_errors = array();

    /**
     * Record an API failure so the run status can surface it to the user.
     *
     * @param string    $context Short label for the failed call.
     * @param Exception $e       The caught exception.
     */
    private function record_api_error( $context, Exception $e ) {
        $this->api_errors[] = $context . ': ' . $e->getMessage();
        $this->debug_log( '[CPA] ' . $context . ': ' . $e->getMessage() );
    }
    /**
     * Get custom table name (unsanitized).
     *
     * @return string
     */
    private function get_urls_table_name() {
        global $wpdb;
        return $wpdb->prefix . 'cpanalyzer_urls';
    }

    /**
     * Get sanitized custom table name for direct SQL usage.
     *
     * @return string
     */
    private function get_sanitized_urls_table_name() {
        return esc_sql( $this->get_urls_table_name() );
    }

    /**
     * Initialize data collector
     */
    public function init() {
        add_action('cpanalyzer_daily_refresh', array($this, 'collect_and_analyse'));
        add_action('cpanalyzer_background_content_analysis', array($this, 'process_background_content_analysis'));

        // Initialize content scraper
        $this->content_scraper = new CPAP_Content_Scraper();
        $this->content_scraper->init();
    }

    /**
     * Placeholder preserved for call-site compatibility; does nothing.
     * set_time_limit / ini_set removed per WordPress Plugin Directory guidelines.
     *
     * @deprecated
     */
    private function maybe_bump_limits( $time_limit = 90, $memory_limit = '256M' ) {
    }

    /**
     * Main data collection and analysis function
     */
    public function collect_and_analyse() {
        // Get settings
        $settings = $this->get_settings();
        if ( ! $this->validate_settings( $settings ) ) {
            return false;
        }
        
        try {
            $this->api_errors = array();

            // Collect GA4 data via OAuth Bearer token
            $ga_data = $this->collect_ga4_data();

            // Collect Search Console data via OAuth Bearer token
            $gsc_data = $this->collect_search_console_data();

            // If every API call failed and no data came back, surface the failure
            // instead of reporting a successful run over an empty dataset.
            if ( ! empty( $this->api_errors ) && empty( $ga_data['current'] ) && empty( $gsc_data ) ) {
                update_option(
                    'cpanalyzer_status_msg',
                    sprintf(
                        /* translators: %s: first API error message */
                        __( 'Data collection failed — no data could be retrieved. First error: %s. Check your Google connection on the Settings page.', 'content-performance-analyzer' ),
                        $this->api_errors[0]
                    )
                );
                return false;
            }

            // Process and flag pages
            $flagged_pages = $this->process_and_flag_pages($ga_data, $gsc_data);
            
            // Save data
            $this->save_collected_data($ga_data, $gsc_data, $flagged_pages);
            
            // Process with AI if needed
            if (!empty($flagged_pages['payload'])) {
                $ai_processor = new CPAP_AI_Processor();
                $ai_processor->process_flagged_pages($flagged_pages['payload']);
            }
            
            $status_msg = sprintf(
                /* translators: %d: Number of pages flagged */
                __('✅ Data collection completed. %d pages flagged for analysis.', 'content-performance-analyzer'),
                count($flagged_pages['flags'])
            );

            if ( ! empty( $this->api_errors ) ) {
                $status_msg .= ' ' . sprintf(
                    /* translators: 1: number of failed API calls; 2: first API error message */
                    __( 'Warning: %1$d API call(s) failed, so some data may be incomplete. First error: %2$s', 'content-performance-analyzer' ),
                    count( $this->api_errors ),
                    $this->api_errors[0]
                );
            }

            update_option('cpanalyzer_status_msg', $status_msg);

            return true;
            
        } catch (Exception $e) {
            update_option('cpanalyzer_status_msg', 
                sprintf(
                    /* translators: %s: Error message */
                    __('Error: %s', 'content-performance-analyzer'),
                    $e->getMessage()
                )
            );
            return false;
        }
    }

    /**
     * Collect content data using the scraper
     *
     * @param array $pages Array of page URLs to scrape
     * @return array Content analysis data
     */
    private function collect_content_data($pages) {
        if (empty($pages)) {
            return array();
        }

        // Initialize content scraper if not already done
        if (!$this->content_scraper) {
            if (!class_exists('CPAP_Content_Scraper')) {
                require_once CPANALYZER_PLUGIN_DIR . 'includes/class-cpa-content-scraper.php';
            }
            $this->content_scraper = new CPAP_Content_Scraper();
            $this->content_scraper->init();
        }

        // Start progressive content analysis
        return $this->start_progressive_content_analysis($pages);
    }

    /**
     * Start progressive content analysis for all pages
     *
     * @param array $pages Array of page URLs to analyze
     * @return array Already analyzed content data (if any)
     */
    private function start_progressive_content_analysis($pages) {
        // Get existing content data to see what's already analyzed
        $existing_content_data = (array) get_option('cpanalyzer_content_data', array());
        
        // Determine which pages need analysis
        $pages_needing_analysis = array();
        $pages_already_analyzed = 0;
        
        foreach ($pages as $page_url) {
            if (isset($existing_content_data[$page_url]) && !empty($existing_content_data[$page_url])) {
                $pages_already_analyzed++;
            } else {
                $pages_needing_analysis[] = $page_url;
            }
        }
        
        // If all pages are already analyzed, return existing data
        if (empty($pages_needing_analysis)) {
            update_option('cpanalyzer_status_msg', 
                /* translators: %d: number of pages */
                sprintf(__('✅ All %d pages already analyzed.', 'content-performance-analyzer'), count($pages))
            );
            return $existing_content_data;
        }
        
        // Analyze a small batch immediately (first 5 pages) to show progress
        $immediate_batch_size = min(5, count($pages_needing_analysis));
        $immediate_batch = array_slice($pages_needing_analysis, 0, $immediate_batch_size);
        $remaining_pages = array_slice($pages_needing_analysis, $immediate_batch_size);
        
        // Analyze immediate batch
        $new_content_data = array();
        foreach ($immediate_batch as $page_url) {
            $content_data = $this->content_scraper->scrape_page_content($page_url);
            if ($content_data) {
                $new_content_data[$page_url] = $content_data;
                $existing_content_data[$page_url] = $content_data;
            }
        }
        
        // Save the updated content data
        update_option('cpanalyzer_content_data', $existing_content_data, false);
        
        // Schedule remaining pages for background analysis
        if (!empty($remaining_pages)) {
            $this->schedule_background_content_analysis($remaining_pages);
            
            $analyzed_count = $pages_already_analyzed + count($new_content_data);
            $total_count = count($pages);
            update_option('cpanalyzer_status_msg', 
                sprintf(
                    /* translators: 1: number of pages analyzed; 2: total number of pages */
                    __('Content analysis in progress: %1$d of %2$d pages analyzed. Background processing continues...', 'content-performance-analyzer'),
                    $analyzed_count,
                    $total_count
                )
            );
        } else {
            $total_analyzed = $pages_already_analyzed + count($new_content_data);
            update_option('cpanalyzer_status_msg', 
                /* translators: %d: number of pages analyzed */
                sprintf(__('✅ Content analysis complete: %d pages analyzed.', 'content-performance-analyzer'), $total_analyzed)
            );
        }
        
        return $existing_content_data;
    }

    /**
     * Schedule background content analysis for remaining pages
     *
     * @param array $remaining_pages Pages still needing analysis
     */
    private function schedule_background_content_analysis($remaining_pages) {
        // Store pages needing analysis in a transient for background processing
        set_transient('cpanalyzer_pending_content_analysis', $remaining_pages, 3600); // 1 hour expiry
        
        // Schedule immediate background processing
        if (!wp_next_scheduled('cpanalyzer_background_content_analysis')) {
            wp_schedule_single_event(time() + 30, 'cpanalyzer_background_content_analysis'); // Start in 30 seconds
        }
    }

    /**
     * Process background content analysis for remaining pages
     */
    public function process_background_content_analysis() {
        // Safely bump limits only during background processing
        $this->maybe_bump_limits( 90, '256M' );

        // Get pending pages from transient
        $pending_pages = get_transient('cpanalyzer_pending_content_analysis');
        
        if (empty($pending_pages)) {
            return; // No pages to process
        }
        
        // Initialize content scraper if not already done
        if (!$this->content_scraper) {
            if (!class_exists('CPAP_Content_Scraper')) {
                require_once CPANALYZER_PLUGIN_DIR . 'includes/class-cpa-content-scraper.php';
            }
            $this->content_scraper = new CPAP_Content_Scraper();
            $this->content_scraper->init();
        }
        
        // Get existing content data
        $existing_content_data = (array) get_option('cpanalyzer_content_data', array());
        
        // Determine batch size based on system performance and page priority
        $flagged_urls = array_keys((array) get_option('cpanalyzer_page_flags', array()));
        $batch_size = $this->calculate_optimal_batch_size($pending_pages, $flagged_urls);
        $processed_pages = 0;
        $successful_analyses = 0;
        $start_time = microtime(true);
        
        // Prioritize flagged pages for faster AI processing
        $prioritized_pages = $this->prioritize_pages_for_scraping($pending_pages, $flagged_urls);
        
        for ($i = 0; $i < count($prioritized_pages) && $i < $batch_size; $i++) {
            // Dynamic timeout check based on actual performance
            $elapsed_time = microtime(true) - $start_time;
            $time_per_page = $i > 0 ? $elapsed_time / $i : 0;
            $estimated_remaining_time = ($batch_size - $i) * max($time_per_page, 3); // Assume minimum 3s per page
            
            if ($elapsed_time + $estimated_remaining_time > 75) { // Stop if we might timeout
                break;
            }
            
            $page_url = $prioritized_pages[$i];
            $processed_pages++;
            
            // Skip if already analyzed (race condition protection)
            if (isset($existing_content_data[$page_url]) && !empty($existing_content_data[$page_url])) {
                continue;
            }
            
            $content_data = $this->content_scraper->scrape_page_content($page_url);
            if ($content_data) {
                $existing_content_data[$page_url] = $content_data;
                $successful_analyses++;
            }
        }
        
        // Save updated content data
        if ($successful_analyses > 0) {
            update_option('cpanalyzer_content_data', $existing_content_data, false);
        }
        
        // Remove processed pages from pending list
        $remaining_pages = array_slice($pending_pages, $processed_pages);
        
        // Update transient with remaining pages
        if (!empty($remaining_pages)) {
            set_transient('cpanalyzer_pending_content_analysis', $remaining_pages, 3600);
            
            // Schedule next batch more aggressively for faster completion
            $next_delay = $this->calculate_next_batch_delay($remaining_pages, $flagged_urls);
            wp_schedule_single_event(time() + $next_delay, 'cpanalyzer_background_content_analysis');
            
            // Check if any flagged pages now have complete content data for AI processing
            $this->check_and_process_pending_ai();
            
            // Update status
            $total_analyzed = count($existing_content_data);
            $total_flagged = count((array) get_option('cpanalyzer_page_flags', array()));
            update_option('cpanalyzer_status_msg', 
                sprintf(
                    /* translators: 1: total pages analyzed; 2: total flagged pages; 3: pages analyzed in this batch */
                    __('Background content analysis: %1$d of %2$d pages analyzed (+%3$d this batch)', 'content-performance-analyzer'),
                    $total_analyzed,
                    $total_flagged,
                    $successful_analyses
                )
            );
        } else {
            // All done - clean up transient and update final status
            delete_transient('cpanalyzer_pending_content_analysis');
            
            $total_analyzed = count($existing_content_data);
            
            // Check if we need to process pending AI suggestions now that content analysis is complete
            $this->check_and_process_pending_ai();
            
            update_option('cpanalyzer_status_msg', 
                /* translators: %d: number of pages analyzed */
                sprintf(__('✅ Background content analysis complete: %d pages analyzed.', 'content-performance-analyzer'), $total_analyzed)
            );
        }
    }

    /**
     * Handle AI processing based on content analysis completeness
     *
     * @param array $flagged_payload Pages flagged for AI analysis
     * @param array $all_pages All pages being analyzed
     * @param array $current_content_data Content data available now
     */
    private function handle_ai_processing($flagged_payload, $all_pages, $current_content_data) {
        // Check if content analysis is complete for all flagged pages
        $flagged_urls = array();
        foreach ($flagged_payload as $page_data) {
            if (isset($page_data['url'])) {
                $flagged_urls[] = $page_data['url'];
            }
        }
        
        $content_complete_for_flagged = true;
        foreach ($flagged_urls as $url) {
            if (!isset($current_content_data[$url]) || empty($current_content_data[$url])) {
                $content_complete_for_flagged = false;
                break;
            }
        }
        
        if ($content_complete_for_flagged) {
            // All flagged pages have content data - process with AI immediately
            $ai_processor = new CPAP_AI_Processor();
            $ai_processor->process_flagged_pages($flagged_payload);
            
            update_option('cpanalyzer_status_msg', 
                /* translators: %d: number of flagged pages */
                sprintf(__('AI analysis complete with full content data for %d flagged pages.', 'content-performance-analyzer'), count($flagged_urls))
            );
        } else {
            // Content analysis incomplete - schedule AI processing for later
            set_transient('cpanalyzer_pending_ai_payload', $flagged_payload, 3600);
            
            update_option('cpanalyzer_status_msg', 
                /* translators: %d: number of flagged pages */
                sprintf(__('AI analysis deferred until content scraping completes for %d flagged pages.', 'content-performance-analyzer'), count($flagged_urls))
            );
        }
    }

    /**
     * Check if pending AI processing can now be completed with updated content data
     */
    private function check_and_process_pending_ai() {
        $pending_ai_payload = get_transient('cpanalyzer_pending_ai_payload');
        
        if (empty($pending_ai_payload)) {
            return; // No pending AI processing
        }
        
        // Get current content data
        $current_content_data = (array) get_option('cpanalyzer_content_data', array());
        
        // Check if all flagged pages now have content data
        $flagged_urls = array();
        foreach ($pending_ai_payload as $page_data) {
            if (isset($page_data['url'])) {
                $flagged_urls[] = $page_data['url'];
            }
        }
        
        $content_complete_for_flagged = true;
        foreach ($flagged_urls as $url) {
            if (!isset($current_content_data[$url]) || empty($current_content_data[$url])) {
                $content_complete_for_flagged = false;
                break;
            }
        }
        
        if ($content_complete_for_flagged) {
            // All flagged pages now have content data - process with AI
            delete_transient('cpanalyzer_pending_ai_payload');
            
            // Rebuild payload with complete content data
            $updated_payload = $this->rebuild_ai_payload_with_content($pending_ai_payload, $current_content_data);
            
            $ai_processor = new CPAP_AI_Processor();
            $ai_processor->process_flagged_pages($updated_payload);
            
            update_option('cpanalyzer_status_msg', 
                /* translators: %d: number of flagged pages */
                sprintf(__('AI analysis now complete with full content data for %d flagged pages!', 'content-performance-analyzer'), count($flagged_urls))
            );
        }
        // If not complete yet, leave the transient and wait for more content analysis
    }

    /**
     * Rebuild AI payload with complete content data
     *
     * @param array $original_payload Original AI payload
     * @param array $content_data Complete content data
     * @return array Updated payload with content data
     */
    private function rebuild_ai_payload_with_content($original_payload, $content_data) {
        $updated_payload = array();
        
        foreach ($original_payload as $page_data) {
            if (isset($page_data['url']) && isset($content_data[$page_data['url']])) {
                // Add content data to the payload
                $page_data['content_data'] = $content_data[$page_data['url']];
            }
            $updated_payload[] = $page_data;
        }
        
        return $updated_payload;
    }

    /**
     * Calculate optimal batch size based on system performance and priorities
     *
     * @param array $pending_pages Pages waiting to be processed
     * @param array $flagged_urls Flagged pages needing priority
     * @return int Optimal batch size
     */
    private function calculate_optimal_batch_size($pending_pages, $flagged_urls) {
        // Base batch size on system performance metrics
        $base_batch_size = 8; // Increased from 5
        
        // Reduce batch size if many flagged pages remain (prioritize speed for AI)
        $flagged_pending = array_intersect($pending_pages, $flagged_urls);
        if (count($flagged_pending) > 10) {
            $base_batch_size = 6; // Smaller batches for faster flagged page completion
        }
        
        // Increase batch size if mostly non-flagged pages remain
        if (count($flagged_pending) < 3 && count($pending_pages) > 20) {
            $base_batch_size = 12; // Larger batches for bulk processing
        }
        
        return min($base_batch_size, count($pending_pages));
    }

    /**
     * Prioritize pages for scraping - flagged pages first
     *
     * @param array $pending_pages All pending pages
     * @param array $flagged_urls Flagged pages needing priority
     * @return array Prioritized pages list
     */
    private function prioritize_pages_for_scraping($pending_pages, $flagged_urls) {
        $flagged_pending = array();
        $non_flagged_pending = array();
        
        foreach ($pending_pages as $page_url) {
            if (in_array($page_url, $flagged_urls)) {
                $flagged_pending[] = $page_url;
            } else {
                $non_flagged_pending[] = $page_url;
            }
        }
        
        // Return flagged pages first, then non-flagged
        return array_merge($flagged_pending, $non_flagged_pending);
    }

    /**
     * Calculate delay for next batch based on priorities
     *
     * @param array $remaining_pages Pages still to process
     * @param array $flagged_urls Flagged pages needing priority
     * @return int Delay in seconds
     */
    private function calculate_next_batch_delay($remaining_pages, $flagged_urls) {
        $flagged_remaining = array_intersect($remaining_pages, $flagged_urls);
        
        // If flagged pages remain, process faster
        if (count($flagged_remaining) > 0) {
            return 20; // 20 seconds delay for flagged pages
        }
        
        // If only non-flagged pages remain, slower processing is OK
        return 45; // 45 seconds delay for non-flagged pages
    }

    /**
     * Get plugin settings — OAuth 2.0 edition (no key file).
     */
    private function get_settings() {
        return array(
            'property_id' => get_option( 'cpanalyzer_property_id', '' ),
            'site_url'    => get_option( 'cpanalyzer_site_url', '' ),
            'ai_ready'    => cpanalyzer_ai_is_configured(),
        );
    }

    /**
     * Validate settings — checks OAuth connection instead of key file.
     */
    private function validate_settings( $settings ) {
        $oauth = new CPAP_Google_OAuth();

        if ( ! $oauth->is_connected() ) {
            update_option(
                'cpanalyzer_status_msg',
                __( 'Google account not connected. Please connect via Settings → Google API Credentials.', 'content-performance-analyzer' )
            );
            return false;
        }

        $required = array( 'property_id', 'site_url' );
        foreach ( $required as $key ) {
            if ( empty( $settings[ $key ] ) ) {
                update_option(
                    'cpanalyzer_status_msg',
                    sprintf(
                        /* translators: %s: Setting key name */
                        __( 'Missing required setting: %s', 'content-performance-analyzer' ),
                        $key
                    )
                );
                return false;
            }
        }

        // Strip "properties/" prefix if present and ensure numeric.
        $property_id = $settings['property_id'];
        if ( strpos( $property_id, 'properties/' ) === 0 ) {
            $property_id = str_replace( 'properties/', '', $property_id );
            update_option( 'cpanalyzer_property_id', $property_id );
        }
        if ( ! is_numeric( $property_id ) ) {
            update_option( 'cpanalyzer_status_msg', __( 'GA4 Property ID must be numeric (e.g., 123456789)', 'content-performance-analyzer' ) );
            return false;
        }

        return true;
    }

    /**
     * Collect GA4 data via the Analytics Data REST API (no SDK).
     *
     * @return array{ current: array, previous: array }
     */
    private function collect_ga4_data() {
        $oauth       = new CPAP_Google_OAuth();
        $token       = $oauth->get_valid_access_token();
        $property_id = get_option( 'cpanalyzer_property_id', '' );

        $ga_data = array( 'current' => array(), 'previous' => array() );

        if ( empty( $token ) || empty( $property_id ) ) {
            return $ga_data;
        }

        $endpoint = 'https://analyticsdata.googleapis.com/v1beta/properties/' . rawurlencode( $property_id ) . ':runReport';
        $headers  = array(
            'Authorization' => 'Bearer ' . $token,
            'Content-Type'  => 'application/json',
        );

        $uploaded_urls = $this->get_uploaded_urls();

        // ---- Current period ------------------------------------------------
        $key_events_total = 0;

        try {
            $body = wp_json_encode( array(
                'dateRanges' => array( array( 'startDate' => '30daysAgo', 'endDate' => 'today' ) ),
                'dimensions' => array( array( 'name' => 'pagePath' ) ),
                'metrics'    => array(
                    array( 'name' => 'screenPageViews' ),
                    array( 'name' => 'activeUsers' ),
                    array( 'name' => 'engagementRate' ),
                    array( 'name' => 'averageSessionDuration' ),
                    array( 'name' => 'keyEvents' ),
                ),
                'limit'      => 2500,
            ) );

            $response = wp_remote_post( $endpoint, array( 'headers' => $headers, 'body' => $body, 'timeout' => 30 ) );
            $data     = $this->parse_api_response( $response, 'GA4 current' );

            foreach ( ( $data['rows'] ?? array() ) as $row ) {
                $page = cpanalyzer_normalize_path( $row['dimensionValues'][0]['value'] ?? '' );
                if ( ! empty( $uploaded_urls ) && ! in_array( $page, $uploaded_urls, true ) ) {
                    continue;
                }
                $m          = $row['metricValues'] ?? array();
                $key_events = (int) ( $m[4]['value'] ?? 0 );

                $ga_data['current'][ $page ] = array(
                    'views'      => (int) ( $m[0]['value'] ?? 0 ),
                    'uniques'    => (int) ( $m[1]['value'] ?? 0 ),
                    'engagement' => (float) ( $m[2]['value'] ?? 0 ),
                    'duration'   => (float) ( $m[3]['value'] ?? 0 ),
                    'form_fills' => $key_events,
                    'views_prev' => 0,
                );

                $key_events_total += $key_events;
            }
        } catch ( Exception $e ) {
            $this->record_api_error( 'GA4 current', $e );
        }

        // ---- Previous period -----------------------------------------------
        try {
            $body = wp_json_encode( array(
                'dateRanges' => array( array( 'startDate' => '60daysAgo', 'endDate' => '31daysAgo' ) ),
                'dimensions' => array( array( 'name' => 'pagePath' ) ),
                'metrics'    => array( array( 'name' => 'screenPageViews' ) ),
                'limit'      => 2500,
            ) );

            $response = wp_remote_post( $endpoint, array( 'headers' => $headers, 'body' => $body, 'timeout' => 30 ) );
            $data     = $this->parse_api_response( $response, 'GA4 previous' );

            foreach ( ( $data['rows'] ?? array() ) as $row ) {
                $page  = cpanalyzer_normalize_path( $row['dimensionValues'][0]['value'] ?? '' );
                $views = (int) ( $row['metricValues'][0]['value'] ?? 0 );
                if ( ! empty( $uploaded_urls ) && ! in_array( $page, $uploaded_urls, true ) ) {
                    continue;
                }
                $ga_data['previous'][ $page ] = array( 'views' => $views );
                if ( isset( $ga_data['current'][ $page ] ) ) {
                    $ga_data['current'][ $page ]['views_prev'] = $views;
                }
            }
        } catch ( Exception $e ) {
            $this->record_api_error( 'GA4 previous', $e );
        }

        // ---- Conversions -----------------------------------------------------
        // Priority: custom event names (setting) > GA4 key events (already on the
        // current-period query) > enhanced-measurement form_submit fallback.
        $custom_events = get_option( 'cpanalyzer_conversion_events', '' );
        $custom_events = array_values( array_filter( array_map( 'trim', explode( ',', (string) $custom_events ) ) ) );

        if ( ! empty( $custom_events ) ) {
            $counts = $this->collect_event_counts( $endpoint, $headers, $custom_events );
            foreach ( $ga_data['current'] as $page => $row ) {
                $ga_data['current'][ $page ]['form_fills'] = $counts[ $page ] ?? 0;
            }
        } elseif ( 0 === $key_events_total ) {
            $counts = $this->collect_event_counts( $endpoint, $headers, array( 'form_submit' ) );
            foreach ( $counts as $page => $count ) {
                if ( isset( $ga_data['current'][ $page ] ) ) {
                    $ga_data['current'][ $page ]['form_fills'] = $count;
                }
            }
        }

        return $ga_data;
    }

    /**
     * Count occurrences of the given GA4 events per page path (last 30 days).
     *
     * @param string $endpoint    GA4 runReport endpoint.
     * @param array  $headers     Authenticated request headers.
     * @param array  $event_names GA4 event names to count.
     * @return array  Keyed by normalised page path.
     */
    private function collect_event_counts( $endpoint, $headers, $event_names ) {
        $counts = array();

        try {
            $body = wp_json_encode( array(
                'dateRanges'      => array( array( 'startDate' => '30daysAgo', 'endDate' => 'today' ) ),
                'dimensions'      => array( array( 'name' => 'pagePath' ) ),
                'metrics'         => array( array( 'name' => 'eventCount' ) ),
                'dimensionFilter' => array(
                    'filter' => array(
                        'fieldName'    => 'eventName',
                        'inListFilter' => array( 'values' => array_values( $event_names ) ),
                    ),
                ),
                'limit' => 2500,
            ) );

            $response = wp_remote_post( $endpoint, array( 'headers' => $headers, 'body' => $body, 'timeout' => 30 ) );
            $data     = $this->parse_api_response( $response, 'GA4 events' );

            foreach ( ( $data['rows'] ?? array() ) as $row ) {
                $page = cpanalyzer_normalize_path( $row['dimensionValues'][0]['value'] ?? '' );
                $counts[ $page ] = ( $counts[ $page ] ?? 0 ) + (int) ( $row['metricValues'][0]['value'] ?? 0 );
            }
        } catch ( Exception $e ) {
            $this->record_api_error( 'GA4 events', $e );
        }

        return $counts;
    }

    /**
     * Collect Search Console data via the Search Console REST API (no SDK).
     *
     * @return array  Keyed by normalised page path.
     */
    private function collect_search_console_data() {
        $oauth    = new CPAP_Google_OAuth();
        $token    = $oauth->get_valid_access_token();
        $site_url = get_option( 'cpanalyzer_site_url', '' );

        $gsc_data = array();

        if ( empty( $token ) || empty( $site_url ) ) {
            return $gsc_data;
        }

        $endpoint = 'https://searchconsole.googleapis.com/webmasters/v3/sites/'
                    . rawurlencode( $site_url ) . '/searchAnalytics/query';
        $headers  = array(
            'Authorization' => 'Bearer ' . $token,
            'Content-Type'  => 'application/json',
        );

        $uploaded_urls = $this->get_uploaded_urls();

        // ---- Current period ------------------------------------------------
        try {
            $body = wp_json_encode( array(
                'startDate'  => gmdate( 'Y-m-d', strtotime( '-30 days' ) ),
                'endDate'    => gmdate( 'Y-m-d' ),
                'dimensions' => array( 'page' ),
                'rowLimit'   => 2500,
            ) );

            $response = wp_remote_post( $endpoint, array( 'headers' => $headers, 'body' => $body, 'timeout' => 30 ) );
            $data     = $this->parse_api_response( $response, 'GSC current' );

            foreach ( ( $data['rows'] ?? array() ) as $row ) {
                $page = cpanalyzer_normalize_path( $row['keys'][0] ?? '' );
                if ( ! empty( $uploaded_urls ) && ! in_array( $page, $uploaded_urls, true ) ) {
                    continue;
                }
                $gsc_data[ $page ] = array(
                    'impressions'      => (int) ( $row['impressions'] ?? 0 ),
                    'clicks'           => (int) ( $row['clicks'] ?? 0 ),
                    'ctr'              => round( (float) ( $row['ctr'] ?? 0 ) * 100, 2 ),
                    'position'         => round( (float) ( $row['position'] ?? 0 ), 1 ),
                    'impressions_prev' => 0,
                    'keywords'         => array(),
                );
            }
        } catch ( Exception $e ) {
            $this->record_api_error( 'GSC current', $e );
        }

        // ---- Previous period -----------------------------------------------
        try {
            $body = wp_json_encode( array(
                'startDate'  => gmdate( 'Y-m-d', strtotime( '-60 days' ) ),
                'endDate'    => gmdate( 'Y-m-d', strtotime( '-31 days' ) ),
                'dimensions' => array( 'page' ),
                'rowLimit'   => 2500,
            ) );

            $response = wp_remote_post( $endpoint, array( 'headers' => $headers, 'body' => $body, 'timeout' => 30 ) );
            $data     = $this->parse_api_response( $response, 'GSC previous' );

            foreach ( ( $data['rows'] ?? array() ) as $row ) {
                $page = cpanalyzer_normalize_path( $row['keys'][0] ?? '' );
                if ( isset( $gsc_data[ $page ] ) ) {
                    $gsc_data[ $page ]['impressions_prev'] = (int) ( $row['impressions'] ?? 0 );
                }
            }
        } catch ( Exception $e ) {
            $this->record_api_error( 'GSC previous', $e );
        }

        // ---- Keywords per page ---------------------------------------------
        try {
            $body = wp_json_encode( array(
                'startDate'  => gmdate( 'Y-m-d', strtotime( '-30 days' ) ),
                'endDate'    => gmdate( 'Y-m-d' ),
                'dimensions' => array( 'page', 'query' ),
                'rowLimit'   => 2500,
            ) );

            $response = wp_remote_post( $endpoint, array( 'headers' => $headers, 'body' => $body, 'timeout' => 30 ) );
            $data     = $this->parse_api_response( $response, 'GSC keywords' );

            foreach ( ( $data['rows'] ?? array() ) as $row ) {
                $page    = cpanalyzer_normalize_path( $row['keys'][0] ?? '' );
                $keyword = sanitize_text_field( $row['keys'][1] ?? '' );
                if ( isset( $gsc_data[ $page ] ) && ! empty( $keyword ) ) {
                    $gsc_data[ $page ]['keywords'][] = array(
                        'keyword'     => $keyword,
                        'impressions' => (int) ( $row['impressions'] ?? 0 ),
                        'clicks'      => (int) ( $row['clicks'] ?? 0 ),
                        'ctr'         => round( (float) ( $row['ctr'] ?? 0 ) * 100, 2 ),
                        'position'    => round( (float) ( $row['position'] ?? 0 ), 1 ),
                    );
                }
            }

            // Sort and trim keywords per page.
            foreach ( $gsc_data as &$page_data ) {
                if ( ! empty( $page_data['keywords'] ) ) {
                    usort( $page_data['keywords'], static function ( $a, $b ) {
                        $clicks_diff = (int) ( $b['clicks'] ?? 0 ) - (int) ( $a['clicks'] ?? 0 );
                        if ( $clicks_diff !== 0 ) {
                            return $clicks_diff;
                        }
                        return (int) ( $b['impressions'] ?? 0 ) - (int) ( $a['impressions'] ?? 0 );
                    } );
                    $page_data['keywords'] = array_slice( $page_data['keywords'], 0, 3 );
                }
            }
            unset( $page_data );
        } catch ( Exception $e ) {
            $this->record_api_error( 'GSC keywords', $e );
        }

        $gsc_data = $this->recover_missing_search_console_keywords( $gsc_data, $endpoint, $headers, $site_url );

        return $gsc_data;
    }

    /**
     * Recover missing GSC keywords for uploaded URLs that already have page metrics.
     *
     * @param array  $gsc_data Existing GSC data keyed by normalized path.
     * @param string $endpoint Search Console query endpoint.
     * @param array  $headers  Auth headers for Search Console.
     * @param string $site_url Configured Search Console property.
     * @return array
     */
    private function recover_missing_search_console_keywords( $gsc_data, $endpoint, $headers, $site_url ) {
        $uploaded_urls = array();
        foreach ( $this->get_uploaded_urls() as $uploaded_url ) {
            $normalized_path = cpanalyzer_normalize_path( (string) $uploaded_url );
            if ( '' !== $normalized_path ) {
                $uploaded_urls[ $normalized_path ] = true;
            }
        }
        $missing_pages = array();
        $recovered     = 0;
        $failed        = 0;
        $limit         = (int) apply_filters( 'cpanalyzer_gsc_keyword_recovery_limit', 0 );

        if ( empty( $gsc_data ) || empty( $uploaded_urls ) ) {
            return $gsc_data;
        }

        foreach ( $gsc_data as $page => $page_data ) {
            $has_metrics = (int) ( $page_data['impressions'] ?? 0 ) > 0 || (int) ( $page_data['clicks'] ?? 0 ) > 0;
            if ( empty( $uploaded_urls[ $page ] ) || ! $has_metrics || ! empty( $page_data['keywords'] ) ) {
                continue;
            }

            $missing_pages[] = $page;
        }

        if ( empty( $missing_pages ) ) {
            return $gsc_data;
        }

        if ( $limit > 0 && count( $missing_pages ) > $limit ) {
            $missing_pages = array_slice( $missing_pages, 0, $limit );
        }

        foreach ( $missing_pages as $page ) {
            $full_url = $this->build_full_url( $page, $site_url );

            try {
                $body = wp_json_encode( array(
                    'startDate'             => gmdate( 'Y-m-d', strtotime( '-30 days' ) ),
                    'endDate'               => gmdate( 'Y-m-d' ),
                    'dimensions'            => array( 'query' ),
                    'rowLimit'              => 10,
                    'dimensionFilterGroups' => array(
                        array(
                            'filters' => array(
                                array(
                                    'dimension'  => 'page',
                                    'operator'   => 'equals',
                                    'expression' => $full_url,
                                ),
                            ),
                        ),
                    ),
                ) );

                $response = wp_remote_post( $endpoint, array( 'headers' => $headers, 'body' => $body, 'timeout' => 15 ) );
                $data     = $this->parse_api_response( $response, 'GSC keyword recovery' );
                $keywords = array();

                foreach ( ( $data['rows'] ?? array() ) as $row ) {
                    $keyword = sanitize_text_field( $row['keys'][0] ?? '' );
                    if ( empty( $keyword ) ) {
                        continue;
                    }

                    $keywords[] = array(
                        'keyword'     => $keyword,
                        'impressions' => (int) ( $row['impressions'] ?? 0 ),
                        'clicks'      => (int) ( $row['clicks'] ?? 0 ),
                        'ctr'         => round( (float) ( $row['ctr'] ?? 0 ) * 100, 2 ),
                        'position'    => round( (float) ( $row['position'] ?? 0 ), 1 ),
                    );
                }

                if ( empty( $keywords ) ) {
                    continue;
                }

                usort( $keywords, static function ( $a, $b ) {
                    $clicks_diff = (int) ( $b['clicks'] ?? 0 ) - (int) ( $a['clicks'] ?? 0 );
                    if ( $clicks_diff !== 0 ) {
                        return $clicks_diff;
                    }
                    return (int) ( $b['impressions'] ?? 0 ) - (int) ( $a['impressions'] ?? 0 );
                } );

                $gsc_data[ $page ]['keywords'] = array_slice( $keywords, 0, 3 );
                ++$recovered;
            } catch ( Exception $e ) {
                ++$failed;
                $this->debug_log( '[CPA] GSC keyword recovery for ' . $page . ': ' . $e->getMessage() );
            }
        }

        $this->debug_log(
            sprintf(
                '[CPA] GSC keyword recovery attempted=%d recovered=%d failed=%d',
                count( $missing_pages ),
                $recovered,
                $failed
            )
        );

        return $gsc_data;
    }

    /**
     * Parse a wp_remote_post() response from a Google REST API.
     * Throws Exception on HTTP / JSON errors so callers can catch cleanly.
     *
     * @param  mixed  $response wp_remote_post return value.
     * @param  string $context  Used in error messages for clarity.
     * @return array  Decoded response body.
     * @throws Exception
     */
    private function parse_api_response( $response, $context = 'API' ) {
        if ( is_wp_error( $response ) ) {
            throw new Exception( esc_html( $context ) . ': ' . esc_html( $response->get_error_message() ) );
        }
        $code = (int) wp_remote_retrieve_response_code( $response );
        $body = json_decode( wp_remote_retrieve_body( $response ), true );

        if ( $code < 200 || $code >= 300 ) {
            $msg = isset( $body['error']['message'] ) ? $body['error']['message'] : "HTTP {$code}";
            throw new Exception( esc_html( $context ) . ': ' . esc_html( $msg ) );
        }
        return is_array( $body ) ? $body : array();
    }

    /**
     * Build a full URL for Search Console page filters.
     *
     * @param string $path     Normalized page path.
     * @param string $site_url Search Console property URL.
     * @return string
     */
    private function build_full_url( $path, $site_url ) {
        if ( strpos( $site_url, 'sc-domain:' ) === 0 ) {
            return 'https://' . substr( $site_url, 10 ) . $path;
        }

        return rtrim( $site_url, '/' ) . '/' . ltrim( $path, '/' );
    }

    /**
     * Get uploaded URLs from the database
     *
     * @return array Array of uploaded URLs
     */
    private function get_uploaded_urls() {
        global $wpdb;
        $cpanalyzer_urls_table = $this->get_urls_table_name();

        // Check if table exists
        // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching -- Table existence check for uploaded URLs
        if ($wpdb->get_var($wpdb->prepare("SHOW TABLES LIKE %s", $cpanalyzer_urls_table)) !== $cpanalyzer_urls_table) {
            return array(); // No uploaded URLs if table doesn't exist
        }

        // Get all uploaded URLs from the database
        // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching, WordPress.DB.PreparedSQLPlaceholders.UnsupportedIdentifierPlaceholder -- Custom table query with %i placeholder (requires WP 6.2+, our minimum version)
        $uploaded_urls = $wpdb->get_col(
            $wpdb->prepare(
                "SELECT url FROM %i ORDER BY priority ASC, id ASC",
                $cpanalyzer_urls_table
            )
        );

        return $uploaded_urls ?: array();
    }

    /**
     * Process and flag pages based on performance criteria (enhanced with content data)
     */
    private function process_and_flag_pages($ga_data, $gsc_data) {
        $flags   = array();
        $payload = array();

        // Performance thresholds
        $thresholds = array(
            'min_impressions'    => 500,
            'low_ctr'            => 2,
            'decay_threshold'    => 25,
            'min_previous_views' => 1000,
            'high_views'         => 800,
            'low_conversions'    => 5
        );

        // Get uploaded URLs from database instead of auto-discovering from GA4/GSC
        $uploaded_urls = $this->get_uploaded_urls();
        
        // If no uploaded URLs, return empty results (no auto-discovery)
        if (empty($uploaded_urls)) {
            update_option('cpanalyzer_status_msg', 
                __('No uploaded URLs found. Please upload URLs via URL Management → Upload URLs to analyze specific pages. Auto-discovery is now disabled.', 'content-performance-analyzer')
            );
            return array(
                'flags'   => array(),
                'payload' => array()
            );
        }
        
        $all_pages = $uploaded_urls;

        foreach ($all_pages as $page) {
            $ga_current = $ga_data['current'][$page] ?? array();
            $ga_previous = $ga_data['previous'][$page] ?? array();
            $gsc = $gsc_data[$page] ?? array();
            
            // Set defaults
            $ga_current = array_merge(array(
                'views' => 0,
                'views_prev' => 0,
                'form_fills' => 0
            ), $ga_current);
            
            $gsc = array_merge(array(
                'impressions' => 0,
                'ctr' => 0,
                'keywords' => array()
            ), $gsc);
            
            // Get content data for PSI checking
            $content_data = get_option('cpanalyzer_content_data', array());
            $page_content_data = $content_data[$page] ?? array();
            
            $flag = $this->determine_page_flag($ga_current, $gsc, $thresholds, $page_content_data);
            
            if ($flag) {
                $flags[$page] = $flag;
                
                // Check if we need to refresh AI suggestion
                if (cpanalyzer_should_refresh_suggestion($page)) {
                    $payload[] = array(
                        'url' => $page,
                        'flag' => $flag,
                        'impressions' => $gsc['impressions'],
                        'ctr' => $gsc['ctr'],
                        'views' => $ga_current['views'],
                        'form_fills' => $ga_current['form_fills'],
                        'keywords' => array_slice($gsc['keywords'], 0, 3)
                    );
                }
            }
        }

        return array(
            'flags'   => $flags,
            'payload' => $payload
        );
    }

    /**
     * Determine flag for a page based on performance criteria
     */
    private function determine_page_flag($ga_data, $gsc_data, $thresholds, $content_data = array()) {
        // Calculate decay percentage
        $decay = 0;
        if ($ga_data['views_prev'] > 0) {
            $decay = round(100 * ($ga_data['views_prev'] - $ga_data['views']) / $ga_data['views_prev'], 1);
        }
        
        // Check for decay flag
        if ($decay >= $thresholds['decay_threshold'] && $ga_data['views_prev'] >= $thresholds['min_previous_views']) {
            return 'decay';
        }
        
        // Check for SEO flag
        if ($gsc_data['impressions'] >= $thresholds['min_impressions'] && $gsc_data['ctr'] <= $thresholds['low_ctr']) {
            return 'seo';
        }
        
        // Check for conversion flag
        if ($ga_data['views'] >= $thresholds['high_views'] && $ga_data['form_fills'] <= $thresholds['low_conversions']) {
            return 'conversion';
        }
        
        // Check for performance flag (PSI-based)
        if (!empty($content_data['psi'])) {
            $psi_data = $content_data['psi'];
            $primary_strategy = $psi_data['primary_strategy'] ?? 'mobile';
            $strategy_data = $psi_data[$primary_strategy] ?? null;
            
            if ($strategy_data) {
                $score = $strategy_data['score'] ?? null;
                $cwv_pass = $strategy_data['core_web_vitals_pass'] ?? true;
                
                // Flag if fails Core Web Vitals OR score is below 50
                if (!$cwv_pass || ($score !== null && $score < 50)) {
                    return 'performance';
                }
            }
        }
        
        return false;
    }

    /**
     * Save collected data to WordPress options
     */
    private function save_collected_data($ga_data, $gsc_data, $flagged_pages) {
        update_option('cpanalyzer_ga_rows', $ga_data['current'], false);
        update_option('cpanalyzer_ga_prev_rows', $ga_data['previous'], false);
        update_option('cpanalyzer_gsc_rows', $gsc_data, false);
        update_option('cpanalyzer_page_flags', $flagged_pages['flags'], false);

        CPAP_Snapshots::maybe_record(
            $flagged_pages['flags'],
            $ga_data['current'],
            $gsc_data,
            (array) get_option('cpanalyzer_content_data', array())
        );
    }
    
    /**
     * Save discovered URLs to batch table for unified processing
     */
    private function save_urls_to_batch_table($ga_data, $gsc_data, $content_data) {
        global $wpdb;
        $cpanalyzer_urls_table = $this->get_urls_table_name();

        // Get all URLs from GA and GSC data
        $all_urls = array_unique(array_merge(
            array_keys($ga_data['current']),
            array_keys($gsc_data)
        ));

        foreach ($all_urls as $url) {
            // Check if URL already exists in batch table
            // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching, WordPress.DB.PreparedSQLPlaceholders.UnsupportedIdentifierPlaceholder -- URL existence check with %i placeholder (requires WP 6.2+, our minimum version)
            $existing = $wpdb->get_var($wpdb->prepare(
                "SELECT id FROM %i WHERE url = %s",
                $cpanalyzer_urls_table,
                $url
            ));
            
            if (!$existing) {
                // Add URL to batch table with the collected data
                $ga_url_data = $ga_data['current'][$url] ?? array();
                $gsc_url_data = $gsc_data[$url] ?? array();
                $content_url_data = $content_data[$url] ?? array();
                
                // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching -- Insert new URLs to custom table
                $wpdb->insert(
                    $cpanalyzer_urls_table,
                    array(
                        'url' => $url,
                        'status' => 'success',
                        'ga_data' => wp_json_encode($ga_url_data),
                        'gsc_data' => wp_json_encode($gsc_url_data),
                        'content_data' => wp_json_encode($content_url_data),
                        'priority' => 0, // Regular discovery = priority 0
                        'last_scanned' => current_time('mysql')
                    )
                );
            } else {
                // Update existing URL with new data
                $ga_url_data = $ga_data['current'][$url] ?? array();
                $gsc_url_data = $gsc_data[$url] ?? array();
                $content_url_data = $content_data[$url] ?? array();
                
                // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching -- Update priority for existing URLs
                $wpdb->update(
                    $cpanalyzer_urls_table,
                    array(
                        'status' => 'success',
                        'ga_data' => wp_json_encode($ga_url_data),
                        'gsc_data' => wp_json_encode($gsc_url_data),
                        'content_data' => wp_json_encode($content_url_data),
                        'last_scanned' => current_time('mysql')
                    ),
                    array('id' => $existing)
                );
            }
        }
    }
    
    /**
     * Schedule AI processing for discovered URLs that need it
     */
    private function schedule_ai_processing_for_discovered_urls($ai_payload) {
        global $wpdb;
        $cpanalyzer_urls_table = $this->get_urls_table_name();

        // Update URLs in batch table to trigger AI processing
        foreach ($ai_payload as $url_data) {
            $url = $url_data['url'] ?? '';
            if (empty($url)) continue;

            // Check if URL exists and doesn't have AI suggestions
            // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching, WordPress.DB.PreparedSQLPlaceholders.UnsupportedIdentifierPlaceholder -- AI suggestions check with %i placeholder (requires WP 6.2+, our minimum version)
            $existing = $wpdb->get_row($wpdb->prepare(
                "SELECT id, ai_suggestions FROM %i WHERE url = %s",
                $cpanalyzer_urls_table,
                $url
            ));
            
            if ($existing && empty($existing->ai_suggestions)) {
                // Get AI suggestions
                $ai_processor = new CPAP_AI_Processor();
                $ai_suggestions = $ai_processor->get_ai_suggestions($url_data);
                
                if (!empty($ai_suggestions)) {
                    // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching -- AI suggestions update to custom table
                    $wpdb->update(
                        $cpanalyzer_urls_table,
                        array('ai_suggestions' => wp_json_encode($ai_suggestions)),
                        array('id' => $existing->id)
                    );
                }
            }
        }
    }
    
    /**
     * Trigger insights sync to ensure data appears immediately
     */
    private function trigger_insights_sync() {
        // Ensure admin class is loaded
        if (!class_exists('CPAP_Admin')) {
            require_once CPANALYZER_PLUGIN_DIR . 'includes/class-cpa-admin.php';
        }
        
        $admin = new CPAP_Admin();
        $admin->sync_batch_data_to_insights();
    }
    
    /**
     * Debug logging method with WP_DEBUG_LOG protection
     *
     * @param string $message The message to log
     */
    private function debug_log($message) {
        if (defined('WP_DEBUG_LOG') && WP_DEBUG_LOG) {
            // phpcs:ignore WordPress.PHP.DevelopmentFunctions.error_log_error_log -- Protected debug logging
            error_log($message);
        }
    }
}
