<?php

/**
 * Serper.dev competitor research collector
 * Fetches top-ranking competitor pages for a keyword and extracts
 * their headings + body text for real gap analysis.
 *
 * @package ContentPerformanceAnalyzer
 * @since 1.2.0
 */
if ( !defined( 'ABSPATH' ) ) {
    exit;
}
if ( class_exists( 'CPAP_Serper_Collector' ) ) {
    return;
}
class CPAP_Serper_Collector {
    /**
     * Serper.dev API endpoint
     */
    const SERPER_API_URL = 'https://google.serper.dev/search';

    /**
     * Chrome-like User-Agent to avoid bot blocks
     */
    const USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36';

    /**
     * Max characters of body text to extract per competitor (token budget)
     */
    const MAX_BODY_CHARS = 2000;

    /**
     * Max number of competitor URLs to fetch
     */
    const MAX_COMPETITORS = 3;

    /**
     * Organic results requested per Serper query.
     */
    const RESULTS_PER_QUERY = 20;

    /**
     * Hard cap on Serper API calls per keyword analysis.
     */
    const MAX_SERPER_QUERIES = 2;

    /**
     * HTTP timeout in seconds for fetching competitor pages
     */
    const FETCH_TIMEOUT = 8;

    /**
     * Get and sanitize the Serper API key from WP options
     *
     * @return string
     */
    public function get_api_key() {
        $key = get_option( 'cpanalyzer_serper_api_key', '' );
        $key = trim( preg_replace( '/[^\\x20-\\x7E]/', '', (string) $key ) );
        return $key;
    }

    /**
     * Check if Serper is configured.
     *
     * A personal key always counts. With an active Pro license the hosted
     * relay provides the searches, so Pro is configured with no key at all.
     *
     * @return bool
     */
    public function is_configured() {
        $key = $this->get_api_key();
        if ( !empty( $key ) && strlen( $key ) > 10 ) {
            return true;
        }
        return false;
    }

    /**
     * POST one search to Serper — directly with the user's key, or through
     * the hosted relay when a Pro license provides the searches.
     *
     * Every Serper call in this class goes through here so the two
     * transports can never drift apart.
     *
     * @param array $payload Serper request body (q, num, gl, hl, ...).
     * @param int   $timeout HTTP timeout in seconds.
     * @return array|WP_Error wp_remote_post() result.
     */
    private function serper_request( $payload, $timeout = 15 ) {
        $endpoint = self::SERPER_API_URL;
        $headers = array(
            'X-API-KEY'    => $this->get_api_key(),
            'Content-Type' => 'application/json',
        );
        return wp_remote_post( $endpoint, array(
            'headers' => $headers,
            'body'    => wp_json_encode( $payload ),
            'timeout' => $timeout,
        ) );
    }

    /**
     * Send a temporary Serper debug event into the plugin log stream.
     *
     * @param string $event
     * @param string $message
     * @param array  $data
     * @return void
     */
    private function log_serper_event( $event, $message, $data = array() ) {
        if ( class_exists( 'CPAP_Core' ) && method_exists( 'CPAP_Core', 'get_instance' ) ) {
            CPAP_Core::get_instance()->log_event( $event, $message, ( is_array( $data ) ? $data : array() ) );
        }
    }

    /**
     * Fetch SERP feature intelligence for one keyword — used to confirm
     * zero-click capture (answer box above the organic results) and to spot
     * intent drift (top results shifting to video/forum/commerce formats).
     *
     * One Serper call per keyword, cached for 7 days.
     *
     * @param string $keyword Keyword to check on the live SERP.
     * @param string $own_url The user's own URL, to record its live position.
     * @return array|WP_Error {
     *   'keyword'         => string,
     *   'answer_box'      => bool,
     *   'paa_count'       => int,
     *   'knowledge_graph' => bool,
     *   'video_present'   => bool,
     *   'non_article_top' => int,   // video/forum/commerce results in top 10
     *   'own_position'    => int|null,
     *   'top_domains'     => string[],
     *   'retrieved_at'    => string,
     * }
     */
    public function get_serp_features( $keyword, $own_url = '' ) {
        $keyword = sanitize_text_field( (string) $keyword );
        if ( '' === $keyword || !$this->is_configured() ) {
            return new WP_Error('serper_unavailable', 'Serper is not configured or the keyword is empty.');
        }
        $cache_key = 'cpanalyzer_serp_feat_' . md5( strtolower( $keyword ) );
        $cached = get_transient( $cache_key );
        if ( is_array( $cached ) ) {
            return $cached;
        }
        $response = $this->serper_request( array(
            'q'   => $keyword,
            'num' => 10,
            'gl'  => 'us',
            'hl'  => 'en',
        ), 15 );
        if ( is_wp_error( $response ) ) {
            return new WP_Error('serper_request_failed', 'Serper SERP feature request failed: ' . $response->get_error_message());
        }
        $code = (int) wp_remote_retrieve_response_code( $response );
        if ( 200 !== $code ) {
            return new WP_Error('serper_error', 'Serper returned HTTP ' . $code . ' for SERP feature check.');
        }
        $json = json_decode( wp_remote_retrieve_body( $response ), true );
        if ( !is_array( $json ) ) {
            return new WP_Error('serper_error', 'Serper returned an unreadable SERP feature response.');
        }
        $non_article_hosts = array(
            'youtube.com',
            'youtu.be',
            'vimeo.com',
            'tiktok.com',
            // video
            'reddit.com',
            'quora.com',
            'stackexchange.com',
            'stackoverflow.com',
            // forum
            'amazon.com',
            'ebay.com',
            'etsy.com',
        );
        $own_path = ( '' !== $own_url ? cpanalyzer_normalize_path( $own_url ) : '' );
        $own_host = strtolower( (string) wp_parse_url( home_url(), PHP_URL_HOST ) );
        $own_position = null;
        $non_article_top = 0;
        $top_domains = array();
        foreach ( array_slice( (array) ($json['organic'] ?? array()), 0, 10 ) as $index => $result ) {
            $link = (string) ($result['link'] ?? '');
            $host = strtolower( (string) wp_parse_url( $link, PHP_URL_HOST ) );
            $host = preg_replace( '/^www\\./', '', $host );
            if ( '' !== $host && count( $top_domains ) < 5 && !in_array( $host, $top_domains, true ) ) {
                $top_domains[] = $host;
            }
            foreach ( $non_article_hosts as $flagged_host ) {
                if ( $host === $flagged_host || '' !== $host && substr( $host, -strlen( '.' . $flagged_host ) ) === '.' . $flagged_host ) {
                    $non_article_top++;
                    break;
                }
            }
            if ( null === $own_position && '' !== $own_host && false !== strpos( $host, preg_replace( '/^www\\./', '', $own_host ) ) ) {
                if ( '' === $own_path || cpanalyzer_normalize_path( $link ) === $own_path ) {
                    $own_position = ( isset( $result['position'] ) ? (int) $result['position'] : $index + 1 );
                }
            }
        }
        $features = array(
            'keyword'         => $keyword,
            'answer_box'      => !empty( $json['answerBox'] ),
            'ai_overview'     => !empty( $json['aiOverview'] ),
            'paa_count'       => count( (array) ($json['peopleAlsoAsk'] ?? array()) ),
            'knowledge_graph' => !empty( $json['knowledgeGraph'] ),
            'video_present'   => !empty( $json['videos'] ),
            'non_article_top' => $non_article_top,
            'own_position'    => $own_position,
            'top_domains'     => $top_domains,
            'retrieved_at'    => gmdate( 'Y-m-d H:i:s' ),
        );
        set_transient( $cache_key, $features, 7 * DAY_IN_SECONDS );
        $this->log_serper_event( 'serper_serp_features', 'Collected SERP features for decay diagnosis.', array(
            'keyword'         => $keyword,
            'answer_box'      => $features['answer_box'],
            'non_article_top' => $features['non_article_top'],
            'own_position'    => $features['own_position'],
        ) );
        return $features;
    }

    /**
     * Main entry point: get competitor context for a keyword.
     * Returns an array ready to be merged into the AI prompt payload.
     *
     * @param string $keyword  The primary keyword to search.
     * @param string $page_url The user's own URL (excluded from competitors).
     * @return array {
     *   'competitors'     => array of { url, headings[], body_excerpt },
     *   'paa'             => array of question strings,
     *   'related_keywords'=> array of related query strings,
     *   'serper_error'    => string|null  (null on success)
     * }
     */
    public function get_competitor_context( $keyword, $page_url = '' ) {
        if ( !$this->is_configured() ) {
            return array(
                'competitors'      => array(),
                'paa'              => array(),
                'related_keywords' => array(),
                'serper_error'     => 'Serper API key not configured.',
            );
        }
        if ( empty( $keyword ) ) {
            return array(
                'competitors'      => array(),
                'paa'              => array(),
                'related_keywords' => array(),
                'serper_error'     => 'No keyword provided.',
            );
        }
        // Step 1: Get Google results from Serper.dev
        $serp_intelligence = $this->fetch_serp_results( $keyword );
        if ( is_wp_error( $serp_intelligence ) ) {
            return array(
                'competitors'      => array(),
                'paa'              => array(),
                'related_keywords' => array(),
                'serper_error'     => $serp_intelligence->get_error_message(),
            );
        }
        // Step 2: Fetch and extract content from each competitor URL
        $competitors = array();
        $user_domain = $this->extract_domain( $page_url );
        foreach ( $serp_intelligence['organic'] as $result_url ) {
            if ( count( $competitors ) >= self::MAX_COMPETITORS ) {
                break;
            }
            // Skip if it's the user's own domain
            if ( !empty( $user_domain ) && $this->extract_domain( $result_url ) === $user_domain ) {
                continue;
            }
            $extracted = $this->fetch_and_extract( $result_url );
            if ( null !== $extracted ) {
                $competitors[] = $extracted;
            }
        }
        return array(
            'competitors'      => $competitors,
            'paa'              => $serp_intelligence['paa'],
            'related_keywords' => $serp_intelligence['related_keywords'],
            'serper_error'     => ( empty( $competitors ) ? 'Could not fetch competitor content.' : null ),
        );
    }

    /**
     * Call Serper.dev API and return aggregated SERP intelligence.
     *
     * @param string $keyword
     * @return array|WP_Error {
     *   @type string[] $organic
     *   @type string[] $paa
     *   @type string[] $related_keywords
     * }
     */
    private function fetch_serp_results( $keyword ) {
        $queries = $this->generate_search_queries( $keyword );
        $all_results = array();
        $paa_questions = array();
        $related_keywords = array();
        $this->log_serper_event( 'serper_queries', 'Prepared Serper queries for keyword analysis.', array(
            'keyword' => $keyword,
            'queries' => $queries,
            'mode'    => $this->get_query_mode( $keyword ),
        ) );
        foreach ( $queries as $query ) {
            $query_results = $this->fetch_serp_results_for_query( $query );
            if ( is_wp_error( $query_results ) ) {
                return $query_results;
            }
            if ( !empty( $query_results['organic'] ) ) {
                $all_results = array_merge( $all_results, $query_results['organic'] );
            }
            if ( !empty( $query_results['paa'] ) ) {
                $paa_questions = array_merge( $paa_questions, $query_results['paa'] );
            }
            if ( !empty( $query_results['related_keywords'] ) ) {
                $related_keywords = array_merge( $related_keywords, $query_results['related_keywords'] );
            }
        }
        $organic_urls = $this->deduplicate_urls( $all_results );
        if ( empty( $organic_urls ) ) {
            return new WP_Error('serper_no_results', 'No organic results returned by Serper.');
        }
        return array(
            'organic'          => $organic_urls,
            'paa'              => array_values( array_unique( array_filter( $paa_questions ) ) ),
            'related_keywords' => array_values( array_unique( array_filter( $related_keywords ) ) ),
        );
    }

    /**
     * Generate Serper queries for a keyword. Default mode keeps one query; deep mode adds one intent-expansion query.
     *
     * @param string $keyword
     * @return string[]
     */
    private function generate_search_queries( $keyword ) {
        $keyword = sanitize_text_field( $keyword );
        $mode = $this->get_query_mode( $keyword );
        $queries = array($keyword);
        if ( 'deep' === $mode ) {
            $secondary_query = $this->build_secondary_query( $keyword );
            if ( !empty( $secondary_query ) && strtolower( $secondary_query ) !== strtolower( $keyword ) ) {
                $queries[] = $secondary_query;
            }
        }
        $queries = array_slice( array_values( array_unique( array_filter( $queries ) ) ), 0, self::MAX_SERPER_QUERIES );
        return ( !empty( $queries ) ? $queries : array($keyword) );
    }

    /**
     * Resolve Serper query mode.
     *
     * @param string $keyword
     * @return string
     */
    private function get_query_mode( $keyword ) {
        $mode = apply_filters( 'cpanalyzer_serper_query_mode', 'basic', $keyword );
        $mode = ( is_string( $mode ) ? strtolower( trim( $mode ) ) : 'basic' );
        return ( in_array( $mode, array('basic', 'deep'), true ) ? $mode : 'basic' );
    }

    /**
     * Build one bounded intent-expansion query.
     *
     * @param string $keyword
     * @return string
     */
    private function build_secondary_query( $keyword ) {
        $keyword = trim( (string) $keyword );
        if ( '' === $keyword ) {
            return '';
        }
        $secondary_query = ( preg_match( '/\\b(how to|guide|tutorial|beginners?|introduction|basics?)\\b/i', $keyword ) ? $keyword . ' list' : $keyword . ' guide' );
        /**
         * Filter the secondary Serper query used in deep mode.
         *
         * @param string $secondary_query Generated secondary query.
         * @param string $keyword         Base keyword.
         */
        $secondary_query = apply_filters( 'cpanalyzer_serper_secondary_query', $secondary_query, $keyword );
        return sanitize_text_field( $secondary_query );
    }

    /**
     * Call Serper.dev for one query and normalize the useful response slices.
     *
     * @param string $query
     * @return array|WP_Error
     */
    private function fetch_serp_results_for_query( $query ) {
        $request_payload = array(
            'q'           => sanitize_text_field( $query ),
            'num'         => self::RESULTS_PER_QUERY,
            'gl'          => 'us',
            'hl'          => 'en',
            'autocorrect' => true,
        );
        $this->log_serper_event( 'serper_request', 'Dispatching Serper search request.', array(
            'payload' => $request_payload,
        ) );
        $response = $this->serper_request( $request_payload, 15 );
        if ( is_wp_error( $response ) ) {
            $this->log_serper_event( 'serper_error', 'Serper request failed before a response was returned.', array(
                'query' => $query,
                'error' => $response->get_error_message(),
            ) );
            return new WP_Error('serper_request_failed', 'Serper API request failed: ' . $response->get_error_message());
        }
        $code = (int) wp_remote_retrieve_response_code( $response );
        if ( $code !== 200 ) {
            // Both transports return a readable reason: Serper puts it at
            // `message`, the hosted relay at `error.message` (which already
            // distinguishes a bad key, an invalid license, and spent quota).
            $error_json = json_decode( wp_remote_retrieve_body( $response ), true );
            $error_detail = '';
            if ( is_array( $error_json ) ) {
                if ( isset( $error_json['error']['message'] ) && is_string( $error_json['error']['message'] ) ) {
                    $error_detail = $error_json['error']['message'];
                } elseif ( isset( $error_json['message'] ) && is_string( $error_json['message'] ) ) {
                    $error_detail = $error_json['message'];
                }
            }
            $this->log_serper_event( 'serper_error', 'Serper search returned a non-success HTTP status.', array(
                'query'     => $query,
                'http_code' => $code,
                'detail'    => $error_detail,
            ) );
            $message = 'Search API returned HTTP ' . $code . (( '' !== $error_detail ? ': ' . $error_detail : '' ));
            return new WP_Error(( 401 === $code || 403 === $code ? 'serper_auth_failed' : 'serper_error' ), $message);
        }
        $json = json_decode( wp_remote_retrieve_body( $response ), true );
        $organic = array();
        foreach ( (array) ($json['organic'] ?? array()) as $result ) {
            $link = $result['link'] ?? '';
            if ( filter_var( $link, FILTER_VALIDATE_URL ) !== false ) {
                $organic[] = $link;
            }
        }
        $paa_questions = array();
        foreach ( (array) ($json['peopleAlsoAsk'] ?? array()) as $paa_item ) {
            $question = sanitize_text_field( $paa_item['question'] ?? '' );
            if ( '' !== $question ) {
                $paa_questions[] = $question;
            }
        }
        $related_keywords = array();
        foreach ( (array) ($json['relatedSearches'] ?? array()) as $related_item ) {
            $related_query = sanitize_text_field( $related_item['query'] ?? '' );
            if ( '' !== $related_query ) {
                $related_keywords[] = $related_query;
            }
        }
        $this->log_serper_event( 'serper_response', 'Received Serper response.', array(
            'query'            => $query,
            'http_code'        => $code,
            'organic_count'    => count( $organic ),
            'paa_count'        => count( $paa_questions ),
            'related_count'    => count( $related_keywords ),
            'first_organic'    => array_slice( $organic, 0, 3 ),
            'paa'              => array_slice( $paa_questions, 0, 5 ),
            'related_keywords' => array_slice( $related_keywords, 0, 5 ),
        ) );
        return array(
            'organic'          => $organic,
            'paa'              => $paa_questions,
            'related_keywords' => $related_keywords,
        );
    }

    /**
     * Deduplicate organic result URLs while preserving order.
     *
     * @param array $urls
     * @return array
     */
    private function deduplicate_urls( $urls ) {
        $unique_urls = array();
        $seen_urls = array();
        foreach ( (array) $urls as $url ) {
            $url = esc_url_raw( $url );
            if ( empty( $url ) || isset( $seen_urls[$url] ) ) {
                continue;
            }
            $seen_urls[$url] = true;
            $unique_urls[] = $url;
        }
        return $unique_urls;
    }

    /**
     * Fetch a competitor page and extract its headings + clean body text.
     * Returns null if the page is blocked or fails.
     *
     * @param string $url
     * @return array|null { url, headings[], body_excerpt }
     */
    private function fetch_and_extract( $url ) {
        $response = wp_remote_get( $url, array(
            'timeout'    => self::FETCH_TIMEOUT,
            'user-agent' => self::USER_AGENT,
            'headers'    => array(
                'Accept'          => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
                'Accept-Language' => 'en-US,en;q=0.9',
                'Accept-Encoding' => 'gzip, deflate',
                'Connection'      => 'close',
                'Cache-Control'   => 'no-cache',
            ),
            'sslverify'  => get_option( 'cpanalyzer_verify_ssl', 'yes' ) !== 'no',
        ) );
        if ( is_wp_error( $response ) ) {
            return null;
        }
        $code = (int) wp_remote_retrieve_response_code( $response );
        // Treat 403, 429, 503 (Cloudflare/bot blocks) as a graceful skip
        if ( $code === 403 || $code === 429 || $code === 503 || $code < 200 || $code >= 400 ) {
            return null;
        }
        $html = wp_remote_retrieve_body( $response );
        if ( empty( $html ) || strlen( $html ) < 500 ) {
            return null;
        }
        // Detect if we got a CAPTCHA page (Cloudflare challenge, etc.)
        if ( $this->is_captcha_page( $html ) ) {
            return null;
        }
        // Extract structured content
        $headings = $this->extract_headings( $html );
        $body_excerpt = $this->extract_clean_body( $html );
        // Skip if extraction yielded nothing useful
        if ( empty( $headings ) && strlen( $body_excerpt ) < 200 ) {
            return null;
        }
        return array(
            'url'          => $url,
            'headings'     => $headings,
            'body_excerpt' => $body_excerpt,
        );
    }

    /**
     * Detect Cloudflare CAPTCHA / challenge pages.
     *
     * @param string $html
     * @return bool
     */
    private function is_captcha_page( $html ) {
        $captcha_signals = array(
            'cf-browser-verification',
            'challenge-form',
            'Checking your browser',
            'cf_clearance',
            'Enable JavaScript and cookies to continue',
            'Just a moment',
            'DDoS protection by Cloudflare',
            '<title>Attention Required!</title>'
        );
        foreach ( $captcha_signals as $signal ) {
            if ( stripos( $html, $signal ) !== false ) {
                return true;
            }
        }
        return false;
    }

    /**
     * Extract H1–H3 headings from HTML.
     *
     * @param string $html
     * @return string[]
     */
    private function extract_headings( $html ) {
        $headings = array();
        if ( preg_match_all( '/<h[1-3][^>]*>(.*?)<\\/h[1-3]>/is', $html, $matches ) ) {
            foreach ( $matches[1] as $match ) {
                $text = trim( wp_strip_all_tags( $match ) );
                // Skip empty or navigation-style headings
                if ( !empty( $text ) && strlen( $text ) > 3 && strlen( $text ) < 200 ) {
                    $headings[] = $text;
                }
            }
        }
        // Cap at 15 headings to keep token budget sane
        return array_slice( $headings, 0, 15 );
    }

    /**
     * Extract clean body text — removes scripts, styles, nav, header, footer, forms.
     * Returns up to MAX_BODY_CHARS characters.
     *
     * @param string $html
     * @return string
     */
    private function extract_clean_body( $html ) {
        // Remove entire blocks that add noise and tokens
        $noise_tags = array(
            'script',
            'style',
            'noscript',
            'nav',
            'header',
            'footer',
            'aside',
            'form',
            'iframe',
            'svg',
            'figure',
            'figcaption',
            'button',
            'select',
            'textarea',
            'input'
        );
        foreach ( $noise_tags as $tag ) {
            $html = preg_replace( '/<' . $tag . '[^>]*>.*?<\\/' . $tag . '>/is', '', $html );
        }
        // Remove all remaining HTML tags
        $text = wp_strip_all_tags( $html );
        // Collapse whitespace
        $text = preg_replace( '/\\s+/', ' ', $text );
        $text = trim( $text );
        // Truncate to token budget
        if ( strlen( $text ) > self::MAX_BODY_CHARS ) {
            $text = substr( $text, 0, self::MAX_BODY_CHARS ) . '…';
        }
        return $text;
    }

    /**
     * Extract domain from a URL for same-domain filtering.
     *
     * @param string $url
     * @return string
     */
    private function extract_domain( $url ) {
        $parsed = wp_parse_url( $url );
        return ( isset( $parsed['host'] ) ? strtolower( $parsed['host'] ) : '' );
    }

    /**
     * Cache wrapper: get competitor context with transient caching.
     * Cached for 24 hours per keyword to avoid burning Serper quota.
     *
     * @param string $keyword
     * @param string $page_url
     * @return array
     */
    public function get_competitor_context_cached( $keyword, $page_url = '' ) {
        $mode = $this->get_query_mode( $keyword );
        $cache_key = 'cpanalyzer_serper_' . md5( strtolower( trim( $keyword ) ) . '|' . $mode );
        $cached = get_transient( $cache_key );
        if ( false !== $cached && is_array( $cached ) ) {
            $this->log_serper_event( 'serper_cache_hit', 'Using cached Serper result.', array(
                'keyword'          => $keyword,
                'mode'             => $mode,
                'competitor_count' => count( $cached['competitors'] ?? array() ),
                'paa_count'        => count( $cached['paa'] ?? array() ),
                'related_count'    => count( $cached['related_keywords'] ?? array() ),
            ) );
            return $cached;
        }
        $result = $this->get_competitor_context( $keyword, $page_url );
        // Only cache successful results
        if ( !empty( $result['competitors'] ) ) {
            set_transient( $cache_key, $result, DAY_IN_SECONDS );
            $this->log_serper_event( 'serper_cache_store', 'Stored Serper result in cache.', array(
                'keyword'          => $keyword,
                'mode'             => $mode,
                'competitor_count' => count( $result['competitors'] ?? array() ),
                'paa_count'        => count( $result['paa'] ?? array() ),
                'related_count'    => count( $result['related_keywords'] ?? array() ),
            ) );
        }
        return $result;
    }

}
