<?php
/**
 * Content scraper class for analyzing page content and layout
 *
 * @package ContentPerformanceAnalyzer
 * @since 1.0.0
 */

// Prevent direct access
if ( ! defined( 'ABSPATH' ) ) {
	exit;
}

class CPAP_Content_Scraper {

	private $user_agent    = 'Mozilla/5.0 (compatible; CPA-Bot/1.0; WordPress Content Performance Analyzer)';
	private $timeout       = 10; // Optimized timeout for faster processing
	private $max_redirects = 2; // Reduced redirects to speed up processing

	/**
	 * Initialize content scraper
	 */
	public function init() {
		// Hook for any initialization needed
	}

	/**
	 * Scrape and analyze multiple pages
	 *
	 * @param array $pages Array of pages to analyze
	 * @return array Analyzed content data
	 */
	public function analyze_pages( $pages ) {
		$analyzed_data = array();

		foreach ( $pages as $page_url ) {
			$content_data = $this->scrape_page_content( $page_url );
			if ( $content_data ) {
				$analyzed_data[ $page_url ] = $content_data;
			}

			// Optimized delay for faster bulk processing
			usleep( 250000 ); // 0.25 second delay (reduced from 0.5s)
		}

		return $analyzed_data;
	}

	/**
	 * Scrape and analyze a single page
	 *
	 * @param string $page_url Page URL to scrape
	 * @return array|false Content analysis data or false on failure
	 */
	public function scrape_page_content( $page_url ) {
		$full_url = $this->build_full_url( $page_url );

		if ( ! $full_url ) {
			return false;
		}

		$html = $this->fetch_page_html( $full_url );
		if ( ! $html ) {
			return false;
		}

		return $this->analyze_html_content( $html, $page_url );
	}

	/**
	 * Build full URL from page path
	 *
	 * @param string $page_url Page URL or path
	 * @return string|false Full URL or false if invalid
	 */
	private function build_full_url( $page_url ) {
		$site_url = get_site_url();

		// If it's already a full URL, validate it's from our domain
		if ( filter_var( $page_url, FILTER_VALIDATE_URL ) ) {
			$parsed      = wp_parse_url( $page_url );
			$site_parsed = wp_parse_url( $site_url );

			if ( $parsed['host'] !== $site_parsed['host'] ) {
				return false; // External URL, skip
			}

			return $page_url;
		}

		// Build full URL from path
		$page_url = ltrim( $page_url, '/' );
		return rtrim( $site_url, '/' ) . '/' . $page_url;
	}

	/**
	 * Fetch HTML content from URL
	 *
	 * @param string $url Full URL to fetch
	 * @return string|false HTML content or false on failure
	 */
	private function fetch_page_html( $url ) {
		$verify_ssl_opt = get_option( 'cpanalyzer_verify_ssl', 'yes' ) === 'yes';

		$args = array(
			'timeout'     => $this->timeout,
			'redirection' => $this->max_redirects,
			'user-agent'  => $this->user_agent,
			'headers'     => array(
				'Accept'          => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
				'Accept-Language' => 'en-US,en;q=0.5',
				'Accept-Encoding' => 'gzip, deflate',
				'Cache-Control'   => 'no-cache',
			),
			'sslverify'   => $verify_ssl_opt,
		);

		$response = wp_remote_get( $url, $args );

		if ( is_wp_error( $response ) ) {
			return false;
		}

		$response_code = wp_remote_retrieve_response_code( $response );
		if ( $response_code !== 200 ) {
			return false;
		}

		return wp_remote_retrieve_body( $response );
	}

	/**
	 * Analyze HTML content and extract insights
	 *
	 * @param string $html HTML content
	 * @param string $page_url Original page URL
	 * @return array Content analysis data
	 */
	private function analyze_html_content( $html, $page_url ) {
		// Create DOMDocument
		libxml_use_internal_errors( true );
		$dom = new DOMDocument();
		$dom->loadHTML( '<?xml encoding="UTF-8">' . $html );
		libxml_clear_errors();

		$xpath = new DOMXPath( $dom );

		$analysis = array(
			'url'                => $page_url,
			'timestamp'          => time(),

			// SEO Elements
			'title'              => $this->extract_title( $xpath ),
			'meta_description'   => $this->extract_meta_description( $xpath ),
			'h1_tags'            => $this->extract_headings( $xpath, 'h1' ),
			'h2_tags'            => $this->extract_headings( $xpath, 'h2' ),
			'h3_tags'            => $this->extract_headings( $xpath, 'h3' ),

			// Deep content extraction (for Content Refresh Brief)
			'body_text'          => $this->extract_body_text( $xpath ),
			'meta_keywords'      => $this->extract_meta_keywords( $xpath ),
			'og_data'            => $this->extract_og_data( $xpath ),

			// Content Analysis
			'word_count'         => $this->count_words( $html ),
			'content_length'     => $this->get_content_length( $xpath ),
			'paragraph_count'    => $this->count_elements( $xpath, '//p' ),
			'image_count'        => $this->count_elements( $xpath, '//img' ),
			'internal_links'     => $this->count_internal_links( $xpath ),
			'external_links'     => $this->count_external_links( $xpath ),

			// Technical SEO
			'has_noindex'        => $this->detect_noindex( $xpath ),
			'canonical_url'      => $this->extract_canonical_url( $xpath ),
			'canonical_mismatch' => $this->detect_canonical_mismatch( $xpath, $page_url ),
			'has_schema'         => $this->check_schema_markup( $html ),
			'schema_types'       => $this->extract_schema_types( $html ),
			'images_without_alt' => $this->count_images_without_alt( $xpath ),
			'images_missing_alt_list' => array_slice( $this->get_images_without_alt( $xpath ), 0, 20 ),
			'page_size'          => strlen( $html ),

			// Answer-engine readiness signals
			'question_analysis'  => $this->analyze_question_headings( $xpath ),
			'list_count'         => $this->count_elements( $xpath, '//ul' ) + $this->count_elements( $xpath, '//ol' ),
			'table_count'        => $this->count_elements( $xpath, '//table' ),
			'has_date_signal'    => $this->detect_date_signal( $xpath, $html ),
			'stale_dates'        => $this->detect_stale_dates( $xpath ),

			// Content Quality Indicators
			'has_forms'          => $this->check_forms( $xpath ),
			'has_cta_buttons'    => $this->check_cta_buttons( $xpath ),
			'has_video'          => $this->check_video_content( $xpath ),
			'reading_level'      => $this->estimate_reading_level( $xpath ),

			// Layout Analysis
			'layout_structure'   => $this->analyze_layout_structure( $xpath ),
			'content_gaps'       => $this->identify_content_gaps( $xpath ),

			// Enhanced Semantic Analysis (Phase 1)
			'semantic_analysis'  => $this->analyze_content_semantics( $html, $xpath ),
		);

		return $analysis;
	}

	/**
	 * Detect a robots noindex directive in the page head. A noindex on a
	 * traffic-losing page IS the diagnosis — no content fix matters until
	 * it is removed or confirmed intentional.
	 *
	 * @param DOMXPath $xpath
	 * @return bool
	 */
	private function detect_noindex( DOMXPath $xpath ) {
		$nodes = $xpath->query( '//meta[translate(@name, "ROBOTS", "robots")="robots" or translate(@name, "GOLEBT", "golebt")="googlebot"]' );
		if ( ! $nodes ) {
			return false;
		}

		foreach ( $nodes as $node ) {
			$content = strtolower( (string) $node->getAttribute( 'content' ) );
			if ( false !== strpos( $content, 'noindex' ) ) {
				return true;
			}
		}

		return false;
	}

	/**
	 * Extract the rel=canonical URL, if declared.
	 *
	 * @param DOMXPath $xpath
	 * @return string Empty string when no canonical is declared.
	 */
	private function extract_canonical_url( DOMXPath $xpath ) {
		$nodes = $xpath->query( '//link[translate(@rel, "CANOICL", "canoicl")="canonical"]' );
		if ( $nodes && $nodes->length > 0 ) {
			return esc_url_raw( (string) $nodes->item( 0 )->getAttribute( 'href' ) );
		}
		return '';
	}

	/**
	 * True when the page declares a canonical pointing at a DIFFERENT path —
	 * its ranking signals are being consolidated elsewhere, which explains
	 * ranking loss no rewrite can fix.
	 *
	 * @param DOMXPath $xpath
	 * @param string   $page_url
	 * @return bool
	 */
	private function detect_canonical_mismatch( DOMXPath $xpath, $page_url ) {
		$canonical = $this->extract_canonical_url( $xpath );
		if ( '' === $canonical || ! function_exists( 'cpanalyzer_normalize_path' ) ) {
			return false;
		}

		return cpanalyzer_normalize_path( $canonical ) !== cpanalyzer_normalize_path( $page_url );
	}

	/**
	 * Extract plain text from the <body> element (stripped of all tags).
	 * Capped at 5 000 characters to keep AI prompts manageable.
	 *
	 * @param DOMXPath $xpath
	 * @return string
	 */
	private function extract_body_text( DOMXPath $xpath ) {
		$body_nodes = $xpath->query( '//body' );
		if ( $body_nodes->length === 0 ) {
			return '';
		}

		// Remove <script>, <style>, <nav>, <header>, <footer> noise first.
		$remove_tags = array( 'script', 'style', 'nav', 'header', 'footer', 'aside' );
		foreach ( $remove_tags as $tag ) {
			$nodes = $xpath->query( '//' . $tag );
			foreach ( $nodes as $node ) {
				$node->parentNode->removeChild( $node );
			}
		}

		$text = $body_nodes->item( 0 )->textContent;
		// Collapse whitespace and trim.
		$text = preg_replace( '/\s+/', ' ', $text );
		$text = trim( $text );

		// Limit to 5 000 chars to keep prompts lean.
		return mb_substr( $text, 0, 5000 );
	}

	/**
	 * Extract <meta name="keywords"> content.
	 *
	 * @param DOMXPath $xpath
	 * @return string
	 */
	private function extract_meta_keywords( DOMXPath $xpath ) {
		$nodes = $xpath->query( '//meta[@name="keywords"]/@content' );
		if ( $nodes->length > 0 ) {
			return sanitize_text_field( trim( $nodes->item( 0 )->textContent ) );
		}
		return '';
	}

	/**
	 * Extract Open Graph meta tags (og:title, og:description, og:type).
	 *
	 * @param DOMXPath $xpath
	 * @return array
	 */
	private function extract_og_data( DOMXPath $xpath ) {
		$og = array();
		$og_properties = array( 'og:title', 'og:description', 'og:type', 'og:image' );

		foreach ( $og_properties as $prop ) {
			$nodes = $xpath->query( '//meta[@property="' . $prop . '"]/@content' );
			if ( $nodes->length > 0 ) {
				$short_key       = str_replace( 'og:', '', $prop );
				$og[ $short_key ] = sanitize_text_field( trim( $nodes->item( 0 )->textContent ) );
			}
		}

		return $og;
	}

	/**
	 * Extract page title
	 */
	private function extract_title( $xpath ) {
		$title_nodes = $xpath->query( '//title' );
		if ( $title_nodes->length > 0 ) {
			$title = trim( $title_nodes->item( 0 )->textContent );
			return array(
				'text'   => $title,
				'length' => strlen( $title ),
				'words'  => str_word_count( $title ),
			);
		}
		return array(
			'text'   => '',
			'length' => 0,
			'words'  => 0,
		);
	}

	/**
	 * Extract meta description
	 */
	private function extract_meta_description( $xpath ) {
		$meta_nodes = $xpath->query( '//meta[@name="description"]/@content' );
		if ( $meta_nodes->length > 0 ) {
			$description = trim( $meta_nodes->item( 0 )->textContent );
			return array(
				'text'   => $description,
				'length' => strlen( $description ),
				'words'  => str_word_count( $description ),
			);
		}
		return array(
			'text'   => '',
			'length' => 0,
			'words'  => 0,
		);
	}

	/**
	 * Extract heading tags
	 */
	private function extract_headings( $xpath, $tag ) {
		$headings = array();
		$nodes    = $xpath->query( '//' . $tag );

		foreach ( $nodes as $node ) {
			$text = trim( $node->textContent );
			if ( ! empty( $text ) ) {
				$headings[] = array(
					'text'   => $text,
					'length' => strlen( $text ),
					'words'  => str_word_count( $text ),
				);
			}
		}

		return $headings;
	}

	/**
	 * Count words in content
	 */
	private function count_words( $html ) {
		// Remove scripts, styles, and other non-content
		$text = preg_replace( '/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/mi', '', $html );
		$text = preg_replace( '/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/mi', '', $text );
		$text = wp_strip_all_tags( $text );
		$text = preg_replace( '/\s+/', ' ', $text );

		return str_word_count( trim( $text ) );
	}

	/**
	 * Get main content length
	 */
	private function get_content_length( $xpath ) {
		// Try to find main content areas
		$content_selectors = array(
			'//main',
			'//article',
			'//*[@class="content"]',
			'//*[@id="content"]',
			'//body',
		);

		foreach ( $content_selectors as $selector ) {
			$nodes = $xpath->query( $selector );
			if ( $nodes->length > 0 ) {
				$content = trim( $nodes->item( 0 )->textContent );
				return strlen( $content );
			}
		}

		return 0;
	}

	/**
	 * Count specific elements
	 */
	private function count_elements( $xpath, $query ) {
		return $xpath->query( $query )->length;
	}

	/**
	 * Count internal links
	 */
	private function count_internal_links( $xpath ) {
		$site_domain    = wp_parse_url( get_site_url(), PHP_URL_HOST );
		$links          = $xpath->query( '//a[@href]' );
		$internal_count = 0;

		foreach ( $links as $link ) {
			$href = $link->getAttribute( 'href' );

			// Relative links are internal
			if ( strpos( $href, '/' ) === 0 && strpos( $href, '//' ) !== 0 ) {
				++$internal_count;
				continue;
			}

			// Check if absolute URL is internal
			$link_domain = wp_parse_url( $href, PHP_URL_HOST );
			if ( $link_domain === $site_domain ) {
				++$internal_count;
			}
		}

		return $internal_count;
	}

	/**
	 * Count external links
	 */
	private function count_external_links( $xpath ) {
		$site_domain    = wp_parse_url( get_site_url(), PHP_URL_HOST );
		$links          = $xpath->query( '//a[@href]' );
		$external_count = 0;

		foreach ( $links as $link ) {
			$href = $link->getAttribute( 'href' );

			// Skip relative links and anchors
			if ( strpos( $href, '/' ) === 0 || strpos( $href, '#' ) === 0 ) {
				continue;
			}

			$link_domain = wp_parse_url( $href, PHP_URL_HOST );
			if ( $link_domain && $link_domain !== $site_domain ) {
				++$external_count;
			}
		}

		return $external_count;
	}

	/**
	 * Check for schema markup
	 */
	private function check_schema_markup( $html ) {
		return (
		strpos( $html, 'application/ld+json' ) !== false ||
		strpos( $html, 'itemscope' ) !== false ||
		strpos( $html, 'vocab="http://schema.org"' ) !== false
		);
	}

	/**
	 * Collect the schema.org @type values declared in JSON-LD blocks,
	 * including nested @graph entries. Answer engines favor pages with
	 * FAQPage/Article/HowTo markup, so the AEO score checks for them.
	 *
	 * @param  string $html
	 * @return string[] Unique type names, e.g. ['Article', 'FAQPage'].
	 */
	private function extract_schema_types( $html ) {
		$types = array();

		if ( ! preg_match_all( '#<script[^>]*type=["\']application/ld\+json["\'][^>]*>(.*?)</script>#is', $html, $matches ) ) {
			return $types;
		}

		foreach ( $matches[1] as $json ) {
			$data = json_decode( trim( $json ), true );
			if ( ! is_array( $data ) ) {
				continue;
			}

			$stack = array( $data );
			while ( ! empty( $stack ) ) {
				$item = array_pop( $stack );
				if ( ! is_array( $item ) ) {
					continue;
				}
				if ( isset( $item['@type'] ) ) {
					foreach ( (array) $item['@type'] as $type ) {
						if ( is_string( $type ) && '' !== $type ) {
							$types[ $type ] = $type;
						}
					}
				}
				if ( isset( $item['@graph'] ) && is_array( $item['@graph'] ) ) {
					foreach ( $item['@graph'] as $graph_item ) {
						$stack[] = $graph_item;
					}
				}
				// Numeric arrays of entities at the top level.
				if ( isset( $item[0] ) ) {
					foreach ( $item as $sub_item ) {
						$stack[] = $sub_item;
					}
				}
			}
		}

		return array_values( $types );
	}

	/**
	 * Find question-phrased H2/H3 headings and count how many are followed
	 * directly by a concise answer paragraph (20-90 words) — the block shape
	 * answer engines and featured snippets lift.
	 *
	 * @param  DOMXPath $xpath
	 * @return array{ question_headings: string[], answered_count: int }
	 */
	private function analyze_question_headings( $xpath ) {
		$pattern   = '/^(what|how|why|when|where|which|who|can|could|does|do|is|are|should|will)\b|\?\s*$/i';
		$questions = array();
		$answered  = 0;

		foreach ( $xpath->query( '//h2|//h3' ) as $node ) {
			$text = trim( preg_replace( '/\s+/', ' ', (string) $node->textContent ) );
			if ( '' === $text || ! preg_match( $pattern, $text ) ) {
				continue;
			}
			$questions[] = $text;

			// Walk to the next element sibling (PHP 7.4-safe).
			$sibling = $node->nextSibling;
			while ( $sibling && XML_ELEMENT_NODE !== $sibling->nodeType ) {
				$sibling = $sibling->nextSibling;
			}
			if ( $sibling && 'p' === strtolower( $sibling->nodeName ) ) {
				$words = str_word_count( wp_strip_all_tags( (string) $sibling->textContent ) );
				if ( $words >= 20 && $words <= 90 ) {
					$answered++;
				}
			}
		}

		return array(
			'question_headings' => array_slice( $questions, 0, 20 ),
			'answered_count'    => $answered,
		);
	}

	/**
	 * Detect a machine-readable freshness signal: a <time> element or
	 * article published/modified meta tags.
	 *
	 * @param  DOMXPath $xpath
	 * @param  string   $html
	 * @return bool
	 */
	private function detect_date_signal( $xpath, $html ) {
		if ( $xpath->query( '//time' )->length > 0 ) {
			return true;
		}
		return (bool) preg_match( '#property=["\']article:(published|modified)_time["\']#i', $html );
	}

	/**
	 * Find years written into the visible copy that now read as out of date.
	 *
	 * A page can carry a perfectly fresh modified date and still say "the
	 * 2023 study" or "as of 2024" in its body, which undermines it with both
	 * readers and answer engines. Those references are detectable, cheap to
	 * fix, and a real refresh trigger independent of traffic movement.
	 *
	 * Only years strictly before the current one count, and only where the
	 * surrounding phrasing implies currency ("in 2024", "as of 2024",
	 * "2024 study", "2024 guide"). A bare year is ignored: historical
	 * statements ("founded in 1998", "the 2008 crash") are correct as
	 * written and must not be flagged. Copyright lines, times, and
	 * four-digit numbers that merely look like years are excluded too.
	 *
	 * @param DOMXPath $xpath Document under analysis.
	 * @return array{years: int[], phrases: string[], oldest_year: int, count: int}
	 */
	private function detect_stale_dates( DOMXPath $xpath ) {
		$empty = array( 'years' => array(), 'phrases' => array(), 'oldest_year' => 0, 'count' => 0 );

		$body = $this->extract_body_text( $xpath );
		if ( ! is_string( $body ) || '' === trim( $body ) ) {
			return $empty;
		}

		$current_year = (int) gmdate( 'Y' );
		$body         = preg_replace( '/\s+/', ' ', $body );

		// Currency-implying constructions only. Each alternative keeps the
		// year in group 1 so the matched phrase can be reported verbatim.
		$patterns = array(
			// "in 2024", "as of 2024", "since 2024", "for 2024", "by 2024"
			'/\b(?:in|as\s+of|since|for|by|during|throughout)\s+((?:19|20)\d{2})\b/i',
			// "2024 study", "2024 report", "2024 guide", "2024 data" ...
			'/\b((?:19|20)\d{2})\s+(?:study|studies|report|reports|survey|surveys|data|statistics|stats|guide|guides|edition|update|updates|version|research|benchmark|benchmarks|trends?|figures?)\b/i',
			// "latest 2024", "current 2024", "this year (2024)"
			'/\b(?:latest|current|newest|most\s+recent|updated)\s+((?:19|20)\d{2})\b/i',
		);

		$years   = array();
		$phrases = array();

		foreach ( $patterns as $pattern ) {
			if ( ! preg_match_all( $pattern, $body, $matches, PREG_SET_ORDER ) ) {
				continue;
			}
			foreach ( $matches as $match ) {
				$year = (int) $match[1];

				// Only years that have actually passed, and nothing so old
				// that it is obviously a historical reference.
				if ( $year >= $current_year || $year < $current_year - 12 ) {
					continue;
				}

				$phrase = trim( (string) $match[0] );
				if ( '' === $phrase ) {
					continue;
				}

				$years[] = $year;
				if ( count( $phrases ) < 5 && ! in_array( $phrase, $phrases, true ) ) {
					$phrases[] = $phrase;
				}
			}
		}

		if ( empty( $years ) ) {
			return $empty;
		}

		$unique = array_values( array_unique( $years ) );
		sort( $unique );

		return array(
			'years'       => $unique,
			'phrases'     => $phrases,
			'oldest_year' => (int) $unique[0],
			'count'       => count( $years ),
		);
	}

	/**
	 * Count images without alt text.
	 *
	 * Counts unique real images rather than raw <img> nodes — lazy-load
	 * plugins (e.g. WP Rocket) render each image twice (placeholder +
	 * noscript fallback), which double-counted before.
	 */
	private function count_images_without_alt( $xpath ) {
		return count( $this->get_images_without_alt( $xpath ) );
	}

	/**
	 * Collect the source URLs of images missing alt text.
	 *
	 * Resolves lazy-load attributes (data-lazy-src/data-src) before src,
	 * skips inline data: placeholder images, and dedupes by resolved URL so
	 * placeholder + noscript pairs report as one image.
	 *
	 * @param  DOMXPath $xpath
	 * @return string[] Unique image URLs, in document order.
	 */
	private function get_images_without_alt( $xpath ) {
		$nodes  = $xpath->query( '//img[not(@alt) or @alt=""]' );
		$images = array();

		foreach ( $nodes as $node ) {
			$src = trim( (string) $node->getAttribute( 'data-lazy-src' ) );
			if ( '' === $src ) {
				$src = trim( (string) $node->getAttribute( 'data-src' ) );
			}
			if ( '' === $src ) {
				$src = trim( (string) $node->getAttribute( 'src' ) );
			}

			// Inline data: URIs are lazy-load placeholders, not real images.
			if ( '' === $src || 0 === stripos( $src, 'data:' ) ) {
				continue;
			}

			$images[ $src ] = $src;
		}

		return array_values( $images );
	}

	/**
	 * Check for forms
	 */
	private function check_forms( $xpath ) {
		$forms     = $xpath->query( '//form' );
		$form_data = array();

		foreach ( $forms as $form ) {
			$inputs    = $xpath->query( './/input[@type="email" or @type="text" or @type="tel"]', $form );
			$textareas = $xpath->query( './/textarea', $form );
			$selects   = $xpath->query( './/select', $form );

			$form_data[] = array(
				'inputs'       => $inputs->length,
				'textareas'    => $textareas->length,
				'selects'      => $selects->length,
				'total_fields' => $inputs->length + $textareas->length + $selects->length,
			);
		}

		return array(
			'count' => $forms->length,
			'forms' => $form_data,
		);
	}

	/**
	 * Check for CTA buttons
	 */
	private function check_cta_buttons( $xpath ) {
		$cta_patterns = array(
			'//button[contains(translate(text(), "ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz"), "sign up")]',
			'//button[contains(translate(text(), "ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz"), "get started")]',
			'//button[contains(translate(text(), "ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz"), "free trial")]',
			'//button[contains(translate(text(), "ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz"), "demo")]',
			'//a[contains(translate(text(), "ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz"), "sign up")]',
			'//a[contains(translate(text(), "ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz"), "get started")]',
			'//a[contains(translate(text(), "ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz"), "free trial")]',
			'//a[contains(translate(text(), "ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz"), "demo")]',
		);

		$cta_count = 0;
		$cta_texts = array();

		foreach ( $cta_patterns as $pattern ) {
			$nodes      = $xpath->query( $pattern );
			$cta_count += $nodes->length;

			foreach ( $nodes as $node ) {
				$text = trim( $node->textContent );
				if ( ! empty( $text ) && ! in_array( $text, $cta_texts ) ) {
					$cta_texts[] = $text;
				}
			}
		}

		return array(
			'count' => $cta_count,
			'texts' => array_slice( $cta_texts, 0, 5 ), // Limit to top 5
		);
	}

	/**
	 * Check for video content
	 */
	private function check_video_content( $xpath ) {
		$video_count  = $xpath->query( '//video' )->length;
		$iframe_count = $xpath->query( '//iframe[contains(@src, "youtube") or contains(@src, "vimeo") or contains(@src, "wistia")]' )->length;

		return array(
			'native_videos'   => $video_count,
			'embedded_videos' => $iframe_count,
			'total'           => $video_count + $iframe_count,
		);
	}

	/**
	 * Estimate reading level (simplified)
	 */
	private function estimate_reading_level( $xpath ) {
		$content = $xpath->query( '//p | //div[contains(@class, "content")]' )->item( 0 );
		if ( ! $content ) {
			return array(
				'level' => 'unknown',
				'score' => 0,
			);
		}

		$text      = trim( $content->textContent );
		$sentences = preg_split( '/[.!?]+/', $text );
		$words     = str_word_count( $text );
		$syllables = $this->count_syllables( $text );

		if ( count( $sentences ) == 0 || $words == 0 ) {
			return array(
				'level' => 'unknown',
				'score' => 0,
			);
		}

		// Simplified Flesch Reading Ease
		$avg_sentence_length    = $words / max( 1, count( $sentences ) );
		$avg_syllables_per_word = $syllables / max( 1, $words );

		$score = 206.835 - ( 1.015 * $avg_sentence_length ) - ( 84.6 * $avg_syllables_per_word );

		if ( $score >= 60 ) {
			$level = 'easy';
		} elseif ( $score >= 30 ) {
			$level = 'moderate';
		} else {
			$level = 'difficult';
		}

		return array(
			'level'                  => $level,
			'score'                  => round( $score, 1 ),
			'avg_sentence_length'    => round( $avg_sentence_length, 1 ),
			'avg_syllables_per_word' => round( $avg_syllables_per_word, 2 ),
		);
	}

	/**
	 * Count syllables in text (simplified)
	 */
	private function count_syllables( $text ) {
		$text      = strtolower( $text );
		$syllables = 0;
		$words     = str_word_count( $text, 1 );

		foreach ( $words as $word ) {
			$syllables += max( 1, preg_match_all( '/[aeiouy]+/', $word ) );
		}

		return $syllables;
	}

	/**
	 * Analyze layout structure
	 */
	private function analyze_layout_structure( $xpath ) {
		return array(
			'has_header'    => $xpath->query( '//header' )->length > 0,
			'has_nav'       => $xpath->query( '//nav' )->length > 0,
			'has_main'      => $xpath->query( '//main' )->length > 0,
			'has_aside'     => $xpath->query( '//aside' )->length > 0,
			'has_footer'    => $xpath->query( '//footer' )->length > 0,
			'div_count'     => $xpath->query( '//div' )->length,
			'section_count' => $xpath->query( '//section' )->length,
			'article_count' => $xpath->query( '//article' )->length,
		);
	}

	/**
	 * Identify potential content gaps
	 */
	private function identify_content_gaps( $xpath ) {
		$gaps = array();

		// Check for missing elements
		if ( $xpath->query( '//h1' )->length === 0 ) {
			$gaps[] = 'missing_h1';
		}

		if ( $xpath->query( '//h2' )->length === 0 ) {
			$gaps[] = 'no_subheadings';
		}

		if ( $xpath->query( '//img' )->length === 0 ) {
			$gaps[] = 'no_images';
		}

		if ( $xpath->query( '//ul | //ol' )->length === 0 ) {
			$gaps[] = 'no_lists';
		}

		if ( $xpath->query( '//a[@href]' )->length < 3 ) {
			$gaps[] = 'insufficient_links';
		}

		return $gaps;
	}

	/**
	 * Generate content recommendations based on analysis
	 */
	public function generate_content_recommendations( $content_data ) {
		$recommendations = array();

		if ( empty( $content_data ) ) {
			return $recommendations;
		}

		// Title recommendations
		if ( $content_data['title']['length'] < 30 ) {
			$recommendations[] = 'Expand title - currently too short (' . $content_data['title']['length'] . ' chars)';
		} elseif ( $content_data['title']['length'] > 60 ) {
			$recommendations[] = 'Shorten title - currently too long (' . $content_data['title']['length'] . ' chars)';
		}

		// Meta description recommendations
		if ( $content_data['meta_description']['length'] === 0 ) {
			$recommendations[] = 'Add meta description for better search snippets';
		} elseif ( $content_data['meta_description']['length'] < 120 ) {
			$recommendations[] = 'Expand meta description to 150-160 characters';
		}

		// Content length recommendations
		if ( $content_data['word_count'] < 300 ) {
			$recommendations[] = 'Add more content - currently ' . $content_data['word_count'] . ' words (aim for 500+)';
		}

		// Heading structure
		if ( empty( $content_data['h1_tags'] ) ) {
			$recommendations[] = 'Add H1 tag for better SEO structure';
		} elseif ( count( $content_data['h1_tags'] ) > 1 ) {
			$recommendations[] = 'Use only one H1 tag per page';
		}

		if ( empty( $content_data['h2_tags'] ) ) {
			$recommendations[] = 'Add H2 subheadings to improve content structure';
		}

		// Image optimization
		if ( $content_data['images_without_alt'] > 0 ) {
			$recommendations[] = 'Add alt text to ' . $content_data['images_without_alt'] . ' images';
		}

		// Internal linking
		if ( $content_data['internal_links'] < 3 ) {
			$recommendations[] = 'Add more internal links (currently ' . $content_data['internal_links'] . ')';
		}

		// CTA recommendations
		if ( $content_data['has_cta_buttons']['count'] === 0 ) {
			$recommendations[] = 'Add call-to-action buttons to improve conversions';
		}

		// Form recommendations
		if ( $content_data['has_forms']['count'] === 0 ) {
			$recommendations[] = 'Consider adding a lead capture form';
		}

		// Content gaps
		foreach ( $content_data['content_gaps'] as $gap ) {
			switch ( $gap ) {
				case 'no_images':
					$recommendations[] = 'Add relevant images to improve engagement';
					break;
				case 'no_lists':
					$recommendations[] = 'Add bullet points or numbered lists for better readability';
					break;
				case 'insufficient_links':
					$recommendations[] = 'Add more internal and external links';
					break;
			}
		}

		return array_slice( $recommendations, 0, 5 ); // Limit to top 5 recommendations
	}

	/**
	 * Enhanced Semantic Content Analysis (Phase 1)
	 *
	 * @param string   $html Full HTML content
	 * @param DOMXPath $xpath XPath object for DOM queries
	 * @return array Semantic analysis data
	 */
	private function analyze_content_semantics( $html, $xpath ) {
		$main_content = $this->extract_main_content_text( $xpath );

		$semantic_analysis = array(
			'topic_keywords'               => $this->extract_topic_keywords( $main_content ),
			'keyword_density'              => $this->analyze_keyword_density( $main_content ),
			'content_depth_score'          => $this->calculate_content_depth_score( $main_content, $xpath ),
			'content_freshness_indicators' => $this->detect_content_freshness( $html ),
			'semantic_structure'           => $this->analyze_semantic_structure( $main_content ),
			'content_themes'               => $this->identify_content_themes( $main_content ),
		);

		/**
		 * Filter the semantic analysis results.
		 *
		 * @since 1.0.0
		 *
		 * @param array  $semantic_analysis The semantic analysis data.
		 * @param string $html              The full HTML content.
		 * @param string $main_content      The extracted main content text.
		 */
		return apply_filters( 'cpanalyzer_semantic_analysis', $semantic_analysis, $html, $main_content );
	}

	/**
	 * Extract main content text from page
	 *
	 * @param DOMXPath $xpath XPath object
	 * @return string Clean main content text
	 */
	private function extract_main_content_text( $xpath ) {
		// Try to find main content areas in order of preference
		$content_selectors = array(
			'//main//text()',
			'//article//text()',
			'//*[@class="content"]//text()',
			'//*[@id="content"]//text()',
			'//body//p//text()',
		);

		foreach ( $content_selectors as $selector ) {
			$text_nodes = $xpath->query( $selector );
			if ( $text_nodes->length > 0 ) {
				$content = '';
				foreach ( $text_nodes as $node ) {
					$content .= ' ' . trim( $node->textContent );
				}

				// Clean and return if substantial content found
				$content = $this->clean_text( $content );
				if ( str_word_count( $content ) > 50 ) {
					return $content;
				}
			}
		}

		// Fallback to body text
		$body_text = $xpath->query( '//body//text()' );
		$content   = '';
		foreach ( $body_text as $node ) {
			$content .= ' ' . trim( $node->textContent );
		}

		return $this->clean_text( $content );
	}

	/**
	 * Extract topic keywords using TF-IDF approach
	 *
	 * @param string $content Main content text
	 * @return array Top topic keywords with scores
	 */
	private function extract_topic_keywords( $content ) {
		if ( empty( $content ) || str_word_count( $content ) < 20 ) {
			return array();
		}

		// Get word frequency
		$words     = $this->tokenize_content( $content );
		$word_freq = array_count_values( $words );

		// Remove stop words and short words
		$stop_words     = $this->get_stop_words();
		$filtered_words = array();

		foreach ( $word_freq as $word => $freq ) {
			// Sanitize and validate word
			$clean_word = sanitize_text_field( $word );
			if ( strlen( $clean_word ) >= 3 && ! in_array( strtolower( $clean_word ), $stop_words ) && $freq >= 2 ) {
				$filtered_words[ $clean_word ] = absint( $freq );
			}
		}

		// Calculate TF scores and sort
		$total_words = array_sum( $word_freq );
		$tf_scores   = array();

		foreach ( $filtered_words as $word => $freq ) {
			$tf_scores[ $word ] = ( $freq / $total_words ) * 100; // Percentage
		}

		// Sort by relevance and return top 10
		arsort( $tf_scores );
		return array_slice( $tf_scores, 0, 10, true );
	}

	/**
	 * Analyze keyword density for important terms
	 *
	 * @param string $content Main content text
	 * @return array Keyword density analysis
	 */
	private function analyze_keyword_density( $content ) {
		if ( empty( $content ) ) {
			return array(
				'total_words'   => 0,
				'top_densities' => array(),
			);
		}

		$words       = $this->tokenize_content( $content );
		$total_words = count( $words );
		$word_freq   = array_count_values( $words );

		// Calculate densities for words appearing 3+ times
		$densities = array();
		foreach ( $word_freq as $word => $freq ) {
			if ( $freq >= 3 && strlen( $word ) >= 4 ) {
				$densities[ $word ] = round( ( $freq / $total_words ) * 100, 2 );
			}
		}

		// Sort and get top densities
		arsort( $densities );

		return array(
			'total_words'    => $total_words,
			'top_densities'  => array_slice( $densities, 0, 8, true ),
			'over_optimized' => array_filter(
				$densities,
				function ( $density ) {
					return $density > 3.0; // Flag densities over 3%
				}
			),
		);
	}

	/**
	 * Calculate content depth score
	 *
	 * @param string   $content Main content text
	 * @param DOMXPath $xpath XPath object for structure analysis
	 * @return array Content depth metrics
	 */
	private function calculate_content_depth_score( $content, $xpath ) {
		$word_count     = str_word_count( $content );
		$sentence_count = $this->count_sentences( $content );

		// Structural depth indicators
		$headings_count = $xpath->query( '//h1 | //h2 | //h3 | //h4 | //h5 | //h6' )->length;
		$list_items     = $xpath->query( '//li' )->length;
		$tables         = $xpath->query( '//table' )->length;
		$blockquotes    = $xpath->query( '//blockquote' )->length;

		// Calculate depth score (0-100)
		$depth_score = 0;

		// Word count contribution (0-30 points)
		if ( $word_count >= 1500 ) {
			$depth_score += 30;
		} elseif ( $word_count >= 1000 ) {
			$depth_score += 20;
		} elseif ( $word_count >= 500 ) {
			$depth_score += 10;
		}

		// Structure contribution (0-25 points)
		$depth_score += min( 25, $headings_count * 3 );

		// Content variety (0-20 points)
		if ( $list_items > 0 ) {
			$depth_score += 5;
		}
		if ( $tables > 0 ) {
			$depth_score += 5;
		}
		if ( $blockquotes > 0 ) {
			$depth_score += 5;
		}
		if ( $xpath->query( '//img' )->length > 0 ) {
			$depth_score += 5;
		}

		// Readability contribution (0-25 points)
		$avg_sentence_length = $sentence_count > 0 ? $word_count / $sentence_count : 0;
		if ( $avg_sentence_length >= 15 && $avg_sentence_length <= 25 ) {
			$depth_score += 15; // Good readability
		} elseif ( $avg_sentence_length >= 10 && $avg_sentence_length <= 30 ) {
			$depth_score += 10; // Acceptable readability
		}

		if ( $sentence_count >= 50 ) {
			$depth_score += 10; // Sufficient detail
		}

		return array(
			'depth_score'           => min( 100, $depth_score ),
			'word_count'            => $word_count,
			'sentence_count'        => $sentence_count,
			'avg_sentence_length'   => round( $avg_sentence_length, 1 ),
			'structural_elements'   => $headings_count + $list_items + $tables,
			'content_variety_score' => min( 20, ( $list_items > 0 ? 5 : 0 ) + ( $tables > 0 ? 5 : 0 ) + ( $blockquotes > 0 ? 5 : 0 ) + ( $xpath->query( '//img' )->length > 0 ? 5 : 0 ) ),
		);
	}

	/**
	 * Detect content freshness indicators
	 *
	 * @param string $html Full HTML content
	 * @return array Freshness indicators
	 */
	private function detect_content_freshness( $html ) {
		$freshness_indicators = array();

		// Look for date patterns in content
		$date_patterns = array(
			'/(\d{4}[-\/]\d{1,2}[-\/]\d{1,2})/', // YYYY-MM-DD or YYYY/MM/DD
			'/(\d{1,2}[-\/]\d{1,2}[-\/]\d{4})/', // MM-DD-YYYY or MM/DD/YYYY
			'/(January|February|March|April|May|June|July|August|September|October|November|December)\s+\d{1,2},?\s+\d{4}/i',
			'/(updated?|published?|modified?|last\s+updated?|as\s+of)[\s:]+([^<\n]{1,50})/i',
		);

		$found_dates = array();
		foreach ( $date_patterns as $pattern ) {
			if ( preg_match_all( $pattern, $html, $matches ) ) {
				$found_dates = array_merge( $found_dates, $matches[0] );
			}
		}

		// Look for current year mentions.
		$current_year          = gmdate( 'Y' );
		$current_year_mentions = preg_match_all( '/' . $current_year . '/', $html );

		// Look for "fresh" language.
		$fresh_terms      = array( 'new', 'latest', 'updated', 'recent', '2024', '2025', 'current' );
		$fresh_term_count = 0;
		foreach ( $fresh_terms as $term ) {
			$fresh_term_count += substr_count( strtolower( $html ), $term );
		}

		return array(
			'found_dates'           => array_slice( array_unique( $found_dates ), 0, 5 ),
			'current_year_mentions' => $current_year_mentions,
			'fresh_terms_count'     => $fresh_term_count,
			'freshness_score'       => min( 100, ( $current_year_mentions * 5 ) + ( $fresh_term_count * 2 ) ),
		);
	}

	/**
	 * Analyze semantic structure of content
	 *
	 * @param string $content Main content text
	 * @return array Semantic structure analysis
	 */
	private function analyze_semantic_structure( $content ) {
		if ( empty( $content ) ) {
			return array();
		}

		// Analyze question patterns
		$question_count = preg_match_all( '/\?/', $content );
		$question_words = preg_match_all( '/\b(what|how|why|when|where|who|which)\b/i', $content );

		// Analyze action words
		$action_words = array( 'get', 'use', 'create', 'build', 'make', 'develop', 'implement', 'start', 'begin', 'learn' );
		$action_count = 0;
		foreach ( $action_words as $word ) {
			$action_count += substr_count( strtolower( $content ), $word );
		}

		// Analyze list/step indicators
		$list_indicators = preg_match_all( '/\b(first|second|third|next|then|finally|step\s+\d+)\b/i', $content );

		return array(
			'question_count'       => $question_count,
			'question_words'       => $question_words,
			'action_words_count'   => $action_count,
			'list_indicators'      => $list_indicators,
			'has_how_to_structure' => $question_words > 0 && $list_indicators > 0,
			'content_type_signals' => $this->detect_content_type_signals( $content ),
		);
	}

	/**
	 * Identify content themes using keyword clustering
	 *
	 * @param string $content Main content text
	 * @return array Identified themes
	 */
	private function identify_content_themes( $content ) {
		if ( empty( $content ) || str_word_count( $content ) < 50 ) {
			return array();
		}

		$words      = $this->tokenize_content( $content );
		$stop_words = $this->get_stop_words();

		// Filter and get significant words
		$significant_words = array();
		$word_freq         = array_count_values( $words );

		foreach ( $word_freq as $word => $freq ) {
			if ( strlen( $word ) >= 4 && ! in_array( strtolower( $word ), $stop_words ) && $freq >= 2 ) {
				$significant_words[ $word ] = $freq;
			}
		}

		// Simple theme detection based on word co-occurrence
		$themes = array();

		// Technology theme
		$tech_words = array( 'software', 'app', 'platform', 'tool', 'system', 'technology', 'digital', 'online', 'web', 'api' );
		$tech_score = $this->calculate_theme_score( $significant_words, $tech_words );
		if ( $tech_score > 0 ) {
			$themes['technology'] = $tech_score;
		}

		// Business theme
		$business_words = array( 'business', 'company', 'customer', 'sales', 'marketing', 'revenue', 'growth', 'strategy', 'service' );
		$business_score = $this->calculate_theme_score( $significant_words, $business_words );
		if ( $business_score > 0 ) {
			$themes['business'] = $business_score;
		}

		// Educational theme
		$edu_words = array( 'learn', 'guide', 'tutorial', 'help', 'tips', 'course', 'training', 'education', 'teach' );
		$edu_score = $this->calculate_theme_score( $significant_words, $edu_words );
		if ( $edu_score > 0 ) {
			$themes['educational'] = $edu_score;
		}

		// Sort themes by relevance
		arsort( $themes );

		return array(
			'primary_themes' => array_slice( $themes, 0, 3, true ),
			'theme_keywords' => array_slice( $significant_words, 0, 15, true ),
		);
	}

	/**
	 * Helper methods for semantic analysis
	 */

	/**
	 * Clean and sanitize text for analysis.
	 *
	 * @param string $text Text to clean.
	 * @return string Cleaned text.
	 */
	private function clean_text( $text ) {
		// Remove extra whitespace and clean up text
		$text = sanitize_text_field( $text );
		$text = preg_replace( '/\s+/', ' ', $text );
		$text = preg_replace( '/[^\p{L}\p{N}\s\-\'\.]/u', ' ', $text );
		return trim( $text );
	}

	/**
	 * Tokenize content into individual words.
	 *
	 * @param string $content Content to tokenize.
	 * @return array Array of words.
	 */
	private function tokenize_content( $content ) {
		// Convert to lowercase and extract words.
		$content = strtolower( $this->clean_text( $content ) );
		$words   = preg_split( '/\s+/', $content );

		// Filter out very short words and common patterns.
		return array_filter(
			$words,
			function ( $word ) {
				return strlen( $word ) >= 3 && ! is_numeric( $word );
			}
		);
	}

	/**
	 * Get stop words for semantic analysis.
	 *
	 * @return array Stop words array.
	 */
	private function get_stop_words() {
		$default_stop_words = array(
			'the',
			'and',
			'for',
			'are',
			'but',
			'not',
			'you',
			'all',
			'can',
			'had',
			'her',
			'was',
			'one',
			'our',
			'out',
			'day',
			'get',
			'has',
			'him',
			'his',
			'how',
			'its',
			'may',
			'new',
			'now',
			'old',
			'see',
			'two',
			'who',
			'boy',
			'did',
			'man',
			'men',
			'put',
			'say',
			'she',
			'too',
			'use',
			'way',
			'been',
			'call',
			'come',
			'each',
			'find',
			'from',
			'have',
			'into',
			'like',
			'look',
			'made',
			'make',
			'more',
			'much',
			'must',
			'over',
			'said',
			'same',
			'such',
			'take',
			'than',
			'that',
			'them',
			'this',
			'time',
			'very',
			'were',
			'what',
			'when',
			'with',
			'word',
			'work',
			'your',
			'about',
			'after',
			'again',
			'back',
			'could',
			'every',
			'first',
			'good',
			'great',
			'just',
			'know',
			'last',
			'life',
			'long',
			'many',
			'most',
			'name',
			'only',
			'other',
			'part',
			'place',
			'right',
			'seem',
			'small',
			'sound',
			'still',
			'such',
			'take',
			'think',
			'through',
			'turn',
			'want',
			'water',
			'well',
			'where',
			'will',
			'year',
			'also',
			'being',
			'best',
			'both',
			'came',
			'come',
			'does',
			'down',
			'even',
			'form',
			'give',
			'good',
			'hand',
			'here',
			'home',
			'keep',
			'large',
			'left',
			'line',
			'little',
			'never',
			'next',
			'open',
			'play',
			'point',
			'public',
			'really',
			'should',
			'state',
			'these',
			'things',
			'times',
			'under',
			'until',
			'using',
			'while',
			'years',
		);

		/**
		 * Filter the stop words used in semantic analysis.
		 *
		 * @since 1.0.0
		 *
		 * @param array $default_stop_words Default stop words array.
		 */
		return apply_filters( 'cpanalyzer_stop_words', $default_stop_words );
	}

	/**
	 * Count sentences in text.
	 *
	 * @param string $text Text to analyze.
	 * @return int Number of sentences.
	 */
	private function count_sentences( $text ) {
		// Count sentences by looking for sentence-ending punctuation.
		return preg_match_all( '/[.!?]+/', $text );
	}

	/**
	 * Detect content type signals in text.
	 *
	 * @param string $content Content to analyze.
	 * @return array Content type signals.
	 */
	private function detect_content_type_signals( $content ) {
		$signals = array();

		// How-to signals.
		if ( preg_match( '/\bhow\s+to\b/i', $content ) || preg_match( '/step\s+\d+/i', $content ) ) {
			$signals[] = 'how_to_guide';
		}

		// List/Tips signals.
		if ( preg_match( '/\b\d+\s+(tips|ways|methods|steps|reasons)\b/i', $content ) ) {
			$signals[] = 'listicle';
		}

		// Review signals.
		if ( preg_match( '/\b(review|pros|cons|rating|stars)\b/i', $content ) ) {
			$signals[] = 'review';
		}

		// Comparison signals.
		if ( preg_match( '/\b(vs|versus|compared?|alternative)\b/i', $content ) ) {
			$signals[] = 'comparison';
		}

		return $signals;
	}

	/**
	 * Calculate theme score based on word frequency and theme words
	 *
	 * @param array $word_freq Word frequency array.
	 * @param array $theme_words Theme-specific words to look for.
	 * @return int Theme score.
	 */
	private function calculate_theme_score( $word_freq, $theme_words ) {
		$score = 0;
		foreach ( $theme_words as $theme_word ) {
			foreach ( $word_freq as $word => $freq ) {
				if ( strpos( strtolower( $word ), $theme_word ) !== false ) {
					$score += $freq;
				}
			}
		}
		return $score;
	}
}
