<?php
/**
 * AI access (free): the verified AI-crawler token table, the site's own
 * robots.txt fetched + parsed, and the per-provider access matrix that backs
 * the "AI access" fact panel on the Site row. Findings (severity-scored) are
 * premium and live in class-ai-readiness__premium_only.php; this class only
 * states facts.
 *
 * HONESTY CONTRACT (docs/SPEC-ai-readiness.md s.2.2, s.3.1 - the two refuted
 * claims constrain every string that leaves this class):
 * - Tier language only: "documented to honor" / "documented advisory" -
 *   never "guaranteed", never "removes you from {product}".
 * - Fetch-vantage rule: robots.txt is fetched over the PUBLIC path only (the
 *   front-of-CDN view, so edge-injected rules are seen). There is no origin/
 *   loopback fallback for this check - if the public fetch fails we report
 *   "could not read robots.txt", never a silently-wrong origin view.
 * - Token list re-verification: every release touching this feature
 *   re-checks the four vendor doc pages cited on each token below
 *   (RELEASE-CHECKLIST gate; vendors revised these docs twice in the six
 *   months before 2026-07).
 *
 * @package search-social-appearance-audit
 */

if ( ! defined( 'ABSPATH' ) ) {
	exit;
}

class SSAA_AI_Access {

	/** Transient cache for the fetched+evaluated matrix (per scan, cheap). */
	const CACHE_KEY = 'ssaa_ai_access_matrix';
	const CACHE_TTL = 300;

	/**
	 * The VERIFIED token table (deep-research 2026-07-22; every row traces to
	 * a live vendor doc). tier: 'respected' = vendor documents the token as
	 * honoring robots.txt; 'advisory' = vendor documents it may/does ignore
	 * robots.txt for user-triggered fetches. OAI-AdsBot deliberately omitted
	 * (edge case); CCBot / Applebot-Extended / meta-externalagent / Bytespider
	 * deliberately omitted pending a verification round - add via the
	 * ssaa_ai_crawler_tokens filter only with sourced semantics.
	 *
	 * @return array[] token => { provider, purpose, tier, consequence, doc }
	 */
	public static function tokens() {
		$tokens = array(
			'GPTBot'           => array(
				'provider'    => 'OpenAI',
				'purpose'     => __( 'AI model training', 'search-social-appearance-audit' ),
				'tier'        => 'respected',
				'consequence' => __( 'OpenAI documents a GPTBot disallow as the signal that content should not be used to train its models. It does not affect ChatGPT search visibility (that is OAI-SearchBot).', 'search-social-appearance-audit' ),
				'doc'         => 'https://developers.openai.com/api/docs/bots',
			),
			'OAI-SearchBot'    => array(
				'provider'    => 'OpenAI',
				'purpose'     => __( 'ChatGPT search visibility', 'search-social-appearance-audit' ),
				'tier'        => 'respected',
				'consequence' => __( 'OpenAI documents that sites opted out of OAI-SearchBot are not shown in ChatGPT search answers.', 'search-social-appearance-audit' ),
				'doc'         => 'https://developers.openai.com/api/docs/bots',
			),
			'ChatGPT-User'     => array(
				'provider'    => 'OpenAI',
				'purpose'     => __( 'User-requested page fetches', 'search-social-appearance-audit' ),
				'tier'        => 'advisory',
				'consequence' => __( 'OpenAI documents that robots.txt rules "may not apply" to user-initiated fetches - a disallow here is advisory.', 'search-social-appearance-audit' ),
				'doc'         => 'https://developers.openai.com/api/docs/bots',
			),
			'ClaudeBot'        => array(
				'provider'    => 'Anthropic',
				'purpose'     => __( 'AI model training', 'search-social-appearance-audit' ),
				'tier'        => 'respected',
				'consequence' => __( 'Anthropic documents ClaudeBot as honoring robots.txt directives.', 'search-social-appearance-audit' ),
				'doc'         => 'https://support.claude.com/en/articles/8896518',
			),
			'Claude-SearchBot' => array(
				'provider'    => 'Anthropic',
				'purpose'     => __( 'Claude search indexing', 'search-social-appearance-audit' ),
				'tier'        => 'respected',
				'consequence' => __( 'Anthropic documents Claude-SearchBot (search indexing) as honoring robots.txt directives.', 'search-social-appearance-audit' ),
				'doc'         => 'https://support.claude.com/en/articles/8896518',
			),
			'Claude-User'      => array(
				'provider'    => 'Anthropic',
				'purpose'     => __( 'User-requested page fetches', 'search-social-appearance-audit' ),
				'tier'        => 'respected',
				'consequence' => __( 'Anthropic documents Claude-User as honoring robots.txt - unusually for a user-triggered fetcher.', 'search-social-appearance-audit' ),
				'doc'         => 'https://support.claude.com/en/articles/8896518',
			),
			'PerplexityBot'    => array(
				'provider'    => 'Perplexity',
				'purpose'     => __( 'Perplexity search surfacing', 'search-social-appearance-audit' ),
				'tier'        => 'respected',
				'consequence' => __( 'Perplexity states PerplexityBot surfaces sites in its search results and is not used for model training.', 'search-social-appearance-audit' ),
				'doc'         => 'https://docs.perplexity.ai/docs/resources/perplexity-crawlers',
			),
			'Perplexity-User'  => array(
				'provider'    => 'Perplexity',
				'purpose'     => __( 'User-requested page fetches', 'search-social-appearance-audit' ),
				'tier'        => 'advisory',
				'consequence' => __( 'Perplexity documents that this fetcher "generally ignores" robots.txt because a user requested the page - a disallow here is advisory.', 'search-social-appearance-audit' ),
				'doc'         => 'https://docs.perplexity.ai/docs/resources/perplexity-crawlers',
			),
			'Google-Extended'  => array(
				'provider'    => 'Google',
				'purpose'     => __( 'Gemini training and grounding', 'search-social-appearance-audit' ),
				'tier'        => 'respected',
				'consequence' => __( 'Google documents Google-Extended as controlling Gemini model training and grounding only - it has no effect on Google Search inclusion, ranking, or AI Overviews.', 'search-social-appearance-audit' ),
				'doc'         => 'https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers',
			),
		);

		/**
		 * Extend/adjust the audited token table. Only add tokens with sourced,
		 * current semantics (see the class docblock re-verification rule).
		 *
		 * @param array $tokens token => descriptor.
		 */
		return (array) apply_filters( 'ssaa_ai_crawler_tokens', $tokens );
	}

	/**
	 * Fetch + evaluate the site's robots.txt against the token table, plus the
	 * llms.txt presence fact. Cached briefly (one scan hits this once per site
	 * row, but re-scans and the Exclusions/panel reads share it).
	 *
	 * @param bool $bypass_cache Force a fresh fetch (per-row Re-scan).
	 * @return array {
	 *   fetched:     bool   robots.txt was readable over the public path,
	 *   http_status: int,
	 *   robots_url:  string,
	 *   matrix:      array  token => { provider, purpose, tier, consequence,
	 *                        access: allowed|partial|blocked, via: token|wildcard|none,
	 *                        lines: string[] (the matched robots.txt lines) },
	 *   llms_txt:    bool|null  present / absent / null = could not check,
	 * }
	 */
	public static function evaluate( $bypass_cache = false ) {
		if ( ! $bypass_cache ) {
			$cached = get_transient( self::CACHE_KEY );
			if ( is_array( $cached ) ) {
				return $cached;
			}
		}

		$robots_url = home_url( '/robots.txt' );
		$result     = array(
			'fetched'     => false,
			// 'ok' = 200 with rules; 'absent' = 404/410 (RFC 9309: crawlers
			// treat a missing robots.txt as NO restrictions - the matrix is
			// all-allowed); 'unreachable' = transport error / 5xx (unknown -
			// the matrix is suppressed rather than guessed).
			'status'      => 'unreachable',
			'http_status' => 0,
			'robots_url'  => $robots_url,
			'matrix'      => array(),
			// Real-world anomaly (seen live on staging 2026-07-23): a 404
			// response that nevertheless carries robots rules in the body -
			// rules crawlers are documented to disregard. Surfaced as a fact.
			'served_with_404' => false,
			'llms_txt'    => null,
			'cdn'         => '',
		);

		// PUBLIC path only - see the fetch-vantage rule in the class docblock.
		$response = wp_remote_get( $robots_url, array(
			'timeout'     => 15,
			'redirection' => 3,
			'user-agent'  => 'SSAA-Audit/' . SSAA_VERSION . '; ' . home_url( '/' ),
		) );

		if ( ! is_wp_error( $response ) ) {
			$result['http_status'] = (int) wp_remote_retrieve_response_code( $response );

			// CDN detection (the A6 advisory's factual basis): this fetch has
			// the response headers, unlike the engine's page fetches. Presence
			// only - no claims about the CDN's bot policy.
			$server = strtolower( (string) wp_remote_retrieve_header( $response, 'server' ) );
			if ( false !== strpos( $server, 'cloudflare' ) || '' !== (string) wp_remote_retrieve_header( $response, 'cf-ray' ) ) {
				$result['cdn'] = 'Cloudflare';
			}

			$body = (string) wp_remote_retrieve_body( $response );

			if ( 200 === $result['http_status'] ) {
				$result['fetched'] = true;
				$result['status']  = 'ok';
				$groups            = self::parse_robots( $body );
				foreach ( self::tokens() as $token => $meta ) {
					$result['matrix'][ $token ] = array_merge( $meta, self::evaluate_token( $token, $groups ) );
				}
			} elseif ( in_array( $result['http_status'], array( 404, 410 ), true ) ) {
				// RFC 9309: a missing robots.txt means no restrictions - the
				// matrix is definitively ALL-ALLOWED, not unknown.
				$result['fetched'] = true;
				$result['status']  = 'absent';
				foreach ( self::tokens() as $token => $meta ) {
					$result['matrix'][ $token ] = array_merge( $meta, array( 'access' => 'allowed', 'via' => 'none', 'lines' => array() ) );
				}
				// The anomaly: a 404 that still serves robots rules (a real
				// misconfiguration - those rules likely have no effect).
				if ( false !== stripos( $body, 'user-agent' ) && ( false !== stripos( $body, 'disallow' ) || false !== stripos( $body, 'allow' ) ) ) {
					$result['served_with_404'] = true;
				}
			}
		}

		// llms.txt: presence FACT only (no finding - no verified consumer).
		$llms = wp_remote_head( home_url( '/llms.txt' ), array( 'timeout' => 10, 'redirection' => 3 ) );
		if ( ! is_wp_error( $llms ) ) {
			$result['llms_txt'] = ( 200 === (int) wp_remote_retrieve_response_code( $llms ) );
		}

		set_transient( self::CACHE_KEY, $result, self::CACHE_TTL );
		return $result;
	}

	/**
	 * FREE site-defect findings from the evaluated access structure. This is
	 * deliberately NOT the premium detector: a robots.txt served with HTTP 404
	 * is a plain server misconfiguration whose rules have no effect for ANY
	 * crawler (RFC 9309: 404 = no restrictions) - search engines and AI
	 * crawlers alike - so it belongs with the free basic issue detection
	 * (noindex, missing_title), not the premium interpretive layer. Decided
	 * 2026-07-23 (Steve): the Site row must never read OK while its own fact
	 * panel describes a misconfiguration.
	 *
	 * @param array $access SSAA_AI_Access::evaluate() structure.
	 * @return array Analyzer-shaped issues.
	 */
	public static function site_defect_issues( array $access ) {
		$issues = array();

		if ( ! empty( $access['served_with_404'] ) ) {
			$issues[] = array(
				'code'     => 'ai_robots_rules_ineffective',
				'area'     => 'ai',
				'severity' => 'warning',
				'message'  => __( 'robots.txt returns HTTP 404 while still serving rules. Crawlers treat a 404 robots.txt as no restrictions, so the rules in the body have no effect for any crawler - search engines and AI crawlers alike. Fix the server so robots.txt returns HTTP 200.', 'search-social-appearance-audit' ),
				'evidence' => array(),
			);
		}

		return $issues;
	}

	/**
	 * Parse robots.txt into UA groups. A group = one or more consecutive
	 * User-agent lines followed by its rules; rules are kept verbatim for
	 * evidence. Comments stripped; matching is case-insensitive on the UA
	 * token per the robots.txt spec.
	 *
	 * @param string $body robots.txt body.
	 * @return array[] [ { agents: string[] (lowercased), rules: [ {type: allow|disallow, path, line} ] } ]
	 */
	public static function parse_robots( $body ) {
		$groups  = array();
		$current = null;
		$in_ua   = false;

		foreach ( preg_split( '/\r\n|\r|\n/', $body ) as $raw ) {
			$line = trim( preg_replace( '/#.*$/', '', $raw ) );
			if ( '' === $line || false === strpos( $line, ':' ) ) {
				continue;
			}
			list( $field, $value ) = array_map( 'trim', explode( ':', $line, 2 ) );
			$field                 = strtolower( $field );

			if ( 'user-agent' === $field ) {
				if ( ! $in_ua || null === $current ) {
					// A new group starts at the first UA line after rules.
					if ( null !== $current ) {
						$groups[] = $current;
					}
					$current = array( 'agents' => array(), 'rules' => array() );
				}
				$current['agents'][] = strtolower( $value );
				$in_ua               = true;
			} elseif ( in_array( $field, array( 'allow', 'disallow' ), true ) ) {
				if ( null === $current ) {
					continue; // Rules before any UA line - invalid, skip.
				}
				$in_ua              = false;
				$current['rules'][] = array(
					'type' => $field,
					'path' => $value,
					'line' => trim( $raw ),
				);
			} else {
				// sitemap / crawl-delay / unknown fields end a UA run.
				$in_ua = false;
			}
		}
		if ( null !== $current ) {
			$groups[] = $current;
		}
		return $groups;
	}

	/**
	 * Evaluate one token against the parsed groups: the most specific matching
	 * group wins (exact token match beats '*', per the robots.txt spec), then
	 * within it the site-wide verdict is derived from the rules:
	 * - 'blocked': an effective "Disallow: /" (and no overriding "Allow: /").
	 * - 'partial': some paths disallowed, the site root allowed.
	 * - 'allowed': no matching group, an empty Disallow, or no disallows.
	 *
	 * @param string  $token  Crawler token (case preserved for display).
	 * @param array[] $groups Parsed groups.
	 * @return array { access, via, lines }
	 */
	public static function evaluate_token( $token, array $groups ) {
		$needle   = strtolower( $token );
		$matched  = null;
		$via      = 'none';

		// Exact token group first...
		foreach ( $groups as $group ) {
			if ( in_array( $needle, $group['agents'], true ) ) {
				$matched = $group;
				$via     = 'token';
				break;
			}
		}
		// ...else the wildcard group.
		if ( null === $matched ) {
			foreach ( $groups as $group ) {
				if ( in_array( '*', $group['agents'], true ) ) {
					$matched = $group;
					$via     = 'wildcard';
					break;
				}
			}
		}

		if ( null === $matched ) {
			return array( 'access' => 'allowed', 'via' => 'none', 'lines' => array() );
		}

		$root_disallowed = false;
		$root_allowed    = false;
		$has_disallow    = false;
		$lines           = array();

		foreach ( $matched['rules'] as $rule ) {
			$lines[] = $rule['line'];
			if ( 'disallow' === $rule['type'] ) {
				if ( '/' === $rule['path'] ) {
					$root_disallowed = true;
				}
				if ( '' !== $rule['path'] ) {
					$has_disallow = true;
				}
			} elseif ( 'allow' === $rule['type'] && '/' === $rule['path'] ) {
				$root_allowed = true;
			}
		}

		// Longest-match on the root path: "Allow: /" and "Disallow: /" are the
		// same length; the spec resolves the tie in favour of Allow.
		if ( $root_disallowed && ! $root_allowed ) {
			$access = 'blocked';
		} elseif ( $has_disallow ) {
			$access = 'partial';
		} else {
			$access = 'allowed';
		}

		return array( 'access' => $access, 'via' => $via, 'lines' => $lines );
	}
}
