<?php
if ( ! defined( 'ABSPATH' ) ) {
	exit;
}

/**
 * REST API: /wp-json/myaio/v1/technical-seo-scan
 *
 * Combined output:
 * - check_sitemap
 * - check_robots_meta
 * - schema_scan
 *
 * Optional query params:
 * - url=https://example.com/about-us
 * - path=/about-us
 *
 * Auth:
 * - Single shared auth callback:
 *   myaio_sac_rest_permission_check
 */

////////////////////////////////////////////////////////////////////////////////////////////////////
// ROUTE REGISTRATION
////////////////////////////////////////////////////////////////////////////////////////////////////

add_action(
	'rest_api_init',
	function () {
		register_rest_route(
			'myaio/v1',
			'/technical-seo-scan',
			array(
				'methods'             => WP_REST_Server::READABLE,
				'callback'            => 'myaio_technical_seo_scan_combined',
				'permission_callback' => 'myaio_sac_rest_permission_check',
				'args'                => array(
					'url' => array(
						'type'              => 'string',
						'required'          => false,
						'sanitize_callback' => 'sanitize_text_field',
					),
					'path' => array(
						'type'              => 'string',
						'required'          => false,
						'sanitize_callback' => 'sanitize_text_field',
					),
				),
			)
		);
	}
);

/**
 * Main combined callback.
 *
 * @param WP_REST_Request $request Request.
 * @return WP_REST_Response
 */
function myaio_technical_seo_scan_combined( WP_REST_Request $request ) {
	$target_url = myaio_tss_get_target_url_optional( $request );

	$response_data = array(
		'check_sitemap'     => myaio_tss_check_sitemap( $target_url ),
		'check_robots_meta' => myaio_tss_check_robots_meta( $target_url ),
		'schema_scan'       => myaio_tss_schema_scan( $target_url ),
	);

	return rest_ensure_response( $response_data );
}

////////////////////////////////////////////////////////////////////////////////////////////////////
// SHARED URL HELPERS
////////////////////////////////////////////////////////////////////////////////////////////////////

/**
 * Optional target URL resolver.
 *
 * Behavior:
 * - If url/path given => same-host validated URL string
 * - If nothing given => empty string
 *
 * @param WP_REST_Request $request Request.
 * @return string|WP_Error
 */
function myaio_tss_get_target_url_optional( WP_REST_Request $request ) {
	$url  = trim( (string) $request->get_param( 'url' ) );
	$path = trim( (string) $request->get_param( 'path' ) );

	if ( '' === $url && '' === $path ) {
		return '';
	}

	if ( '' !== $path && '' === $url ) {
		$path = ltrim( $path, '/' );
		$url  = home_url( '/' . $path );
	}

	$url = esc_url_raw( $url );

	if ( '' === $url || ! wp_http_validate_url( $url ) ) {
		return new WP_Error( 'invalid_url', 'Invalid URL provided.', array( 'status' => 400 ) );
	}

	$site_host = wp_parse_url( home_url( '/' ), PHP_URL_HOST );
	$url_host  = wp_parse_url( $url, PHP_URL_HOST );

	if ( ! $site_host || ! $url_host || 0 !== strcasecmp( (string) $site_host, (string) $url_host ) ) {
		return new WP_Error( 'forbidden_host', 'Only URLs on this site are allowed.', array( 'status' => 403 ) );
	}

	return rtrim( $url, '/' );
}

/**
 * Normalize URL for comparisons.
 *
 * @param string $u URL.
 * @return string
 */
function myaio_tss_norm_url( $u ) {
	$u     = trim( (string) $u );
	$parts = wp_parse_url( $u );

	if ( ! $parts ) {
		return rtrim( $u, '/' );
	}

	$scheme = isset( $parts['scheme'] ) ? strtolower( $parts['scheme'] ) : 'http';
	$host   = isset( $parts['host'] ) ? strtolower( $parts['host'] ) : '';
	$port   = isset( $parts['port'] ) ? ':' . $parts['port'] : '';
	$path   = isset( $parts['path'] ) ? $parts['path'] : '';
	$query  = isset( $parts['query'] ) ? '?' . $parts['query'] : '';
	$frag   = isset( $parts['fragment'] ) ? '#' . $parts['fragment'] : '';

	if ( '/' !== $path ) {
		$path = rtrim( $path, '/' );
	}

	return $scheme . '://' . $host . $port . $path . $query . $frag;
}

/**
 * ISO8601 helper.
 *
 * @param int $ts Timestamp.
 * @return string|null
 */
function myaio_tss_iso8601( $ts ) {
	$ts = (int) $ts;
	return $ts > 0 ? gmdate( 'c', $ts ) : null;
}

////////////////////////////////////////////////////////////////////////////////////////////////////
// SHARED HTTP / XML HELPERS
////////////////////////////////////////////////////////////////////////////////////////////////////

/**
 * URL existence check: HEAD then small GET.
 *
 * @param string $url URL.
 * @return bool
 */
function myaio_tss_url_exists( $url ) {
	$url = trim( (string) $url );
	if ( '' === $url ) {
		return false;
	}

	$res = wp_remote_head(
		$url,
		array(
			'timeout'     => 4,
			'redirection' => 2,
			'sslverify'   => true,
			'headers'     => array(
				'User-Agent' => 'Mozilla/5.0 (Sitemap Checker)',
				'Accept'     => '*/*',
			),
		)
	);

	if ( ! is_wp_error( $res ) ) {
		$code = (int) wp_remote_retrieve_response_code( $res );
		if ( $code >= 200 && $code < 400 ) {
			return true;
		}

		if ( ! in_array( $code, array( 0, 400, 401, 403, 405 ), true ) ) {
			return false;
		}
	}

	$res = wp_remote_get(
		$url,
		array(
			'timeout'     => 4,
			'redirection' => 2,
			'sslverify'   => true,
			'headers'     => array(
				'User-Agent' => 'Mozilla/5.0 (Sitemap Checker)',
				'Accept'     => 'application/xml,text/xml,text/plain;q=0.9,*/*;q=0.8',
				'Range'      => 'bytes=0-2048',
			),
		)
	);

	if ( is_wp_error( $res ) ) {
		return false;
	}

	$code = (int) wp_remote_retrieve_response_code( $res );
	return ( $code >= 200 && $code < 400 );
}

/**
 * Fetch XML.
 *
 * @param string $url URL.
 * @return SimpleXMLElement|null
 */
function myaio_tss_fetch_xml( $url ) {
	$res = wp_remote_get(
		$url,
		array(
			'timeout'     => 8,
			'redirection' => 2,
			'sslverify'   => true,
			'headers'     => array(
				'User-Agent' => 'Mozilla/5.0 (Sitemap XML Fetch)',
				'Accept'     => 'application/xml,text/xml,text/plain;q=0.9,*/*;q=0.8',
			),
		)
	);

	if ( is_wp_error( $res ) ) {
		return null;
	}

	$code = (int) wp_remote_retrieve_response_code( $res );
	if ( $code < 200 || $code >= 400 ) {
		return null;
	}

	$body = (string) wp_remote_retrieve_body( $res );
	if ( '' === $body ) {
		return null;
	}

	$is_gz_url = (bool) preg_match( '/\.gz(\?|$)/i', (string) $url );
	$looks_gz  = ( strlen( $body ) >= 2 && ord( $body[0] ) === 0x1f && ord( $body[1] ) === 0x8b );

	if ( ( $is_gz_url || $looks_gz ) && function_exists( 'gzdecode' ) ) {
		$decoded = @gzdecode( $body );
		if ( false !== $decoded && '' !== $decoded ) {
			$body = $decoded;
		}
	}

	libxml_use_internal_errors( true );
	$xml = simplexml_load_string( $body );
	libxml_clear_errors();

	return $xml ?: null;
}

/**
 * Parse sitemap type.
 *
 * @param SimpleXMLElement|null $xml XML.
 * @return string|null
 */
function myaio_tss_parse_sitemap_type( $xml ) {
	if ( ! $xml ) {
		return null;
	}

	$is_index  = $xml->xpath( '/*[local-name()="sitemapindex"]' );
	$is_urlset = $xml->xpath( '/*[local-name()="urlset"]' );

	if ( $is_index && ! empty( $is_index ) ) {
		return 'index';
	}

	if ( $is_urlset && ! empty( $is_urlset ) ) {
		return 'urlset';
	}

	return null;
}

/**
 * Extract child sitemap URLs.
 *
 * @param SimpleXMLElement|null $xml XML.
 * @return array
 */
function myaio_tss_extract_index_children( $xml ) {
	if ( ! $xml ) {
		return array();
	}

	$nodes = $xml->xpath( '/*[local-name()="sitemapindex"]/*[local-name()="sitemap"]/*[local-name()="loc"]' );
	$out   = array();

	if ( is_array( $nodes ) ) {
		foreach ( $nodes as $n ) {
			$loc = trim( (string) $n );
			if ( '' !== $loc ) {
				$out[] = $loc;
			}
		}
	}

	return array_values( array_unique( $out ) );
}

/**
 * Extract URLset URLs.
 *
 * @param SimpleXMLElement|null $xml XML.
 * @return array
 */
function myaio_tss_extract_urlset_urls( $xml ) {
	if ( ! $xml ) {
		return array();
	}

	$nodes = $xml->xpath( '/*[local-name()="urlset"]/*[local-name()="url"]/*[local-name()="loc"]' );
	$out   = array();

	if ( is_array( $nodes ) ) {
		foreach ( $nodes as $n ) {
			$loc = trim( (string) $n );
			if ( '' !== $loc ) {
				$out[] = $loc;
			}
		}
	}

	return array_values( array_unique( $out ) );
}

////////////////////////////////////////////////////////////////////////////////////////////////////
// SHARED CRAWLABLE URL HELPER
////////////////////////////////////////////////////////////////////////////////////////////////////

/**
 * Get crawlable URLs, optionally filtered to a single URL.
 *
 * Output shape remains compatible with old crawlable URL providers:
 * [
 *   [
 *     'url' => '...',
 *     'type' => '...',
 *     'category' => '...'
 *   ]
 * ]
 *
 * @param string $requested_url Requested URL.
 * @return array
 */
function myaio_tss_get_filtered_crawlable_urls( $requested_url = '' ) {
	if ( function_exists( 'myaio_sac_get_all_crawlable_urls' ) ) {
		$urls = myaio_sac_get_all_crawlable_urls();
	} elseif ( function_exists( 'myaio_sac_get_all_site_urls' ) ) {
		$urls = myaio_sac_get_all_site_urls();
	} else {
		return array();
	}

	if ( ! is_array( $urls ) ) {
		return array();
	}

	if ( '' === $requested_url ) {
		return $urls;
	}

	$requested_norm = myaio_tss_norm_url( $requested_url );
	$matched        = null;

	foreach ( $urls as $item ) {
		if ( ! is_array( $item ) || empty( $item['url'] ) ) {
			continue;
		}

		if ( myaio_tss_norm_url( (string) $item['url'] ) === $requested_norm ) {
			$matched = $item;
			break;
		}
	}

	if ( null !== $matched ) {
		return array( $matched );
	}

	return array(
		array(
			'url'      => $requested_url,
			'type'     => null,
			'category' => null,
		),
	);
}

////////////////////////////////////////////////////////////////////////////////////////////////////
// CHECK SITEMAP
////////////////////////////////////////////////////////////////////////////////////////////////////

/**
 * Parse robots.txt and extract Sitemap lines.
 *
 * @param string $robots_url Robots URL.
 * @return array
 */
function myaio_tss_get_sitemaps_from_robots( $robots_url ) {
	$out = array(
		'ok'       => false,
		'status'   => 0,
		'sitemaps' => array(),
		'error'    => null,
	);

	$res = wp_remote_get(
		$robots_url,
		array(
			'timeout'     => 5,
			'redirection' => 2,
			'sslverify'   => true,
			'headers'     => array(
				'User-Agent' => 'Mozilla/5.0 (WordPress REST Sitemap Checker)',
				'Accept'     => 'text/plain',
			),
		)
	);

	if ( is_wp_error( $res ) ) {
		$out['error'] = $res->get_error_message();
		return $out;
	}

	$out['ok']     = true;
	$out['status'] = (int) wp_remote_retrieve_response_code( $res );

	$body = (string) wp_remote_retrieve_body( $res );
	if ( '' === $body ) {
		return $out;
	}

	if ( preg_match_all( '/^\s*sitemap\s*:\s*(\S+)\s*$/im', $body, $m ) ) {
		foreach ( (array) ( $m[1] ?? array() ) as $u ) {
			$u = trim( $u );
			if ( '' === $u ) {
				continue;
			}

			if ( 0 !== strpos( $u, 'http://' ) && 0 !== strpos( $u, 'https://' ) ) {
				$u = home_url( '/' . ltrim( $u, '/' ) );
			}

			$out['sitemaps'][] = esc_url_raw( $u );
		}
	}

	$out['sitemaps'] = array_values( array_unique( array_filter( $out['sitemaps'] ) ) );

	return $out;
}

/**
 * Resolve sitemap directory for compatibility fields.
 *
 * @return array
 */
function myaio_tss_resolve_sitemaps_dir() {
	global $wp_filesystem;

	if ( ! function_exists( 'WP_Filesystem' ) ) {
		require_once ABSPATH . 'wp-admin/includes/file.php';
	}

	WP_Filesystem();

	$root_dir = trailingslashit( ABSPATH ) . 'sitemaps/';
	$root_url = home_url( '/sitemaps/' );

	if ( ! $wp_filesystem->is_dir( $root_dir ) ) {
		$wp_filesystem->mkdir( $root_dir );
	}

	if ( $wp_filesystem->is_writable( $root_dir ) ) {
		return array(
			'dir' => $root_dir,
			'url' => trailingslashit( $root_url ),
		);
	}

	$uploads = wp_get_upload_dir();
	$u_dir   = trailingslashit( $uploads['basedir'] ) . 'sitemaps/';
	$u_url   = trailingslashit( $uploads['baseurl'] ) . 'sitemaps/';

	if ( ! $wp_filesystem->is_dir( $u_dir ) ) {
		$wp_filesystem->mkdir( $u_dir );
	}

	return array(
		'dir' => $u_dir,
		'url' => $u_url,
	);
}

/**
 * Locate canonical sitemap file.
 *
 * @return array
 */
function myaio_tss_locate_sitemap_file() {
	$where   = myaio_tss_resolve_sitemaps_dir();
	$root    = $where['dir'] . 'sitemap.xml';
	$uploads = wp_get_upload_dir();
	$u_path  = trailingslashit( $uploads['basedir'] ) . 'sitemaps/sitemap.xml';

	$path = null;

	if ( is_readable( $root ) ) {
		$path = $root;
	} elseif ( is_readable( $u_path ) ) {
		$path = $u_path;
	}

	return array(
		'path'  => $path,
		'mtime' => ( $path && file_exists( $path ) ) ? filemtime( $path ) : 0,
		'url'   => home_url( '/sitemaps/sitemap.xml' ),
	);
}

/**
 * Same output shape as old check-sitemap API.
 * URL param does not change sitemap discovery logic, but is accepted at merged level.
 *
 * @param string|WP_Error $requested_url Requested URL.
 * @return array
 */
function myaio_tss_check_sitemap( $requested_url = '' ) {
	if ( is_wp_error( $requested_url ) ) {
		return array(
			'available'               => false,
			'message'                 => $requested_url->get_error_message(),
			'sitemap_url'             => home_url( '/sitemaps/sitemap.xml' ),
			'previous_sitemap_url'    => get_option( 'myaio_sac_prev_sitemap_url' ) ?: ( get_option( 'myaio_sac_last_sitemap_url' ) ?: null ),
			'last_modified'           => null,
			'previous_last_modified'  => myaio_tss_iso8601( (int) ( get_option( 'myaio_sac_prev_sitemap_mtime' ) ?: get_option( 'myaio_sac_last_sitemap_mtime' ) ) ),
			'checked'                 => array(),
			'sitemap_files_count'     => 0,
			'sitemaps_found'          => array(),
			'total_url_count'         => 0,
			'unique_url_count'        => 0,
			'folder_children_count'   => 0,
			'folder_children_samples' => array(),
			'robots'                  => array(
				'url'      => home_url( '/robots.txt' ),
				'ok'       => false,
				'status'   => 0,
				'sitemaps' => array(),
				'error'    => $requested_url->get_error_message(),
			),
		);
	}

	$cache_key = 'myaio_sac_check_sitemap_cache_v1';
	$cached    = get_transient( $cache_key );

	if ( is_array( $cached ) ) {
		return $cached;
	}

	$canonical_url = home_url( '/sitemaps/sitemap.xml' );
	$saved         = get_option( 'myaio_sac_last_sitemap_url' );

	$checked         = array();
	$exists          = array();
	$guessed_children = array();

	$robots_url  = home_url( '/robots.txt' );
	$robots_info = myaio_tss_get_sitemaps_from_robots( $robots_url );

	if ( ! empty( $robots_info['sitemaps'] ) ) {
		foreach ( $robots_info['sitemaps'] as $u ) {
			$checked[] = $u;
			if ( myaio_tss_url_exists( $u ) ) {
				$exists[] = $u;
			}
		}
	}

	if ( empty( $exists ) ) {
		$fallbacks = array(
			home_url( '/sitemap.xml' ),
			home_url( '/sitemap_index.xml' ),
			home_url( '/wp-sitemap.xml' ),
		);

		foreach ( $fallbacks as $u ) {
			$checked[] = $u;
			if ( myaio_tss_url_exists( $u ) ) {
				$exists[] = $u;
				break;
			}
		}
	}

	if ( empty( $exists ) ) {
		$checked[] = $canonical_url;
		if ( myaio_tss_url_exists( $canonical_url ) ) {
			$exists[] = $canonical_url;
		}
	}

	$max_candidate_checks = count( $checked );

	if ( empty( $exists ) ) {
		$resp = array(
			'available'               => false,
			'message'                 => 'No sitemap found at known locations (including robots.txt).',
			'sitemap_url'             => $canonical_url,
			'previous_sitemap_url'    => get_option( 'myaio_sac_prev_sitemap_url' ) ?: ( $saved ?: null ),
			'last_modified'           => null,
			'previous_last_modified'  => myaio_tss_iso8601( (int) ( get_option( 'myaio_sac_prev_sitemap_mtime' ) ?: get_option( 'myaio_sac_last_sitemap_mtime' ) ) ),
			'checked'                 => $checked,
			'sitemap_files_count'     => 0,
			'sitemaps_found'          => array(),
			'total_url_count'         => 0,
			'unique_url_count'        => 0,
			'folder_children_count'   => count( $guessed_children ),
			'folder_children_samples' => array_slice( $guessed_children, 0, 20 ),
			'robots'                  => array(
				'url'      => $robots_url,
				'ok'       => (bool) ( $robots_info['ok'] ?? false ),
				'status'   => (int) ( $robots_info['status'] ?? 0 ),
				'sitemaps' => array_values( array_unique( $robots_info['sitemaps'] ?? array() ) ),
				'error'    => $robots_info['error'] ?? null,
			),
		);

		set_transient( $cache_key, $resp, 10 * MINUTE_IN_SECONDS );
		return $resp;
	}

	$visited    = array();
	$queue      = $exists;
	$sitemaps   = array();
	$all_urls   = array();
	$safe_loops = 0;

	$max_nodes = 60;
	$max_urls  = 20000;
	$partial   = false;

	while ( ! empty( $queue ) && $safe_loops < $max_nodes ) {
		$next = array_shift( $queue );
		$norm = myaio_tss_norm_url( $next );

		if ( isset( $visited[ $norm ] ) ) {
			$safe_loops++;
			continue;
		}

		$visited[ $norm ] = true;

		$xml = myaio_tss_fetch_xml( $next );
		if ( ! $xml ) {
			$sitemaps[] = array(
				'url'       => $next,
				'type'      => 'fetch-failed',
				'url_count' => null,
			);
			$safe_loops++;
			continue;
		}

		$type = myaio_tss_parse_sitemap_type( $xml );

		if ( 'index' === $type ) {
			$children   = myaio_tss_extract_index_children( $xml );
			$sitemaps[] = array(
				'url'       => $next,
				'type'      => 'index',
				'url_count' => null,
			);

			foreach ( $children as $child ) {
				$cn = myaio_tss_norm_url( $child );
				if ( ! isset( $visited[ $cn ] ) ) {
					$queue[] = $child;
				}
			}
		} elseif ( 'urlset' === $type ) {
			$urls       = myaio_tss_extract_urlset_urls( $xml );
			$sitemaps[] = array(
				'url'       => $next,
				'type'      => 'urlset',
				'url_count' => count( $urls ),
			);

			foreach ( $urls as $u ) {
				$all_urls[ myaio_tss_norm_url( $u ) ] = true;
				if ( count( $all_urls ) >= $max_urls ) {
					$partial = true;
					break;
				}
			}

			if ( $partial ) {
				break;
			}
		} else {
			$sitemaps[] = array(
				'url'       => $next,
				'type'      => 'unknown',
				'url_count' => null,
			);
		}

		$safe_loops++;
	}

	if ( $safe_loops >= $max_nodes ) {
		$partial = true;
	}

	$sum_counts   = 0;
	$file_count   = count( $sitemaps );
	$unique_total = count( $all_urls );

	foreach ( $sitemaps as $s ) {
		if ( 'urlset' === $s['type'] && isset( $s['url_count'] ) ) {
			$sum_counts += (int) $s['url_count'];
		}
	}

	$loc        = myaio_tss_locate_sitemap_file();
	$curr_mtime = (int) ( get_option( 'myaio_sac_last_sitemap_mtime' ) ?: 0 );
	if ( ! empty( $loc['mtime'] ) && (int) $loc['mtime'] > 0 ) {
		$curr_mtime = (int) $loc['mtime'];
	}

	$prev_url   = get_option( 'myaio_sac_prev_sitemap_url' ) ?: null;
	$prev_mtime = (int) ( get_option( 'myaio_sac_prev_sitemap_mtime' ) ?: 0 );

	$resp = array(
		'available'               => true,
		'partial'                 => $partial,
		'message'                 => $partial
			? sprintf( 'Found %d sitemap file(s) (partial crawl due to safety limits); total URLs (non-deduped across files): %d; unique URLs: %d.', $file_count, $sum_counts, $unique_total )
			: sprintf( 'Found %d sitemap file(s); total URLs (non-deduped across files): %d; unique URLs: %d.', $file_count, $sum_counts, $unique_total ),
		'sitemap_url'             => $canonical_url,
		'previous_sitemap_url'    => $prev_url ?: ( $saved ?: null ),
		'last_modified'           => myaio_tss_iso8601( $curr_mtime ),
		'previous_last_modified'  => myaio_tss_iso8601( $prev_mtime ?: (int) get_option( 'myaio_sac_last_sitemap_mtime' ) ),
		'sitemap_files_count'     => $file_count,
		'sitemaps_found'          => $sitemaps,
		'total_url_count'         => $sum_counts,
		'unique_url_count'        => $unique_total,
		'folder_children_count'   => count( $guessed_children ),
		'folder_children_samples' => array_slice( $guessed_children, 0, 20 ),
		'checked'                 => $checked,
		'robots'                  => array(
			'url'      => $robots_url,
			'ok'       => (bool) ( $robots_info['ok'] ?? false ),
			'status'   => (int) ( $robots_info['status'] ?? 0 ),
			'sitemaps' => array_values( array_unique( $robots_info['sitemaps'] ?? array() ) ),
			'error'    => $robots_info['error'] ?? null,
		),
		'limits'                  => array(
			'max_candidate_checks' => $max_candidate_checks,
			'max_nodes'            => $max_nodes,
			'max_urls'             => $max_urls,
		),
	);

	set_transient( $cache_key, $resp, 10 * MINUTE_IN_SECONDS );

	return $resp;
}

////////////////////////////////////////////////////////////////////////////////////////////////////
// CHECK ROBOTS META
////////////////////////////////////////////////////////////////////////////////////////////////////

/**
 * Same output shape as old check-robots-meta API.
 * If no URL is passed, defaults to homepage.
 *
 * @param string|WP_Error $target_url Target URL.
 * @return array
 */
function myaio_tss_check_robots_meta( $target_url = '' ) {
	if ( is_wp_error( $target_url ) ) {
		return array(
			'error'   => 'URL unreachable',
			'details' => $target_url->get_error_message(),
		);
	}

	if ( '' === $target_url ) {
		$target_url = home_url( '/' );
	}

	$response = wp_remote_get(
		$target_url,
		array(
			'timeout'     => 15,
			'redirection' => 5,
			'user-agent'  => 'myaio-robots-check/1.1.0',
			'headers'     => array(
				'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
			),
		)
	);

	if ( is_wp_error( $response ) ) {
		return array(
			'error'   => 'URL unreachable',
			'details' => $response->get_error_message(),
		);
	}

	$status       = (int) wp_remote_retrieve_response_code( $response );
	$html         = (string) wp_remote_retrieve_body( $response );
	$content_type = (string) wp_remote_retrieve_header( $response, 'content-type' );
	$x_robots_tag = (string) wp_remote_retrieve_header( $response, 'x-robots-tag' );

	if ( $status < 200 || $status >= 400 ) {
		return array(
			'url'            => $target_url,
			'meta_available' => false,
			'message'        => 'Non-success HTTP status returned.',
			'http_status'    => $status,
			'x_robots_tag'   => '' !== $x_robots_tag ? $x_robots_tag : null,
		);
	}

	if ( '' !== $content_type && false === stripos( $content_type, 'text/html' ) ) {
		return array(
			'url'            => $target_url,
			'meta_available' => false,
			'message'        => 'Response is not HTML, cannot reliably scan <meta> tags.',
			'http_status'    => $status,
			'content_type'   => $content_type,
			'x_robots_tag'   => '' !== $x_robots_tag ? $x_robots_tag : null,
		);
	}

	libxml_use_internal_errors( true );
	$dom = new DOMDocument();

	// phpcs:ignore WordPress.PHP.NoSilencedErrors.Discouraged
	@$dom->loadHTML( $html );

	$metas    = $dom->getElementsByTagName( 'meta' );
	$found    = false;
	$contents = array();

	foreach ( $metas as $meta ) {
		if ( 0 === strcasecmp( $meta->getAttribute( 'name' ), 'robots' ) ) {
			$found = true;

			$meta_content = trim( (string) $meta->getAttribute( 'content' ) );
			if ( '' !== $meta_content ) {
				$contents[] = $meta_content;
			}
		}
	}

	if ( $found && ! empty( $contents ) ) {
		$directives = array();

		foreach ( $contents as $content_string ) {
			foreach ( explode( ',', (string) $content_string ) as $part ) {
				$d = trim( (string) $part );
				if ( '' !== $d ) {
					$directives[] = $d;
				}
			}
		}

		$directives     = array_values( array_unique( $directives ) );
		$merged_content = implode( ', ', $directives );

		return array(
			'url'            => $target_url,
			'meta_available' => true,
			'content'        => $merged_content,
			'http_status'    => $status,
			'x_robots_tag'   => '' !== $x_robots_tag ? $x_robots_tag : null,
		);
	}

	return array(
		'url'            => $target_url,
		'meta_available' => false,
		'message'        => 'No robots meta tag found in this page <head>.',
		'http_status'    => $status,
		'x_robots_tag'   => '' !== $x_robots_tag ? $x_robots_tag : null,
	);
}

////////////////////////////////////////////////////////////////////////////////////////////////////
// SCHEMA SCAN
////////////////////////////////////////////////////////////////////////////////////////////////////

/**
 * Same output shape as old schema-scan API, but pagination removed.
 * If URL is passed, only scans that one URL.
 *
 * @param string|WP_Error $requested_url Requested URL.
 * @return array
 */
function myaio_tss_schema_scan( $requested_url = '' ) {
	$http_timeout = 30;

	if ( is_wp_error( $requested_url ) ) {
		return array(
			'error' => $requested_url->get_error_message(),
		);
	}

	$all_urls = myaio_tss_get_filtered_crawlable_urls( $requested_url );

	if ( ! is_array( $all_urls ) ) {
		$all_urls = array();
	}

	$results           = array();
	$total_with_schema = 0;

	if ( ! class_exists( 'DOMDocument' ) || ! class_exists( 'DOMXPath' ) ) {
		return array(
			'error' => 'DOM extension not available',
		);
	}

	foreach ( $all_urls as $url_data ) {
		$page_url = '';
		if ( is_array( $url_data ) && isset( $url_data['url'] ) ) {
			$page_url = (string) $url_data['url'];
		}

		$page_url = esc_url_raw( trim( $page_url ) );
		if ( '' === $page_url ) {
			continue;
		}

		$type     = ( is_array( $url_data ) && isset( $url_data['type'] ) ) ? sanitize_text_field( (string) $url_data['type'] ) : '';
		$category = ( is_array( $url_data ) && isset( $url_data['category'] ) ) ? sanitize_text_field( (string) $url_data['category'] ) : '';

		$response = wp_remote_get(
			$page_url,
			array(
				'timeout'     => $http_timeout,
				'redirection' => 5,
				'headers'     => array(
					'User-Agent' => 'MYAIO-Schema-Scanner/1.0; ' . home_url(),
					'Accept'     => 'text/html,application/xhtml+xml',
				),
			)
		);

		if ( is_wp_error( $response ) ) {
			$results[] = array(
				'url'      => $page_url,
				'type'     => $type,
				'category' => $category,
				'error'    => $response->get_error_message(),
			);
			continue;
		}

		$code = (int) wp_remote_retrieve_response_code( $response );
		if ( $code < 200 || $code >= 300 ) {
			$results[] = array(
				'url'      => $page_url,
				'type'     => $type,
				'category' => $category,
				'error'    => 'HTTP ' . $code,
			);
			continue;
		}

		$html = (string) wp_remote_retrieve_body( $response );
		if ( '' === $html ) {
			$results[] = array(
				'url'      => $page_url,
				'type'     => $type,
				'category' => $category,
				'error'    => 'Empty HTML',
			);
			continue;
		}

		libxml_use_internal_errors( true );

		$dom    = new DOMDocument();
		$loaded = $dom->loadHTML( '<?xml encoding="utf-8" ?>' . $html, LIBXML_NOWARNING | LIBXML_NOERROR );

		if ( ! $loaded ) {
			libxml_clear_errors();
			$results[] = array(
				'url'      => $page_url,
				'type'     => $type,
				'category' => $category,
				'error'    => 'Failed to parse HTML',
			);
			continue;
		}

		$jsonld_blocks = array();
		$scripts       = $dom->getElementsByTagName( 'script' );

		foreach ( $scripts as $script ) {
			if ( ! $script instanceof DOMElement ) {
				continue;
			}

			$type_attr = trim( (string) $script->getAttribute( 'type' ) );
			if ( 'application/ld+json' === strtolower( $type_attr ) ) {
				$content = trim( (string) $script->textContent );
				if ( '' !== $content ) {
					$jsonld_blocks[] = $content;
				}
			}
		}

		$xpath = new DOMXPath( $dom );

		$microdata_nodes = $xpath->query( '//*[@itemscope or @itemtype]' );
		$has_microdata   = ( $microdata_nodes && $microdata_nodes->length > 0 );

		$rdfa_nodes = $xpath->query( '//*[@typeof or @vocab]' );
		$has_rdfa   = ( $rdfa_nodes && $rdfa_nodes->length > 0 );

		libxml_clear_errors();

		$has_schema = ( ! empty( $jsonld_blocks ) || $has_microdata || $has_rdfa );

		if ( $has_schema ) {
			$total_with_schema++;
		}

		$results[] = array(
			'url'           => $page_url,
			'type'          => $type,
			'category'      => $category,
			'has_schema'    => $has_schema,
			'has_jsonld'    => ! empty( $jsonld_blocks ),
			'has_microdata' => $has_microdata,
			'has_rdfa'      => $has_rdfa,
			'jsonld'        => $jsonld_blocks,
		);
	}

	$total_available = count( $all_urls );

	return array(
		'summary' => array(
			'total_urls_available' => $total_available,
			'limit'                => $total_available,
			'offset'               => 0,
			'scanned_this_call'    => $total_available,
			'pages_with_schema'    => $total_with_schema,
		),
		'pages'   => $results,
		'next'    => array(
			'offset' => null,
			'limit'  => $total_available,
		),
	);
}