<?php
/**
 * Shard Manager for AI Store Index
 *
 * Orchestrates sharded index storage with intelligent read/write coordination.
 * Handles shard allocation during builds and intelligent shard selection during reads.
 *
 * @package AIPT\Api\StoreIndex
 * @since 2.5.0
 */

namespace AIPT\Api\StoreIndex;

/**
 * ShardManager class
 *
 * Coordinates BUILD operations (shard assignment, map creation) and
 * READ operations (map loading, intelligent shard selection, LRU caching).
 */
class ShardManager {

	/**
	 * Maximum products per shard (hard limit).
	 * Sequential assignment: first 10K products -> shard 0, next 10K -> shard 1.
	 */
	public const MAX_PRODUCTS_PER_SHARD = 10000;

	/**
	 * Maximum compressed shard size in bytes (5 MB).
	 * Triggers new shard creation during build when exceeded.
	 */
	public const MAX_SHARD_SIZE_BYTES = 5 * 1024 * 1024;

	/**
	 * Estimated compression ratio for TOON+gzip.
	 *
	 * Conservative value of 4 chosen to prevent shard overflow.
	 * Actual ratio varies 5-7x depending on data, but 4x provides
	 * safe margin for worst-case scenarios (short text, lots of numbers).
	 */
	private const ESTIMATED_COMPRESSION_RATIO = 4;

	/**
	 * Safety threshold for shard size (80% of max).
	 *
	 * Triggers shard split at 80% to leave buffer for:
	 * - Compression ratio variance
	 * - Category metadata overhead
	 * - Final encoding overhead
	 */
	private const SIZE_SAFETY_THRESHOLD = 0.80;

	/**
	 * Index directory relative to uploads.
	 */
	private const INDEX_DIR = 'ai-product-tools/ai-index';

	/**
	 * Profile ID.
	 *
	 * @var int
	 */
	private $profile_id;

	/**
	 * ToonEncoder instance.
	 *
	 * @var ToonEncoder
	 */
	private $encoder;

	/**
	 * ToonDecoder instance.
	 *
	 * @var ToonDecoder
	 */
	private $decoder;

	/**
	 * Shard cache for read operations.
	 *
	 * @var ShardCache
	 */
	private $shard_cache;

	/**
	 * Loaded map data (cached after first read).
	 *
	 * @var array|null
	 */
	private $map = null;

	/**
	 * Current build state (only used during BUILD operations).
	 *
	 * @var array{
	 *   shards: array,
	 *   current_shard_id: int,
	 *   category_distribution: array
	 * }|null
	 */
	private $build_state = null;

	/**
	 * Constructor.
	 *
	 * @param int $profile_id Profile ID.
	 */
	public function __construct( int $profile_id ) {
		$this->profile_id  = $profile_id;
		$this->encoder     = new ToonEncoder();
		$this->decoder     = new ToonDecoder();
		$this->shard_cache = new ShardCache();
	}

	/**
	 * Initialize build state for a new sharded index build.
	 *
	 * Called at the start of IndexBuilder batch processing.
	 * Resets all shard state and prepares for sequential product assignment.
	 *
	 * @return void
	 */
	public function initBuild(): void {
		$this->build_state = array(
			'shards'                => array(),
			'current_shard_id'      => 0,
			'category_distribution' => array(), 
		);

		$this->build_state['shards'][0] = array(
			'products'       => array(),
			'categories'     => array(),
			'estimated_size' => 0, 
		);
	}

	/**
	 * Add a product to the appropriate shard during build.
	 *
	 * Automatically creates new shards when limits are reached.
	 * Uses incremental size tracking (O(1)) to prevent shard overflow.
	 * Tracks category distribution for map file.
	 *
	 * @param array $product Product profile data with 'id', 'categories', etc.
	 * @return int Shard ID where product was added.
	 * @throws \RuntimeException If build not initialized.
	 */
	public function addProduct( array $product ): int {
		if ( $this->build_state === null ) {

			throw new \RuntimeException( 'Build not initialized. Call initBuild() first.' );
		}

		$shard_id      = $this->build_state['current_shard_id'];
		$current_shard = &$this->build_state['shards'][ $shard_id ];

		$product_estimated_size = $this->estimateProductSize( $product );

		$size_threshold = (int) ( self::MAX_SHARD_SIZE_BYTES * self::SIZE_SAFETY_THRESHOLD );

		$current_estimated_size = $current_shard['estimated_size'] ?? 0;

		$needs_new_shard = false;

		if ( count( $current_shard['products'] ) >= self::MAX_PRODUCTS_PER_SHARD ) {
			$needs_new_shard = true;
		}

		if ( ! $needs_new_shard && ( $current_estimated_size + $product_estimated_size ) > $size_threshold ) {

			if ( count( $current_shard['products'] ) > 0 ) {
				$needs_new_shard = true;
			}
		}

		if ( $needs_new_shard ) {
			++$this->build_state['current_shard_id'];
			$shard_id = $this->build_state['current_shard_id'];

			$this->build_state['shards'][ $shard_id ] = array(
				'products'       => array(),
				'categories'     => array(),
				'estimated_size' => 0,
			);
			$current_shard = &$this->build_state['shards'][ $shard_id ];
		}

		$product_id                               = (string) $product['id'];
		$current_shard['products'][ $product_id ] = $product;

		$current_shard['estimated_size'] = ( $current_shard['estimated_size'] ?? 0 ) + $product_estimated_size;

		if ( ! empty( $product['categories'] ) && is_array( $product['categories'] ) ) {
			foreach ( $product['categories'] as $cat_id ) {
				$cat_key = (string) $cat_id;

				if ( ! isset( $current_shard['categories'][ $cat_key ] ) ) {
					$current_shard['categories'][ $cat_key ] = array();
				}
				$current_shard['categories'][ $cat_key ][] = (int) $product['id'];

				if ( ! isset( $this->build_state['category_distribution'][ $cat_id ] ) ) {
					$this->build_state['category_distribution'][ $cat_id ] = array();
				}
				if ( ! isset( $this->build_state['category_distribution'][ $cat_id ][ $shard_id ] ) ) {
					$this->build_state['category_distribution'][ $cat_id ][ $shard_id ] = 0;
				}
				++$this->build_state['category_distribution'][ $cat_id ][ $shard_id ];
			}
		}

		return $shard_id;
	}

	/**
	 * Get current build state for saving to TempStorage.
	 *
	 * @return array|null Build state or null if not building.
	 */
	public function getBuildState(): ?array {
		return $this->build_state;
	}

	/**
	 * Get minimal build state for manifest storage.
	 *
	 * Returns only counters and shard references, not the full product data.
	 * This prevents memory exhaustion and timeout when saving large builds.
	 * The full product data is written directly to temp shard files.
	 *
	 * @return array|null Minimal build state or null if not building.
	 */
	public function getMinimalBuildState(): ?array {
		if ( $this->build_state === null ) {
			return null;
		}

		$shard_summaries = array();
		$total_products = 0;

		foreach ( $this->build_state['shards'] as $shard_id => $shard_data ) {
			$product_count = count( $shard_data['products'] ?? array() );
			$total_products += $product_count;

			$shard_summaries[ $shard_id ] = array(
				'product_count'  => $product_count,
				'category_count' => count( $shard_data['categories'] ?? array() ),
				'estimated_size' => $shard_data['estimated_size'] ?? 0,
			);
		}

		return array(
			'current_shard_id'      => $this->build_state['current_shard_id'],
			'shard_summaries'       => $shard_summaries,
			'total_products'        => $total_products,
			'category_distribution' => $this->build_state['category_distribution'],
		);
	}

	/**
	 * Restore build state from TempStorage.
	 *
	 * Ensures backward compatibility by recalculating estimated_size
	 * for shards that don't have it (from older builds).
	 *
	 * @param array $state Previously saved build state.
	 * @return void
	 */
	public function restoreBuildState( array $state ): void {
		$this->build_state = $state;

		if ( isset( $this->build_state['shards'] ) ) {
			foreach ( $this->build_state['shards'] as $shard_id => &$shard_data ) {
				if ( ! isset( $shard_data['estimated_size'] ) ) {

					$shard_data['estimated_size'] = $this->recalculateShardEstimatedSize( $shard_data );
				}
			}
			unset( $shard_data ); 
		}
	}

	/**
	 * Recalculate estimated_size for a shard from its products.
	 *
	 * Used for backward compatibility when restoring state without estimated_size.
	 *
	 * @param array $shard_data Shard data with products.
	 * @return int Total estimated compressed size in bytes.
	 */
	private function recalculateShardEstimatedSize( array $shard_data ): int {
		$total_size = 0;

		if ( ! empty( $shard_data['products'] ) ) {
			foreach ( $shard_data['products'] as $product ) {
				$total_size += $this->estimateProductSize( $product );
			}
		}

		return $total_size;
	}

	/**
	 * Get current build statistics.
	 *
	 * @return array{shard_count: int, total_products: int, current_shard_products: int}|null
	 */
	public function getBuildStats(): ?array {
		if ( $this->build_state === null ) {
			return null;
		}

		$total_products = 0;
		foreach ( $this->build_state['shards'] as $shard ) {
			$total_products += count( $shard['products'] );
		}

		$current_shard_id       = $this->build_state['current_shard_id'];
		$current_shard_products = count( $this->build_state['shards'][ $current_shard_id ]['products'] ?? array() );

		return array(
			'shard_count'            => count( $this->build_state['shards'] ),
			'total_products'         => $total_products,
			'current_shard_products' => $current_shard_products,
		);
	}

	/**
	 * Get total products across all shards during build.
	 *
	 * @return int Total product count.
	 */
	public function getTotalProducts(): int {
		if ( $this->build_state === null ) {
			return 0;
		}

		$total = 0;
		foreach ( $this->build_state['shards'] as $shard ) {
			$total += count( $shard['products'] );
		}

		return $total;
	}

	/**
	 * Finalize build: write all shards and create map file.
	 *
	 * Called at the end of IndexBuilder batch processing.
	 * Writes shard files atomically and generates the map.
	 *
	 * @param array $global_meta Global index metadata (version, built_at, etc.).
	 * @return array Map data structure with total_size and shard_count.
	 * @throws \RuntimeException If write operations fail.
	 */
	public function finalizeBuild( array $global_meta ): array {
		if ( $this->build_state === null ) {

			throw new \RuntimeException( 'No build state to finalize.' );
		}

		$this->ensureDirectory();

		$shard_metadata   = array();
		$total_products   = 0;
		$total_categories = 0;
		$total_size       = 0;
		$temp_files       = array();
		$backup_files     = array();
		$is_first_build   = false; 
		$category_meta    = array(); 

		try {

			foreach ( $this->build_state['shards'] as $shard_id => $shard_data ) {

				if ( empty( $shard_data['products'] ) ) {
					continue;
				}

				$enriched_categories = $this->enrichCategoryMetadata( $shard_data['categories'] );

				foreach ( $enriched_categories as $cat_id => $cat_data ) {
					$cat_key = (string) $cat_id;
					if ( ! isset( $category_meta[ $cat_key ] ) ) {
						$category_meta[ $cat_key ] = array(
							'name' => $cat_data['name'] ?? '',
							'slug' => $cat_data['slug'] ?? '',
						);
					}
				}

				$shard_index = array(
					'meta'       => array_merge(
						$global_meta,
						array(
							'shard_id'       => $shard_id,
							'product_count'  => count( $shard_data['products'] ),
							'category_count' => count( $enriched_categories ),
						)
					),
					'products'   => $shard_data['products'],
					'categories' => $enriched_categories,
				);

				$temp_path              = $this->writeShardToTemp( $shard_id, $shard_index );
				$temp_files[ $shard_id ] = $temp_path;

				$file_size_result = filesize( $temp_path );
				if ( $file_size_result === false ) {

					error_log(
						sprintf(
							'[AIPT] ShardManager: Failed to get file size for shard %d at %s',
							$shard_id,
							$temp_path
						)
					);
					$file_size = 0;
				} else {
					$file_size = (int) $file_size_result;
				}

				$product_ids_in_shard = array_map( 'intval', array_keys( $shard_data['products'] ) );
				$shard_metadata[ $shard_id ] = array(
					'shard_id'       => $shard_id,
					'product_count'  => count( $shard_data['products'] ),
					'category_count' => count( $enriched_categories ),
					'file_size'      => $file_size,
					'min_product_id' => ! empty( $product_ids_in_shard ) ? min( $product_ids_in_shard ) : 0,
					'max_product_id' => ! empty( $product_ids_in_shard ) ? max( $product_ids_in_shard ) : 0,
				);

				$total_products   += count( $shard_data['products'] );
				$total_categories += count( $enriched_categories );
				$total_size       += $file_size;
			}

			$category_map = $this->buildCategoryMap();

			$map_data = array(
				'meta'          => array_merge(
					$global_meta,
					array(
						'total_shards'     => count( $shard_metadata ),
						'total_products'   => $total_products,
						'total_categories' => $total_categories,
						'total_size'       => $total_size,
						'shards'           => $shard_metadata,
					)
				),
				'products'      => array(), 
				'categories'    => $category_map,
				'category_meta' => $category_meta, 
			);

			$map_temp_path           = $this->writeMapToTemp( $map_data );
			$temp_files['map']       = $map_temp_path;

			$map_size_result = filesize( $map_temp_path );
			if ( $map_size_result !== false ) {
				$total_size += (int) $map_size_result;
			}

			$pattern = $this->getDirectory() . '/products-ai-index-' . $this->profile_id . '-*.toon.gz';
			$existing     = glob( $pattern ) ?: array();
			$is_first_build = empty( $existing ); 

			foreach ( $existing as $file ) {
				$backup = $file . '.bak';

				if ( @rename( $file, $backup ) ) {
					$backup_files[] = $backup;
				} else {

					foreach ( $backup_files as $created_backup ) {
						$original = preg_replace( '/\.bak$/', '', $created_backup );

						@rename( $created_backup, $original );
					}

					foreach ( $temp_files as $temp_path ) {
						if ( file_exists( $temp_path ) ) {

							@unlink( $temp_path );
						}
					}
					throw new \RuntimeException(
						sprintf(
							'Failed to create backup for %s. Aborting finalization to prevent data loss.',
							$file
						)
					);
				}
			}

			foreach ( $temp_files as $id => $temp_path ) {
				if ( $id === 'map' ) {
					$final_path = $this->getMapFilePath();
				} else {
					$final_path = $this->getShardFilePath( (int) $id );
				}

				if ( ! $this->atomicRename( $temp_path, $final_path ) ) {

					foreach ( $backup_files as $backup ) {
						$original = preg_replace( '/\.bak$/', '', $backup );

						@rename( $backup, $original );
					}
					throw new \RuntimeException( "Failed to finalize file: {$id}" );
				}
			}

			foreach ( $backup_files as $backup ) {

				@unlink( $backup );
			}

		} catch ( \Exception $e ) {

			foreach ( $temp_files as $temp_path ) {
				if ( file_exists( $temp_path ) ) {

					@unlink( $temp_path );
				}
			}

			if ( ! empty( $backup_files ) ) {
				foreach ( $backup_files as $backup ) {
					$original = preg_replace( '/\.bak$/', '', $backup );

					@rename( $backup, $original );
				}
			} elseif ( $is_first_build ) {

				$this->deleteAllShards();
			} else {

				error_log(
					'[AIPT] ShardManager: Finalize failed and backup restoration not possible. ' .
					'Live index preserved but may be in inconsistent state. Manual cleanup may be required.'
				);
			}

			throw $e;
		}

		$this->build_state = null;

		$this->clearCache();

		return array(
			'total_size'       => $total_size,
			'shard_count'      => count( $shard_metadata ),
			'total_products'   => $total_products,
			'total_categories' => $total_categories,
			'map_data'         => $map_data,
		);
	}

	/**
	 * Load the map file.
	 *
	 * Cached after first read. Provides shard metadata and category hints.
	 *
	 * @return array|null Map data or null if unavailable.
	 */
	public function loadMap(): ?array {
		if ( $this->map !== null ) {
			return $this->map;
		}

		$map_path = $this->getMapFilePath();
		if ( ! file_exists( $map_path ) ) {
			return null;
		}

		try {

			$compressed = file_get_contents( $map_path );
			if ( $compressed === false ) {
				return null;
			}

			$toon = @gzdecode( $compressed );
			if ( $toon === false ) {
				return null;
			}

			$this->map = $this->decoder->decode( $toon );
			return $this->map;

		} catch ( \Exception $e ) {

			return null;
		}
	}

	/**
	 * Check if sharded index is available.
	 *
	 * @return bool True if map file exists and is readable.
	 */
	public function isAvailable(): bool {
		$map = $this->loadMap();
		return $map !== null && ! empty( $map['meta']['shards'] );
	}

	/**
	 * Get total shard count from map.
	 *
	 * @return int Number of shards or 0 if unavailable.
	 */
	public function getShardCount(): int {
		$map = $this->loadMap();
		return $map['meta']['total_shards'] ?? 0;
	}

	/**
	 * Load a shard with LRU caching.
	 *
	 * @param int $shard_id Shard ID to load.
	 * @return array|null Shard data or null if unavailable.
	 */
	public function loadShard( int $shard_id ): ?array {

		$cached = $this->shard_cache->get( $shard_id );
		if ( $cached !== null ) {
			return $cached;
		}

		$shard_path = $this->getShardFilePath( $shard_id );
		if ( ! file_exists( $shard_path ) ) {
			return null;
		}

		try {

			$compressed = file_get_contents( $shard_path );
			if ( $compressed === false ) {
				return null;
			}

			$toon = @gzdecode( $compressed );
			if ( $toon === false ) {
				return null;
			}

			$shard_data = $this->decoder->decode( $toon );

			$this->shard_cache->put( $shard_id, $shard_data );

			return $shard_data;

		} catch ( \Exception $e ) {

			return null;
		}
	}

	/**
	 * Get shards that contain products from specified categories.
	 *
	 * Uses the category distribution map for intelligent shard selection.
	 *
	 * @param array $category_ids Category term IDs to find.
	 * @return array Array of shard IDs sorted by relevance (most products first).
	 */
	public function getShardsForCategories( array $category_ids ): array {
		$map = $this->loadMap();
		if ( $map === null ) {
			return array();
		}

		$shard_scores = array();

		foreach ( $category_ids as $cat_id ) {
			$cat_key = (string) $cat_id;
			if ( isset( $map['categories'][ $cat_key ] ) ) {
				$distribution = $map['categories'][ $cat_key ];
				foreach ( $distribution as $shard_id => $count ) {
					if ( ! isset( $shard_scores[ $shard_id ] ) ) {
						$shard_scores[ $shard_id ] = 0;
					}
					$shard_scores[ $shard_id ] += $count;
				}
			}
		}

		arsort( $shard_scores );

		return array_keys( $shard_scores );
	}

	/**
	 * Get default shards to load when no category hint is available.
	 *
	 * Returns a randomized selection from the top shards (by product count)
	 * to provide variety while still preferring larger shards.
	 *
	 * @param int $limit Maximum shards to return.
	 * @return array Array of shard IDs.
	 */
	public function getDefaultShards( int $limit = 3 ): array {
		$map = $this->loadMap();
		if ( $map === null || empty( $map['meta']['shards'] ) ) {
			return array();
		}

		$shards = $map['meta']['shards'];
		uasort(
			$shards,
			function ( $a, $b ) {
				return ( $b['product_count'] ?? 0 ) - ( $a['product_count'] ?? 0 );
			}
		);

		$shard_keys = array_keys( $shards );

		$pool_size  = min( $limit * 2, count( $shard_keys ) );
		$shard_pool = array_slice( $shard_keys, 0, $pool_size );

		shuffle( $shard_pool );

		return array_slice( $shard_pool, 0, $limit );
	}

	/**
	 * Analyze a search query and return relevant shard IDs.
	 *
	 * Extracts category hints from the query and maps to shards.
	 *
	 * @param string $query Search query.
	 * @return array{shard_ids: int[], confidence: float}
	 */
	public function getShardsForQuery( string $query ): array {
		$map = $this->loadMap();
		if ( $map === null ) {
			return array(
				'shard_ids'  => array(),
				'confidence' => 0.0,
			);
		}

		$query_lower = strtolower( trim( $query ) );
		$words       = array_filter(
			explode( ' ', preg_replace( '/[^a-z0-9\s]/', '', $query_lower ) ),
			function ( $word ) {
				return strlen( $word ) >= 3;
			}
		);

		if ( empty( $words ) ) {
			return array(
				'shard_ids'  => $this->getDefaultShards(),
				'confidence' => 0.0,
			);
		}

		$matched_categories = array();
		$categories         = $map['categories'] ?? array();

		$category_meta_raw = $map['category_meta'] ?? array();
		$category_meta     = array();

		foreach ( $category_meta_raw as $cat_id => $cat_data ) {
			if ( ! empty( $cat_data['name'] ) ) {
				$category_meta[ $cat_id ] = strtolower( $cat_data['name'] );
			}
			if ( ! empty( $cat_data['slug'] ) ) {
				$category_meta[ $cat_id . '_slug' ] = strtolower( $cat_data['slug'] );
			}
		}

		foreach ( $words as $word ) {
			foreach ( $category_meta as $key => $name ) {
				if ( strpos( $name, $word ) !== false || strpos( $word, $name ) !== false ) {

					$cat_id = (int) preg_replace( '/_slug$/', '', $key );
					if ( isset( $categories[ (string) $cat_id ] ) ) {
						$matched_categories[ $cat_id ] = true;
					}
				}
			}
		}

		if ( empty( $matched_categories ) ) {
			return array(
				'shard_ids'  => $this->getDefaultShards(),
				'confidence' => 0.0,
			);
		}

		$shard_ids  = $this->getShardsForCategories( array_keys( $matched_categories ) );
		$confidence = min( 1.0, count( $matched_categories ) * 0.3 );

		return array(
			'shard_ids'  => $shard_ids,
			'confidence' => $confidence,
		);
	}

	/**
	 * Get products by category with intelligent shard selection.
	 *
	 * @param int $category_id Category term ID.
	 * @return array{product_ids: int[], products: array}
	 */
	public function getProductsByCategory( int $category_id ): array {
		$map = $this->loadMap();
		if ( $map === null ) {
			return array(
				'product_ids' => array(),
				'products'    => array(),
			);
		}

		$cat_key      = (string) $category_id;
		$product_ids  = array();
		$products     = array();

		$shard_hints = $map['categories'][ $cat_key ] ?? null;

		if ( $shard_hints !== null ) {

			foreach ( array_keys( $shard_hints ) as $shard_id ) {
				$shard_data = $this->loadShard( (int) $shard_id );
				if ( $shard_data === null ) {
					continue;
				}

				if ( isset( $shard_data['categories'][ $cat_key ]['productIds'] ) ) {
					$ids         = $shard_data['categories'][ $cat_key ]['productIds'];
					$product_ids = array_merge( $product_ids, $ids );

					foreach ( $ids as $pid ) {
						$pid_key = (string) $pid;
						if ( isset( $shard_data['products'][ $pid_key ] ) ) {
							$products[ $pid ] = $shard_data['products'][ $pid_key ];
						}
					}
				}
			}
		} else {

			$shard_count = $map['meta']['total_shards'] ?? 0;
			for ( $i = 0; $i < $shard_count; $i++ ) {
				$shard_data = $this->loadShard( $i );
				if ( $shard_data === null ) {
					continue;
				}

				if ( isset( $shard_data['categories'][ $cat_key ]['productIds'] ) ) {
					$ids         = $shard_data['categories'][ $cat_key ]['productIds'];
					$product_ids = array_merge( $product_ids, $ids );

					foreach ( $ids as $pid ) {
						$pid_key = (string) $pid;
						if ( isset( $shard_data['products'][ $pid_key ] ) ) {
							$products[ $pid ] = $shard_data['products'][ $pid_key ];
						}
					}
				}
			}
		}

		return array(
			'product_ids' => $product_ids,
			'products'    => $products,
		);
	}

	/**
	 * Get products by IDs with intelligent shard selection.
	 *
	 * Uses min/max product ID metadata per shard to skip shards that
	 * cannot contain any of the requested IDs. Products are stored in
	 * ID-sorted order, so each shard contains a contiguous ID range.
	 *
	 * @param int[] $product_ids Product IDs to retrieve.
	 * @return array<int, array> Map of product_id => profile.
	 */
	public function getProductsByIds( array $product_ids ): array {
		$map = $this->loadMap();
		if ( $map === null || empty( $product_ids ) ) {
			return array();
		}

		$results        = array();
		$shard_metadata = $map['meta']['shards'] ?? array();

		$product_ids_int = array_map( 'intval', $product_ids );
		$min_requested   = min( $product_ids_int );
		$max_requested   = max( $product_ids_int );

		foreach ( $shard_metadata as $shard_id => $meta ) {

			if ( isset( $meta['min_product_id'], $meta['max_product_id'] ) ) {
				$shard_min = (int) $meta['min_product_id'];
				$shard_max = (int) $meta['max_product_id'];

				if ( $shard_max < $min_requested || $shard_min > $max_requested ) {
					continue;
				}
			}

			$shard_data = $this->loadShard( (int) $shard_id );
			if ( $shard_data === null ) {
				continue;
			}

			foreach ( $product_ids as $product_id ) {
				$product_key = (string) $product_id;
				if ( isset( $shard_data['products'][ $product_key ] ) ) {
					$results[ $product_id ] = $shard_data['products'][ $product_key ];
				}
			}

			if ( count( $results ) === count( $product_ids ) ) {
				break;
			}
		}

		return $results;
	}

	/**
	 * Load multiple shards and merge their products.
	 *
	 * @param array $shard_ids Shard IDs to load.
	 * @return array Merged products array.
	 */
	public function loadAndMergeShards( array $shard_ids ): array {
		$merged = array(
			'products'   => array(),
			'categories' => array(),
		);

		foreach ( $shard_ids as $shard_id ) {
			$shard_data = $this->loadShard( (int) $shard_id );
			if ( $shard_data === null ) {
				continue;
			}

			foreach ( $shard_data['products'] ?? array() as $pid => $product ) {
				$merged['products'][ $pid ] = $product;
			}

			foreach ( $shard_data['categories'] ?? array() as $cat_id => $cat_data ) {
				if ( ! isset( $merged['categories'][ $cat_id ] ) ) {
					$merged['categories'][ $cat_id ] = $cat_data;
				} else {

					if ( isset( $cat_data['productIds'] ) ) {
						$merged['categories'][ $cat_id ]['productIds'] = array_merge(
							$merged['categories'][ $cat_id ]['productIds'] ?? array(),
							$cat_data['productIds']
						);
					}
				}
			}
		}

		return $merged;
	}

	/**
	 * Delete all shard files and map file for this profile.
	 *
	 * Uses DirectoryIterator for memory-efficient file listing instead of glob().
	 * This prevents memory exhaustion on sites with many shard files.
	 *
	 * @return bool True if all files deleted or didn't exist.
	 */
	public function deleteAllShards(): bool {
		$dir = $this->getDirectory();

		if ( ! is_dir( $dir ) ) {
			return true;
		}

		$prefix = 'products-ai-index-' . $this->profile_id . '-';

		try {
			$iterator = new \DirectoryIterator( $dir );

			foreach ( $iterator as $file ) {
				if ( $file->isDot() || ! $file->isFile() ) {
					continue;
				}

				$filename = $file->getFilename();

				if ( strpos( $filename, $prefix ) === 0 && substr( $filename, -8 ) === '.toon.gz' ) {

					@unlink( $file->getPathname() );
				}
			}
		} catch ( \Exception $e ) {

			$pattern = $dir . '/' . $prefix . '*.toon.gz';
			$files   = glob( $pattern );

			if ( $files ) {
				foreach ( $files as $file ) {

					@unlink( $file );
				}
			}
		}

		return true;
	}

	/**
	 * Clear all caches (map and shard cache).
	 *
	 * @return void
	 */
	public function clearCache(): void {
		$this->map = null;
		$this->shard_cache->clear();
	}

	/**
	 * Get cache statistics.
	 *
	 * @return array Cache stats.
	 */
	public function getCacheStats(): array {
		return $this->shard_cache->getStats();
	}

	/**
	 * Check if the storage directory is writable.
	 *
	 * @return bool True if directory is writable.
	 */
	public function isWritable(): bool {
		$dir = $this->getDirectory();

		if ( ! is_dir( $dir ) ) {

			$upload_dir = wp_upload_dir();
			return \wp_is_writable( $upload_dir['basedir'] );
		}

		return \wp_is_writable( $dir );
	}

	/**
	 * Get the base directory for index files.
	 *
	 * @return string Absolute path to index directory.
	 */
	private function getDirectory(): string {
		$upload_dir = wp_upload_dir();
		return $upload_dir['basedir'] . '/' . self::INDEX_DIR;
	}

	/**
	 * Get the shard directory path for this profile.
	 *
	 * Returns the directory where all shard files for this profile are stored.
	 * Profile isolation is achieved via filename prefix (products-ai-index-{profile_id}-*),
	 * not via subdirectory, to maintain backward compatibility with existing installations.
	 *
	 * @return string Absolute path to shard directory.
	 */
	public function getShardDirectory(): string {
		return $this->getDirectory();
	}

	/**
	 * Get shard file path.
	 *
	 * @param int $shard_id Shard ID.
	 * @return string Absolute path.
	 */
	private function getShardFilePath( int $shard_id ): string {
		return $this->getDirectory() . '/products-ai-index-' . $this->profile_id . '-shard-' . $shard_id . '.toon.gz';
	}

	/**
	 * Get map file path.
	 *
	 * @return string Absolute path.
	 */
	private function getMapFilePath(): string {
		return $this->getDirectory() . '/products-ai-index-' . $this->profile_id . '-map.toon.gz';
	}

	/**
	 * Ensure the storage directory exists.
	 *
	 * @return bool True if directory exists or was created.
	 */
	private function ensureDirectory(): bool {
		$dir = $this->getDirectory();

		if ( is_dir( $dir ) ) {
			return \wp_is_writable( $dir );
		}

		if ( ! wp_mkdir_p( $dir ) ) {
			return false;
		}

		$index_file = $dir . '/index.php';
		if ( ! file_exists( $index_file ) ) {

			file_put_contents( $index_file, '<?php exit;' );
		}

		$htaccess_file = $dir . '/.htaccess';
		if ( ! file_exists( $htaccess_file ) ) {

			file_put_contents( $htaccess_file, 'deny from all' );
		}

		return \wp_is_writable( $dir );
	}

	/**
	 * Estimate a single product's compressed size contribution.
	 *
	 * Uses lightweight JSON length estimation (O(1) vs full TOON encode).
	 * Conservative multiplier accounts for TOON overhead and compression variance.
	 *
	 * @param array $product Product profile data.
	 * @return int Estimated compressed size in bytes.
	 */
	private function estimateProductSize( array $product ): int {

		$json = wp_json_encode( $product, JSON_UNESCAPED_UNICODE );

		if ( $json === false ) {

			return 2048;
		}

		$json_length = strlen( $json );

		$toon_estimate = (int) ( $json_length * 0.9 );

		$compressed_estimate = (int) ceil( ( $toon_estimate / self::ESTIMATED_COMPRESSION_RATIO ) * 1.1 );

		return max( 100, $compressed_estimate );
	}

	/**
	 * Estimate shard size (TOON + gzip).
	 *
	 * Returns a conservative high estimate on error to fail-safe
	 * (trigger shard split rather than allowing oversized shards).
	 *
	 * @param array $shard_data Shard data with products and categories.
	 * @return int Estimated bytes (returns MAX_SHARD_SIZE_BYTES on error).
	 */
	private function estimateShardSize( array $shard_data ): int {
		try {
			$toon = $this->encoder->encode(
				array(
					'meta'       => array(),
					'products'   => $shard_data['products'],
					'categories' => $shard_data['categories'],
				)
			);
			return (int) ceil( strlen( $toon ) / self::ESTIMATED_COMPRESSION_RATIO );
		} catch ( \Exception $e ) {

			error_log(
				sprintf(
					'[AIPT] ShardManager: Failed to estimate shard size (products: %d). Error: %s',
					count( $shard_data['products'] ?? array() ),
					$e->getMessage()
				)
			);

			return self::MAX_SHARD_SIZE_BYTES;
		}
	}

	/**
	 * Write shard to temp file.
	 *
	 * @param int   $shard_id   Shard ID.
	 * @param array $shard_data Shard index data.
	 * @return string Temp file path.
	 * @throws \RuntimeException If write fails.
	 */
	private function writeShardToTemp( int $shard_id, array $shard_data ): string {
		$temp_path = $this->getShardFilePath( $shard_id ) . '.tmp.' . wp_generate_password( 8, false );

		$toon       = $this->encoder->encode( $shard_data );
		$compressed = gzencode( $toon, 9 );

		if ( $compressed === false ) {

			throw new \RuntimeException( 'Failed to compress shard ' . $shard_id );
		}

		if ( strlen( $compressed ) > self::MAX_SHARD_SIZE_BYTES ) {
			throw new \RuntimeException(
				sprintf(
					'Shard %d exceeds size limit: %d bytes (max: %d bytes)',
					$shard_id, 
					strlen( $compressed ), 
					self::MAX_SHARD_SIZE_BYTES 
				)
			);
		}

		$written = file_put_contents( $temp_path, $compressed );
		if ( $written === false ) {

			throw new \RuntimeException( 'Failed to write shard temp file: ' . $shard_id );
		}

		return $temp_path;
	}

	/**
	 * Write map file to temp location.
	 *
	 * @param array $map_data Map data structure.
	 * @return string Temp file path.
	 * @throws \RuntimeException If write fails.
	 */
	private function writeMapToTemp( array $map_data ): string {
		$temp_path = $this->getMapFilePath() . '.tmp.' . wp_generate_password( 8, false );

		$toon       = $this->encoder->encode( $map_data );
		$compressed = gzencode( $toon, 9 );

		if ( $compressed === false ) {

			throw new \RuntimeException( 'Failed to compress map file' );
		}

		$written = file_put_contents( $temp_path, $compressed );
		if ( $written === false ) {

			throw new \RuntimeException( 'Failed to write map temp file' );
		}

		return $temp_path;
	}

	/**
	 * Build category→shard distribution map.
	 *
	 * @return array Category ID => [shard_id => product_count], sorted by count desc.
	 */
	private function buildCategoryMap(): array {
		if ( $this->build_state === null ) {
			return array();
		}

		$map = array();

		foreach ( $this->build_state['category_distribution'] as $cat_id => $shard_counts ) {

			arsort( $shard_counts );
			$map[ (string) $cat_id ] = $shard_counts;
		}

		return $map;
	}

	/**
	 * Enrich category metadata with term names and slugs.
	 *
	 * @param array $categories Category map with product ID arrays.
	 * @return array Enriched category map.
	 */
	private function enrichCategoryMetadata( array $categories ): array {
		if ( empty( $categories ) ) {
			return array();
		}

		$category_ids = array_map( 'intval', array_keys( $categories ) );

		$terms = get_terms(
			array(
				'taxonomy'   => 'product_cat',
				'include'    => $category_ids,
				'hide_empty' => false,
			)
		);

		$term_map = array();
		if ( ! is_wp_error( $terms ) ) {
			foreach ( $terms as $term ) {
				$term_map[ $term->term_id ] = array(
					'name' => $term->name,
					'slug' => $term->slug,
				);
			}
		}

		$enriched = array();
		foreach ( $categories as $cat_id => $product_ids ) {
			$int_id                       = (int) $cat_id;
			$enriched[ (string) $cat_id ] = array(
				'id'         => $int_id,
				'name'       => $term_map[ $int_id ]['name'] ?? null,
				'slug'       => $term_map[ $int_id ]['slug'] ?? null,
				'productIds' => $product_ids,
			);
		}

		return $enriched;
	}

	/**
	 * Atomic rename with Windows compatibility.
	 *
	 * @param string $source Source file path.
	 * @param string $target Target file path.
	 * @return bool True on success.
	 */
	private function atomicRename( string $source, string $target ): bool {

		if ( @rename( $source, $target ) ) {
			return true;
		}

		if ( file_exists( $target ) ) {
			$max_retries    = 3;
			$retry_delay_ms = 100;

			for ( $i = 0; $i < $max_retries; $i++ ) {

				if ( @unlink( $target ) ) {

					break;
				}
				if ( $i < $max_retries - 1 ) {
					usleep( $retry_delay_ms * 1000 );
				}
			}
		}

		return @rename( $source, $target );
	}
}
