import {useEffect, forwardRef, useImperativeHandle, useRef, useState, useCallback, useMemo} from "@wordpress/element";
import {__, sprintf} from "@wordpress/i18n";
import {handleError, handleSuccess, handleWarning} from "../../../../../../../helpers/notifications";
import {UpdateSetting} from "../../../../../../../rest/settings";
import Button from "../../../../../../button/_";
import {ONBOARDING_KEY_PREFIX, updateOnboardingPositions} from "../../index";
import {DATASET_PROCESSES_INFO} from "../../../../pages/knowledge/sources/_data";
import {areAllProcessesDone} from "../../utils/process";
import useProgressDocumentTitle from "../../../../../../../../helpers/hooks/use-progress-document-title";
import {useBackgroundProcess} from "../../../../../../contexts/background-process";
import {GetJob, CancelJob} from "../../../../../../../rest/jobs";
import AddSources from "../../../../pages/knowledge/sources/add-sources";
import Table from "../../../../../../sections/table";
import {DATASET_TYPE_KNOWLEDGE, GetDatasets} from "../../../../../../../rest/datasets";
import {
    renderName,
    renderSource,
    renderStorage,
    renderAIProvider
} from "../../../../pages/knowledge/sources/_renderers";
import {DGEN_PROCESS_KEY_PREFIX, DIN_PROCESS_KEY_PREFIX} from "../../../../../../../data/bg-processes";
import EmbeddingSettings from "./embedding-settings";

/**
 * Update onboarding process state (fire and forget - doesn't wait for response)
 *
 * @param {object} state Object of the process state
 * @param {function} addNotification Notifications state manager (not used, kept for compatibility)
 * @param {object} saveStateRefs Refs object containing saveInProgressRef and lastSavedStateRef
 * @return {void}
 */
const updateOnboardingProcessesState = (state, addNotification, saveStateRefs = null) => {
    // Normalize state: filter out jobs without status, sort by job_id for consistent ordering
    // job_id and status are required fields
    const normalizedState = Array.isArray(state) ? state
            .filter(job => job && job.job_id && job.status) // Remove invalid jobs or jobs without required fields
            .sort((a, b) => a.job_id - b.job_id) // Sort by job_id for consistent ordering
        : state;

    // If saveStateRefs are provided, use them to prevent duplicate/concurrent saves
    if (saveStateRefs) {
        const {saveInProgressRef, lastSavedStateRef} = saveStateRefs;

        // Serialize normalized state for comparison
        const stateString = JSON.stringify(normalizedState);

        // Check if a save is already in progress
        if (saveInProgressRef.current) {
            // A save is in progress, check if the state is the same as what's being saved
            // If different, queue it to run after current save completes
            saveInProgressRef.current.then(() => {
                // After current save completes, check if state is still different
                if (lastSavedStateRef.current !== stateString) {
                    // State changed, save it
                    const savePromise = (async () => {
                        try {
                            await UpdateSetting(LimbChatbot.rest.url, LimbChatbot.rest.nonce, ONBOARDING_KEY_PREFIX + 'processes', normalizedState);
                            lastSavedStateRef.current = stateString;
                        } catch (e) {
                            handleError(e);
                        } finally {
                            saveInProgressRef.current = null;
                        }
                    })();
                    saveInProgressRef.current = savePromise;
                }
            }).catch(() => {
                // Previous save failed, try to save this one
                if (lastSavedStateRef.current !== stateString) {
                    const savePromise = (async () => {
                        try {
                            await UpdateSetting(LimbChatbot.rest.url, LimbChatbot.rest.nonce, ONBOARDING_KEY_PREFIX + 'processes', normalizedState);
                            lastSavedStateRef.current = stateString;
                        } catch (e) {
                            handleError(e);
                        } finally {
                            saveInProgressRef.current = null;
                        }
                    })();
                    saveInProgressRef.current = savePromise;
                }
            });
            return; // Don't start new save, queued above
        }

        // No save in progress, check if state is the same as last saved
        if (lastSavedStateRef.current === stateString) {
            // Same data, skip save
            return;
        }

        // Create save promise and store it (fire and forget)
        const savePromise = (async () => {
            try {
                await UpdateSetting(LimbChatbot.rest.url, LimbChatbot.rest.nonce, ONBOARDING_KEY_PREFIX + 'processes', normalizedState);
                // Update last saved state only on success
                lastSavedStateRef.current = stateString;
            } catch (e) {
                handleError(e);
            } finally {
                // Clear the save in progress flag
                saveInProgressRef.current = null;
            }
        })();

        saveInProgressRef.current = savePromise;
        // Don't await - fire and forget
        return;
    }

    // Fallback to original behavior if no refs provided (fire and forget)
    UpdateSetting(LimbChatbot.rest.url, LimbChatbot.rest.nonce, ONBOARDING_KEY_PREFIX + 'processes', normalizedState)
        .catch(e => {
            handleError(e);
        });
}

const Step3 = forwardRef(({
                              config,
                              processesStateRef,
                              setIsReady,
                              setIsDataReady,
                              collecting,
                              setCollecting,
                              setLearnButton,
                              close,
                              setLoading,
                              setAllProcessesDone,
                              hasUnsavedChanges,
                              setHasUnsavedChanges,
                              notifications
                          }, ref) => {
    // Status constants - shared across the component
    const finishedStatuses = ['completed', 'failed', 'canceled'];
    const activeStatuses = ['pending', 'generating_tasks', 'processing', 'paused'];
    const unfinishedStatuses = ['pending', 'generating_tasks', 'processing', 'paused'];

    // Status messages object - shared for all followJob calls
    // Don't show complete message here - we handle it in onComplete callback to prevent duplicates
    // Use empty string instead of null to prevent fallback message
    const statusMessagesObj = useMemo(() => ({
        complete: '',
        pause: sprintf(__("%s process has been paused.", 'limb-chatbot'), DATASET_PROCESSES_INFO['dataset_generating'].processName),
        cancel: sprintf(__("%s process has been cancelled.", 'limb-chatbot'), DATASET_PROCESSES_INFO['dataset_generating'].processName)
    }), []);
    const [isDataFetched, setIsDataFetched] = useState(false);

    const [learnProgress, setLearnProgress] = useState(false);
    const [allProcessesDoneLocal, setAllProcessesDoneLocal] = useState(false);

    const collectKnowledgeRef = useRef(null);
    const embeddingSettingsRef = useRef(null);

    // Embedding settings state (updated via callback from EmbeddingSettings component)
    const [hasEmbeddingSupport, setHasEmbeddingSupport] = useState(false);
    const [isEmbeddingSettingsValid, setIsEmbeddingSettingsValid] = useState(false);
    const saveEmbeddingSettingsRef = useRef(null);
    const [showEmbeddingSettingsUI, setShowEmbeddingSettingsUI] = useState(true); // Show by default, will be set based on initial check

    // Track if form is valid to learn (managed by AddSources component)
    const [isFormValid, setIsFormValid] = useState(false);

    // Form and datasets state
    const addSourcesFormRef = useRef(null);
    const [datasets, setDatasets] = useState([]);
    const [datasetsPagination, setDatasetsPagination] = useState({
        page: 1,
        perPage: 3,
        total: 0
    });
    const [datasetsOrder, setDatasetsOrder] = useState({
        orderBy: 'updated_at',
        order: 'desc',
    });

    // Refs to prevent duplicate/concurrent saves
    const saveInProgressRef = useRef(null);
    const lastSavedStateRef = useRef(null);
    const saveStateRefs = useMemo(() => ({saveInProgressRef, lastSavedStateRef}), []);

    const [startCollectingKnowledge, setStartCollectingKnowledge] = useState(false);

    // Track if processes have ended (set to true when processes end, false when selections change)
    const [processesEnded, setProcessesEnded] = useState(false);

    const {updateTitle: updateDocumentProgressTitle} = useProgressDocumentTitle();
    const {createJob, followJob} = useBackgroundProcess();

    /**
     * Fetch datasets
     */
    const fetchDatasets = useCallback(async (page = 1, perPage = 3, orderParams = {}) => {
        try {
            const orderby = orderParams.orderBy || 'updated_at';
            const order = orderParams.order || 'desc';
            const reqParams = {
                page: page,
                per_page: perPage,
                orderby,
                order,
                type: DATASET_TYPE_KNOWLEDGE,
                include: ['metas', 'storage', 'source_object', 'source_url', 'ai_provider_id', 'dataset_entries_count']
            };
            const res = await GetDatasets(LimbChatbot.rest.url, LimbChatbot.rest.nonce, reqParams);
            if (res.items?.length) {
                setDatasets(res.items);
            } else {
                setDatasets([]);
            }
            // Only update pagination/order if they actually changed to prevent loops
            setDatasetsPagination(prevState => {
                if (prevState.page === page && prevState.perPage === perPage && prevState.total === +res.total) {
                    return prevState; // No change, return same object
                }
                return {
                    ...prevState,
                    page: page,
                    perPage: perPage,
                    total: +res.total,
                };
            });
            setDatasetsOrder(prevState => {
                if (prevState.orderBy === orderby && prevState.order === order) {
                    return prevState; // No change, return same object
                }
                return {
                    orderBy: orderby,
                    order: order,
                };
            });
        } catch (e) {
            handleError(e, notifications.set, {
                title: __("Failed to fetch datasets.", 'limb-chatbot'),
                description: e.message ? __(e.message, 'limb-chatbot') : __("Please check your connection and try again.", 'limb-chatbot'),
            });
        }
    }, [notifications]);

    /**
     * Check if all processes are done (not running) and update parent state
     */
    const checkAndUpdateAllProcessesDone = useCallback(() => {
        if (!setAllProcessesDone || !processesStateRef.current) {
            return;
        }

        const allDone = areAllProcessesDone(processesStateRef);
        setAllProcessesDone(allDone);
        setAllProcessesDoneLocal(allDone);

        // Set step as ready when all processes are done
        if (allDone) {
            setIsReady(true);
        }
    }, [setAllProcessesDone, setIsReady]);

    useImperativeHandle(ref, () => ({
        discardChanges() {
            // Discard AddSources form changes
            if (addSourcesFormRef.current?.discardChanges) {
                addSourcesFormRef.current.discardChanges();
            }
        },
        async checkAndStopProcesses() {
            try {
                // Cancel all incomplete jobs from process state
                if (processesStateRef.current && Array.isArray(processesStateRef.current)) {
                    const updatedJobs = [...processesStateRef.current];

                    // Cancel all unfinished jobs
                    for (let i = 0; i < updatedJobs.length; i++) {
                        const job = updatedJobs[i];
                        if (job.job_id && unfinishedStatuses.includes(job.status)) {
                            try {
                                await CancelJob(job.job_id);
                            } catch (e) {
                                // Job might already be canceled or completed, continue
                            }
                            // Update status to canceled
                            updatedJobs[i] = {...job, status: 'canceled'};
                        }
                    }

                    // Save the updated state (fire and forget)
                    updateOnboardingProcessesState(updatedJobs, notifications.set, saveStateRefs);
                    processesStateRef.current = updatedJobs;
                }

                return true;
            } catch (e) {
                handleError(e, notifications.set, {
                    title: __("Failed to cancel ongoing processes.", 'limb-chatbot'),
                    description: e.message ? __(e.message, 'limb-chatbot') : __("Please check and try again.", 'limb-chatbot'),
                });
                return false;
            }
        },
    }));

    /**
     * Handle form unsaved changes
     */
    const handleFormUnsavedChanges = useCallback((hasChanges) => {
        // Update parent state (for Discard button)
        if (setHasUnsavedChanges) {
            setHasUnsavedChanges(hasChanges);
        }

        // If form has changes and processes have ended, reset processesEnded to allow new process
        if (hasChanges && processesEnded) {
            setProcessesEnded(false);
            // No need to clear job IDs - new jobs will be added to the array when created
        }
    }, [processesEnded, setHasUnsavedChanges]);

    /**
     * Handle collect knowledge button click
     */
    const handleCollectKnowledge = useCallback(async () => {
        if (collecting) {
            return;
        }

        // Embedding settings should already be saved (auto-saved on config selection)
        // If for some reason they're not saved, we can't proceed
        if (!hasEmbeddingSupport) {
            return;
        }

        // Validate form data
        const dataToSave = await addSourcesFormRef.current?.getDataToSave();
        if (!dataToSave) {
            // Form validation failed, errors are already shown
            return;
        }

        // Use ref to access the latest collectKnowledge function
        if (collectKnowledgeRef.current) {
            collectKnowledgeRef.current();
        }
    }, [collecting, hasEmbeddingSupport]);

    // Keep collectKnowledge ref updated
    useEffect(() => {
        collectKnowledgeRef.current = collectKnowledge;
    });

    useEffect(() => {
        // Check if we have jobs array
        const hasJobs = processesStateRef?.current && Array.isArray(processesStateRef.current) && processesStateRef.current.length > 0;

        // Check if all jobs are finished (completed, failed, or canceled)
        const allJobsFinished = hasJobs && processesStateRef.current.every(job => finishedStatuses.includes(job.status));

        // Check if at least one process is completed
        const hasAtLeastOneCompleted = hasJobs && processesStateRef.current.some(job => job.status === 'completed');

        // Determine if Learn button should be enabled
        // Enabled when: embedding settings saved AND AddSources valid
        const isLearnButtonEnabled = hasEmbeddingSupport && isFormValid;

        // Show Done button when:
        // 1. There are jobs AND
        // 2. All jobs are finished AND
        // 3. Not currently collecting AND
        // 4. Form hasn't changed (hasUnsavedChanges = false) - if form changed, user wants to learn more
        // OR
        // 1. At least one process is completed AND
        // 2. Form is not valid to learn AND
        // 3. Not currently collecting
        const shouldShowDone = (hasJobs && allJobsFinished && !collecting && !hasUnsavedChanges) ||
            (hasAtLeastOneCompleted && !isFormValid && !collecting);

        if (shouldShowDone) {
            // Show Done button when all processes are done and form hasn't changed
            // OR when at least one process is completed and form is not valid
            // This allows user to escape even if processes failed or form is invalid
            setLearnButton(
                <Button
                    label={__("Done", 'limb-chatbot')}
                    onClick={() => close()}
                />
            );
        } else {
            // Show Learn button - disabled when:
            // - Currently collecting, OR
            // - Embedding settings not saved AND fields not filled, OR
            // - AddSources form not valid
            setLearnButton(
                <Button
                    label={collecting ? __("Learning", 'limb-chatbot') : __("Learn", 'limb-chatbot')}
                    icon="run"
                    onClick={handleCollectKnowledge}
                    loading={collecting}
                    progress={learnProgress > 0 ? learnProgress : undefined}
                    disabled={collecting || !isLearnButtonEnabled}
                />
            );
        }

        return () => {
            // Reset the button
            setLearnButton(null);
        }
    }, [collecting, learnProgress, allProcessesDoneLocal, processesEnded, hasUnsavedChanges, isFormValid, close, handleCollectKnowledge, hasEmbeddingSupport, isEmbeddingSettingsValid]);

    /**
     * Convert form data to job config
     *
     * @param {object[]} settingsData Settings data from form
     * @return {object} Job config object
     */
    const convertFormDataToJobConfig = useCallback((settingsData) => {
        const config = {};

        // Extract generating config
        const generatingSettings = settingsData.filter(item => item.key.startsWith(DGEN_PROCESS_KEY_PREFIX));
        generatingSettings.forEach(setting => {
            const key = setting.key.replace(DGEN_PROCESS_KEY_PREFIX, '');
            config[key] = setting.value;
        });

        // Extract indexing config
        const indexingSettings = settingsData.filter(item => item.key.startsWith(DIN_PROCESS_KEY_PREFIX));
        indexingSettings.forEach(setting => {
            const key = setting.key.replace(DIN_PROCESS_KEY_PREFIX, '');
            // Map indexing config keys with indexing_ prefix for generating job
            if (key === 'ai_model_id') {
                config.indexing_ai_model_id = setting.value;
            } else if (key === 'config_id') {
                config.indexing_config_id = setting.value;
            } else if (key === 'dimension') {
                config.indexing_dimension = setting.value;
            } else if (key === 'vector_index_type') {
                config.indexing_vector_index_type = setting.value;
            } else if (key === 'vector_index_config_id') {
                config.indexing_vector_index_config_id = setting.value;
            }
        });

        return config;
    }, []);

    /**
     * Collect knowledge
     *
     * @return {Promise<void>}
     */
    const collectKnowledge = async () => {
        // Status messages object - shared for all followJob calls
        // Don't show complete message here - we handle it in onComplete callback to prevent duplicates
        // Use empty string instead of null to prevent fallback message
        const statusMessagesObj = {
            complete: '',
            pause: sprintf(__("%s process has been paused.", 'limb-chatbot'), DATASET_PROCESSES_INFO['dataset_generating'].processName),
            cancel: sprintf(__("%s process has been cancelled.", 'limb-chatbot'), DATASET_PROCESSES_INFO['dataset_generating'].processName)
        };

        // Get form data
        const dataToSave = await addSourcesFormRef.current?.getDataToSave();
        if (!dataToSave) {
            // Form validation failed, errors are already shown
            return;
        }

        setCollecting(true);
        setLearnProgress(0);
        // Reset processesEnded when starting new processes
        setProcessesEnded(false);

        // Start the process
        try {
            await UpdateSetting(LimbChatbot.rest.url, LimbChatbot.rest.nonce, ONBOARDING_KEY_PREFIX + 'status', 'start');
        } catch (e) {
            handleError(e, notifications.set, {
                title: __("Failed to start onboarding process.", 'limb-chatbot'),
                description: e.message ? __(e.message, 'limb-chatbot') : __("Please check and try again.", 'limb-chatbot'),
            });
            setCollecting(false);
            return;
        }

        // Convert form data to job config
        const generatingJobConfig = convertFormDataToJobConfig(dataToSave.data);

        // Initialize processes state as array if needed
        if (!processesStateRef.current || !Array.isArray(processesStateRef.current)) {
            processesStateRef.current = [];
        }

        const jobs = processesStateRef.current;

        // Declare jobId and job variables
        let jobId = null;
        let job = null;

        // Check if there's a paused job - resume it instead of creating new one
        const pausedJob = jobs.find(job => job.job_id && job.status === 'paused');
        if (pausedJob) {
            // Resume the paused job
            jobId = pausedJob.job_id;

            // Skip if we've already checked this job ID
            if (checkedJobIdsRef.current.has(jobId)) {
                // Already checked this job, use existing status from array
                const existingJob = jobs.find(j => j.job_id === jobId);
                if (existingJob && (existingJob.status === 'paused' || activeStatuses.includes(existingJob.status))) {
                    // Job can be resumed
                    job = {id: jobId, status: existingJob.status};
                } else {
                    // Job is finished, create new one
                    jobId = null;
                }
            } else {
                try {
                    const jobData = await GetJob(jobId);
                    // Mark as checked
                    checkedJobIdsRef.current.add(jobId);

                    if (jobData && (jobData.status === 'paused' || activeStatuses.includes(jobData.status))) {
                        // Job exists and can be resumed, update status and follow it
                        const jobIndex = jobs.findIndex(j => j.job_id === jobId);
                        if (jobIndex !== -1 && jobs[jobIndex].status !== jobData.status) {
                            jobs[jobIndex] = {
                                job_id: jobId,
                                status: jobData.status
                            };
                            // Update the ref with normalized array
                            processesStateRef.current = jobs;
                            updateOnboardingProcessesState(jobs, notifications.set, saveStateRefs);
                        }

                        // Follow the resumed job (will continue below)
                        job = jobData;
                    } else {
                        // Job is finished or doesn't exist, create new one
                        jobId = null;
                    }
                } catch (e) {
                    // Mark as checked even on error to prevent retry
                    checkedJobIdsRef.current.add(jobId);
                    // Job might not exist, create new one
                    jobId = null;
                }
            }
        }

        // Cancel any other existing incomplete jobs before creating a new one
        if (!jobId) {
            const incompleteJobs = jobs.filter(job =>
                job.job_id && activeStatuses.includes(job.status)
            );

            for (const incompleteJob of incompleteJobs) {
                try {
                    await CancelJob(incompleteJob.job_id);
                } catch (e) {
                    // Job might already be canceled, continue
                }
                // Update status to canceled
                const jobIndex = jobs.findIndex(j => j.job_id === incompleteJob.job_id);
                if (jobIndex !== -1 && jobs[jobIndex].status !== 'canceled') {
                    jobs[jobIndex] = {
                        job_id: incompleteJob.job_id,
                        status: 'canceled'
                    };
                }
            }

            // Save updated state after canceling
            if (incompleteJobs.length > 0) {
                updateOnboardingProcessesState(jobs, notifications.set, saveStateRefs);
            }
        }

        // Create new job if we don't have one to resume
        if (!jobId) {
            try {
                // Create generating job
                const newJob = await createJob({
                    type: 'dataset_generating',
                    sub_type: 'informational',
                    config: generatingJobConfig,
                    chatbot_uuid: 'default'
                });

                jobId = newJob.id;
                job = newJob;

                // Add job to array immediately with its current status
                jobs.push({
                    job_id: jobId,
                    status: job.status || 'pending'
                });

                // Update the ref with normalized array
                processesStateRef.current = jobs;
                // Save updated state (fire and forget)
                updateOnboardingProcessesState(jobs, notifications.set, saveStateRefs);
            } catch (e) {
                handleError(e, notifications.set, {
                    title: __("Failed to create job.", 'limb-chatbot'),
                    description: e.message ? __(e.message, 'limb-chatbot') : __("Please check and try again.", 'limb-chatbot'),
                });
                setCollecting(false);
                return;
            }
        } else {
            // Resuming existing job - show notification
        }

        // Follow the job (both new and resumed)
        if (jobId) {
            try {
                await followJob({
                    jobId,
                    processType: 'dataset_generating',
                    setProgressState: (v) => {
                        setLearnProgress(v);
                    },
                    onProgressUpdate: async () => {
                        // Fetch datasets on every progress update (fire and forget)
                        fetchDatasets(1, datasetsPagination.perPage, {
                            orderBy: 'updated_at',
                            order: 'desc'
                        }).catch(() => {
                            // Ignore errors when fetching datasets during progress
                        });

                        // Update job status by checking current job status
                        // Only update if status actually changed to avoid unnecessary saves
                        // Skip if we've already checked this job recently (to prevent duplicate calls)
                        if (checkedJobIdsRef.current.has(jobId)) {
                            return;
                        }

                        // Skip if job is already marked as finished in our state
                        const currentJobInState = jobs.find(j => j.job_id === jobId);
                        if (currentJobInState && finishedStatuses.includes(currentJobInState.status)) {
                            return;
                        }

                        try {
                            const currentJob = await GetJob(jobId);
                            // Mark as checked
                            checkedJobIdsRef.current.add(jobId);

                            if (currentJob && currentJob.status) {
                                const jobIndex = jobs.findIndex(j => j.job_id === jobId);
                                if (jobIndex !== -1 && jobs[jobIndex].status !== currentJob.status) {
                                    jobs[jobIndex] = {
                                        job_id: jobId,
                                        status: currentJob.status
                                    };
                                    // Update the ref with normalized array
                                    processesStateRef.current = jobs;
                                    updateOnboardingProcessesState(jobs, notifications.set, saveStateRefs);
                                }
                            }
                        } catch (e) {
                            // Mark as checked even on error to prevent retry
                            checkedJobIdsRef.current.add(jobId);
                            // Ignore errors when checking job status
                        }
                    },
                    onComplete: async () => {
                        // Update job status to completed
                        const jobIndex = jobs.findIndex(j => j.job_id === jobId);
                        if (jobIndex !== -1 && jobs[jobIndex].status !== 'completed') {
                            jobs[jobIndex] = {
                                job_id: jobId,
                                status: 'completed'
                            };
                            // Update the ref with normalized array
                            processesStateRef.current = jobs;
                            updateOnboardingProcessesState(jobs, notifications.set, saveStateRefs);

                            // After job completes, mark embedding support as available
                            if (!hasEmbeddingSupport) {
                                setHasEmbeddingSupport(true);
                            }
                            // Hide embedding settings UI after first job completion
                            setShowEmbeddingSettingsUI(false);
                        }

                        // Reset form selections
                        if (addSourcesFormRef.current?.processCompleted) {
                            addSourcesFormRef.current.processCompleted();
                        }

                        // Refresh datasets once
                        await fetchDatasets(1, datasetsPagination.perPage, {orderBy: 'updated_at', order: 'desc'});

                        setCollecting(false);
                        setIsReady(true);
                        checkAndUpdateAllProcessesDone();
                        setProcessesEnded(true);
                        setHasUnsavedChanges(false);

                        // Check if all jobs are finished
                        const allFinished = jobs.every(job => finishedStatuses.includes(job.status));
                        if (allFinished) {
                            // Only show notification if we haven't shown it for this job yet
                            // AND this job didn't complete on page load (checkAndLoadExistingProcesses handles those)
                            const alreadyShown = completedNotificationShownRef.current.has(jobId);
                            const completedOnLoad = jobsCompletedOnPageLoadRef.current.has(jobId);
                            if (!alreadyShown && !completedOnLoad) {
                                // Mark as shown IMMEDIATELY before calling handleSuccess to prevent duplicate calls
                                completedNotificationShownRef.current.add(jobId);
                                handleSuccess(notifications.set, {
                                    title: DATASET_PROCESSES_INFO['dataset_generating'].success,
                                });
                            }
                        }
                    },
                    statusMessages: statusMessagesObj,
                    resetProgress: () => {
                        setLearnProgress(0);
                    },
                });
            } catch (e) {
                // Update job status to failed on error
                const jobIndex = jobs.findIndex(j => j.job_id === jobId);
                if (jobIndex !== -1 && jobs[jobIndex].status !== 'failed') {
                    jobs[jobIndex] = {
                        job_id: jobId,
                        status: 'failed'
                    };
                    // Update the ref with normalized array
                    processesStateRef.current = jobs;
                    updateOnboardingProcessesState(jobs, notifications.set, saveStateRefs);
                }

                handleError(e, notifications.set, {
                    title: __("Failed to follow process.", 'limb-chatbot'),
                    description: e.message ? __(e.message, 'limb-chatbot') : __("Please check and try again.", 'limb-chatbot'),
                });
                setCollecting(false);
            }
        }
    }

    useEffect(() => {
        fetchData();
        fetchDatasets(1, datasetsPagination.perPage, {orderBy: 'updated_at', order: 'desc'});
        // eslint-disable-next-line react-hooks/exhaustive-deps
    }, []);

    useEffect(() => {
        updateDocumentProgressTitle(learnProgress, (clampedProgress, originalTitle) =>
            (clampedProgress > 0 ? sprintf(__("%d%% Ready", 'limb-chatbot'), clampedProgress) + ' - ' : '') + originalTitle
        );
    }, [learnProgress, updateDocumentProgressTitle]);

    // Track if we've already checked existing processes to prevent infinite loops
    const hasCheckedExistingProcessesRef = useRef(false);
    // Track which job IDs we've already checked to prevent duplicate GetJob calls
    const checkedJobIdsRef = useRef(new Set());
    // Track which jobs have already shown completion notifications
    const completedNotificationShownRef = useRef(new Set());
    // Track jobs that were completed when page loaded (to distinguish from jobs that complete during session)
    const jobsCompletedOnPageLoadRef = useRef(new Set());

    /**
     * Check and load existing processes
     * Processes state is now an array: [{job_id: 123, status: 'processing'}, ...]
     */
    const checkAndLoadExistingProcesses = useCallback(async () => {
        if (!isDataFetched || hasCheckedExistingProcessesRef.current) {
            return;
        }

        hasCheckedExistingProcessesRef.current = true;

        // Initialize as array if not already
        if (!processesStateRef.current || !Array.isArray(processesStateRef.current)) {
            processesStateRef.current = [];
        }

        const jobs = processesStateRef.current;

        // Find incomplete jobs
        const incompleteJobs = jobs.filter(job =>
            job.job_id && activeStatuses.includes(job.status)
        );

        // If multiple incomplete jobs, cancel all but the most recent one
        if (incompleteJobs.length > 1) {
            // Sort by job_id (assuming higher ID = more recent) or keep last in array
            incompleteJobs.sort((a, b) => {
                const aIndex = jobs.findIndex(j => j.job_id === a.job_id);
                const bIndex = jobs.findIndex(j => j.job_id === b.job_id);
                return bIndex - aIndex; // Most recent first
            });

            // Cancel all except the most recent
            const jobsToCancel = incompleteJobs.slice(1);
            for (const job of jobsToCancel) {
                try {
                    await CancelJob(job.job_id);
                } catch (e) {
                    // Job might already be canceled, continue
                }
                // Update status in array
                const jobIndex = jobs.findIndex(j => j.job_id === job.job_id);
                if (jobIndex !== -1 && jobs[jobIndex].status !== 'canceled') {
                    jobs[jobIndex] = {
                        job_id: job.job_id,
                        status: 'canceled'
                    };
                }
            }

            // Update the ref with normalized array
            processesStateRef.current = jobs;
            // Save updated state (fire and forget)
            updateOnboardingProcessesState(jobs, notifications.set, saveStateRefs);
        }

        // Check and update status of incomplete jobs
        const jobsToCheck = incompleteJobs.length > 0 ? [incompleteJobs[0]] : [];
        let hasActiveJob = false;
        let mostRecentJob = null;

        for (const job of jobsToCheck) {
            // Skip if job is already marked as finished in our state (no need to check)
            if (finishedStatuses.includes(job.status)) {
                continue;
            }

            // Skip if we've already checked this job ID
            if (checkedJobIdsRef.current.has(job.job_id)) {
                // Already checked, use existing status
                if (activeStatuses.includes(job.status)) {
                    hasActiveJob = true;
                    mostRecentJob = job;
                }
                continue;
            }

            try {
                const jobData = await GetJob(job.job_id);
                // Mark as checked
                checkedJobIdsRef.current.add(job.job_id);

                if (jobData && jobData.completed === true) {
                    // Job is completed and removed
                    const jobIndex = jobs.findIndex(j => j.job_id === job.job_id);
                    if (jobIndex !== -1 && jobs[jobIndex].status !== 'completed') {
                        jobs[jobIndex] = {
                            job_id: job.job_id,
                            status: 'completed'
                        };
                    }
                } else if (jobData && finishedStatuses.includes(jobData.status)) {
                    // Job is finished
                    const jobIndex = jobs.findIndex(j => j.job_id === job.job_id);
                    if (jobIndex !== -1 && jobs[jobIndex].status !== jobData.status) {
                        jobs[jobIndex] = {
                            job_id: job.job_id,
                            status: jobData.status
                        };
                    }
                } else if (jobData && activeStatuses.includes(jobData.status)) {
                    // Job is still active
                    const jobIndex = jobs.findIndex(j => j.job_id === job.job_id);
                    if (jobIndex !== -1 && jobs[jobIndex].status !== jobData.status) {
                        jobs[jobIndex] = {
                            job_id: job.job_id,
                            status: jobData.status
                        };
                    }
                    hasActiveJob = true;
                    mostRecentJob = job;
                }
            } catch (e) {
                // Mark as checked even on error to prevent retry
                checkedJobIdsRef.current.add(job.job_id);
                // Job might not exist, mark as failed
                const jobIndex = jobs.findIndex(j => j.job_id === job.job_id);
                if (jobIndex !== -1 && jobs[jobIndex].status !== 'failed') {
                    jobs[jobIndex] = {
                        job_id: job.job_id,
                        status: 'failed'
                    };
                }
            }
        }

        // Save updated state (fire and forget)
        if (jobsToCheck.length > 0) {
            processesStateRef.current = jobs;
            updateOnboardingProcessesState(jobs, notifications.set, saveStateRefs);
        }

        // Check if all jobs are finished
        const allFinished = jobs.length > 0 && jobs.every(job => finishedStatuses.includes(job.status));
        if (allFinished) {
            // All jobs are finished - set processesEnded to true
            setProcessesEnded(true);
            checkAndUpdateAllProcessesDone();
            // Refresh datasets once
            await fetchDatasets(1, datasetsPagination.perPage, {orderBy: 'updated_at', order: 'desc'});

            // Show notification only on initial page load when jobs are already completed
            // Don't show if there's an active job (it will be resumed and onComplete will handle notification)
            // This prevents duplicate notifications when onComplete callbacks also fire
            const completedJobs = jobs.filter(job => job.status === 'completed');
            const newCompletedJobs = completedJobs.filter(job => !completedNotificationShownRef.current.has(job.job_id));

            if (newCompletedJobs.length > 0 && !hasActiveJob) {
                // Only show on initial page load (not when jobs are currently completing)
                // The onComplete callbacks will handle notifications for jobs that complete during this session
                handleSuccess(notifications.set, {
                    title: DATASET_PROCESSES_INFO['dataset_generating'].success,
                });
                // Mark all completed jobs as having shown notification
                // Also mark them as completed on page load to distinguish from jobs that complete during session
                newCompletedJobs.forEach(job => {
                    completedNotificationShownRef.current.add(job.job_id);
                    jobsCompletedOnPageLoadRef.current.add(job.job_id);
                });
            }
        } else if (hasActiveJob && mostRecentJob) {
            // Has active job - resume it
            handleWarning({
                data: processesStateRef.current,
                message: "Not all onboard processes are complete."
            }, notifications.set, {
                title: __("Not all processes are complete.", 'limb-chatbot'),
                description: __("The processes will be resumed automatically.", 'limb-chatbot'),
            });

            // Resume the job
            const jobToResume = mostRecentJob;
            const jobStatus = jobs.find(j => j.job_id === jobToResume.job_id)?.status;

            // For pending, generating_tasks, processing - resume immediately
            // For paused - resume when Learn button is clicked
            if (jobStatus === 'pending' || jobStatus === 'generating_tasks' || jobStatus === 'processing') {
                setCollecting(true);
                setLearnProgress(0);
                setProcessesEnded(false);

                // Follow the existing job
                try {
                    await followJob({
                        jobId: jobToResume.job_id,
                        processType: 'dataset_generating',
                        setProgressState: (v) => {
                            setLearnProgress(v);
                        },
                        onProgressUpdate: async () => {
                            // Fetch datasets on every progress update (fire and forget)
                            fetchDatasets(1, datasetsPagination.perPage, {
                                orderBy: 'updated_at',
                                order: 'desc'
                            }).catch(() => {
                                // Ignore errors when fetching datasets during progress
                            });

                            // Update job status by checking current job status
                            // Skip if we've already checked this job recently (to prevent duplicate calls)
                            if (checkedJobIdsRef.current.has(jobToResume.job_id)) {
                                return;
                            }

                            // Skip if job is already marked as finished in our state
                            const currentJobInState = jobs.find(j => j.job_id === jobToResume.job_id);
                            if (currentJobInState && finishedStatuses.includes(currentJobInState.status)) {
                                return;
                            }

                            try {
                                const currentJob = await GetJob(jobToResume.job_id);
                                // Mark as checked
                                checkedJobIdsRef.current.add(jobToResume.job_id);

                                if (currentJob) {
                                    const jobIndex = jobs.findIndex(j => j.job_id === jobToResume.job_id);
                                    if (jobIndex !== -1 && jobs[jobIndex].status !== currentJob.status) {
                                        jobs[jobIndex] = {
                                            job_id: jobToResume.job_id,
                                            status: currentJob.status
                                        };
                                        // Update the ref with normalized array
                                        processesStateRef.current = jobs;
                                        updateOnboardingProcessesState(jobs, notifications.set, saveStateRefs);
                                    }
                                }
                            } catch (e) {
                                // Mark as checked even on error to prevent retry
                                checkedJobIdsRef.current.add(jobToResume.job_id);
                                // Ignore errors when checking job status
                            }
                        },
                        onComplete: async () => {
                            // Update job status to completed
                            const jobIndex = jobs.findIndex(j => j.job_id === jobToResume.job_id);
                            if (jobIndex !== -1 && jobs[jobIndex].status !== 'completed') {
                                jobs[jobIndex] = {
                                    job_id: jobToResume.job_id,
                                    status: 'completed'
                                };
                                // Update the ref with normalized array
                                processesStateRef.current = jobs;
                                updateOnboardingProcessesState(jobs, notifications.set, saveStateRefs);

                                // After job completes, if embedding settings were not saved, mark them as saved
                                // This will hide the embedding settings UI
                                if (!hasEmbeddingSupport) {
                                    setHasEmbeddingSupport(true);
                                }
                            }

                            // Reset form selections
                            if (addSourcesFormRef.current?.processCompleted) {
                                addSourcesFormRef.current.processCompleted();
                            }

                            // Refresh datasets once
                            await fetchDatasets(1, datasetsPagination.perPage, {orderBy: 'updated_at', order: 'desc'});

                            setCollecting(false);
                            setIsReady(true);
                            checkAndUpdateAllProcessesDone();
                            setProcessesEnded(true);
                            setHasUnsavedChanges(false);

                            // Check if all jobs are finished
                            const allFinished = jobs.every(job => finishedStatuses.includes(job.status));
                            if (allFinished) {
                                // Only show notification if we haven't shown it for this job yet
                                // AND this job didn't complete on page load (checkAndLoadExistingProcesses handles those)
                                const alreadyShown = completedNotificationShownRef.current.has(jobToResume.job_id);
                                const completedOnLoad = jobsCompletedOnPageLoadRef.current.has(jobToResume.job_id);
                                if (!alreadyShown && !completedOnLoad) {
                                    handleSuccess(notifications.set, {
                                        title: DATASET_PROCESSES_INFO['dataset_generating'].success,
                                    });
                                    completedNotificationShownRef.current.add(jobToResume.job_id);
                                }
                            }
                        },
                        statusMessages: statusMessagesObj,
                        resetProgress: () => {
                            setLearnProgress(0);
                        },
                    });
                } catch (e) {
                    // Update job status to failed on error
                    const jobIndex = jobs.findIndex(j => j.job_id === jobToResume.job_id);
                    if (jobIndex !== -1 && jobs[jobIndex].status !== 'failed') {
                        jobs[jobIndex] = {
                            job_id: jobToResume.job_id,
                            status: 'failed'
                        };
                        // Update the ref with normalized array
                        processesStateRef.current = jobs;
                        updateOnboardingProcessesState(jobs, notifications.set, saveStateRefs);
                    }

                    handleError(e, notifications.set, {
                        title: __("Failed to resume process.", 'limb-chatbot'),
                        description: e.message ? __(e.message, 'limb-chatbot') : __("Please check and try again.", 'limb-chatbot'),
                    });
                    setCollecting(false);
                }
            } else if (jobStatus === 'paused') {
                // For paused jobs, resume when Learn button is clicked
                setStartCollectingKnowledge(true);
            }
        }

        checkAndUpdateAllProcessesDone();
    }, [isDataFetched, notifications, checkAndUpdateAllProcessesDone, saveStateRefs]);

    useEffect(() => {
        checkAndLoadExistingProcesses();
        // eslint-disable-next-line react-hooks/exhaustive-deps
    }, [isDataFetched]);

    useEffect(() => {
        if (startCollectingKnowledge) {
            collectKnowledge();
        }
    }, [startCollectingKnowledge]);

    /**
     * Fetch data
     *
     * @return {Promise<void>}
     */
    const fetchData = async () => {
        setLoading(prev => prev + 1);
        setTimeout(() => {
            setIsDataFetched(true);
            setLoading(prev => prev - 1);
            setIsDataReady(true);
        }, 200);
    }

    // Create table structure without checkbox and actions columns, no details/accordion
    const tableStructure = useMemo(() => ({
        columns: [
            {
                id: 'name',
                label: __("Name", 'limb-chatbot'),
                sortable: true,
                className: 'lbaic-settings-table-card-header-input',
                render: null,
                value: {
                    className: 'lbaic-settings-table-card-body-input',
                    render: (row, index) => renderName(row, index, {
                        preparing: false,
                        processState: null
                    }),
                },
            },
            {
                id: 'type',
                label: __("Type", 'limb-chatbot'),
                className: 'lbaic-settings-table-card-header-type',
                render: null,
                value: {
                    className: 'lbaic-settings-table-card-body-type',
                    render: (row, index) => renderSource(row),
                },
            },
            {
                id: 'ai',
                label: __("AI", 'limb-chatbot'),
                className: 'lbaic-settings-table-card-header-ai',
                render: null,
                value: {
                    className: 'lbaic-settings-table-card-body-ai',
                    render: (row, index) => renderAIProvider(row),
                },
            },
            {
                id: 'storage',
                label: __("Storage", 'limb-chatbot'),
                className: 'lbaic-settings-table-card-header-storage',
                render: null,
                value: {
                    className: 'lbaic-settings-table-card-body-storage',
                    render: (row, index) => renderStorage(row)
                },
            },
        ],
        row: {
            className: (row, index) => {
                const classes = [];
                // Errors
                const errors = row.metas?.length ? row.metas.filter(item => item.meta_key === 'error').map(item => item.meta_value) : [];
                if (errors.length) {
                    classes.push('error');
                }
                return classes.join(' ');
            },
            details: {
                show: false, // No details/accordion
            },
        },
    }), []);

    // ResizeObserver to watch for height changes in .lbaic-settings-popup-body
    useEffect(() => {
        if (!isDataFetched) {
            return;
        }

        const popupBodyElement = document.querySelector('.lbaic-settings-mp-popup .lbaic-settings-popup-body');
        if (!popupBodyElement) {
            return;
        }

        const resizeObserver = new ResizeObserver(() => {
            // Recalculate chatbot position when popup body height changes
            window.requestAnimationFrame(() => {
                updateOnboardingPositions();
            });
        });

        resizeObserver.observe(popupBodyElement);

        return () => {
            resizeObserver.disconnect();
        };
    }, [isDataFetched]);

    if (!isDataFetched) {
        return null;
    }

    return (
        <>
            <EmbeddingSettings
                ref={embeddingSettingsRef}
                config={config}
                hasEmbeddingSupport={hasEmbeddingSupport}
                setHasEmbeddingSupport={setHasEmbeddingSupport}
                setLoading={setLoading}
                notifications={notifications}
                showEmbeddingSettingsUI={showEmbeddingSettingsUI}
                setShowEmbeddingSettingsUI={setShowEmbeddingSettingsUI}
                onStateChange={(state) => {
                    setIsEmbeddingSettingsValid(state.isEmbeddingSettingsValid);
                    saveEmbeddingSettingsRef.current = state.saveEmbeddingSettings;

                    // When settings are saved, update AddSources data indexing settings
                    if (state.settingsSaved && addSourcesFormRef.current?.updateDataIndexingSettings) {
                        addSourcesFormRef.current.updateDataIndexingSettings();
                    }
                }}
                collecting={collecting}
            />
            {hasEmbeddingSupport && (
                <div className='lbaic-settings-a-ei-body'>
                    <AddSources
                        ref={addSourcesFormRef}
                        saving={collecting}
                        setLoading={setLoading}
                        setShowLoading={() => {
                        }}
                        updateHasUnsavedChanges={handleFormUnsavedChanges}
                        setIsFormValid={setIsFormValid}
                        notifications={notifications}
                        excludeSourceTypes={['sitemap']}
                    />
                </div>
            )}
            {datasets.length > 0 && (
                <div className='lbaic-settings-mp-popup-content'>
                    <Table
                        className='lbaic-settings-a-es-table lbaic-settings-scroll-style lbaic-settings-scroll-x lbaic-settings-table-card-kb-items'
                        structure={tableStructure}
                        data={datasets}
                        loading={false}
                        pagination={datasetsPagination}
                        order={{get: datasetsOrder, set: setDatasetsOrder}}
                        _callback={fetchDatasets}>
                    </Table>
                </div>
            )}
        </>
    );
});

export default Step3;