package Whostmgr::API::1::CometBackup;

#                                      Copyright 2025 WebPros International, LLC
#                                                           All rights reserved.
# copyright@cpanel.net                                         http://cpanel.net
# This code is subject to the cPanel license. Unauthorized copying is prohibited.

# This module provides API endpoints for managing Comet Backup

=encoding utf-8

=head1 NAME

Whostmgr::API::1::CometBackup - WHM API endpoints for Comet Backup integration

=head1 DESCRIPTION

This module provides the WHM API v1 endpoints for managing Comet Backup operations,
including server registration, backup job management, restore operations, and
credential management.

=head1 API METHODS

=cut

use cPstrict;
use Cpanel::JSON                          ();
use Time::HiRes                           ();
use Whostmgr::API::1::Utils               ();
use Whostmgr::CometBackup::BackupJobs     ();
use Whostmgr::CometBackup::BackupRunLogs  ();
use Whostmgr::CometBackup::CometBackupAPI ();
use Whostmgr::CometBackup::Constants      ();
use Data::Dumper                          qw(Dumper);

use Whostmgr::CometBackup::Logger;
my $logger = Whostmgr::CometBackup::Logger->new();

=head2 register_server_with_comet_backup

Registers this server with Comet Backup system. Creates a backup user account and
obtains API credentials if not already registered.

=head3 Parameters

None required.

=head3 Returns

A hash reference containing:

=over 4

=item * C<server> - Hash with C<user_id> and C<api_key> for the registered server

=back

=head3 Example

    my $result = register_server_with_comet_backup({}, $metadata, $api_info);

=cut

=head2 initialize_comet_backup

Initializes the server with Comet Backup. This function is idempotent and will only
perform the steps needed based on the current configuration status.

=head3 Parameters

None required.

=head3 Returns

A hash reference containing:

=over 4

=item * C<initialized> - Boolean indicating success (1) or failure (0)

=item * C<message> - Status message describing the result

=back

=head3 Example

    my $result = initialize_comet_backup({}, $metadata, $api_info);

=cut

sub initialize_comet_backup {
    my ( $args, $metadata, $api_info_hr ) = @_;

    my $result;
    eval {
        require Whostmgr::CometBackup::Setup;
        Whostmgr::CometBackup::Setup::initialize();
        Whostmgr::API::1::Utils::set_metadata_ok($metadata);
        $result = {
            initialized => 1,
            message     => 'Comet Backup initialization completed successfully',
        };
    };

    if ($@) {
        my $error = $@;
        $logger->error("Failed to initialize Comet Backup: $error");
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "Failed to initialize Comet Backup: $error" );
        return {
            initialized => 0,
            message     => "Initialization failed: $error",
        };
    }

    return $result;
}

sub register_server_with_comet_backup {
    my ( $args, $metadata, $api_info_hr ) = @_;
    require Whostmgr::CometBackup::Setup;

    my $result;
    eval {
        # If the server is already set up, we can skip the rest. This also lets us know what steps we
        # need to target to get it all up and running.
        my $comet_backup_config_status = get_comet_backup_configuration_status( {}, $metadata, $api_info_hr );

        # my $comet_backup_config_status = get_comet_backup_configuration_status( { json => Cpanel::JSON::Dump( { progress_number => 100 } ) }, $metadata, $api_info_hr );
        $logger->error( Dumper($comet_backup_config_status) );
        if ( $comet_backup_config_status->{configuration}->{status} eq 'configured' ) {
            Whostmgr::API::1::Utils::set_metadata_ok($metadata);
            $result = { server => { status => 'registered' } };
            return;
        }

        # Now we look step by step..

        # See if we have a license
        unless ( $comet_backup_config_status->{configuration}->{has_license} ) {
            $result = { status => 'unlicensed' };
            die "Could not find Comet Backup license for server";
        }

        # See if agent is installed
        unless ( $comet_backup_config_status->{configuration}->{agent_installed} ) {
            my $result = Whostmgr::CometBackup::Setup::install_comet_agent();
            if ( !$result ) {
                die "Failed to install Comet Backup agent";
            }
        }

        # See if agent is running
        unless ( $comet_backup_config_status->{configuration}->{agent_running} ) {
            require Whostmgr::CometBackup::AgentManager;
            my ( $status, $message ) = Whostmgr::CometBackup::AgentManager::start();
            unless ($status) {
                die "Failed to start Comet Backup agent: $message";
            }
        }

        # If we have credentials already, log the backup tool agent in
        if ( $comet_backup_config_status->{configuration}->{has_credentials} ) {

            # See if we have logged the agent in to the comet server using credentials
            unless ( $comet_backup_config_status->{configuration}->{agent_logged_in} ) {
                Whostmgr::CometBackup::Setup::setup_comet_agent();
            }
        }
        else {
            $result = { status => 'credentials_missing' };
            die "Server credentials for Comet Backup are missing";
        }

        Whostmgr::API::1::Utils::set_metadata_ok($metadata);
        $result = { server => { status => 'registered' } };
    };

    if ($@) {
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "Failed to register server with Comet backup: $@" );
        return { server => $result->{status} };
    }

    return $result;
}

=head2 disable_comet_backup

Disables Comet Backup on this server by creating a touchfile that prevents backup operations.

=head3 Parameters

None required.

=head3 Returns

A hash reference containing:

=over 4

=item * C<disabled> - Boolean indicating if the operation succeeded (1) or failed (0)

=back

=head3 Example

    my $result = disable_comet_backup({}, $metadata, $api_info);

=cut

sub disable_comet_backup {
    my ( $args, $metadata, $api_info_hr ) = @_;

    eval {
        require Cpanel::FileUtils::TouchFile;
        require Cpanel::Mkdir;

        # Ensure the comet directory exists
        my $comet_dir = '/var/cpanel/comet';
        Cpanel::Mkdir::ensure_directory_existence_and_mode( $comet_dir, 0755 ) unless -d $comet_dir;

        # Create the touchfile to indicate Comet Backup is disabled
        my $touchfile = $Whostmgr::CometBackup::Constants::DISABLED_TOUCHFILE;
        Cpanel::FileUtils::TouchFile::touchfile($touchfile);

        $logger->debug("Comet Backup disabled - touchfile created at $touchfile");
    };

    if ($@) {
        $logger->error("Failed to disable Comet Backup: $@");
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "Failed to disable Comet Backup: $@" );
        return { disabled => 0 };
    }

    Whostmgr::API::1::Utils::set_metadata_ok($metadata);
    return { disabled => 1 };
}

=head2 enable_comet_backup

Enables Comet Backup on this server by removing the disabled touchfile.

=head3 Parameters

None required.

=head3 Returns

A hash reference containing:

=over 4

=item * C<enabled> - Boolean indicating if the operation succeeded (1) or failed (0)

=back

=head3 Example

    my $result = enable_comet_backup({}, $metadata, $api_info);

=cut

sub enable_comet_backup {
    my ( $args, $metadata, $api_info_hr ) = @_;

    eval {
        # Remove the touchfile to indicate Comet Backup is enabled
        my $touchfile = $Whostmgr::CometBackup::Constants::DISABLED_TOUCHFILE;

        if ( -f $touchfile ) {
            unlink $touchfile or die "Failed to remove touchfile: $!";
            $logger->debug("Comet Backup enabled - touchfile removed from $touchfile");
        }
        else {
            $logger->debug("Comet Backup enabled - touchfile did not exist");
        }
    };

    if ($@) {
        $logger->error("Failed to enable Comet Backup: $@");
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "Failed to enable Comet Backup: $@" );
        return { enabled => 0 };
    }

    Whostmgr::API::1::Utils::set_metadata_ok($metadata);
    return { enabled => 1 };
}

=head2 is_comet_backup_enabled

Checks if Comet Backup is currently enabled on this server.

=head3 Parameters

None required.

=head3 Returns

A hash reference containing:

=over 4

=item * C<enabled> - Boolean indicating if Comet Backup is enabled (1) or disabled (0)

=item * C<disabled> - Boolean indicating if Comet Backup is disabled (1) or enabled (0)

=item * C<touchfile> - Path to the disabled touchfile

=back

=head3 Example

    my $result = is_comet_backup_enabled({}, $metadata, $api_info);

=cut

sub is_comet_backup_enabled {
    my ( $args, $metadata, $api_info_hr ) = @_;

    # Check if the touchfile exists
    # If it exists, Comet Backup is disabled; if not, it's enabled
    my $touchfile  = $Whostmgr::CometBackup::Constants::DISABLED_TOUCHFILE;
    my $is_enabled = !-f $touchfile;

    Whostmgr::API::1::Utils::set_metadata_ok($metadata);
    return {
        enabled   => $is_enabled,
        disabled  => !$is_enabled,
        touchfile => $touchfile,
    };
}

sub get_comet_license_status {
    my ( $args, $metadata, $api_info_hr ) = @_;

    # TODO : implement this in earnst once license information is available from upstream

    Whostmgr::API::1::Utils::set_metadata_ok($metadata);
    return {
        license => {
            status       => 'placeholder',
            expires_date => undef,
        }
    };
}

=head2 get_allowed_region_hints

Fetches the list of allowed region hints for cPanel Hosted Comet licenses from the Comet Account Portal.
Reads the WebPros Account JWT id_token from the stored JWT file for identity verification.

=head3 Parameters

None required - the id_token is read from the stored WPA JWT file.

=head3 Returns

A hash reference containing:

=over 4

=item * C<console> - Array of available console/management regions with id and description

=item * C<storage> - Array of available storage regions with id and description

=back

=head3 Example

    my $result = get_allowed_region_hints({}, $metadata );

=cut

sub get_allowed_region_hints {
    my ( $args, $metadata ) = @_;

    my $result;
    eval {
        require Whostmgr::CometBackup::Setup;
        $result = Whostmgr::CometBackup::Setup::get_allowed_region_hints();
    };

    if ( $@ || !$result ) {
        my $error = $@ || "Failed to fetch region hints from Comet Account Portal";
        chomp $error;
        $logger->error("get_allowed_region_hints failed: $error");
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, $error );
        return {};
    }

    Whostmgr::API::1::Utils::set_metadata_ok($metadata);
    return $result;
}

=head2 get_server_credentials_from_cap

Fetches server credentials from Comet Access Portal (CAP) using the license activation code,
region information and WPA JWT. On success, saves the credentials to the credentials file.

=head3 Parameters

JSON object containing:

=over 4

=item * C<region_id> - The region ID for the storage server (required)

=back

=head3 Returns

A hash reference containing:

=over 4

=item * C<success> - Boolean indicating if credentials were fetched and saved successfully

=back

=head3 Example

    my $result = get_server_credentials_from_cap({
        json => encode_json({ region_id => 'us-west-1' })
    }, $metadata, $api_info);

=cut

sub get_server_credentials_from_cap {
    my ( $args, $metadata, $api_info_hr ) = @_;

    my ( $status, $decoded_args ) = _decode_json_args( $metadata, $args );
    if ( $status < 1 ) {
        return { success => 0 };
    }

    my $region_id = $decoded_args->{region_id};
    unless ($region_id) {
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "region_id parameter is required" );
        return { success => 0 };
    }

    my ( $result, $credentials_json );
    eval {
        require Whostmgr::CometBackup::Setup;

        # Fetches and saves credentials, returns 1 on success
        ( $result, $credentials_json ) = Whostmgr::CometBackup::Setup::get_server_credentials_from_cap( { region_id => $region_id } );
    };

    if ($@) {
        my $error = $@;
        chomp $error;
        $logger->error("get_server_credentials_from_cap failed: $error");
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, $error );
        return { success => 0 };
    }

    if ($result) {
        Whostmgr::API::1::Utils::set_metadata_ok($metadata);
        return { success => 1, credentials => $credentials_json };
    }
    else {
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "Failed to fetch and save server credentials" );
        return { success => 0 };
    }
}

sub get_comet_backup_configuration_status {
    my ( $args, $metadata, $api_info_hr ) = @_;

    my $progress        = 0;
    my $status          = 'not_started';
    my $status_message  = '';
    my $agent_installed = 0;
    my $agent_running   = 0;
    my $agent_logged_in = 0;
    my $has_credentials = 0;
    my $is_monitored    = 0;
    my $has_license     = 0;

    my $result = {
        configuration => {
            progress        => $progress,
            status          => $status,
            status_message  => $status_message,
            agent_installed => $agent_installed,
            agent_running   => $agent_running,
            agent_logged_in => $agent_logged_in,
            has_credentials => $has_credentials,
            has_license     => $has_license,
        }
    };

    eval {
        require Whostmgr::CometBackup::Setup;
        require Whostmgr::CometBackup::AgentManager;
        require Whostmgr::CometBackup::CometBackupAPI;
        require Cpanel::Server::Type;
        require Cpanel::PwCache;

        # License check (10% progress)
        # TODO - Currently we have an override flag to get past this step on sandboxes. This allows for some testing in a dev environment.
        # Even if left it for release, it won't be a problem since the API calls will return failure.
        # Note that we don't return at this point since we just want to add the license status to the return info.
        if ( -f '/var/cpanel/dev_sandbox' && -f '/var/cpanel/comet/license' ) {
            $has_license = 1;
        }
        else {
            my @products = Cpanel::Server::Type::get_products();
            if ( grep { $_ eq 'comet_backup' } @products ) {
                $has_license = 1;
            }
        }
        if ($has_license) {
            $progress       = 10;
            $status         = 'licensed';
            $status_message = 'Comet Backup license is valid';
        }
        else {
            $status         = 'unlicensed';
            $status_message = 'Comet Backup license is not present or invalid';
        }

        $result->{configuration}{progress}       = 10;
        $result->{configuration}{status}         = $status;
        $result->{configuration}{status_message} = $status_message;
        $result->{configuration}{has_license}    = $has_license;

        # Check if agent is installed
        $agent_installed = Whostmgr::CometBackup::Setup::is_comet_agent_installed();
        if ($agent_installed) {
            $result->{configuration}{agent_installed} = 1;
            $result->{configuration}{progress}        = 50;
            $result->{configuration}{status}          = 'agent_installed';
            $result->{configuration}{status_message}  = 'Comet Backup agent is installed';
        }
        else {
            $result->{configuration}{status}         = 'agent_not_installed';
            $result->{configuration}{status_message} = 'Comet Backup agent is not installed';
            Whostmgr::API::1::Utils::set_metadata_ok($metadata);
            return;
        }

        # Check if agent is running
        my ( $status, $agent_status ) = Whostmgr::CometBackup::AgentManager::get_status();
        if ( !$status ) {
            die "Failed to check agent status: $agent_status";
        }
        $agent_running = ( $agent_status eq 'active' ) ? 1 : 0;
        if ($agent_running) {
            $result->{configuration}{agent_running}  = 1;
            $result->{configuration}{progress}       = 80;
            $result->{configuration}{status}         = 'agent_running';
            $result->{configuration}{status_message} = 'Comet Backup agent is running';
        }
        else {
            $result->{configuration}{status}         = 'agent_not_running';
            $result->{configuration}{status_message} = 'Comet Backup agent is installed but not running';
            Whostmgr::API::1::Utils::set_metadata_ok($metadata);
            return;
        }

        # Check if server credentials are configured
        eval {
            my $server_config = Whostmgr::CometBackup::CometBackupAPI::get_server_config();
            if ( $server_config && $server_config->{server_url} && $server_config->{username} && $server_config->{password} ) {
                $result->{configuration}{has_credentials} = 1;
                $result->{configuration}{progress}        = 25;
                $result->{configuration}{status}          = 'credentials_configured';
                $result->{configuration}{status_message}  = 'Server credentials are configured';
            }
        };
        if ($@) {

            # Credentials not available yet
            $result->{configuration}{status}         = 'credentials_missing';
            $result->{configuration}{status_message} = 'Server credentials not configured';
            Whostmgr::API::1::Utils::set_metadata_ok($metadata);
            return;
        }

        # Check if agent is logged into Comet server
        $agent_logged_in = Whostmgr::CometBackup::Setup::is_agent_logged_in();
        if ($agent_logged_in) {
            $result->{configuration}{agent_logged_in} = 1;
            $result->{configuration}{progress}        = 100;
            $result->{configuration}{status}          = 'configured';
            $result->{configuration}{status_message}  = 'Server is fully configured with Comet Backup';
        }
        else {
            $result->{configuration}{progress}       = 90;
            $result->{configuration}{status}         = 'agent_not_logged_in';
            $result->{configuration}{status_message} = 'Agent is not logged into Comet server';
            Whostmgr::API::1::Utils::set_metadata_ok($metadata);
            return;
        }

        # All checks passed - server is fully configured (100% progress)
        Whostmgr::API::1::Utils::set_metadata_ok($metadata);
    };

    if ($@) {
        $logger->error("Error checking server configuration: $@");
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "Failed to check server configuration: $@" );
        $result->{configuration}{status}         = 'error';
        $result->{configuration}{status_message} = "Error: $@";
    }

    return $result;
}

=head2 create_backup_job

Creates a new backup job configuration for specified users with scheduling and retention settings.

=head3 Parameters

JSON object containing:

=over 4

=item * C<users> - Array of usernames to back up

=item * C<schedule> - Backup schedule configuration

=item * C<retention> - Retention policy settings

=item * C<destinations> - Array of backup destination IDs

=item * C<name> - Name for the backup job

=back

=head3 Returns

A hash reference containing:

=over 4

=item * C<job_uuid> - UUID of the created backup job

=item * C<created> - Boolean indicating success

=back

=head3 Example

    my $result = create_backup_job({
        json => encode_json({
            users => ['user1', 'user2'],
            name => 'Daily Backup',
            schedule => { frequency => 'daily', hour => 2 }
        })
    }, $metadata, $api_info);

=cut

sub create_backup_job {
    my ( $args, $metadata, $api_info_hr ) = @_;
    $logger->debug( "create_backup_job called with args: " . Dumper($args) . " ,  metadata: " . Dumper($metadata) );

    my ( $status, $decoded_args ) = _decode_json_args( $metadata, $args );
    if ( $status < 1 ) {
        return;
    }

    # Extract job configuration from decoded arguments
    $logger->debug( "Decoded args: " . Dumper($decoded_args) );
    my $job_config = $decoded_args->{job_config};
    unless ( $job_config && ref $job_config eq 'HASH' ) {
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "job_config parameter required and must be a hash reference" );
        return { job => {} };
    }

    # Validate required fields
    my @required_fields = qw(name schedule_type backup_options destinations all_accounts);
    for my $field (@required_fields) {
        unless ( exists $job_config->{$field} ) {
            $logger->warn("Missing required field: $field");
            Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "job_config.$field is required" );
            return { job => {} };
        }
    }

    # Additional validation for specific account backup
    if ( !$job_config->{all_accounts} && ( !$job_config->{accounts} || ref $job_config->{accounts} ne 'ARRAY' || @{ $job_config->{accounts} } == 0 ) ) {
        $logger->debug("When all_accounts is 0, accounts array must be provided and non-empty");
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "When all_accounts is 0, accounts must be a non-empty array" );
        return { job => {} };
    }

    # Normalize backup_options for simplified frontend API
    _normalize_backup_options( $job_config->{backup_options} );

    # Validate server config before creating backup job
    require Whostmgr::CometBackup::CometBackupAPI;
    my $server_config;
    $logger->debug("Validating server configuration before creating backup job");
    eval {
        require Whostmgr::CometBackup::CometBackupAPI;
        $server_config = Whostmgr::CometBackup::CometBackupAPI::get_server_config();
    };
    if ($@) {
        my $error = "Cannot create backup job - server not configured: $@";
        $logger->error($error);
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, $error );
        return { job => {} };
    }

    $logger->debug("All required fields present, attempting to create backup job");

    my $job_uuid;
    my $error;
    eval {
        require Whostmgr::CometBackup::BackupJobs;
        require Whostmgr::CometBackup::ProtectedItemMap;
        require Whostmgr::CometBackup::CometBackupAPI::ProtectedItemBatch;

        my $jobs = Whostmgr::CometBackup::BackupJobs->new();

        $logger->debug( "About to call create_backup_job with: " . Dumper($job_config) );
        $job_uuid = $jobs->create_backup_job(%$job_config);

        $logger->debug("Successfully created backup job with UUID: $job_uuid");
        1;
    } or do {
        $error = $@ || 'Unknown error occurred';

        # Clean up error message for end users by removing file paths and line numbers
        my $user_friendly_error = $error;
        $user_friendly_error =~ s/\s+at\s+\/[^\s]+\s+line\s+\d+\.?\s*$//;

        $logger->error("Error creating backup job: $error");
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "Failed to create backup job: $user_friendly_error" );
        return { job => {} };
    };

    # TODO : break this out into something that can take longer to run without affecting API response
    # Create Protected Items for selected users
    $logger->debug("Creating Protected Items for backup job: $job_uuid");
    my @users_to_backup = _get_users_to_backup($job_config);
    $logger->debug( "Users to backup: " . join( ', ', @users_to_backup ) );

    my $map = Whostmgr::CometBackup::ProtectedItemMap->new();

    # Get target user from server configuration
    my $target_user = $server_config->{username} or die "Unable to determine target_user from server config";

    my $batch = Whostmgr::CometBackup::CometBackupAPI::ProtectedItemBatch->new( target_user => $target_user );

    my @all_item_uids;
    my @all_protected_items;    # Store all items for potential retry
    for my $username (@users_to_backup) {
        my $user_start_time = Time::HiRes::time();

        # Create local Protected Item tracking records
        my $create_start = Time::HiRes::time();
        my @item_uids    = $map->create_protected_items_for_user_job(
            job_uuid    => $job_uuid,
            system_user => $username,

            # item_types auto-determined from job config
        );
        my $create_elapsed = Time::HiRes::time() - $create_start;
        push @all_item_uids, @item_uids;

        $logger->debug( sprintf( "[TIMING] create_protected_items_for_user_job took %.3fs for user: $username", $create_elapsed ) );

        # Get the created Protected Items with full details
        my $get_start       = Time::HiRes::time();
        my @protected_items = $map->get_protected_items_for_user(
            system_user => $username,
            job_uuid    => $job_uuid,
        );
        my $get_elapsed = Time::HiRes::time() - $get_start;
        $logger->debug( sprintf( "[TIMING] get_protected_items_for_user took %.3fs for user: $username", $get_elapsed ) );

        # Store for potential retry and add to current batch
        my $batch_start = Time::HiRes::time();
        for my $item (@protected_items) {
            push @all_protected_items, {
                item_uid      => $item->{protected_item_uid},
                item_type     => $item->{item_type},
                system_user   => $username,
                configuration => $item->{item_configuration} || {},
            };

            $batch->add_protected_item(
                item_uid      => $item->{protected_item_uid},
                item_type     => $item->{item_type},
                system_user   => $username,
                configuration => $item->{item_configuration} || {},
            );
        }
        my $batch_elapsed = Time::HiRes::time() - $batch_start;
        $logger->debug( sprintf( "[TIMING] batch add_protected_item loop took %.3fs for user: $username", $batch_elapsed ) );

        my $user_total_elapsed = Time::HiRes::time() - $user_start_time;
        $logger->debug( sprintf( "[TIMING] TOTAL for user $username: %.3fs (created %d items)", $user_total_elapsed, scalar(@item_uids) ) );
    }

    # Submit all Protected Items to Comet server with retry logic for hash mismatch
    my $success;
    my $max_retries = 3;
    my $retry_count = 0;

    # Debug: Check initial hash state
    $logger->debug("DEBUG: About to submit batch. Checking initial hash state...");
    my $initial_hash = eval { $batch->_get_current_user_hash() };
    $logger->debug( "DEBUG: Initial hash from batch: " . ( $initial_hash || 'FAILED TO GET HASH: ' . $@ ) );

    while ( $retry_count <= $max_retries ) {
        $logger->debug("DEBUG: Retry attempt $retry_count (max: $max_retries)");

        eval { $success = $batch->submit(); };

        if ($@) {

            # Capture error immediately to prevent it from being cleared
            my $error_text = "$@";
            $logger->debug("DEBUG: Submit failed with error: $error_text");

            # Debug: Check the exact error content and pattern matching
            my $contains_hash_mismatch = ( $error_text =~ /Hash Mismatch/i ) ? 'YES' : 'NO';
            my $contains_status_409    = ( $error_text =~ /Status:\s*409/ )  ? 'YES' : 'NO';
            $logger->debug("DEBUG: Error analysis - Hash Mismatch pattern: $contains_hash_mismatch, Status 409: $contains_status_409");
            $logger->debug("DEBUG: Full error text: '$error_text'");

            # Check if it's a hash mismatch error (HTTP 409)
            if ( $error_text =~ /Hash Mismatch/i && $error_text =~ /Status:\s*409/ ) {
                $retry_count++;
                $logger->debug("DEBUG: Hash mismatch detected. Current retry count: $retry_count, max: $max_retries");

                if ( $retry_count <= $max_retries ) {
                    $logger->warn("Hash mismatch detected on attempt $retry_count. Retrying with fresh hash...");

                    # Debug: Check what hash we had before clearing cache
                    my $old_hash = eval { $batch->_get_current_user_hash() };
                    $logger->debug( "DEBUG: Hash before forcing fresh fetch: " . ( $old_hash || 'FAILED: ' . $@ ) );

                    # Force the batch to clear any cached profile data and fetch fresh
                    # Create completely new batch instance and force fresh profile fetch
                    $batch = Whostmgr::CometBackup::CometBackupAPI::ProtectedItemBatch->new(
                        target_user => $target_user,
                    );

                    # Force fetch fresh profile immediately to see if hash changes
                    $logger->debug("DEBUG: Forcing fresh profile fetch to get latest hash...");
                    my $fresh_profile = eval { $batch->_get_user_profile() };
                    my $new_hash      = $fresh_profile ? $fresh_profile->{ProfileHash} : 'FAILED_TO_GET_PROFILE';
                    $logger->debug( "DEBUG: Fresh profile fetch result - Hash: " . ( $new_hash || 'UNDEFINED' ) );

                    # Also check via direct hash method
                    my $direct_hash = eval { $batch->_get_current_user_hash() };
                    $logger->debug( "DEBUG: Direct hash method result: " . ( $direct_hash || 'FAILED: ' . $@ ) );

                    if ( $old_hash eq $new_hash ) {
                        $logger->debug("DEBUG: WARNING - Hash did not change after fresh fetch! Server may not be updating hash correctly.");
                    }
                    else {
                        $logger->debug("DEBUG: Hash successfully changed from '$old_hash' to '$new_hash'");
                    }

                    # Re-add all Protected Items to the new batch
                    $logger->debug( "DEBUG: Re-adding " . scalar(@all_protected_items) . " protected items to new batch" );
                    for my $item (@all_protected_items) {
                        $batch->add_protected_item(
                            item_uid      => $item->{item_uid},
                            item_type     => $item->{item_type},
                            system_user   => $item->{system_user},
                            configuration => $item->{configuration},
                        );
                    }

                    # Debug: Final hash check before retry
                    my $final_hash = eval { $batch->_get_current_user_hash() };
                    $logger->debug( "DEBUG: Final hash before retry: " . ( $final_hash || 'FAILED: ' . $@ ) );

                    # Sleep briefly before retry
                    $logger->debug("DEBUG: Sleeping $retry_count seconds before retry...");
                    sleep($retry_count);
                    next;
                }
                else {
                    $logger->debug("DEBUG: Maximum retries ($max_retries) exceeded for hash mismatch");
                    die "Failed to create Protected Items on Comet server after $max_retries hash mismatch retries: $error_text";
                }
            }
            else {
                $logger->debug("DEBUG: Error is not a hash mismatch, re-throwing: $error_text");

                # Re-throw the error if it's not a hash mismatch
                die $error_text;
            }
        }
        else {
            $logger->debug("DEBUG: Submit succeeded on attempt $retry_count");
        }

        # Success - break out of loop
        last if $success;
    }

    unless ($success) {
        die "Failed to create Protected Items on Comet server after $max_retries retries";
    }

    # Mark items as synced
    $map->mark_items_synced_to_comet( \@all_item_uids );
    $logger->debug( "DEBUG: Successfully created and synced " . scalar(@all_item_uids) . " Protected Items to Comet server" );

    if ($error) {

        # Clean up error message for end users by removing file paths and line numbers
        my $user_friendly_error = $error;
        $user_friendly_error =~ s/\s+at\s+\/[^\s]+\s+line\s+\d+\.?\s*$//;

        $logger->warn("Error creating backup job: $error");
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "Failed to create backup job: $user_friendly_error" );
        return { job => {} };
    }

    # If run_now is set to 1, trigger the backup job to run immediately
    if ( $job_config->{run_now} ) {
        $logger->debug("run_now flag is set, triggering immediate backup for job: $job_uuid");

        # Call run_backup_job internally
        my $run_metadata = {};
        my $run_args     = {
            json => Cpanel::JSON::Dump( { job_uuid => $job_uuid } ),
        };

        eval {
            run_backup_job( $run_args, $run_metadata, {} );
            $logger->debug("Successfully triggered immediate backup for job: $job_uuid");
        };

        if ($@) {
            $logger->warn("Failed to trigger immediate backup for job $job_uuid: $@");

            # Don't fail the entire create operation, just log the warning
        }
    }

    # Success case
    $logger->debug("Returning successful response with UUID: $job_uuid");
    Whostmgr::API::1::Utils::set_metadata_ok($metadata);
    return {
        job => {
            job_uuid => $job_uuid,
        }
    };
}

sub _get_users_to_backup ($job_config) {
    my @users;

    if ( $job_config->{all_accounts} ) {
        require Cpanel::Config::Users;
        @users = Cpanel::Config::Users::getcpusers();
    }
    else {
        @users = @{ $job_config->{accounts} || [] };
    }

    return @users;
}

=head2 get_backup_jobs

Retrieves all configured backup jobs for this server.

=head3 Parameters

None required (optional JSON parameters for filtering may be supported).

=head3 Returns

An array reference of backup job configurations, each containing job UUID, name, schedule,
users to back up, destinations, retention settings, and status.

=head3 Example

    my $jobs = get_backup_jobs({}, $metadata, $api_info);

=cut

sub get_backup_jobs {
    my ( $args, $metadata, $api_info_hr ) = @_;

    my $decoded_args;

    # Support both GET requests (with job_ids query param) and POST requests (with json body)
    if ( ref $args eq 'HASH' && defined $args->{'job_ids'} ) {

        # GET request: job_ids is a JSON-encoded array in query parameters
        eval {
            my $job_ids_json  = $args->{'job_ids'};
            my $job_ids_array = Cpanel::JSON::Load($job_ids_json);

            # Normalize parameter name from job_ids to job_uuids for consistency
            $decoded_args = { job_uuids => $job_ids_array };
        };
        if ($@) {
            Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "Failed to decode job_ids parameter: $@" );
            return { jobs => [] };
        }
    }
    elsif ( ref $args eq 'HASH' && defined $args->{'json'} ) {

        # POST request: traditional json parameter with full request body
        ( my $status, $decoded_args ) = _decode_json_args( $metadata, $args );
        if ( $status < 1 ) {
            return;
        }
    }

    my $result;

    eval {
        require Whostmgr::CometBackup::BackupJobs;
        my $jobs = Whostmgr::CometBackup::BackupJobs->new();

        # Extract job_uuids/job_ids from args if provided, otherwise get all jobs
        # Support both job_ids and job_uuids for flexibility
        my $job_uuids = [];
        my %options;
        if ( ref $decoded_args eq 'HASH' ) {
            $job_uuids = $decoded_args->{job_ids} || $decoded_args->{job_uuids} || [];

            # Extract sorting parameters if provided
            if ( $decoded_args->{sort_by} ) {
                $options{sort_by} = $decoded_args->{sort_by};
            }
            if ( $decoded_args->{sort_direction} ) {
                $options{sort_direction} = $decoded_args->{sort_direction};
            }
        }

        # Validate job_uuids is an array reference if provided
        if ( defined $job_uuids && ref $job_uuids ne 'ARRAY' ) {
            Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "job_ids/job_uuids parameter must be an array reference" );
            $result = { jobs => [] };
            return;
        }

        $logger->debug( "Fetching jobs for UUIDs: " . Dumper($job_uuids) );
        $logger->debug( "Sort options: " . Dumper( \%options ) ) if %options;
        my @job_list = $jobs->get_backup_jobs( $job_uuids, \%options );

        # Check the next_run time for each job, if it is less than 5 minutes from the current system time, replace the time stamp with "Pending".
        # This helps the user understand that the job is ready to be run and is just waiting on the cron job to start the backup runner.
        my $current_time = time();
        foreach my $job (@job_list) {
            if ( defined $job->{next_run} ) {
                my $next_run_time = $job->{next_run};
                if ( $next_run_time - $current_time < 300 ) {
                    $job->{next_run} = "Pending";
                }
            }
        }

        $logger->debug("Job list retrieved.");

        # Check if any jobs have currently running backups and add run_uuid if so
        if ( -f $Whostmgr::CometBackup::Constants::RUN_LOG_DB ) {
            require Whostmgr::CometBackup::BackupRunLogs;
            my $logs = Whostmgr::CometBackup::BackupRunLogs->new();

            for my $job (@job_list) {
                my $job_uuid = $job->{job_uuid};
                next unless $job_uuid;

                $job->{updated_timestamp} = $job->{last_run};

                # Check for running backups for this job
                eval {
                    my @running_runs = $logs->get_recent_runs(
                        job_uuid => $job_uuid,
                        status   => 'running',
                        limit    => 1,
                    );

                    # If there's a running backup, add the run_uuid to the job data
                    if ( @running_runs && $running_runs[0]->{run_uuid} ) {
                        $job->{run_uuid} = $running_runs[0]->{run_uuid};
                        $logger->debug("Found running backup for job $job_uuid: run_uuid=$job->{run_uuid}");
                    }
                };
                if ($@) {
                    $logger->warn("Failed to check running status for job $job_uuid: $@");
                }
            }
        }

        Whostmgr::API::1::Utils::set_metadata_ok($metadata);
        $result = { jobs => \@job_list };
    };

    if ($@) {
        $logger->warn("get_backup_jobs error: $@");
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "Failed to retrieve backup jobs: $@" );
        $result = { jobs => [] };
    }

    return $result;
}

sub _decode_json_args {
    my ( $metadata, $args_hr ) = @_;

    if ( !defined $args_hr->{'json'} ) {
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "Missing required 'json' parameter" );
        return ( 0, undef );
    }

    my $decoded_args;
    eval { $decoded_args = Cpanel::JSON::Load( $args_hr->{'json'} ); };
    if ($@) {

        # Extract clean error message from JSON parse error
        my $error       = $@;
        my $clean_error = "Invalid JSON syntax";

        # Try to extract the specific parse error message
        if ( $error =~ /because of an error:\s*(.+?)\s+at\s+\/usr\/local\/cpanel/s ) {
            $clean_error = $1;

            # Clean up the error message further
            $clean_error =~ s/\s+$//;    # Remove trailing whitespace
        }
        elsif ( $error =~ /because of an error:\s*(.+?)$/s ) {
            $clean_error = $1;
            $clean_error =~ s/\s+$//;
        }

        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "Invalid JSON: $clean_error" );
        return ( 0, undef );
    }
    delete $args_hr->{'json'};
    return ( 1, $decoded_args );
}

=head2 delete_backup_job

Deletes a backup job configuration by UUID.

=head3 Parameters

JSON object containing:

=over 4

=item * C<job_uuid> - (Required) UUID of the backup job to delete

=back

=head3 Returns

A hash reference containing:

=over 4

=item * C<deleted> - Hash with C<job_uuid> of the deleted job on success

=back

=head3 Example

    my $result = delete_backup_job({
        json => encode_json({
            job_uuid => '550e8400-e29b-41d4-a716-446655440000'
        })
    }, $metadata, $api_info);

=cut

sub delete_backup_job {
    my ( $args, $metadata, $api_info_hr ) = @_;

    $logger->debug( "delete_backup_job called with args: " . Dumper($args) );

    my ( $status, $decoded_args ) = _decode_json_args( $metadata, $args );
    if ( $status < 1 ) {
        return;
    }

    # Extract job configuration from decoded arguments
    $logger->debug( "Decoded args: " . Dumper($decoded_args) );

    my $job_uuid = $decoded_args->{job_uuid};
    $logger->debug("Attempting to delete job with UUID: $job_uuid");
    unless ($job_uuid) {
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "job_uuid parameter required" );
        return { deleted => {} };
    }

    my $delete_response;
    eval {
        require Whostmgr::CometBackup::BackupJobs;

        my $jobs    = Whostmgr::CometBackup::BackupJobs->new();
        my $success = $jobs->delete_backup_job($job_uuid);

        if ($success) {
            Whostmgr::API::1::Utils::set_metadata_ok($metadata);
            $delete_response = {
                deleted => {
                    job_uuid => $job_uuid,
                },
            };
        }
        else {
            Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "Backup job not found or already deleted" );
            $delete_response = { deleted => {} };
        }
    };

    if ($@) {
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "Failed to delete backup job: $@" );
        return { deleted => {} };
    }

    $delete_response //= { deleted => {} };
    return $delete_response;
}

=head2 disable_backup_job

Disables a backup job so it will not run on its schedule.

=head3 Parameters

JSON object containing:

=over 4

=item * C<job_uuid> - (Required) UUID of the backup job to disable

=back

=head3 Returns

A hash reference containing C<disabled> hash with C<job_uuid> on success.

=head3 Example

    my $result = disable_backup_job({
        json => encode_json({ job_uuid => $uuid })
    }, $metadata, $api_info);

=cut

sub disable_backup_job {
    my ( $args, $metadata, $api_info_hr ) = @_;

    $logger->debug( "disable_backup_job called with args: " . Dumper($args) );

    my ( $status, $decoded_args ) = _decode_json_args( $metadata, $args );
    if ( $status < 1 ) {
        return;
    }

    # Extract job configuration from decoded arguments
    $logger->debug( "Decoded args: " . Dumper($decoded_args) );

    my $job_uuid = $decoded_args->{job_uuid};
    $logger->debug("Attempting to disable job with UUID: $job_uuid");
    unless ($job_uuid) {
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "job_uuid parameter required" );
        return { disabled => {} };
    }

    my $disable_response;
    eval {
        require Whostmgr::CometBackup::BackupJobs;

        my $jobs    = Whostmgr::CometBackup::BackupJobs->new();
        my $success = $jobs->disable_backup_job($job_uuid);

        if ($success) {
            Whostmgr::API::1::Utils::set_metadata_ok($metadata);
            $disable_response = {
                disabled => {
                    job_uuid => $job_uuid,
                },
            };
        }
        else {
            Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "Backup job not found or already disabled" );
            $disable_response = { disabled => {} };
        }
    };

    if ($@) {
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "Failed to disable backup job: $@" );
        return { disabled => {} };
    }

    $disable_response //= { disabled => {} };
    return $disable_response;
}

=head2 enable_backup_job

Enables a previously disabled backup job to run on its schedule.

=head3 Parameters

JSON object containing:

=over 4

=item * C<job_uuid> - (Required) UUID of the backup job to enable

=back

=head3 Returns

A hash reference containing C<enabled> hash with C<job_uuid> on success.

=head3 Example

    my $result = enable_backup_job({
        json => encode_json({ job_uuid => $uuid })
    }, $metadata, $api_info);

=cut

sub enable_backup_job {
    my ( $args, $metadata, $api_info_hr ) = @_;

    $logger->debug( "enable_backup_job called with args: " . Dumper($args) );

    my ( $status, $decoded_args ) = _decode_json_args( $metadata, $args );
    if ( $status < 1 ) {
        return;
    }

    # Extract job configuration from decoded arguments
    $logger->debug( "Decoded args: " . Dumper($decoded_args) );

    my $job_uuid = $decoded_args->{job_uuid};
    $logger->debug("Attempting to enable job with UUID: $job_uuid");
    unless ($job_uuid) {
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "job_uuid parameter required" );
        return { enabled => {} };
    }

    my $enable_response;
    eval {
        require Whostmgr::CometBackup::BackupJobs;

        my $jobs    = Whostmgr::CometBackup::BackupJobs->new();
        my $success = $jobs->enable_backup_job($job_uuid);

        if ($success) {
            Whostmgr::API::1::Utils::set_metadata_ok($metadata);
            $enable_response = {
                enabled => {
                    job_uuid => $job_uuid,
                },
            };
        }
        else {
            Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "Backup job not found or already enabled" );
            $enable_response = { enabled => {} };
        }
    };

    if ($@) {
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "Failed to enable backup job: $@" );
        return { enabled => {} };
    }

    $enable_response //= { enabled => {} };
    return $enable_response;
}

=head2 modify_backup_job

Modifies an existing backup job configuration.

=head3 Parameters

JSON object containing:

=over 4

=item * C<job_uuid> - (Required) UUID of the backup job to modify

=item * Other parameters - Same as create_backup_job for settings to update

=back

=head3 Returns

A hash reference containing C<modified> hash with C<job_uuid> on success.

=head3 Example

    my $result = modify_backup_job({
        json => encode_json({
            job_uuid => $uuid,
            name => 'Updated Job Name'
        })
    }, $metadata, $api_info);

=cut

sub modify_backup_job {
    my ( $args, $metadata, $api_info_hr ) = @_;

    $logger->debug( "modify_backup_job called with args: " . Dumper($args) );

    my ( $status, $decoded_args ) = _decode_json_args( $metadata, $args );
    if ( $status < 1 ) {
        return;
    }

    # Extract job UUID and configuration from decoded arguments
    $logger->debug( "Decoded args: " . Dumper($decoded_args) );
    my $job_uuid   = $decoded_args->{job_uuid};
    my $job_config = $decoded_args->{job_config};

    unless ($job_uuid) {
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "job_uuid parameter required" );
        return { job => {} };
    }

    unless ( $job_config && ref $job_config eq 'HASH' ) {
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "job_config parameter required and must be a hash reference" );
        return { job => {} };
    }

    # Validate required fields in job_config
    my @required_fields = qw(name schedule_type backup_options destinations all_accounts);
    for my $field (@required_fields) {
        unless ( exists $job_config->{$field} ) {
            $logger->warn("Missing required field: $field");
            Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "job_config.$field is required" );
            return { job => {} };
        }
    }

    # Additional validation for specific account backup
    if ( !$job_config->{all_accounts} && ( !$job_config->{accounts} || ref $job_config->{accounts} ne 'ARRAY' || @{ $job_config->{accounts} } == 0 ) ) {
        $logger->warn("When all_accounts is 0, accounts array must be provided and non-empty");
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "When all_accounts is 0, accounts must be a non-empty array" );
        return { job => {} };
    }

    # Normalize backup_options for simplified frontend API
    _normalize_backup_options( $job_config->{backup_options} );

    $logger->debug("All required fields present, attempting to modify backup job");

    my $success;
    eval {
        require Whostmgr::CometBackup::BackupJobs;

        my $jobs = Whostmgr::CometBackup::BackupJobs->new();

        $logger->debug( "About to call modify_backup_job with UUID: $job_uuid and config: " . Dumper($job_config) );
        $success = $jobs->modify_backup_job( $job_uuid, %$job_config );

        if ($success) {
            $logger->debug("Successfully modified backup job with UUID: $job_uuid");
        }
        else {
            $logger->debug("modify_backup_job returned false for UUID: $job_uuid");
        }
    };

    if ($@) {
        $logger->warn("Error modifying backup job: $@");
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "Failed to modify backup job: $@" );
        return { job => {} };
    }

    if ( !$success ) {
        $logger->warn("Backup job not found or modification failed");
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "Backup job not found or modification failed" );
        return { job => {} };
    }

    # If run_now is set to 1, trigger the backup job to run immediately
    if ( $job_config->{run_now} ) {
        $logger->debug("run_now flag is set, triggering immediate backup for job: $job_uuid");

        # Call run_backup_job internally
        my $run_metadata = {};
        my $run_args     = {
            json => Cpanel::JSON::Dump( { job_uuid => $job_uuid } ),
        };

        eval {
            run_backup_job( $run_args, $run_metadata, {} );
            $logger->debug("Successfully triggered immediate backup for job: $job_uuid");
        };

        if ($@) {
            $logger->warn("Failed to trigger immediate backup for job $job_uuid: $@");

            # Don't fail the entire modify operation, just log the warning
        }
    }

    # Success case
    $logger->debug("Returning successful response for modified job UUID: $job_uuid");
    Whostmgr::API::1::Utils::set_metadata_ok($metadata);
    return {
        job => {
            job_uuid => $job_uuid,
            modified => 1,
        }
    };
}

# Takes the UUID of an existing backup job and creates a duplicate with a new UUID and potentially new name

=head2 duplicate_backup_job

Creates a copy of an existing backup job with a new UUID.

=head3 Parameters

JSON object containing:

=over 4

=item * C<job_uuid> - (Required) UUID of the backup job to duplicate

=item * C<name> - (Optional) Name for the duplicated job

=back

=head3 Returns

A hash reference containing C<duplicated> hash with new C<job_uuid> on success.

=head3 Example

    my $result = duplicate_backup_job({
        json => encode_json({
            job_uuid => $uuid,
            name => 'Copy of Original'
        })
    }, $metadata, $api_info);

=cut

sub duplicate_backup_job {
    my ( $args, $metadata, $api_info_hr ) = @_;

    $logger->debug( "duplicate_backup_job called with args: " . Dumper($args) );

    my ( $status, $decoded_args ) = _decode_json_args( $metadata, $args );
    if ( $status < 1 ) {
        return;
    }

    # Extract parameters from decoded arguments
    $logger->debug( "Decoded args: " . Dumper($decoded_args) );
    my $source_job_uuid = $decoded_args->{job_uuid};
    my $new_name        = $decoded_args->{new_name};

    # Sanitize new_name if provided
    if ( defined $new_name ) {
        eval { $new_name = Whostmgr::CometBackup::BackupJobs::sanitize_text_input( $new_name, 'new_name' ); };
        if ($@) {
            Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, $@ );
            return { job => {} };
        }
    }

    $logger->debug("Attempting to duplicate job with UUID: $source_job_uuid");
    unless ($source_job_uuid) {
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "job_uuid parameter required" );
        return { job => {} };
    }

    my $new_job_uuid;
    eval {
        require Whostmgr::CometBackup::BackupJobs;

        my $jobs = Whostmgr::CometBackup::BackupJobs->new();
        $new_job_uuid = $jobs->duplicate_backup_job( $source_job_uuid, $new_name );

        unless ($new_job_uuid) {
            Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "Source backup job not found" );
            return { job => {} };
        }

        $logger->debug("Successfully duplicated backup job. New UUID: $new_job_uuid");
    };

    if ($@) {
        $logger->warn("Error duplicating backup job: $@");
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "Failed to duplicate backup job: $@" );
        return { job => {} };
    }

    # Success case
    $logger->debug("Returning successful response with new UUID: $new_job_uuid");
    Whostmgr::API::1::Utils::set_metadata_ok($metadata);
    return {
        job => {
            job_uuid => $new_job_uuid,
        }
    };
}

# TODO - implement this, there is a function in the runner, cancel_backup_job we can use / move / whatever

=head2 cancel_running_backup_job

Cancels a currently running backup job.

=head3 Parameters

JSON object containing:

=over 4

=item * C<job_uuid> - (Required) UUID of the backup job to cancel

=back

=head3 Returns

A hash reference containing C<cancelled> hash with C<job_uuid> on success.

=head3 Example

    my $result = cancel_running_backup_job({
        json => encode_json({ job_uuid => $uuid })
    }, $metadata, $api_info);

=cut

sub cancel_running_backup_job {
    my ( $args, $metadata, $api_info_hr ) = @_;

    $logger->debug( "cancel_running_backup_job called with args: " . Dumper($args) );

    my ( $status, $decoded_args ) = _decode_json_args( $metadata, $args );
    if ( $status < 1 ) {
        return;
    }

    # Extract parameters from decoded arguments
    $logger->debug( "Decoded args: " . Dumper($decoded_args) );
    my $job_uuid = $decoded_args->{job_uuid};
    my $run_uuid = $decoded_args->{run_uuid};

    unless ($job_uuid) {
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "job_uuid parameter required" );
        return { cancelled => {} };
    }

    # run_uuid is optional - we can cancel by job_uuid alone
    # The runner checks for the cancel file by job_uuid

    eval {
        require Whostmgr::CometBackup::BackupJobs;
        require Whostmgr::CometBackup::BackupRunLogs;
        require Whostmgr::CometBackup::Constants;
        require Cpanel::FileUtils::TouchFile;

        # Validate the job exists
        my $jobs     = Whostmgr::CometBackup::BackupJobs->new();
        my @job_list = $jobs->get_backup_jobs( [$job_uuid] );

        unless (@job_list) {
            die "Backup job not found: $job_uuid";
        }

        my $job = $job_list[0];

        # Verify the job is actually running
        unless ( $job->{status} && $job->{status} eq 'running' ) {
            die "Backup job is not currently running (status: " . ( $job->{status} || 'unknown' ) . ")";
        }

        # If run_uuid is provided, verify it matches the current run
        if ( $run_uuid && $job->{run_uuid} && $job->{run_uuid} ne $run_uuid ) {
            die "Provided run_uuid does not match the current running job";
        }

        # Create the cancel file to signal the runner to stop
        my $cancel_file = Whostmgr::CometBackup::Constants::get_cancel_file_path($job_uuid);

        $logger->debug(
            "Creating cancel file for backup job",
            {
                job_uuid    => $job_uuid,
                run_uuid    => $run_uuid || $job->{run_uuid},
                cancel_file => $cancel_file,
            }
        );

        Cpanel::FileUtils::TouchFile::touchfile($cancel_file);

        # Update job status to 'cancelling' in the database
        # This provides an interstitial status during the brief window between cancel request and actual cancellation
        # The runner will update this to 'cancelled' when it detects the cancel file and shuts down
        $jobs->update_job_status( $job_uuid, 'cancelling' );

        $logger->debug("Backup job cancellation requested successfully");
    };

    if ($@) {
        my $error = $@;
        $logger->error("Failed to cancel backup job: $error");
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "Failed to cancel backup job: $error" );
        return { cancelled => {} };
    }

    Whostmgr::API::1::Utils::set_metadata_ok($metadata);
    return {
        cancelled => {
            job_uuid => $job_uuid,
            run_uuid => $run_uuid,
            status   => 'cancelling',
            message  => 'Backup job cancellation requested successfully'
        }
    };
}

=head2 run_backup_job

Manually triggers a backup job to run immediately.

=head3 Parameters

JSON object containing:

=over 4

=item * C<job_uuid> - (Required) UUID of the backup job to run

=back

=head3 Returns

A hash reference containing C<job_uuid> and C<run_uuid> for tracking the backup run.

=head3 Example

    my $result = run_backup_job({
        json => encode_json({ job_uuid => $uuid })
    }, $metadata, $api_info);

=cut

sub run_backup_job {
    my ( $args, $metadata, $api_info_hr ) = @_;

    $logger->debug( "run_backup_job called with args: " . Dumper($args) );

    my ( $status, $decoded_args ) = _decode_json_args( $metadata, $args );
    if ( $status < 1 ) {
        return;
    }

    # Extract job configuration from decoded arguments
    $logger->debug( "Decoded args: " . Dumper($decoded_args) );

    my $job_uuid = $decoded_args->{job_uuid};
    $logger->debug("Attempting to run backup job with UUID: $job_uuid");
    unless ($job_uuid) {
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "job_uuid parameter required" );
        return { started => 0 };
    }

    # Validate UUID format
    require Cpanel::Validate::UUID;
    unless ( Cpanel::Validate::UUID::is_valid_uuid($job_uuid) ) {
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "Invalid UUID format: $job_uuid" );
        return { started => 0 };
    }

    # Validate that the job exists and is active
    eval {
        require Whostmgr::CometBackup::BackupJobs;

        my $jobs     = Whostmgr::CometBackup::BackupJobs->new();
        my @job_list = $jobs->get_backup_jobs( [$job_uuid] );

        unless (@job_list) {
            die "Backup job not found: $job_uuid";
        }

        my $job_config = $job_list[0];

        # Check if job is active
        unless ( $job_config->{is_active} ) {
            die "Backup job is not active: $job_uuid";
        }

        $logger->debug("Validated backup job $job_uuid exists and is active");
    };

    if ($@) {
        my $error = $@;
        $logger->error("Job validation failed: $error");
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "Failed to validate job: $error" );
        return { started => 0 };
    }

    # Fork the backup runner script in the background
    my $backup_runner_path = $Whostmgr::CometBackup::Constants::BACKUP_RUNNER_PATH;

    unless ( -x $backup_runner_path ) {
        $logger->error("Backup runner script not found or not executable: $backup_runner_path");
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "Backup runner script not available" );
        return { started => 0 };
    }

    # Generate a unique run_uuid for this backup run instance
    require Data::UUID;
    my $ug       = Data::UUID->new();
    my $run_uuid = lc( $ug->create_str() );

    $logger->debug("Generated run_uuid: $run_uuid for job $job_uuid");

    # Store the current_run_uuid in the backup_jobs table for immediate access
    eval {
        require Whostmgr::CometBackup::BackupJobs;
        my $jobs = Whostmgr::CometBackup::BackupJobs->new();
        $jobs->set_current_run_uuid( $job_uuid, $run_uuid );
        $logger->debug("Set current_run_uuid in database for job $job_uuid");
    };
    if ($@) {
        $logger->warn("Failed to set current_run_uuid in database: $@");

        # Continue anyway - this is not critical
    }

    my $pid = fork();

    if ( !defined $pid ) {

        # Fork failed
        $logger->error("Failed to fork backup runner process: $!");
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "Failed to start backup process: $!" );
        return { started => 0 };
    }

    if ( $pid == 0 ) {

        # Child process - exec the backup runner
        # Redirect STDOUT and STDERR to /dev/null to detach from parent
        open STDOUT, '>', '/dev/null' or die "Cannot redirect STDOUT: $!";
        open STDERR, '>', '/dev/null' or die "Cannot redirect STDERR: $!";

        # Become session leader to fully detach
        require POSIX;
        POSIX::setsid() or die "Cannot start a new session: $!";

        # Execute the backup runner with run_uuid
        exec( $backup_runner_path, '--force', '--job_uuid', $job_uuid, '--run_uuid', $run_uuid );

        # If exec fails, we die (child process only)
        die "Failed to execute backup runner: $!";
    }

    # Parent process - return immediately
    $logger->info("Started backup runner for job $job_uuid (run $run_uuid) with PID $pid");

    Whostmgr::API::1::Utils::set_metadata_ok($metadata);
    return {
        started  => 1,
        job_uuid => $job_uuid,
        run_uuid => $run_uuid,
        pid      => $pid,
    };
}

# Returns the log of a specific backup job run identified by job_uuid and run_uuid
# Retrieves the log file content and returns it for download

=head2 download_backup_job_log

Downloads the log file for a completed backup job run.

=head3 Parameters

JSON object containing:

=over 4

=item * C<job_uuid> - (Required) UUID of the backup job

=item * C<run_uuid> - (Required) UUID of the specific backup run

=back

=head3 Returns

A hash reference containing C<log> string with the log file content.

=head3 Example

    my $result = download_backup_job_log({
        json => encode_json({
            job_uuid => $job_uuid,
            run_uuid => $run_uuid
        })
    }, $metadata, $api_info);

=cut

sub download_backup_job_log {
    my ( $args, $metadata, $api_info_hr ) = @_;

    $logger->debug( "download_backup_job_log called with args: " . Dumper($args) );

    my ( $status, $decoded_args ) = _decode_json_args( $metadata, $args );
    if ( $status < 1 ) {
        return;
    }

    # Extract parameters from decoded arguments
    $logger->debug( "Decoded args: " . Dumper($decoded_args) );
    my $job_uuid = $decoded_args->{job_uuid};
    my $run_uuid = $decoded_args->{run_uuid};

    unless ($job_uuid) {
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "job_uuid parameter required" );
        return { log => '', error => 'job_uuid parameter required' };
    }

    unless ($run_uuid) {
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "run_uuid parameter required" );
        return { log => '', error => 'run_uuid parameter required' };
    }

    # Validate UUID formats
    require Cpanel::Validate::UUID;
    unless ( Cpanel::Validate::UUID::is_valid_uuid($job_uuid) ) {
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "Invalid job_uuid format" );
        return { log => '', error => 'Invalid job_uuid format' };
    }

    unless ( Cpanel::Validate::UUID::is_valid_uuid($run_uuid) ) {
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "Invalid run_uuid format" );
        return { log => '', error => 'Invalid run_uuid format' };
    }

    # Get the log file path using the Constants helper function
    my $log_file = Whostmgr::CometBackup::Constants::get_text_log_file_path( $job_uuid, $run_uuid );

    $logger->debug( "Looking for log file", { log_file => $log_file } );

    # Check if log file exists
    unless ( -f $log_file ) {
        my $error_msg = "Log file not found for job $job_uuid, run $run_uuid";
        $logger->warn($error_msg);
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, $error_msg );
        return { log => '', error => $error_msg };
    }

    # Check if log file is readable
    unless ( -r $log_file ) {
        my $error_msg = "Log file is not readable: $log_file";
        $logger->error($error_msg);
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, $error_msg );
        return { log => '', error => $error_msg };
    }

    # Read the log file content
    my $log_content;
    eval {
        require Cpanel::LoadFile;
        $log_content = Cpanel::LoadFile::load($log_file);
    };

    if ($@) {
        my $error_msg = "Failed to read log file: $@";
        $logger->error($error_msg);
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, $error_msg );
        return { log => '', error => $error_msg };
    }

    $logger->debug( "Successfully retrieved log file", { log_file => $log_file, size => length($log_content) } );
    Whostmgr::API::1::Utils::set_metadata_ok($metadata);
    return {
        log      => $log_content,
        job_uuid => $job_uuid,
        run_uuid => $run_uuid,
    };
}

# Returns the log of a specific restore job run identified by run_uuid
# Retrieves the log file content and returns it for download
# Note: For restores, run_uuid is the same as job_uuid

=head2 download_restore_job_log

Downloads the log file for a specific restore job.

=head3 Parameters

=over 4

=item * C<run_uuid> - (Required) The UUID of the restore job

=back

=head3 Returns

A hash reference containing:

=over 4

=item * C<log> - The log file content as a string

=item * C<error> - Error message if the operation failed

=back

=head3 Example

    my $result = download_restore_job_log({
        json => encode_json({
            run_uuid => '550e8400-e29b-41d4-a716-446655440000'
        })
    }, $metadata, $api_info);

=cut

sub download_restore_job_log {
    my ( $args, $metadata, $api_info_hr ) = @_;

    $logger->debug( "download_restore_job_log called with args: " . Dumper($args) );

    my ( $status, $decoded_args ) = _decode_json_args( $metadata, $args );
    if ( $status < 1 ) {
        return;
    }

    # Extract parameters from decoded arguments
    $logger->debug( "Decoded args: " . Dumper($decoded_args) );
    my $run_uuid = $decoded_args->{run_uuid};

    unless ($run_uuid) {
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "run_uuid parameter required" );
        return { log => '', error => 'run_uuid parameter required' };
    }

    # Validate UUID format
    require Cpanel::Validate::UUID;
    unless ( Cpanel::Validate::UUID::is_valid_uuid($run_uuid) ) {
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "Invalid run_uuid format" );
        return { log => '', error => 'Invalid run_uuid format' };
    }

    # Get the log file path using the Constants helper function
    # Note: For restores, job_uuid and run_uuid are the same, so we pass run_uuid twice
    my $log_file = Whostmgr::CometBackup::Constants::get_text_log_file_path( $run_uuid, $run_uuid );

    $logger->debug( "Looking for restore log file", { log_file => $log_file } );

    # Check if log file exists
    unless ( -f $log_file ) {
        my $error_msg = "Restore log file not found for run $run_uuid";
        $logger->warn($error_msg);
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, $error_msg );
        return { log => '', error => $error_msg };
    }

    # Check if log file is readable
    unless ( -r $log_file ) {
        my $error_msg = "Restore log file is not readable: $log_file";
        $logger->error($error_msg);
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, $error_msg );
        return { log => '', error => $error_msg };
    }

    # Read the log file content
    my $log_content;
    eval {
        require Cpanel::LoadFile;
        $log_content = Cpanel::LoadFile::load($log_file);
    };

    if ($@) {
        my $error_msg = "Failed to read restore log file: $@";
        $logger->error($error_msg);
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, $error_msg );
        return { log => '', error => $error_msg };
    }

    $logger->debug( "Successfully retrieved restore log file", { log_file => $log_file, size => length($log_content) } );
    Whostmgr::API::1::Utils::set_metadata_ok($metadata);
    return {
        log      => $log_content,
        run_uuid => $run_uuid,
    };
}

# Returns a list of past and currently running backup job runs with their statuses, storage size, time taken, etc.

=head2 list_backup_run_history

Lists historical backup runs with optional filtering.

=head3 Parameters

JSON object with optional filters:

=over 4

=item * C<job_uuid> - Filter by specific backup job

=item * C<status> - Filter by run status

=item * C<limit> - Maximum number of results

=back

=head3 Returns

An array reference of backup run records with details including status, timing, and users backed up.

=head3 Example

    my $result = list_backup_run_history({
        json => encode_json({ limit => 50 })
    }, $metadata, $api_info);

=cut

sub list_backup_run_history {
    my ( $args, $metadata, $api_info_hr ) = @_;

    $logger->debug( "list_backup_run_history called with args: " . Dumper($args) );

    my $decoded_args;
    if ( ref $args eq 'HASH' && defined $args->{'json'} ) {
        ( my $status, $decoded_args ) = _decode_json_args( $metadata, $args );
        if ( $status < 1 ) {
            return;
        }
    }

    # Extract search and filter parameters from args
    my %filters = ();
    my $limit   = 50;    # Default limit
    if ( ref $decoded_args eq 'HASH' ) {
        $filters{job_uuid}   = $decoded_args->{job_uuid}   if $decoded_args->{job_uuid};
        $filters{name}       = $decoded_args->{name}       if $decoded_args->{name};
        $filters{status}     = $decoded_args->{status}     if $decoded_args->{status};
        $filters{job_type}   = $decoded_args->{job_type}   if $decoded_args->{job_type};
        $filters{time_range} = $decoded_args->{time_range} if $decoded_args->{time_range};
        $filters{start_date} = $decoded_args->{start_date} if $decoded_args->{start_date};
        $filters{end_date}   = $decoded_args->{end_date}   if $decoded_args->{end_date};
        $limit               = $decoded_args->{limit}      if $decoded_args->{limit} && $decoded_args->{limit} > 0;
    }

    $logger->debug( "Filters applied: " . Dumper( \%filters ) );

    # Get backup run history from database using BackupRunLogs
    my @runs;
    my $cutoff_time;

    # Check if database exists before attempting to query it
    # This prevents creating an empty database file on first API call
    unless ( -f $Whostmgr::CometBackup::Constants::RUN_LOG_DB ) {
        $logger->debug("Backup run history database does not exist yet, returning empty result");
        Whostmgr::API::1::Utils::set_metadata_ok($metadata);
        return {
            history         => [],
            total_entries   => 0,
            filters_applied => \%filters
        };
    }

    eval {
        my $logs = Whostmgr::CometBackup::BackupRunLogs->new();

        # Build parameters for get_backup_runs based on filters
        my %run_params = ( limit => $limit );

        # Handle time range filter by converting to start_date
        if ( $filters{time_range} ) {
            my $current_time = time();

            if ( $filters{time_range} eq 'last_24_hours' ) {
                $cutoff_time = $current_time - ( 24 * 60 * 60 );
            }
            elsif ( $filters{time_range} eq 'last_week' ) {
                $cutoff_time = $current_time - ( 7 * 24 * 60 * 60 );
            }
            elsif ( $filters{time_range} eq 'last_month' ) {
                $cutoff_time = $current_time - ( 30 * 24 * 60 * 60 );
            }
            elsif ( $filters{time_range} eq 'last_3_months' ) {
                $cutoff_time = $current_time - ( 90 * 24 * 60 * 60 );
            }
            elsif ( $filters{time_range} eq 'last_6_months' ) {
                $cutoff_time = $current_time - ( 180 * 24 * 60 * 60 );
            }
            elsif ( $filters{time_range} eq 'last_year' ) {
                $cutoff_time = $current_time - ( 365 * 24 * 60 * 60 );
            }

            $run_params{start_date} = $cutoff_time if $cutoff_time;
        }

        # Apply other filters directly
        $run_params{job_uuid}   = $filters{job_uuid}   if $filters{job_uuid};
        $run_params{status}     = $filters{status}     if $filters{status};
        $run_params{start_date} = $filters{start_date} if $filters{start_date};
        $run_params{end_date}   = $filters{end_date}   if $filters{end_date};

        @runs = $logs->get_backup_runs(%run_params);
    };

    if ($@) {

        # If database doesn't exist or table is missing, log warning and return empty result
        if ( $@ =~ /no such table/i || $@ =~ /unable to open database/i ) {
            $logger->debug("Backup run history database not yet initialized, returning empty result");
            @runs = ();
        }
        else {
            # Re-throw unexpected errors
            die $@;
        }
    }

    $logger->debug( "Retrieved " . scalar(@runs) . " runs from database" );

    # Transform database results into the expected format
    my @history = ();
    for my $run (@runs) {

        # Map status values
        my $status = $run->{status};
        if ( $status eq 'completed' ) {
            $status = 'Completed';
        }
        elsif ( $status eq 'failed' ) {
            $status = 'Failed';
        }
        elsif ( $status eq 'running' ) {
            $status = 'Processing';
        }

        # Calculate execution time
        my $execution_time         = '';
        my $execution_time_seconds = 0;
        if ( $run->{duration} && $run->{duration} > 0 ) {
            $execution_time         = $run->{duration_formatted} || _format_duration( $run->{duration} );
            $execution_time_seconds = $run->{duration};
        }
        elsif ( $status eq 'Processing' && $run->{start_time} ) {
            my $current_duration = time() - $run->{start_time};
            $execution_time         = _format_duration($current_duration);
            $execution_time_seconds = $current_duration;
        }

        # Format total size from aggregated backup data
        my $total_size = '';
        if ( $run->{total_size_bytes} && $run->{total_size_bytes} > 0 ) {
            $total_size = _format_bytes( $run->{total_size_bytes} );
        }

        push @history, {
            name                   => $run->{job_name}        || "Job $run->{job_uuid}",                        # From backup_jobs table
            description            => $run->{job_description} || '',                                            # From backup_jobs table
            job_uuid               => $run->{job_uuid},
            run_uuid               => $run->{run_uuid},                                                         # Unique run identifier
            start_time             => $run->{start_time},
            start_time_formatted   => $run->{start_time_formatted} || scalar localtime( $run->{start_time} ),
            job_type               => 'automatic',                                                              # Default to automatic for now - could be added to job config later
            execution_time         => $execution_time,
            execution_time_seconds => $execution_time_seconds,                                                  # Raw seconds for sorting
            total_size             => $total_size,                                                              # Aggregated size from all protected items in this run
            total_size_bytes       => $run->{total_size_bytes} || 0,                                            # Raw bytes for sorting
            status                 => $status,
            error_message          => $run->{error_message},
        };
    }

    # Apply name filter if specified (case-insensitive substring match)
    my @filtered_history = @history;
    if ( $filters{name} ) {
        @filtered_history = grep { $_->{name} =~ /\Q$filters{name}\E/i } @filtered_history;
        $logger->debug( "Filtered by name: $filters{name}, found " . scalar(@filtered_history) . " entries" );
    }

    # Apply status filter
    if ( $filters{status} ) {
        @filtered_history = grep { $_->{status} eq $filters{status} } @filtered_history;
        $logger->debug( "Filtered by status: $filters{status}, found " . scalar(@filtered_history) . " entries" );
    }

    # Apply job_type filter
    if ( $filters{job_type} ) {
        @filtered_history = grep { $_->{job_type} eq $filters{job_type} } @filtered_history;
        $logger->debug( "Filtered by job_type: $filters{job_type}, found " . scalar(@filtered_history) . " entries" );
    }

    # Apply time range filter
    if ( $filters{time_range} ) {
        if ( $cutoff_time > 0 ) {
            @filtered_history = grep { $_->{start_time} >= $cutoff_time } @filtered_history;
            $logger->debug( "Filtered by time_range: $filters{time_range}, found " . scalar(@filtered_history) . " entries" );
        }
    }

    # Apply custom date range filter (start_date and end_date as Unix timestamps)
    if ( $filters{start_date} ) {
        @filtered_history = grep { $_->{start_time} >= $filters{start_date} } @filtered_history;
        $logger->debug( "Filtered by start_date: $filters{start_date}, found " . scalar(@filtered_history) . " entries" );
    }

    if ( $filters{end_date} ) {
        @filtered_history = grep { $_->{start_time} <= $filters{end_date} } @filtered_history;
        $logger->debug( "Filtered by end_date: $filters{end_date}, found " . scalar(@filtered_history) . " entries" );
    }

    # Convert timestamps to readable format for display
    for my $entry (@filtered_history) {
        $entry->{start_time_formatted} = scalar localtime( $entry->{start_time} );
    }

    # Sort by start_time (most recent first)
    @filtered_history = sort { $b->{start_time} <=> $a->{start_time} } @filtered_history;

    $logger->debug( "Returning filtered backup run history with " . scalar(@filtered_history) . " entries" );
    Whostmgr::API::1::Utils::set_metadata_ok($metadata);
    return {
        history         => \@filtered_history,
        total_entries   => scalar(@filtered_history),
        filters_applied => \%filters
    };
}

sub _get_disk_usage_for_path {
    my ($backup_path) = @_;

    # Ensure that df output is consistent, regardless of locale settings
    local $ENV{LC_ALL} = 'C';

    # Get filesystem stats for the backup path using 1-byte blocks
    my $df_output = `df -B 1 '$backup_path' 2>/dev/null | tail -n 1`;

    return $df_output;
}

=head2 get_backup_destinations

Retrieves all configured backup destinations with usage statistics.

=head3 Parameters

None required.

=head3 Returns

A hash reference containing C<destinations> array with destination details including
storage type, location, and capacity information.

=head3 Example

    my $result = get_backup_destinations({}, $metadata, $api_info);

=cut

sub get_backup_destinations {
    my ( $args, $metadata, $api_info_hr ) = @_;

    my @destinations;

    eval {
        require Whostmgr::CometBackup::CometBackupAPI;

        # Get user profile from Comet server which includes all configured destinations
        my $profile_result = Whostmgr::CometBackup::CometBackupAPI::make_server_request(
            "user/web/get-user-profile-and-hash",
            {}
        );

        unless ( $profile_result && $profile_result->{Status} == 200 && $profile_result->{Profile} ) {
            die "Failed to retrieve user profile from Comet server";
        }

        my $profile = $profile_result->{Profile};
        my $dests   = $profile->{Destinations} || {};

        # Get destination metadata from local database to include names
        my %local_metadata;
        eval {
            require Whostmgr::CometBackup::BackupJobs;
            my $jobs         = Whostmgr::CometBackup::BackupJobs->new();
            my @all_metadata = $jobs->get_destination_metadata();
            %local_metadata = map { $_->{destination_id} => $_ } @all_metadata;
        };
        if ($@) {
            $logger->warn("Failed to load destination metadata from local database: $@");

            # Continue without local metadata
        }

        for my $dest_id ( keys %$dests ) {
            my $dest = $dests->{$dest_id};

            # Determine storage type from DestinationType
            # 1000 = Comet Storage (cloud)
            # 1002 = Local Copy
            # Other types: S3, B2, FTP, etc.
            my $dest_type           = $dest->{DestinationType} || 0;
            my $storage_type        = 'unknown';
            my $comet_description   = $dest->{Description} || 'Unnamed Storage';
            my $storage_name        = $comet_description;                          # Default to Comet description
            my $storage_description = $comet_description;                          # Keep original Comet description
            my $recommended         = 0;
            my $storage_path        = '';
            my $size_used           = 0;
            my $size_total          = 0;

            # Use local database metadata if available
            if ( $local_metadata{$dest_id} ) {
                my $local = $local_metadata{$dest_id};

                # Prefer local name over Comet description for display
                if ( $local->{name} ) {
                    $storage_name = $local->{name};
                }

                # Use local description if available, otherwise keep Comet description
                if ( $local->{description} ) {
                    $storage_description = $local->{description};
                }
            }

            if ( $dest_type == 1000 || $dest_type == 1003 ) {

                # Comet Storage (cloud)
                $storage_type = 'cloud';
                $recommended  = 1;

                # Get size from Statistics if available
                if ( $dest->{Statistics} && $dest->{Statistics}->{ClientProvidedSize} ) {
                    $size_used = $dest->{Statistics}->{ClientProvidedSize}->{Size} || 0;
                }

                # For cloud storage, we might not have a total limit or it might be StorageLimitBytes
                if ( $dest->{StorageLimitEnabled} && $dest->{StorageLimitBytes} ) {
                    $size_total = $dest->{StorageLimitBytes};
                }
            }
            elsif ( $dest_type == 1002 ) {

                # Local Copy destination
                $storage_type = 'local';
                $storage_path = $dest->{LocalcopyPath} || '';

                # Get real disk usage for local path from filesystem
                # Note: Statistics.ClientProvidedSize.Size is typically 0 for local destinations
                # so we must use df to get accurate filesystem usage
                if ($storage_path) {
                    eval {
                        my $df_output = _get_disk_usage_for_path($storage_path);

                        # Parse df output: filesystem total used available use% mountpoint
                        if ( $df_output && $df_output =~ /\s+(\d+)\s+(\d+)\s+(\d+)\s+\d+%/ ) {

                            # Explicitly convert to integers to ensure proper type
                            $size_total = int($1);
                            $size_used  = int($2);
                        }
                    };
                    if ($@) {
                        $logger->warn("Failed to get disk usage for local path $storage_path: $@");
                    }
                }
            }
            else {
                # Other destination types (S3, B2, FTP, SFTP, etc.)
                # Determine type based on which fields are populated
                if ( $dest->{S3Server} && $dest->{S3BucketName} ) {
                    $storage_type = 's3';
                }
                elsif ( $dest->{B2}->{AccountID} ) {
                    $storage_type = 'b2';
                }
                elsif ( $dest->{FTPServer} ) {
                    $storage_type = 'ftp';
                }
                elsif ( $dest->{SFTPServer} ) {
                    $storage_type = 'sftp';
                }
                elsif ( $dest->{Swift}->{Username} ) {
                    $storage_type = 'swift';
                }
                elsif ( $dest->{WebDav}->{URL} ) {
                    $storage_type = 'webdav';
                }
                elsif ( $dest->{Storj}->{SatelliteAddress} ) {
                    $storage_type = 'storj';
                }
                elsif ( $dest->{SMB}->{SMBServer} ) {
                    $storage_type = 'smb';
                }

                # Get size from Statistics if available
                if ( $dest->{Statistics} && $dest->{Statistics}->{ClientProvidedSize} ) {
                    $size_used = $dest->{Statistics}->{ClientProvidedSize}->{Size} || 0;
                }

                if ( $dest->{StorageLimitEnabled} && $dest->{StorageLimitBytes} ) {
                    $size_total = $dest->{StorageLimitBytes};
                }
            }

            my $dest_data = {
                id                  => $dest_id,
                storage_name        => $storage_name,           # Local name if available, otherwise Comet description
                storage_description => $storage_description,    # Local description if available, otherwise Comet description
                storage_type        => $storage_type,
                size_used           => $size_used,
                size_total          => $size_total,
                recommended         => $recommended,
            };

            # Add storage_path for local destinations
            $dest_data->{storage_path} = $storage_path if $storage_path;

            push @destinations, $dest_data;
        }
    };

    if ($@) {
        $logger->error("Failed to get destinations from Comet: $@");
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "Failed to retrieve backup destinations: $@" );
        return { destinations => [] };
    }

    # Ensure recovery_data Protected Item is up-to-date with current destinations
    # This runs asynchronously to not block the API response
    eval {
        require Whostmgr::CometBackup::Setup;
        Whostmgr::CometBackup::Setup::ensure_recovery_data_protected_item();
    };
    if ($@) {
        $logger->warn("Failed to update recovery_data Protected Item: $@");

        # Don't fail the API call, just log the warning
    }

    Whostmgr::API::1::Utils::set_metadata_ok($metadata);
    return {
        destinations => \@destinations,
    };
}

=head2 get_backup_destination_regions

Returns a list of available backup destination regions for cloud storage.

=head3 Parameters

None required.

=head3 Returns

A hash reference containing C<regions> array with region details including
id, name, and regionCode.

=head3 Example

    my $result = get_backup_destination_regions({}, $metadata, $api_info);

=cut

sub get_backup_destination_regions {
    my ( $args, $metadata, $api_info_hr ) = @_;

    my @regions;

    eval {
        # Get allowed region hints from Comet Access Portal
        my $region_hints = get_allowed_region_hints( {}, $metadata, $api_info_hr );

        unless ( $region_hints && ref $region_hints eq 'HASH' && $region_hints->{storage} ) {
            die "Failed to retrieve storage region hints";
        }

        # Transform storage regions to the format expected by the API
        my $storage_regions = $region_hints->{storage};
        for my $region (@$storage_regions) {
            push @regions, {
                id         => $region->{id},
                name       => $region->{description},
                regionCode => $region->{id},
                provider   => $region->{provider},
                pingtime   => 0,
            };
        }
    };

    if ($@) {
        $logger->error("Failed to get destination regions: $@");
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "Failed to retrieve backup destination regions: $@" );
        return { regions => [] };
    }

    Whostmgr::API::1::Utils::set_metadata_ok($metadata);
    return {
        regions => \@regions,
    };
}

=head2 create_local_storage

Creates a local storage destination for Comet Backup.

=head3 Parameters

=over 4

=item * C<local_path> - (Required) The filesystem path for the local storage

=item * C<name> - (Required) The name for the storage destination

=item * C<description> - (Optional) A description for the storage destination (defaults to name)

=back

=head3 Returns

A hash reference containing:

=over 4

=item * C<created> - Boolean indicating success (1) or failure (0)

=item * C<destination_id> - The UUID of the created storage destination (on success)

=item * C<name> - The name of the storage destination (on success)

=item * C<local_path> - The path where storage was created (on success)

=back

=head3 Example

    my $result = create_local_storage({
        json => encode_json({
            local_path => '/backup/storage',
            name => 'Local Backup Storage',
            description => 'Primary local backup destination'
        })
    }, $metadata, $api_info);

=cut

sub create_local_storage {
    my ( $args, $metadata, $api_info_hr ) = @_;

    my ( $status, $decoded_args ) = _decode_json_args( $metadata, $args );
    if ( $status < 1 ) {
        return;
    }

    my $local_path  = $decoded_args->{local_path};
    my $name        = $decoded_args->{name};
    my $description = $decoded_args->{description} || $name;

    # Sanitize text input fields to prevent injection attacks
    eval {
        $name        = Whostmgr::CometBackup::BackupJobs::sanitize_text_input( $name,        'name' )        if defined $name;
        $description = Whostmgr::CometBackup::BackupJobs::sanitize_text_input( $description, 'description' ) if defined $description;
    };
    if ($@) {
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, $@ );
        return { created => 0 };
    }

    # Validate required parameters
    unless ( defined $local_path && length $local_path ) {
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "local_path parameter required" );
        return { created => 0 };
    }

    unless ( defined $name && length $name ) {
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "name parameter required" );
        return { created => 0 };
    }

    $logger->debug("Creating local storage destination: $name at $local_path");

    # Ensure the local path exists
    eval {
        unless ( -e $local_path ) {
            require Cpanel::Mkdir;
            Cpanel::Mkdir::ensure_directory_existence_and_mode( $local_path, 0755 );
            $logger->debug("Created local storage directory: $local_path");
        }
        elsif ( !-d $local_path ) {
            die "Path exists but is not a directory: $local_path";
        }
    };

    if ($@) {
        my $error = "Failed to ensure local storage path exists: $@";
        $logger->error($error);
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, $error );
        return { created => 0 };
    }

    my $destination_id;

    eval {
        require Whostmgr::CometBackup::CometBackupAPI;
        require Data::UUID;

        # Generate a UUID for the destination
        my $ug = Data::UUID->new();
        $destination_id = lc( $ug->create_str() );

        $logger->debug("Generated destination UUID: $destination_id");

        # Get current user profile
        my $profile_response = Whostmgr::CometBackup::CometBackupAPI::make_server_request(
            "user/web/get-user-profile-and-hash",
            {}
        );

        unless ( $profile_response && $profile_response->{Status} == 200 && $profile_response->{Profile} ) {
            die "Failed to retrieve user profile from Comet server";
        }

        my $profile_hash = $profile_response->{ProfileHash};
        my $user_profile = $profile_response->{Profile};

        $logger->debug("Retrieved user profile with hash: $profile_hash");

        # Add new destination to the profile
        my $destinations = $user_profile->{Destinations} ||= {};

        my $current_time = time();

        $destinations->{$destination_id} = {
            Description     => $description,
            CreateTime      => $current_time,
            ModifyTime      => $current_time,
            DestinationType => 1002,            # Local Copy
            LocalcopyPath   => $local_path,
        };

        $logger->info( "Added local storage destination to profile", { destination_id => $destination_id, path => $local_path } );

        # Update user profile on server
        my $profile_json = Cpanel::JSON::Dump($user_profile);

        my $result = Whostmgr::CometBackup::CometBackupAPI::make_server_request(
            "user/web/set-profile-hash",
            {
                ProfileData => $profile_json,
                ProfileHash => $profile_hash,
            }
        );

        unless ( $result && $result->{Status} == 200 ) {
            my $error_msg = $result->{Message} || 'Unknown error';
            die "Failed to update user profile: $error_msg (Status: " . ( $result->{Status} || 'unknown' ) . ")";
        }

        $logger->debug("Successfully created local storage destination on Comet server");

        # Save destination metadata to local database
        eval {
            require Whostmgr::CometBackup::BackupJobs;
            my $jobs = Whostmgr::CometBackup::BackupJobs->new();
            my $dbh  = $jobs->_get_database_handle();

            my $sth = $dbh->prepare(
                q{
                INSERT INTO destinations (
                    destination_id, name, description, destination_type,
                    local_path, create_time, modify_time
                ) VALUES (?, ?, ?, ?, ?, ?, ?)
            }
            );

            $sth->execute(
                $destination_id,
                $name,
                $description,
                1002,    # DestinationType for Local Copy
                $local_path,
                $current_time,
                $current_time
            );

            $logger->debug("Saved destination metadata to local database: $destination_id");
        };

        if ($@) {
            $logger->warn("Failed to save destination metadata to local database: $@");

            # Don't fail the entire operation - the destination is already created on Comet server
        }
    };

    if ($@) {
        my $error = "Failed to create local storage destination: $@";
        $logger->error($error);
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, $error );
        return { created => 0 };
    }

    Whostmgr::API::1::Utils::set_metadata_ok($metadata);
    return {
        created        => 1,
        destination_id => $destination_id,
    };
}

=head2 delete_local_storage

Deletes a local storage destination from Comet Backup.

=head3 Parameters

JSON object containing:

=over 4

=item * C<destination_id> - (Required) The UUID of the storage destination to delete

=item * C<remove_directory> - (Optional) Whether to remove the local backup directory (default: 0)

=back

=head3 Returns

A hash reference containing:

=over 4

=item * C<deleted> - Boolean indicating success (1) or failure (0)

=item * C<destination_id> - The UUID of the deleted storage destination (on success)

=item * C<directory_removed> - Boolean indicating if the directory was removed (when remove_directory is 1)

=back

=head3 Example

    my $result = delete_local_storage({
        json => encode_json({
            destination_id => '550e8400-e29b-41d4-a716-446655440000',
            remove_directory => 1
        })
    }, $metadata, $api_info);

=cut

sub delete_local_storage {
    my ( $args, $metadata, $api_info_hr ) = @_;

    my ( $status, $decoded_args ) = _decode_json_args( $metadata, $args );
    if ( $status < 1 ) {
        return;
    }

    my $destination_id   = $decoded_args->{destination_id};
    my $remove_directory = $decoded_args->{remove_directory} || 0;

    # Validate required parameters
    unless ( defined $destination_id && length $destination_id ) {
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "destination_id parameter required" );
        return { deleted => 0 };
    }

    $logger->debug("Deleting local storage destination: $destination_id");

    my $directory_removed = 0;
    my $local_path;

    eval {
        require Whostmgr::CometBackup::CometBackupAPI;

        # Get current user profile with hash
        my $profile_response = Whostmgr::CometBackup::CometBackupAPI::make_server_request(
            "user/web/get-user-profile-and-hash",
            {}
        );

        unless ( $profile_response && $profile_response->{Status} == 200 && $profile_response->{Profile} ) {
            die "Failed to retrieve user profile from Comet server";
        }

        my $profile_hash = $profile_response->{ProfileHash};
        my $user_profile = $profile_response->{Profile};

        $logger->debug("Retrieved user profile with hash: $profile_hash");

        # Check if destination exists
        my $destinations = $user_profile->{Destinations} || {};
        unless ( exists $destinations->{$destination_id} ) {
            die "Storage destination not found: $destination_id";
        }

        # Store local path before removal for optional directory deletion
        $local_path = $destinations->{$destination_id}->{LocalcopyPath};

        # If policy prevents deletion, temporarily disable it to allow this administrative operation
        my $policy_was_preventing_deletion = 0;
        if ( $user_profile->{Policy} && $user_profile->{Policy}{PreventDeleteStorageVault} ) {
            $logger->debug("Policy PreventDeleteStorageVault is enabled, disabling it to allow deletion");
            $user_profile->{Policy}{PreventDeleteStorageVault} = Cpanel::JSON::false;
            $policy_was_preventing_deletion = 1;
        }

        # Remove destination from profile
        delete $destinations->{$destination_id};

        $logger->info("Removed local storage destination from profile: $destination_id");

        # Update user profile on server
        my $profile_json = Cpanel::JSON::Dump($user_profile);

        # Debug: Log the hash we're about to send
        $logger->debug("DEBUG: About to call set-profile-hash with ProfileHash: $profile_hash");
        $logger->debug( "DEBUG: Number of destinations after removal: " . scalar( keys %$destinations ) );

        my $result = Whostmgr::CometBackup::CometBackupAPI::make_server_request(
            "user/web/set-profile-hash",
            {
                ProfileData => $profile_json,
                ProfileHash => $profile_hash,
            }
        );

        $logger->debug( "DEBUG: set-profile-hash returned: Status=" . ( $result->{Status} || 'UNDEFINED' ) . ", Message=" . ( $result->{Message} || 'NO MESSAGE' ) );

        unless ( $result && $result->{Status} == 200 ) {
            my $error_msg = $result->{Message} || 'Unknown error';
            die "Failed to update user profile: $error_msg (Status: " . ( $result->{Status} || 'unknown' ) . ")";
        }

        $logger->info("Successfully deleted local storage destination from Comet server");

        # Wait a moment for the server to propagate the change
        sleep 1;

        # Verify the deletion by fetching the profile again
        $logger->debug("DEBUG: Fetching profile again to verify deletion...");
        my $verify_response = Whostmgr::CometBackup::CometBackupAPI::make_server_request(
            "user/web/get-user-profile-and-hash",
            {}
        );
        if ( $verify_response && $verify_response->{Profile} ) {
            my $verify_destinations = $verify_response->{Profile}->{Destinations} || {};
            my $verify_hash         = $verify_response->{ProfileHash};

            $logger->debug("DEBUG: Verification profile hash: $verify_hash");
            $logger->debug( "DEBUG: Verification destination count: " . scalar( keys %$verify_destinations ) );

            if ( exists $verify_destinations->{$destination_id} ) {
                my $dest_info = $verify_destinations->{$destination_id};
                $logger->warn("WARNING: Destination $destination_id still exists in profile after deletion attempt!");
                $logger->debug( "DEBUG: Destination type: " . ( $dest_info->{DestinationType} || 'UNKNOWN' ) );
                $logger->debug( "DEBUG: Destination path: " . ( $dest_info->{LocalcopyPath}   || 'UNKNOWN' ) );
                $logger->debug("DEBUG: This may indicate a server-side issue or the destination is being re-added by another process");
            }
            else {
                $logger->info("Destination successfully removed from profile");
            }

            # Log all destination IDs for debugging
            if ( keys %$verify_destinations ) {
                my @dest_ids = keys %$verify_destinations;
                $logger->debug( "DEBUG: Remaining destination IDs: " . join( ', ', @dest_ids ) );
            }
        }

        # Remove destination metadata from local database
        eval {
            require Whostmgr::CometBackup::BackupJobs;
            my $jobs = Whostmgr::CometBackup::BackupJobs->new();
            my $dbh  = $jobs->_get_database_handle();

            my $sth = $dbh->prepare('DELETE FROM destinations WHERE destination_id = ?');
            $sth->execute($destination_id);

            $logger->debug("Removed destination metadata from local database: $destination_id");
        };

        if ($@) {
            $logger->warn("Failed to remove destination metadata from local database: $@");

            # Don't fail the entire operation - the destination is already deleted from Comet server
        }

        # Optionally remove the local directory
        if ( $remove_directory && defined $local_path && length $local_path && -d $local_path ) {
            require File::Path;

            eval {
                File::Path::remove_tree( $local_path, { safe => 1, error => \my $errors } );

                if (@$errors) {
                    my @error_msgs;
                    for my $diag (@$errors) {
                        my ( $file, $message ) = %$diag;
                        if ( $file eq '' ) {
                            push @error_msgs, "General error: $message";
                        }
                        else {
                            push @error_msgs, "Problem unlinking $file: $message";
                        }
                    }
                    die "Failed to remove directory: " . join( "; ", @error_msgs );
                }

                $directory_removed = 1;
                $logger->debug("Successfully removed local storage directory: $local_path");
            };

            if ($@) {
                $logger->warn("Failed to remove local storage directory: $@");

                # Don't fail the entire operation if directory removal fails
            }
        }
    };

    if ($@) {
        my $error = "Failed to delete local storage destination: $@";
        $logger->error($error);
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, $error );
        return { deleted => 0 };
    }

    Whostmgr::API::1::Utils::set_metadata_ok($metadata);
    return {
        deleted           => 1,
        destination_id    => $destination_id,
        directory_removed => $directory_removed,
    };
}

# Returns list of accounts available for restore
# mode: 'single' - users with any type of backup available
# mode: 'multi' - users with all three protected items (homedir, databases, panel_config) available

=head2 list_accounts_for_restore

Lists user accounts available for restore operations with their backup status.

=head3 Parameters

None required.

=head3 Returns

An array reference of user account objects with backup availability information.

=head3 Example

    my $result = list_accounts_for_restore({}, $metadata, $api_info);

=cut

sub list_accounts_for_restore {
    my ( $args, $metadata, $api_info_hr ) = @_;

    my ( $status, $decoded_args ) = _decode_json_args( $metadata, $args );
    if ( $status < 1 ) {
        return;
    }

    my $mode = $decoded_args->{mode} || 'single';

    unless ( $mode eq 'single' || $mode eq 'multi' ) {
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "mode parameter must be 'single' or 'multi'" );
        return { accounts => [] };
    }

    $logger->debug("Listing accounts for restore in mode: $mode");

    # Check if database exists before attempting to query it
    # This prevents creating an empty database file on first API call
    unless ( -f $Whostmgr::CometBackup::Constants::RUN_LOG_DB ) {
        $logger->debug("Backup run history database does not exist yet, returning empty accounts list");
        Whostmgr::API::1::Utils::set_metadata_ok($metadata);
        return {
            accounts => [],
            mode     => $mode,
        };
    }

    my @all_accounts_with_backups;

    eval {
        require Whostmgr::CometBackup::BackupRunLogs;
        require Cpanel::Config::Users;

        # Get list of cPanel users to match against
        my %cpanel_users = map { $_ => 1 } Cpanel::Config::Users::getcpusers();

        # Get all restorable items from the local database
        my $logs  = Whostmgr::CometBackup::BackupRunLogs->new();
        my @items = $logs->get_restorable_items( limit => 1000 );

        # Group items by system_user
        my %user_backups;

        for my $item (@items) {
            my $system_user = $item->{system_user};
            next unless $system_user;

            # Initialize user entry if not exists
            unless ( exists $user_backups{$system_user} ) {
                $user_backups{$system_user} = {
                    username         => $system_user,
                    user_type        => 'standard',
                    contact_address  => '',
                    existing_account => exists $cpanel_users{$system_user} ? 1 : 0,
                    protected_items  => [],
                };
            }

            # Determine protected item type from protected_item_type
            my @items     = @{ $user_backups{$system_user}->{protected_items} };
            my $item_type = $item->{protected_item_type};

            # Map database protected_item_type to our API types
            # Note: Each item type maps to a single API type
            # Full backup determination is based on job configuration, not item types
            my $mapped_type;
            if ( $item_type =~ /^pkgacct_program_output$/i ) {

                # pkgacct represents only panel_config (account metadata)
                $mapped_type = 'panel_config';
            }
            elsif ( $item_type =~ /homedir|home_?dir/i ) {
                $mapped_type = 'homedir';
            }
            elsif ( $item_type =~ /databases|database?_program_output/i ) {
                $mapped_type = 'databases';
            }

            # Add mapped type to protected items if not already present
            if ( $mapped_type && !grep { $_ eq $mapped_type } @items ) {
                push @items, $mapped_type;
            }
            $user_backups{$system_user}->{protected_items} = \@items;

            # Store job_uuid for later job configuration lookup
            $user_backups{$system_user}->{job_uuid} ||= $item->{job_uuid};
        }

        @all_accounts_with_backups = values %user_backups;
    };

    if ($@) {

        # If database doesn't exist or table is missing, return empty accounts list
        if ( $@ =~ /no such table/i || $@ =~ /unable to open database/i ) {
            $logger->debug("Backup run history database not yet initialized, returning empty accounts list");
            Whostmgr::API::1::Utils::set_metadata_ok($metadata);
            return {
                accounts => [],
                mode     => $mode,
            };
        }

        $logger->error("Error listing accounts for restore: $@");
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "Failed to list accounts: $@" );
        return { accounts => [] };
    }

    # Filter based on mode
    my @result_accounts;

    eval {
        if ( $mode eq 'multi' ) {

            # multi mode: only users whose backups come from full backup jobs
            # A full backup job has: panel_config=1 AND databases defined AND homedir defined
            require Whostmgr::CometBackup::BackupJobs;
            my $jobs_manager = Whostmgr::CometBackup::BackupJobs->new();

            $logger->debug( "DEBUG multi-mode: Checking " . scalar(@all_accounts_with_backups) . " accounts with backups" );
            $logger->debug( "DEBUG multi-mode: Account list: " . join( ', ', map { $_->{username} } @all_accounts_with_backups ) );

            @result_accounts = grep {
                my $user     = $_;
                my $job_uuid = $user->{job_uuid};

                $logger->debug( "DEBUG multi-mode: === Checking user $user->{username} with job_uuid: " . ( $job_uuid || 'NONE' ) . " ===" );

                # Check if this backup comes from a full backup job
                unless ($job_uuid) {
                    $logger->debug("DEBUG multi-mode: User $user->{username} has no job_uuid, skipping");
                    0;    # Return false to grep, don't use 'return' which exits the function
                }
                else {
                    my $is_full_backup = 0;
                    eval {
                        # get_backup_jobs returns an array, pass job_uuid as arrayref
                        my @jobs = $jobs_manager->get_backup_jobs( [$job_uuid] );

                        $logger->debug( "DEBUG multi-mode: get_backup_jobs returned " . scalar(@jobs) . " jobs for $job_uuid" );

                        if ( @jobs && $jobs[0] && $jobs[0]->{backup_options} ) {
                            my $opts = $jobs[0]->{backup_options};

                            $logger->debug( "DEBUG multi-mode: Job $job_uuid backup_options: " . Dumper($opts) );

                            # Full backup requires all three components:
                            # 1. panel_config must be enabled
                            # 2. databases must be defined (with mysql or postgresql)
                            # 3. homedir must be defined
                            my $has_panel_config = $opts->{panel_config};
                            my $has_databases =
                                 $opts->{databases}
                              && ref( $opts->{databases} ) eq 'HASH'
                              && ( $opts->{databases}->{mysql} || $opts->{databases}->{postgresql} );
                            my $has_homedir = exists $opts->{homedir};

                            $logger->debug( "DEBUG multi-mode: Job $job_uuid has_panel_config=" . ( $has_panel_config ? 1 : 0 ) . " has_databases=" . ( $has_databases ? 1 : 0 ) . " has_homedir=" . ( $has_homedir ? 1 : 0 ) );

                            $is_full_backup = ( $has_panel_config && $has_databases && $has_homedir ) ? 1 : 0;

                            $logger->debug("DEBUG multi-mode: Job $job_uuid is_full_backup=$is_full_backup");
                        }
                        else {
                            $logger->debug("DEBUG multi-mode: Job $job_uuid has no backup_options or empty job list");
                        }
                        1;    # ensure eval returns true if no exception
                    } or do {
                        my $eval_error = $@ || 'unknown eval error';
                        $logger->debug("DEBUG multi-mode: EXCEPTION checking job $job_uuid for user $user->{username}: $eval_error");
                    };

                    $logger->debug("DEBUG multi-mode: User $user->{username} is_full_backup=$is_full_backup, result: $is_full_backup");
                    $is_full_backup;    # Return the value to grep (not using 'return')
                }
            } @all_accounts_with_backups;

            $logger->debug( "DEBUG multi-mode: Filtered to " . scalar(@result_accounts) . " accounts with full backups" );
            $logger->debug( "DEBUG multi-mode: Result accounts: " . join( ', ', map { $_->{username} } @result_accounts ) );
        }
        else {
            # single mode: all users with any type of backup available
            @result_accounts = grep {
                my @pis = @{ $_->{protected_items} || [] };
                scalar(@pis) > 0
            } @all_accounts_with_backups;
        }
    };

    if ($@) {
        $logger->error("Error filtering accounts for mode $mode: $@");
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "Failed to filter accounts: $@" );
        return { accounts => [] };
    }

    # Clean up internal fields before returning
    for my $account (@result_accounts) {
        delete $account->{job_uuid};
    }

    # Sort accounts alphabetically by username
    @result_accounts = sort { lc( $a->{username} ) cmp lc( $b->{username} ) } @result_accounts;

    $logger->debug( "Returning " . scalar(@result_accounts) . " accounts for mode: $mode" );
    Whostmgr::API::1::Utils::set_metadata_ok($metadata);
    return {
        accounts => \@result_accounts,
        mode     => $mode,
    };
}

# Runs a restore job with specified options

=head2 run_restore_job

Initiates a restore operation for one or more user accounts.

=head3 Parameters

JSON object containing:

=over 4

=item * C<users> - Array of usernames to restore

=item * C<restore_type> - Type of restore ('full' or 'custom')

=item * C<snapshot_ids> - Hash of snapshot IDs keyed by protected_item_type (panel_config, homedir, databases)

=item * C<source_ids> - Hash of source IDs keyed by protected_item_type (panel_config, homedir, databases)

=item * C<destination_ids> - Hash of destination IDs keyed by protected_item_type (panel_config, homedir, databases)

=item * C<options> - Additional restore options (can be hash of options per protected_item_type or flat)

=back

=head3 Returns

A hash reference containing C<job_uuid> for tracking the restore operation.

=head3 Example

    my $result = run_restore_job({
        json => encode_json({
            users => ['user1'],
            restore_type => 'full'
        })
    }, $metadata, $api_info);

=cut

sub run_restore_job {
    my ( $args, $metadata, $api_info_hr ) = @_;

    my ( $status, $decoded_args ) = _decode_json_args( $metadata, $args );
    if ( $status < 1 ) {
        return;
    }

    my $restore_type = $decoded_args->{type};
    my $users        = $decoded_args->{users};

    unless ($restore_type) {
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "type parameter required (full or custom)" );
        return { job => {} };
    }

    unless ( $restore_type eq 'full' || $restore_type eq 'custom' ) {
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "type must be 'full' or 'custom'" );
        return { job => {} };
    }

    unless ( $users && ref $users eq 'ARRAY' && @$users > 0 ) {
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "users parameter required and must be a non-empty array" );
        return { job => {} };
    }

    # Get the authenticated user who initiated this restore
    my $initiated_by = $ENV{REMOTE_USER} || 'root';

    my $job_uuid;

    eval {
        require Whostmgr::CometBackup::CometBackupAPI;
        require Whostmgr::CometBackup::CometBackupAPI::ProtectedItemRestore;
        require Whostmgr::CometBackup::RestoreRunLogs;
        require Data::UUID;

        # Get the active connection ID from the dispatcher (not the device_id from config)
        # The connection ID is a temporary live connection identifier that changes when
        # the device reconnects, while device_id is a permanent device identifier
        my $connection_id = Whostmgr::CometBackup::CometBackupAPI::_get_user_connection_id();

        unless ($connection_id) {
            die "No live-connected device available for restore operation - device may be offline\n";
        }

        $logger->debug("Using connection ID $connection_id for restore operation");

        # Create restore manager for this connection
        my $restore_manager = Whostmgr::CometBackup::CometBackupAPI::ProtectedItemRestore->new( target_device => $connection_id );

        if ( $restore_type eq 'custom' ) {

            # Custom restore for single user
            if ( @$users > 1 ) {
                die "custom restore only supports single user";
            }

            # Get snapshot_ids, source_ids, and destination_ids - can be either hash (new format) or scalar (legacy format)
            my $snapshot_ids_param    = $decoded_args->{snapshot_ids}    || $decoded_args->{snapshot_id};
            my $source_ids_param      = $decoded_args->{source_ids}      || $decoded_args->{source_id};
            my $destination_ids_param = $decoded_args->{destination_ids} || $decoded_args->{destination_id};

            # Normalize to hash format
            my %snapshot_ids;
            my %source_ids;
            my %destination_ids;

            if ( ref $snapshot_ids_param eq 'HASH' ) {
                %snapshot_ids = %$snapshot_ids_param;
            }
            elsif ($snapshot_ids_param) {

                # Legacy format: single snapshot_id provided
                # Determine which protected item type based on source_id or assume homedir
                my $item_type = 'homedir';    # Default to homedir for backward compatibility
                $snapshot_ids{$item_type} = $snapshot_ids_param;
            }

            if ( ref $source_ids_param eq 'HASH' ) {
                %source_ids = %$source_ids_param;
            }
            elsif ($source_ids_param) {

                # Legacy format: single source_id provided
                my $item_type = 'homedir';    # Default to homedir for backward compatibility
                $source_ids{$item_type} = $source_ids_param;
            }

            if ( ref $destination_ids_param eq 'HASH' ) {
                %destination_ids = %$destination_ids_param;
            }
            elsif ($destination_ids_param) {

                # Legacy format: single destination_id provided
                my $item_type = 'homedir';    # Default to homedir for backward compatibility
                $destination_ids{$item_type} = $destination_ids_param;
            }

            # Validate we have at least one snapshot_id, source_id, and destination_id
            unless ( %snapshot_ids && %source_ids && %destination_ids ) {
                die "snapshot_ids, source_ids, and destination_ids required for custom restore (as hashes keyed by protected_item_type)";
            }

            my $options = $decoded_args->{options} || {};

            # Generate a proper UUID for the restore job
            require Data::UUID;
            my $ug = Data::UUID->new();
            $job_uuid = lc( $ug->create_str() );

            # Track the restore job in the database
            my $restore_logs = Whostmgr::CometBackup::RestoreRunLogs->new();
            $restore_logs->create_restore_job(
                restore_type => $restore_type,
                users        => $users,
                initiated_by => $initiated_by,
                run_uuid     => $job_uuid,
            );

            $logger->debug("Starting custom restore for user: $users->[0], UUID: $job_uuid");

            # Fork off the custom restore manager to run asynchronously
            require Cpanel::Daemonizer::Tiny;
            my $daemon_pid = Cpanel::Daemonizer::Tiny::run_as_daemon(
                sub {
                    # This code runs in the forked daemon process
                    eval {
                        _run_custom_restore_manager(
                            users           => $users,
                            connection_id   => $connection_id,
                            job_uuid        => $job_uuid,
                            snapshot_ids    => \%snapshot_ids,
                            source_ids      => \%source_ids,
                            destination_ids => \%destination_ids,
                            options         => $options,
                            paths           => $decoded_args->{paths},
                        );
                    };
                    if ($@) {
                        $logger->error("Custom restore manager failed: $@");
                    }
                }
            );

            $logger->debug("Forked custom restore manager process with PID: $daemon_pid, job UUID: $job_uuid");
        }
        else {
            # Full restore for one or more users
            # Fork off a manager process to handle the restore asynchronously
            require Cpanel::Daemonizer::Tiny;

            # Generate a proper UUID for the restore job
            my $ug = Data::UUID->new();
            $job_uuid = lc( $ug->create_str() );

            # Track the restore job in the database
            my $restore_logs = Whostmgr::CometBackup::RestoreRunLogs->new();
            $restore_logs->create_restore_job(
                restore_type => $restore_type,
                users        => $users,
                initiated_by => $initiated_by,
                run_uuid     => $job_uuid,
            );

            $logger->info( "Starting full restore for " . scalar(@$users) . " users with UUID: $job_uuid, initiated by: $initiated_by" );

            # Fork off the restore manager
            my $daemon_pid = Cpanel::Daemonizer::Tiny::run_as_daemon(
                sub {
                    # This code runs in the forked daemon process
                    eval {
                        _run_full_restore_manager(
                            users         => $users,
                            connection_id => $connection_id,
                            job_uuid      => $job_uuid,
                        );
                    };
                    if ($@) {
                        $logger->error("Full restore manager failed: $@");
                    }
                }
            );

            $logger->debug("Forked restore manager process with PID: $daemon_pid, job UUID: $job_uuid");
        }
    };

    if ($@) {
        my $error = $@;    # Capture $@ immediately before any operations that might clear it
        $logger->error("Error running restore job: $error");
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "Failed to run restore job: $error" );
        return { job => {} };
    }

    unless ($job_uuid) {
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "Failed to start restore job" );
        return { job => {} };
    }

    $logger->debug("Created restore job with UUID: $job_uuid");

    Whostmgr::API::1::Utils::set_metadata_ok($metadata);
    return {
        job => {
            job_uuid     => $job_uuid,
            restore_type => $restore_type,
            users        => $users,
            status       => 'running',
            initiated_by => $initiated_by,
        }
    };
}

# Returns a list of past restore operations with status, users restored, who initiated, timing, etc.

=head2 list_restore_history

Returns a list of past restore operations with their status, users, initiator, timing, and other details.

=head3 Parameters

All parameters are optional and provided via JSON:

=over 4

=item * C<status> - Filter by restore status (e.g., 'running', 'completed', 'failed')

=item * C<initiated_by> - Filter by the user who initiated the restore

=item * C<start_date> - Filter restores after this date (ISO 8601 format)

=item * C<end_date> - Filter restores before this date (ISO 8601 format)

=item * C<limit> - Maximum number of results to return (default: 50)

=back

=head3 Returns

A hash reference containing:

=over 4

=item * C<history> - Array of restore job records, each with details about the operation

=item * C<total_entries> - The number of restore records returned

=item * C<filters_applied> - Hash of the filters that were applied to the query

=back

=head3 Example

    my $result = list_restore_history({
        json => encode_json({
            status => 'completed',
            limit => 25
        })
    }, $metadata, $api_info);

=cut

sub list_restore_history {
    my ( $args, $metadata, $api_info_hr ) = @_;

    $logger->debug( "list_restore_history called with args: " . Dumper($args) );

    my $decoded_args;
    if ( ref $args eq 'HASH' && defined $args->{'json'} ) {
        ( my $status, $decoded_args ) = _decode_json_args( $metadata, $args );
        if ( $status < 1 ) {
            return;
        }
    }

    # Extract search and filter parameters from args
    my %filters = ();
    my $limit   = 50;    # Default limit
    if ( ref $decoded_args eq 'HASH' ) {
        $filters{status}       = $decoded_args->{status}       if $decoded_args->{status};
        $filters{initiated_by} = $decoded_args->{initiated_by} if $decoded_args->{initiated_by};
        $filters{start_date}   = $decoded_args->{start_date}   if $decoded_args->{start_date};
        $filters{end_date}     = $decoded_args->{end_date}     if $decoded_args->{end_date};
        $limit                 = $decoded_args->{limit}        if $decoded_args->{limit} && $decoded_args->{limit} > 0;
    }

    $logger->debug( "Filters applied: " . Dumper( \%filters ) );

    my @history;
    eval {
        require Whostmgr::CometBackup::RestoreRunLogs;

        my $logs = Whostmgr::CometBackup::RestoreRunLogs->new();

        @history = $logs->get_restore_history(
            limit        => $limit,
            status       => $filters{status},
            initiated_by => $filters{initiated_by},
            start_date   => $filters{start_date},
            end_date     => $filters{end_date},
        );
    };

    if ($@) {
        $logger->error("Error retrieving restore history: $@");
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "Failed to retrieve restore history: $@" );
        return { history => [], total_entries => 0 };
    }

    $logger->debug( "Returning restore history with " . scalar(@history) . " entries" );
    Whostmgr::API::1::Utils::set_metadata_ok($metadata);
    return {
        history         => \@history,
        total_entries   => scalar(@history),
        filters_applied => \%filters
    };
}

# Gets the log output for a restore job using our internal job_uuid
# This looks up the Comet job ID(s) associated with the restore and fetches their logs

=head2 get_restore_log

Retrieves log entries for a specific restore job, with support for pagination.

=head3 Parameters

=over 4

=item * C<job_uuid> - (Required) The UUID of the restore job

=item * C<n_line> - (Optional) Starting line number for pagination (default: 0)

=back

=head3 Returns

A hash reference containing:

=over 4

=item * C<log> - The log text as a string

=item * C<status> - The current status of the restore job

=item * C<entries> - Array of log entry objects with timestamp and message

=item * C<total_lines> - Total number of log lines available

=back

=head3 Example

    my $result = get_restore_log({
        json => encode_json({
            job_uuid => '550e8400-e29b-41d4-a716-446655440000',
            n_line => 0
        })
    }, $metadata, $api_info);

=cut

sub get_restore_log {
    my ( $args, $metadata, $api_info_hr ) = @_;

    my ( $status, $decoded_args ) = _decode_json_args( $metadata, $args );
    if ( $status < 1 ) {
        return;
    }

    my $job_uuid = $decoded_args->{job_uuid};
    my $n_line   = $decoded_args->{n_line} || 0;

    unless ($job_uuid) {
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "job_uuid parameter required" );
        return { log => '' };
    }

    $logger->debug("Fetching restore log for job: $job_uuid, starting at line: $n_line");

    my $log_text   = '';
    my $job_status = 'unknown';
    my @log_entries;
    my $total_lines = 0;

    eval {
        require Whostmgr::CometBackup::RestoreRunLogs;
        require Whostmgr::CometBackup::Constants;

        # First, look up our internal restore job
        my $restore_logs = Whostmgr::CometBackup::RestoreRunLogs->new();
        my $restore_job  = $restore_logs->get_restore_job($job_uuid);

        unless ($restore_job) {
            die "Restore job not found: $job_uuid";
        }

        $job_status = $restore_job->{status};

        # Read the text log file for this restore job (matching backup behavior)
        # We use job_uuid as both job_uuid and run_uuid since they're the same for restores
        my $log_file = Whostmgr::CometBackup::Constants::get_text_log_file_path( $job_uuid, $job_uuid );

        $logger->debug("Reading restore log from file: $log_file");

        if ( -f $log_file && -r $log_file ) {
            open( my $fh, '<', $log_file ) or die "Cannot open log file $log_file: $!";

            # Read all lines
            my @all_lines = <$fh>;
            close($fh);

            $total_lines = scalar(@all_lines);

            # Apply n_line offset for pagination
            my $start_line = $n_line;
            my $end_line   = $#all_lines;

            if ( $start_line > $end_line ) {

                # No lines to return
                $log_text = "";
            }
            else {
                my @selected_lines = @all_lines[ $start_line .. $end_line ];
                $log_text = join( "", @selected_lines );    # Lines already include newlines
            }

            $logger->debug("Successfully read restore log: $total_lines total lines, returning from line $start_line");
        }
        else {
            # Log file doesn't exist yet or not readable
            $logger->warn("Log file not found or not readable: $log_file");
            $log_text    = "Restore job is running. Log file will be available when the restore completes.";
            $total_lines = 1;
        }
    };

    if ($@) {
        $logger->error("Error fetching restore log: $@");
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "Failed to fetch restore log: $@" );
        return { log => '' };
    }

    my $job_result = 'unknown';
    if ( $log_text =~ m/Job\s+completed\s+\(([a-zA-Z]+)\)/ ) {
        $job_result = lc($1);
    }

    my $num_log_lines = ( $log_text =~ tr/\n// );
    my $response      = {
        log         => $log_text,
        job_uuid    => $job_uuid,
        status      => $job_status,
        result      => $job_result,
        start_line  => $n_line,
        end_line    => $n_line + $num_log_lines,
        total_lines => $total_lines || $num_log_lines,
    };

    $logger->debug("Returning log for job $job_uuid, status: $job_status");
    Whostmgr::API::1::Utils::set_metadata_ok($metadata);
    return $response;
}

# Lists recent backup job runs for a specific user

=head2 list_backups_for_user

Lists available backup snapshots for a specific user account.

=head3 Parameters

JSON object containing:

=over 4

=item * C<username> - (Required) Username to query backups for

=back

=head3 Returns

An array reference of backup snapshot objects with timestamps and details.

=head3 Example

    my $result = list_backups_for_user({
        json => encode_json({ username => 'user1' })
    }, $metadata, $api_info);

=cut

sub list_backups_for_user {
    my ( $args, $metadata, $api_info_hr ) = @_;

    my ( $status, $decoded_args ) = _decode_json_args( $metadata, $args );
    if ( $status < 1 ) {
        return;
    }

    my $username = $decoded_args->{username};

    unless ($username) {
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "username parameter required" );
        return { backups => [] };
    }

    $logger->debug("Listing backups for user: $username");

    # Check if database exists before attempting to query it
    # This prevents creating an empty database file on first API call
    unless ( -f $Whostmgr::CometBackup::Constants::RUN_LOG_DB ) {
        $logger->debug("Backup run history database does not exist yet, returning empty backups list");
        Whostmgr::API::1::Utils::set_metadata_ok($metadata);
        return {
            backups  => [],
            username => $username,
        };
    }

    my @backups = ();

    eval {
        require Whostmgr::CometBackup::BackupRunLogs;
        require Whostmgr::CometBackup::CometBackupAPI;
        require Whostmgr::CometBackup::BackupJobs;

        # Get restorable items from local database for this user
        # The database already contains SnapshotID and DestinationGUID from job properties
        my $logs  = Whostmgr::CometBackup::BackupRunLogs->new();
        my @items = $logs->get_restorable_items(
            system_user => $username,
            limit       => 30,
        );

        $logger->debug( "Retrieved restorable items from local database", { count => scalar(@items) } );

        # Get user profile to retrieve destination descriptions
        my $profile_result = Whostmgr::CometBackup::CometBackupAPI::get_user_profile_and_hash();
        my $destinations   = {};

        if ( $profile_result && $profile_result->{Profile} ) {
            my $profile_data = $profile_result->{Profile};
            $destinations = $profile_data->{Destinations} || {};
        }

        # Cache for backup jobs to avoid redundant lookups
        my %job_cache;
        my $jobs_db = Whostmgr::CometBackup::BackupJobs->new();

        # Transform database items into backup list format
        for my $item (@items) {

            # Get backup job configuration to determine what was included
            my $job_uuid       = $item->{job_uuid};
            my $backup_options = {};

            if ( $job_uuid && !exists $job_cache{$job_uuid} ) {
                eval {
                    my @job_data = $jobs_db->get_backup_jobs( [$job_uuid] );
                    $logger->debug( "Looking up backup job: $job_uuid, found=" . scalar(@job_data) );
                    if ( @job_data && $job_data[0] ) {

                        # get_backup_jobs() merges the decoded job_config JSON into the row
                        # so backup_options is directly accessible in job_data[0]
                        $backup_options = $job_data[0]->{backup_options} || {};
                        my $has_homedir    = exists $backup_options->{homedir}        ? 1 : 0;
                        my $has_email_acct = exists $backup_options->{email_accounts} ? 1 : 0;
                        my $opts_json      = Cpanel::JSON::Dump($backup_options);
                        $logger->debug("Retrieved backup_options for job $job_uuid: has_homedir=$has_homedir, has_email_acct=$has_email_acct, options=$opts_json");
                    }
                    else {
                        $logger->debug("No job data found for job_uuid: $job_uuid");
                    }
                    $job_cache{$job_uuid} = $backup_options;
                };
                if ($@) {
                    $logger->warn( "Failed to retrieve backup job configuration", { job_uuid => $job_uuid, error => $@ } );
                    $job_cache{$job_uuid} = {};
                }
            }
            elsif ( $job_uuid && exists $job_cache{$job_uuid} ) {
                $backup_options = $job_cache{$job_uuid};
                my $num_options = scalar( keys %$backup_options );
                $logger->debug("Using cached backup_options for job $job_uuid: has $num_options options");
            }
            else {
                $logger->warn( "No job_uuid available for item type=" . $item->{protected_item_type} . ", run_uuid=" . $item->{run_uuid} );
            }

            # Build protected_items as an object with details instead of array
            my %protected_items;
            my $item_type = $item->{protected_item_type};

            # Homedir item with email inclusion detection
            if ( $item_type =~ /homedir/i ) {
                my $include_email = 0;

                # Check if email_contents was configured in backup_options
                # Can be in either backup_options->{homedir}->{email_contents} (new structure)
                # or backup_options->{email_accounts}->{email_contents} (old structure)
                if ( $backup_options->{homedir} && ref( $backup_options->{homedir} ) eq 'HASH' ) {

                    # New structure: backup_options->{homedir}->{email_contents}
                    if ( $backup_options->{homedir}->{email_contents} ) {
                        $include_email = 1;
                    }
                }
                elsif ( $backup_options->{email_accounts} && ref( $backup_options->{email_accounts} ) eq 'HASH' ) {

                    # Old structure: backup_options->{email_accounts}->{email_contents}
                    if ( $backup_options->{email_accounts}->{email_contents} ) {
                        $include_email = 1;
                    }
                }

                my $num_opts        = scalar( keys %$backup_options );
                my $has_homedir_cfg = exists $backup_options->{homedir}                                             ? 1                                                     : 0;
                my $email_flag      = ( $backup_options->{homedir} && ref( $backup_options->{homedir} ) eq 'HASH' ) ? ( $backup_options->{homedir}->{email_contents} || 0 ) : 0;

                $logger->debug( "Determined email inclusion for homedir: job=$job_uuid, run=" . $item->{run_uuid} . ", include_email=$include_email, has_opts=$num_opts, has_homedir_cfg=$has_homedir_cfg, email_flag=$email_flag" );
                $protected_items{homedir} = {
                    include_email => $include_email,
                };
            }

            # Database items
            if ( $item_type =~ /database/i ) {
                $protected_items{databases} = 1;
            }

            # Panel configuration
            if ( $item_type =~ /pkgacct_program_output/i ) {
                $protected_items{panel_config} = 1;
            }

            # Full backup fallback
            if ( $item_type =~ /full|complete/i && scalar( keys %protected_items ) == 0 ) {
                $protected_items{full} = 1;
            }

            # Use the protected_item_type as fallback if no mapping matched
            unless ( keys %protected_items ) {
                $protected_items{$item_type} = 1;
            }

            # Determine backup status from run status
            my $status = 'success';
            if ( $item->{run_status} eq 'failed' ) {
                $status = 'failed';
            }
            elsif ( $item->{run_status} eq 'running' ) {
                $status = 'running';
            }

            # Get snapshot ID and destination ID from local database
            # These are already extracted from comet_job_properties JSON in BackupRunLogs
            my $source_id       = $item->{protected_item_uid};
            my $backup_job_uuid = $item->{run_uuid};
            my $snapshot_id     = $item->{comet_snapshot_id}    || '';
            my $destination_id  = $item->{comet_destination_id} || '';

            $logger->debug(
                "Using snapshot data from local database",
                {
                    backup_job_uuid => $backup_job_uuid,
                    snapshot_id     => $snapshot_id    || '[none]',
                    destination_id  => $destination_id || '[none]',
                    source_id       => $source_id      || '[none]',
                }
            );

            # Fallback: if no snapshot_id yet, use backup_job_uuid
            unless ($snapshot_id) {
                $snapshot_id = $backup_job_uuid;
                $logger->debug( "Using backup_job_uuid as snapshot_id fallback", { backup_job_uuid => $backup_job_uuid } );
            }

            # Get human-readable destination name from profile
            my $destination_name = 'Unknown Storage';
            if ( $destination_id && $destinations->{$destination_id} ) {
                $destination_name = $destinations->{$destination_id}->{Description} || 'Unknown Storage';
            }
            elsif ( !$destination_id ) {

                # Fallback: if still no destination_id, use first available destination
                my $destination_ids_ref = Whostmgr::CometBackup::CometBackupAPI::get_destination_ids();
                if ( $destination_ids_ref && ref($destination_ids_ref) eq 'ARRAY' && @$destination_ids_ref ) {
                    $destination_id = $destination_ids_ref->[0];
                    if ( $destinations->{$destination_id} ) {
                        $destination_name = $destinations->{$destination_id}->{Description} || 'Unknown Storage';
                    }
                    $logger->debug( "Using first available destination as fallback", { destination_id => $destination_id } );
                }
            }

            push @backups, {
                backup_run_uuid    => $item->{run_uuid},
                backup_job_uuid    => $job_uuid             || '',
                comet_job_id       => $item->{comet_job_id} || '',
                snapshot_id        => $snapshot_id,
                source_id          => $source_id,
                destination_id     => $destination_id,
                datetime_started   => $item->{start_time}           || 0,
                datetime_formatted => $item->{start_time_formatted} || scalar localtime( $item->{start_time} || 0 ),
                backup_type        => 'scheduled',                        # Could enhance later with job info
                backup_destination => $destination_name,
                protected_items    => \%protected_items,
                size               => $item->{size_formatted} || '0MB',
                status             => $status,
            };
        }
    };

    if ($@) {

        # If database doesn't exist or table is missing, return empty backups list
        if ( $@ =~ /no such table/i || $@ =~ /unable to open database/i ) {
            $logger->debug("Backup run history database not yet initialized, returning empty backups list");
            Whostmgr::API::1::Utils::set_metadata_ok($metadata);
            return {
                backups  => [],
                username => $username,
            };
        }

        $logger->error("Error listing backups for user: $@");
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "Failed to list backups: $@" );
        return { backups => [] };
    }

    $logger->debug( "Returning " . scalar(@backups) . " backups for user: $username" );
    Whostmgr::API::1::Utils::set_metadata_ok($metadata);
    return {
        backups  => \@backups,
        username => $username,
        count    => scalar(@backups),
    };
}

use constant NEEDS_ROLE => {

    get_comet_backup_configuration_status => undef,
    initialize_comet_backup               => undef,
    register_server_with_comet_backup     => undef,
    disable_comet_backup                  => undef,
    enable_comet_backup                   => undef,
    create_backup_job                     => undef,
    get_backup_jobs                       => undef,
    delete_backup_job                     => undef,
    modify_backup_job                     => undef,
    disable_backup_job                    => undef,
    enable_backup_job                     => undef,
    duplicate_backup_job                  => undef,
    cancel_running_backup_job             => undef,
    run_backup_job                        => undef,
    download_backup_job_log               => undef,
    download_restore_job_log              => undef,
    list_backup_run_history               => undef,
    list_restore_history                  => undef,
    get_backup_destinations               => undef,
    get_backup_destination_regions        => undef,
    get_allowed_region_hints              => undef,
    get_server_credentials_from_cap       => undef,
    create_local_storage                  => undef,
    delete_local_storage                  => undef,
    list_accounts_for_restore             => undef,
    run_restore_job                       => undef,
    get_restore_log                       => undef,
    list_backups_for_user                 => undef,
    get_local_info                        => undef,
    download_credentials                  => undef,
    upload_credentials                    => undef,
    webpros_auth_status                   => undef,
    webpros_auth_start                    => undef,
    webpros_auth_callback                 => undef,
    webpros_auth_clear                    => undef,
};

sub APPLIST {
    return {
        initialize_comet_backup => [
            namespace => 'CometBackup',
            method    => 'initialize_comet_backup',
            argsmode  => 'strict',
        ],
        register_server_with_comet_backup => [
            namespace => 'CometBackup',
            method    => 'register_server_with_comet_backup',
            argsmode  => 'strict',
        ],
        disable_comet_backup => [
            namespace => 'CometBackup',
            method    => 'disable_comet_backup',
            argsmode  => 'strict',
        ],
        enable_comet_backup => [
            namespace => 'CometBackup',
            method    => 'enable_comet_backup',
            argsmode  => 'strict',
        ],
        get_comet_backup_configuration_status => [
            namespace => 'CometBackup',
            method    => 'get_comet_backup_configuration_status',
            argsmode  => 'strict',
        ],
        get_backup_jobs => [
            namespace => 'CometBackup',
            method    => 'get_backup_jobs',
            argsmode  => 'strict',
        ],
        create_backup_job => [
            namespace => 'CometBackup',
            method    => 'create_backup_job',
            argsmode  => 'strict',
        ],
        delete_backup_job => [
            namespace => 'CometBackup',
            method    => 'delete_backup_job',
            argsmode  => 'strict',
        ],
        disable_backup_job => [
            namespace => 'CometBackup',
            method    => 'disable_backup_job',
            argsmode  => 'strict',
        ],
        enable_backup_job => [
            namespace => 'CometBackup',
            method    => 'enable_backup_job',
            argsmode  => 'strict',
        ],
        duplicate_backup_job => [
            namespace => 'CometBackup',
            method    => 'duplicate_backup_job',
            argsmode  => 'strict',
        ],
        cancel_running_backup_job => [
            namespace => 'CometBackup',
            method    => 'cancel_running_backup_job',
            argsmode  => 'strict',
        ],
        run_backup_job => [
            namespace => 'CometBackup',
            method    => 'run_backup_job',
            argsmode  => 'strict',
        ],
        download_backup_job_log => [
            namespace => 'CometBackup',
            method    => 'download_backup_job_log',
            argsmode  => 'strict',
        ],
        download_restore_job_log => [
            namespace => 'CometBackup',
            method    => 'download_restore_job_log',
            argsmode  => 'strict',
        ],
        list_backup_run_history => [
            namespace => 'CometBackup',
            method    => 'list_backup_run_history',
            argsmode  => 'strict',
        ],
        list_restore_history => [
            namespace => 'CometBackup',
            method    => 'list_restore_history',
            argsmode  => 'strict',
        ],
        get_local_info => [
            namespace => 'CometBackup',
            method    => 'get_local_info',
            argsmode  => 'strict',
        ],
        modify_backup_job => [
            namespace => 'CometBackup',
            method    => 'modify_backup_job',
            argsmode  => 'strict',
        ],
        get_backup_destinations => [
            namespace => 'CometBackup',
            method    => 'get_backup_destinations',
            argsmode  => 'strict',
        ],
        get_backup_destination_regions => [
            namespace => 'CometBackup',
            method    => 'get_backup_destination_regions',
            argsmode  => 'strict',
        ],
        get_allowed_region_hints => [
            namespace => 'CometBackup',
            method    => 'get_allowed_region_hints',
            argsmode  => 'strict',
        ],
        get_server_credentials_from_cap => [
            namespace => 'CometBackup',
            method    => 'get_server_credentials_from_cap',
            argsmode  => 'strict',
        ],
        create_local_storage => [
            namespace => 'CometBackup',
            method    => 'create_local_storage',
            argsmode  => 'strict',
        ],
        delete_local_storage => [
            namespace => 'CometBackup',
            method    => 'delete_local_storage',
            argsmode  => 'strict',
        ],
        list_accounts_for_restore => [
            namespace => 'CometBackup',
            method    => 'list_accounts_for_restore',
            argsmode  => 'strict',
        ],
        run_restore_job => [
            namespace => 'CometBackup',
            method    => 'run_restore_job',
            argsmode  => 'strict',
        ],
        get_restore_log => [
            namespace => 'CometBackup',
            method    => 'get_restore_log',
            argsmode  => 'strict',
        ],
        list_backups_for_user => [
            namespace => 'CometBackup',
            method    => 'list_backups_for_user',
            argsmode  => 'strict',
        ],
        download_credentials => [
            namespace => 'CometBackup',
            method    => 'download_credentials',
            argsmode  => 'strict',
        ],
        upload_credentials => [
            namespace => 'CometBackup',
            method    => 'upload_credentials',
            argsmode  => 'strict',
        ],
        webpros_auth_status => [
            namespace => 'CometBackup',
            method    => 'webpros_auth_status',
            argsmode  => 'strict',
        ],
        webpros_auth_start => [
            namespace => 'CometBackup',
            method    => 'webpros_auth_start',
            argsmode  => 'strict',
        ],
        webpros_auth_callback => [
            namespace => 'CometBackup',
            method    => 'webpros_auth_callback',
            argsmode  => 'strict',
        ],
        webpros_auth_clear => [
            namespace => 'CometBackup',
            method    => 'webpros_auth_clear',
            argsmode  => 'strict',
        ],
    };
}

# Helper function to format duration in seconds to human readable format
sub _format_duration {
    my ($seconds) = @_;
    return '' unless defined $seconds && $seconds > 0;

    my $hours   = int( $seconds / 3600 );
    my $minutes = int( ( $seconds % 3600 ) / 60 );
    my $secs    = $seconds % 60;

    if ( $hours > 0 ) {
        if ( $minutes > 0 ) {
            return sprintf "%d hour%s %d minute%s",
              $hours,   ( $hours == 1   ? '' : 's' ),
              $minutes, ( $minutes == 1 ? '' : 's' );
        }
        else {
            return sprintf "%d hour%s", $hours, ( $hours == 1 ? '' : 's' );
        }
    }
    elsif ( $minutes > 0 ) {
        return sprintf "%d minute%s", $minutes, ( $minutes == 1 ? '' : 's' );
    }
    else {
        return sprintf "%d second%s", $secs, ( $secs == 1 ? '' : 's' );
    }
}

# Helper function to format bytes into human-readable size
sub _format_bytes {
    my ($bytes) = @_;

    return '' unless defined $bytes;
    return '0B' if $bytes == 0;

    my @units      = ( 'B', 'KB', 'MB', 'GB', 'TB' );
    my $unit_index = 0;

    while ( $bytes >= 1024 && $unit_index < $#units ) {
        $bytes /= 1024;
        $unit_index++;
    }

    return sprintf( "%.1f%s", $bytes, $units[$unit_index] );
}

# Helper function to generate mock total size until the API is implemented
# DEPRECATED: This function is no longer used - real size data comes from backup job completion
sub _mock_total_size {

    # Generate a realistic mock size based on random ranges
    my @size_ranges = (
        [ '500MB', '2GB' ],      # Small backups
        [ '2GB',   '15GB' ],     # Medium backups
        [ '15GB',  '50GB' ],     # Large backups
        [ '50GB',  '200GB' ],    # Very large backups
    );

    my $range = $size_ranges[ rand @size_ranges ];
    my ( $min_str, $max_str ) = @$range;

    # Convert to MB for calculation
    my ( $min_val, $min_unit ) = ( $min_str =~ /^(\d+(?:\.\d+)?)(\w+)$/ );
    my ( $max_val, $max_unit ) = ( $max_str =~ /^(\d+(?:\.\d+)?)(\w+)$/ );

    my $min_mb = $min_unit eq 'GB' ? $min_val * 1024 : $min_val;
    my $max_mb = $max_unit eq 'GB' ? $max_val * 1024 : $max_val;

    my $size_mb = $min_mb + rand( $max_mb - $min_mb );

    # Format back to human readable
    if ( $size_mb >= 1024 ) {
        return sprintf "%.1fGB", $size_mb / 1024;
    }
    else {
        return sprintf "%.0fMB", $size_mb;
    }
}

# Helper function to normalize backup_options from simplified frontend format
# to the full backend format expected by validation
sub _normalize_backup_options {
    my ($backup_options) = @_;

    return unless ref $backup_options eq 'HASH';

    # Expand databases: 1 -> databases: { mysql: 1, postgresql: 1 }
    if ( exists $backup_options->{databases} && !ref $backup_options->{databases} ) {
        my $enabled = $backup_options->{databases};
        if ( $enabled == 1 ) {
            $backup_options->{databases} = {
                mysql      => 1,
                postgresql => 1,
            };
        }
        elsif ( $enabled == 0 ) {
            $backup_options->{databases} = {
                mysql      => 0,
                postgresql => 0,
            };
        }
    }

    return;
}

# Helper function to run full restore for multiple users in a forked manager process
# This queries the backup_run_logs.db to find the most recent snapshots for all 3
# protected item types (homedir, databases, panel_config) and restores them
sub _run_full_restore_manager (%args) {
    my $users         = $args{users}         or die "users parameter required";
    my $connection_id = $args{connection_id} or die "connection_id parameter required";
    my $job_uuid      = $args{job_uuid}      or die "job_uuid parameter required";

    require Whostmgr::CometBackup::BackupRunLogs;
    require Whostmgr::CometBackup::CometBackupAPI::ProtectedItemRestore;
    require Whostmgr::CometBackup::Logger;
    require Whostmgr::CometBackup::RestoreRunLogs;
    require Whostmgr::CometBackup::Constants;

    my $logger = Whostmgr::CometBackup::Logger->new( log_level => $ENV{DEBUG} ? 'debug' : 'info' );

    # Open text log file for this restore job
    my $log_file = Whostmgr::CometBackup::Constants::get_text_log_file_path( $job_uuid, $job_uuid );
    my $log_dir  = $log_file;
    $log_dir =~ s{/[^/]+$}{};    # Remove filename to get directory

    unless ( -d $log_dir ) {
        require File::Path;
        File::Path::make_path($log_dir);
    }

    open( my $log_fh, '>', $log_file ) or die "Cannot open log file $log_file: $!";
    my $orig_fh = select($log_fh);
    $| = 1;                      # Autoflush
    select($orig_fh);

    my $log_msg = sub {
        my ($msg) = @_;
        my $timestamp = scalar localtime();
        print $log_fh "[$timestamp] $msg\n";
        $logger->info($msg);
    };

    $log_msg->("Full restore manager started");
    $log_msg->("Job UUID: $job_uuid");
    $log_msg->("Connection ID: $connection_id");
    $log_msg->( "User count: " . scalar(@$users) );
    $log_msg->( "Users: " . join( ", ", @$users ) );

    # Create restore manager
    my $restore_manager = Whostmgr::CometBackup::CometBackupAPI::ProtectedItemRestore->new( target_device => $connection_id );
    my $runlogs         = Whostmgr::CometBackup::BackupRunLogs->new();
    my $restore_logs    = Whostmgr::CometBackup::RestoreRunLogs->new();

    # Process each user
    for my $user (@$users) {
        $log_msg->("Processing full restore for user: $user");

        eval {
            # Get all completed backup items for this user
            my @items = $runlogs->get_restorable_items(
                system_user => $user,
                limit       => 1000,    # Get enough to find all types
            );

            unless (@items) {
                $log_msg->("WARNING: No backup items found for user: $user");
                return;
            }

            $log_msg->( "Found " . scalar(@items) . " backup items for user: $user" );

            # Group items by type and find the most recent for each
            my %latest_by_type;

            for my $item (@items) {
                my $item_type = $item->{protected_item_type};
                my $end_time  = $item->{end_time} || 0;

                # Keep the most recent item for each type
                if ( !$latest_by_type{$item_type} || $end_time > $latest_by_type{$item_type}->{end_time} ) {
                    $latest_by_type{$item_type} = $item;
                }
            }

            # Log what we found
            for my $type ( sort keys %latest_by_type ) {
                my $item = $latest_by_type{$type};
                $log_msg->( "  Latest $type backup for $user: snapshot " . $item->{comet_snapshot_id} );
            }

            # We need all 3 types for a full restore
            my @required_types = qw( homedir_files_folders databases_program_output pkgacct_program_output );
            my @missing_types  = grep { !$latest_by_type{$_} } @required_types;

            if (@missing_types) {
                $log_msg->( "ERROR: Cannot perform full restore - missing required backup types: " . join( ", ", @missing_types ) );
                $log_msg->( "Found types: " . join( ", ", keys %latest_by_type ) );
                return;
            }

            # Restore all 3 protected items in order:
            # 1. panel_config (pkgacct_program_output) - creates account structure
            # 2. databases (databases_program_output) - restores databases
            # 3. homedir (homedir_files_folders) - restores files
            #
            # IMPORTANT: We must wait for each restore to complete before starting the next one,
            # especially for panel_config which must finish before databases/homedir can restore.

            my @restore_order = (
                { type => 'pkgacct_program_output',   name => 'panel configuration' },
                { type => 'databases_program_output', name => 'databases' },
                { type => 'homedir_files_folders',    name => 'home directory' },
            );

            for my $restore_spec (@restore_order) {
                my $type = $restore_spec->{type};
                my $name = $restore_spec->{name};
                my $item = $latest_by_type{$type};

                $log_msg->("Restoring $name for user $user");

                my $job_id = $restore_manager->restore_protected_item(
                    source_id      => $item->{protected_item_uid},
                    destination_id => $item->{comet_destination_id},
                    snapshot_id    => $item->{comet_snapshot_id},
                    username       => $user,
                );

                # restore_protected_item returns:
                # - job_id (string): Restore dispatched successfully and job ID was returned
                # - 0: Restore dispatch failed
                if ( !$job_id ) {
                    $log_msg->("ERROR: Failed to initiate restore of $name for user $user");
                    die "Failed to restore $name for user $user";
                }

                $log_msg->("Successfully initiated restore of $name for user $user (Job ID: $job_id)");

                # Link the Comet job ID to our internal job UUID for tracking
                $restore_logs->add_comet_job_id( $job_uuid, $job_id );

                # Wait for this restore job to complete before starting the next one
                $log_msg->("Waiting for $name restore to complete...");
                my $completed = _wait_for_restore_completion(
                    job_id  => $job_id,
                    timeout => 3600,       # 1 hour timeout
                    log_msg => $log_msg,
                );

                unless ($completed) {
                    $log_msg->("ERROR: Restore of $name for user $user did not complete successfully");
                    die "Restore of $name for user $user failed or timed out";
                }

                $log_msg->("Restore of $name for user $user completed successfully");

                # If this was the panel_config restore, unlock the account password
                # restorepkg --newuser creates accounts with locked passwords (!! prefix)
                # which causes the account to appear suspended in WHM
                # if ( $type eq 'pkgacct_program_output' ) {
                #     $log_msg->("Unlocking password for restored account: $user");
                #     my $unlock_result = system( '/usr/local/cpanel/scripts/unsuspendacct', $user );
                #     if ( $unlock_result != 0 ) {
                #         $log_msg->("WARNING: Failed to unsuspend account $user (exit code: $unlock_result)");
                #     }
                #     else {
                #         $log_msg->("Successfully unlocked password for account: $user");
                #     }
                # }

                # Add a small delay before starting next restore
                sleep 2;
            }

            $log_msg->("Completed full restore initiation for user: $user");
        };

        if ($@) {
            $log_msg->("ERROR processing full restore for user $user: $@");
            $restore_logs->update_restore_job_status( $job_uuid, 'failed', "$@" );
        }
    }

    $log_msg->("Full restore manager completed for job: $job_uuid");
    $restore_logs->update_restore_job_status( $job_uuid, 'completed' );

    close($log_fh);
    return;
}

# Helper function to run custom restore manager in background daemon
# Handles custom restores for specific protected items with waiting for completion
sub _run_custom_restore_manager (%args) {
    my $users           = $args{users}           or die "users parameter required\n";
    my $connection_id   = $args{connection_id}   or die "connection_id parameter required\n";
    my $job_uuid        = $args{job_uuid}        or die "job_uuid parameter required\n";
    my $snapshot_ids    = $args{snapshot_ids}    or die "snapshot_ids parameter required\n";
    my $source_ids      = $args{source_ids}      or die "source_ids parameter required\n";
    my $destination_ids = $args{destination_ids} or die "destination_ids parameter required\n";
    my $options         = $args{options} || {};
    my $paths           = $args{paths};

    require Whostmgr::CometBackup::CometBackupAPI::ProtectedItemRestore;
    require Whostmgr::CometBackup::Logger;
    require Whostmgr::CometBackup::RestoreRunLogs;
    require Whostmgr::CometBackup::Constants;

    my $logger = Whostmgr::CometBackup::Logger->new( log_level => $ENV{DEBUG} ? 'debug' : 'info' );

    # Open text log file for this restore job
    my $log_file = Whostmgr::CometBackup::Constants::get_text_log_file_path( $job_uuid, $job_uuid );
    my $log_dir  = $log_file;
    $log_dir =~ s{/[^/]+$}{};    # Remove filename to get directory

    unless ( -d $log_dir ) {
        require File::Path;
        File::Path::make_path($log_dir);
    }

    open( my $log_fh, '>', $log_file ) or die "Cannot open log file $log_file: $!\n";
    my $orig_fh = select($log_fh);
    $| = 1;                      # Autoflush
    select($orig_fh);

    my $log_msg = sub {
        my ($msg) = @_;
        my $timestamp = scalar localtime();
        print $log_fh "[$timestamp] $msg\n";
        $logger->info($msg);
    };

    $log_msg->("Custom restore job started");
    $log_msg->("Job UUID: $job_uuid");
    $log_msg->("User: $users->[0]");

    my $restore_logs    = Whostmgr::CometBackup::RestoreRunLogs->new();
    my $restore_manager = Whostmgr::CometBackup::CometBackupAPI::ProtectedItemRestore->new( target_device => $connection_id );

    # Map display names to internal keys (in case frontend sends display names)
    my %display_name_to_key = (
        'Account Configuration' => 'panel_config',
        'Databases'             => 'databases',
        'Home directory'        => 'homedir',
    );

    # Normalize snapshot_ids, source_ids, and destination_ids to use internal keys
    my %normalized_snapshot_ids;
    my %normalized_source_ids;
    my %normalized_destination_ids;

    for my $key ( keys %$snapshot_ids ) {
        my $internal_key = $display_name_to_key{$key} || $key;
        $normalized_snapshot_ids{$internal_key} = $snapshot_ids->{$key};
    }

    for my $key ( keys %$source_ids ) {
        my $internal_key = $display_name_to_key{$key} || $key;
        $normalized_source_ids{$internal_key} = $source_ids->{$key};
    }

    for my $key ( keys %$destination_ids ) {
        my $internal_key = $display_name_to_key{$key} || $key;
        $normalized_destination_ids{$internal_key} = $destination_ids->{$key};
    }

    # Use normalized hashes for the rest of the function
    $snapshot_ids    = \%normalized_snapshot_ids;
    $source_ids      = \%normalized_source_ids;
    $destination_ids = \%normalized_destination_ids;

    # Define the restore order to ensure panel_config is always restored first
    # This is critical because panel_config creates the account structure that
    # databases and homedir restores depend on
    my @restore_order = qw( panel_config databases homedir );

    # Build list of item types to restore, maintaining the correct order
    my @ordered_item_types;
    for my $ordered_type (@restore_order) {
        if ( exists $snapshot_ids->{$ordered_type} ) {
            push @ordered_item_types, $ordered_type;
        }
    }

    # Also include any other types not in the predefined order (for future extensibility)
    for my $item_type ( keys %$snapshot_ids ) {
        unless ( grep { $_ eq $item_type } @restore_order ) {
            push @ordered_item_types, $item_type;
        }
    }

    $log_msg->( "Restore order: " . join( ", ", @ordered_item_types ) );

    # Iterate through each protected item type in the correct order
    my $all_success = 1;
    for my $item_type (@ordered_item_types) {
        my $snapshot_id    = $snapshot_ids->{$item_type};
        my $source_id      = $source_ids->{$item_type};
        my $destination_id = $destination_ids->{$item_type};

        unless ( $snapshot_id && $source_id && $destination_id ) {
            $log_msg->("WARNING: Skipping $item_type - missing snapshot_id, source_id, or destination_id");
            next;
        }

        $log_msg->("Restoring $item_type - Source: $source_id, Destination: $destination_id, Snapshot: $snapshot_id");

        # Determine per-item-type options
        # Options can be either:
        # 1. A flat hash (legacy) - apply to all items
        # 2. A nested hash keyed by protected_item_type - per-item options
        my $item_options = {};
        if ( ref $options eq 'HASH' ) {
            if ( exists $options->{$item_type} && ref $options->{$item_type} eq 'HASH' ) {

                # Per-item-type options
                $item_options = $options->{$item_type};
            }
            else {
                # Flat options - use for all items
                $item_options = $options;
            }
        }

        # Use ProtectedItemRestore to handle the restore
        my $job_id;
        if ( $paths && ref $paths eq 'HASH' && $paths->{$item_type} ) {
            $log_msg->( "Restoring specific paths for $item_type: " . scalar( @{ $paths->{$item_type} } ) . " items" );

            # Restore specific paths for this item type
            $job_id = $restore_manager->restore_specific_paths(
                source_id      => $source_id,
                destination_id => $destination_id,
                snapshot_id    => $snapshot_id,
                paths          => $paths->{$item_type},
                $item_options->{restore_path} ? ( restore_path => $item_options->{restore_path} ) : (),
                options => $item_options,
            );
        }
        elsif ( $item_options->{restore_path} ) {
            $log_msg->( "Restoring $item_type to custom location: " . $item_options->{restore_path} );

            # Restore to custom location
            $job_id = $restore_manager->restore_to_custom_location(
                source_id      => $source_id,
                destination_id => $destination_id,
                snapshot_id    => $snapshot_id,
                restore_path   => $item_options->{restore_path},
                options        => $item_options,
            );
        }
        else {
            $log_msg->("Restoring $item_type to original location");

            # Restore to original location
            $job_id = $restore_manager->restore_protected_item(
                source_id      => $source_id,
                destination_id => $destination_id,
                snapshot_id    => $snapshot_id,
                username       => $users->[0],
                options        => $item_options,
            );
        }

        if ( !$job_id ) {
            $log_msg->("ERROR: Failed to initiate restore for $item_type");
            $all_success = 0;
            last;    # Stop processing remaining items if one fails
        }

        $log_msg->("Successfully initiated restore for $item_type (Job ID: $job_id)");

        # Link the Comet job ID to our internal job UUID for tracking
        $restore_logs->add_comet_job_id( $job_uuid, $job_id );

        # Wait for this restore to complete before starting the next one
        # This is critical for panel_config which must finish before databases/homedir
        $log_msg->("Waiting for $item_type restore to complete...");
        my $completed = _wait_for_restore_completion(
            job_id  => $job_id,
            timeout => 3600,       # 1 hour timeout
            log_msg => $log_msg,
        );

        unless ($completed) {
            $log_msg->("ERROR: Restore of $item_type did not complete successfully");
            $all_success = 0;
            last;                  # Stop processing remaining items if one fails
        }

        $log_msg->("Restore of $item_type completed successfully");
    }

    if ($all_success) {
        $log_msg->("Custom restore completed successfully for all protected item types");
        $restore_logs->update_restore_job_status( $job_uuid, 'completed' );
    }
    else {
        $log_msg->("ERROR: Failed to complete one or more custom restores");
        $restore_logs->update_restore_job_status( $job_uuid, 'failed', 'Failed to complete one or more restores' );
    }

    close($log_fh);
    return;
}

# Helper function to wait for a restore job to complete
# Returns 1 on success, 0 on failure/timeout
sub _wait_for_restore_completion (%args) {
    my $job_id  = $args{job_id} or die "job_id parameter required";
    my $timeout = $args{timeout} || 3600;                             # Default 1 hour
    my $log_msg = $args{log_msg};

    require Whostmgr::CometBackup::CometBackupAPI;
    require Whostmgr::CometBackup::Constants;

    my $start_time    = time();
    my $poll_interval = 5;                                            # Check every 5 seconds
    my $last_status   = '';

    while (1) {
        my $elapsed = time() - $start_time;

        if ( $elapsed > $timeout ) {
            $log_msg->("Timeout waiting for job $job_id to complete (${timeout}s)") if $log_msg;
            return 0;
        }

        # Get job properties
        my $job_props = eval {
            Whostmgr::CometBackup::CometBackupAPI::get_job_properties(
                job_id => $job_id,
            );
        };

        if ($@) {
            $log_msg->("Warning: Failed to get job properties for $job_id: $@") if $log_msg;
            sleep $poll_interval;
            next;
        }

        unless ($job_props) {
            $log_msg->("Warning: No job properties returned for $job_id") if $log_msg;
            sleep $poll_interval;
            next;
        }

        # Get job status and use Constants module for interpretation
        # See: https://docs.cometbackup.com/latest/api/api-constants/#job
        my $status = $job_props->{Status} // 0;

        # Log status changes
        if ( $status ne $last_status && $log_msg ) {
            my $status_name = Whostmgr::CometBackup::Constants::get_job_status_name($status);
            chomp($status_name);
            $log_msg->("Job $job_id status: $status_name");
            $last_status = $status;
        }

        # Check for completion states using Constants helper functions
        if ( Whostmgr::CometBackup::Constants::is_job_successful($status) ) {

            # Job completed successfully (5xxx range)
            return 1;
        }
        elsif ( Whostmgr::CometBackup::Constants::is_job_completed_with_warning($status) ) {

            # Job completed with warnings (7001) - treat as success
            # Warnings are common in restore operations (e.g., symlinks already exist)
            $log_msg->("Job $job_id completed with warnings (warnings not indicative of true failures)") if $log_msg;
            return 1;
        }
        elsif ( Whostmgr::CometBackup::Constants::is_job_failed($status) ) {

            # Job failed (7xxx range, excluding warnings)
            my $status_name = Whostmgr::CometBackup::Constants::get_job_status_name($status);
            $log_msg->("Job $job_id failed with status: $status_name") if $log_msg;
            return 0;
        }

        # Job is still running (6xxx range) or unknown status, wait and check again
        sleep $poll_interval;
    }
}

=head2 get_local_info

Retrieves information about a local storage path, including disk usage and availability.

=head3 Parameters

=over 4

=item * C<local_path> - (Required) The local filesystem path to query

=back

=head3 Returns

A hash reference containing:

=over 4

=item * C<size_used> - Bytes of disk space used on the filesystem

=item * C<size_total> - Total bytes available on the filesystem

=item * C<exists> - Boolean indicating if the path exists (1) or not (0)

=item * C<is_dir> - Boolean indicating if the path is a directory (1) or not (0)

=back

=head3 Example

    my $result = get_local_info({
        json => encode_json({
            local_path => '/backup/storage'
        })
    }, $metadata, $api_info);

=cut

sub get_local_info {
    my ( $args, $metadata, $api_info_hr ) = @_;

    my ( $status, $decoded_args ) = _decode_json_args( $metadata, $args );
    if ( $status < 1 ) {
        return;
    }

    my $local_path = $decoded_args->{local_path};

    unless ( defined $local_path && length $local_path ) {
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "local_path parameter required" );
        return {
            size_used  => 0,
            size_total => 0,
            exists     => 0,
            is_dir     => 0,
        };
    }

    $logger->info("Getting local info for path: $local_path");

    # Initialize response with default values
    my $response = {
        size_used  => 0,
        size_total => 0,
        exists     => 0,
        is_dir     => 0,
    };

    eval {
        # Determine the path to check for filesystem stats
        my $path_to_check = $local_path;

        # Check if path exists and is a directory
        if ( -e $local_path ) {
            $response->{exists} = 1;
            $response->{is_dir} = -d $local_path ? 1 : 0;
        }
        else {
            # Path doesn't exist - walk up the directory tree to find the first existing parent
            # This allows us to report filesystem stats for the partition where the path would exist
            $response->{exists} = 0;
            $response->{is_dir} = 0;

            require File::Spec;

            # Split path into volume, directories, and file
            my ( $volume, $directories, $file ) = File::Spec->splitpath( $local_path, 1 );

            # Get directory components
            my @dirs = File::Spec->splitdir($directories);

            # Build path from root, checking each level until we find one that exists
            my @path_components = grep { length $_ } @dirs;    # Remove empty elements

            # Start from the full path and work backwards
            while (@path_components) {
                my $test_path = File::Spec->catdir( $volume || '/', @path_components );

                if ( -e $test_path ) {
                    $path_to_check = $test_path;
                    $logger->debug("Path $local_path does not exist, using parent $path_to_check for filesystem stats");
                    last;
                }

                # Remove last component and try again
                pop @path_components;
            }

            # If no parent exists, fall back to root
            unless ( -e $path_to_check ) {
                $path_to_check = '/';
                $logger->debug("No parent of $local_path exists, using root / for filesystem stats");
            }
        }

        # Get filesystem statistics for the path (or its first existing parent)
        # Use df command with -B 1 for 1-byte blocks to get precise byte counts
        my $df_output = _get_disk_usage_for_path($path_to_check);

        # Parse df output: filesystem total used available use% mountpoint
        # Format: /dev/sda1 1073741824000 536870912000 536870912000 50% /
        if ( $df_output && $df_output =~ /\s+(\d+)\s+(\d+)\s+(\d+)\s+\d+%/ ) {

            # Capture groups: $1 = total, $2 = used, $3 = available
            # Explicitly convert to integers to ensure proper JSON serialization
            $response->{size_total} = int($1);
            $response->{size_used}  = int($2);

            $logger->debug(
                "Retrieved filesystem stats for $local_path",
                {
                    size_total => $response->{size_total},
                    size_used  => $response->{size_used},
                }
            );
        }
        else {
            # df command failed or returned unexpected output
            $logger->warn("Failed to parse df output for path: $path_to_check");

            # Fall back to using Filesys::Df if available
            eval {
                require Filesys::Df;

                my $df_info = Filesys::Df::df($path_to_check);
                if ($df_info) {

                    # Filesys::Df returns values in blocks, multiply by bavail/blocks to get bytes
                    # Total = blocks * block_size, Used = (blocks - bavail) * block_size
                    my $block_size = $df_info->{bsize} || 1024;
                    $response->{size_total} = int( $df_info->{blocks} * $block_size );
                    $response->{size_used}  = int( ( $df_info->{blocks} - $df_info->{bavail} ) * $block_size );

                    $logger->debug("Retrieved filesystem stats using Filesys::Df for $path_to_check");
                }
            };

            if ($@) {
                $logger->warn("Filesys::Df also failed for path: $path_to_check - $@");

                # Leave size values at 0 if both methods fail
            }
        }
    };

    if ($@) {
        $logger->error("Error getting local info for path $local_path: $@");
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "Failed to get local info: $@" );
        return {
            size_used  => 0,
            size_total => 0,
            exists     => 0,
            is_dir     => 0,
        };
    }

    Whostmgr::API::1::Utils::set_metadata_ok($metadata);
    return $response;
}

=head2 download_credentials

Downloads the server's Comet Backup credentials file.

=head3 Parameters

None required.

=head3 Returns

A hash reference containing:

=over 4

=item * C<credentials> - The credentials file content as a JSON string

=item * C<error> - Error message if the operation failed (when applicable)

=back

=head3 Example

    my $result = download_credentials({}, $metadata, $api_info);

=cut

sub download_credentials {
    my ( $args, $metadata, $api_info_hr ) = @_;

    my $credentials_file = $Whostmgr::CometBackup::Constants::CREDENTIALS_FILE;

    # Check if credentials file exists
    unless ( -f $credentials_file ) {
        my $error_msg = "Server credentials file not found: $credentials_file";
        $logger->warn($error_msg);
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, $error_msg );
        return { credentials => '' };
    }

    # Check if credentials file is readable
    unless ( -r $credentials_file ) {
        my $error_msg = "Server credentials file is not readable: $credentials_file";
        $logger->error($error_msg);
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, $error_msg );
        return { credentials => '' };
    }

    # Read the credentials file content
    my $credentials_content;
    eval {
        require Cpanel::LoadFile;
        $credentials_content = Cpanel::LoadFile::load($credentials_file);
    };

    if ($@) {
        my $error_msg = "Failed to read credentials file: $@";
        $logger->error($error_msg);
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, $error_msg );
        return { credentials => '' };
    }

    $logger->debug("Successfully retrieved credentials file");
    Whostmgr::API::1::Utils::set_metadata_ok($metadata);
    return {
        credentials => $credentials_content,
    };
}

=head2 upload_credentials

Uploads and validates Comet Backup server credentials. Optionally triggers disaster recovery
to restore the recovery_data Protected Item when the disaster_recovery parameter is set.

=head3 Parameters

=over 4

=item * C<credentials> - (Required) JSON string containing server credentials with fields:
username, password, server_url

=item * C<disaster_recovery> - (Optional) Boolean flag (1 or 0) to trigger disaster recovery
restore of /var/cpanel/comet/ data after credentials are uploaded

=back

=head3 Returns

A hash reference containing:

=over 4

=item * C<uploaded> - Boolean indicating success (1) or failure (0)

=item * C<credentials_valid> - Boolean indicating if credentials were validated (on success)

=item * C<recovery_initiated> - Boolean indicating if disaster recovery was started (when applicable)

=back

=head3 Example

    my $result = upload_credentials({
        json => encode_json({
            credentials => encode_json({
                username => 'backup_user',
                password => 'secure_password',
                server_url => 'https://backup.example.com'
            }),
            disaster_recovery => 1
        })
    }, $metadata, $api_info);

=cut

sub upload_credentials {
    my ( $args, $metadata, $api_info_hr ) = @_;

    my ( $status, $decoded_args ) = _decode_json_args( $metadata, $args );
    if ( $status < 1 ) {
        return;
    }

    my $credentials_json = $decoded_args->{credentials};

    unless ( defined $credentials_json && length $credentials_json ) {
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "credentials parameter required" );
        return { uploaded => 0 };
    }

    # Validate that credentials is valid JSON
    my $credentials_data;
    eval { $credentials_data = Cpanel::JSON::Load($credentials_json); };

    if ($@) {
        my $error       = $@;
        my $clean_error = "Invalid JSON in credentials parameter";

        # Try to extract specific parse error message
        if ( $error =~ /because of an error:\s*(.+?)\s+at\s+\/usr\/local\/cpanel/s ) {
            $clean_error = $1;
            $clean_error =~ s/\s+$//;
        }
        elsif ( $error =~ /because of an error:\s*(.+?)$/s ) {
            $clean_error = $1;
            $clean_error =~ s/\s+$//;
        }

        $logger->error("Failed to validate credentials JSON: $error");
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "Invalid credentials JSON: $clean_error" );
        return { uploaded => 0 };
    }

    # Validate required fields in credentials
    my @required_fields = qw(username password server_url);
    my @missing_fields;
    for my $field (@required_fields) {
        unless ( exists $credentials_data->{$field} && defined $credentials_data->{$field} && length $credentials_data->{$field} ) {
            push @missing_fields, $field;
        }
    }

    if (@missing_fields) {
        my $error_msg = "Missing required credential fields: " . join( ", ", @missing_fields );
        $logger->error($error_msg);
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, $error_msg );
        return { uploaded => 0 };
    }

    # Test credentials by making an API call to the Comet server
    $logger->debug("Validating credentials by testing connection to Comet server");

    my $test_result;

    eval {
        require Whostmgr::CometBackup::CometBackupAPI;

        $logger->debug("Testing credentials with list-active API call");

        # Make the API call using standard make_server_request with credential overrides
        $test_result = Whostmgr::CometBackup::CometBackupAPI::make_server_request(
            "user/web/dispatcher/list-active",
            {
                username   => $credentials_data->{username},
                password   => $credentials_data->{password},
                server_url => $credentials_data->{server_url},
            }
        );

        # Check response status
        my $status = $test_result->{Status} || 0;

        if ( $status == 401 || $status == 403 ) {
            my $error_msg = $test_result->{Message} || 'Authentication failed';
            die "Authentication failed: $error_msg (Status: $status)";
        }
        elsif ( $status == 200 ) {
            $logger->debug("Credentials validated successfully with list-active");
        }
        else {
            # Other errors are acceptable - we're just testing auth
            # Destination not found, etc. means auth worked
            $logger->debug("Credentials validated successfully (Status: $status)");
        }
    };

    if ($@) {
        my $error               = $@;
        my $user_friendly_error = "Failed to validate credentials with Comet server";

        # Extract user-friendly error message
        if ( $error =~ /Authentication failed: (.+)/ ) {
            $user_friendly_error = "Invalid credentials: $1";
        }
        elsif ( $error =~ /Failed to connect|Connection refused|timeout|HTTP request failed/i ) {
            $user_friendly_error = "Unable to connect to Comet server - check server URL and network connectivity";
        }
        elsif ( $error =~ /Server API Error: (.+)/ ) {
            $user_friendly_error = "Comet API error: $1";
        }

        $logger->error("Credential validation failed: $error");

        my $error_response = {
            uploaded           => 0,
            error              => $user_friendly_error,
            comet_api_response => $test_result,
            detailed_error     => "$error",
        };

        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, $user_friendly_error );
        return $error_response;
    }

    $logger->info("Credentials validated successfully");

    my $credentials_file = $Whostmgr::CometBackup::Constants::CREDENTIALS_FILE;
    my $backup_file;

    eval {
        # Ensure the directory exists
        require Cpanel::Mkdir;
        my $credentials_dir = $credentials_file;
        $credentials_dir =~ s{/[^/]+$}{};
        Cpanel::Mkdir::ensure_directory_existence_and_mode( $credentials_dir, 0755 ) unless -d $credentials_dir;

        # If file exists, create backup with timestamp
        if ( -f $credentials_file ) {
            my $timestamp = time();
            $backup_file = "${credentials_file}-${timestamp}";

            $logger->debug("Backing up existing credentials file to: $backup_file");

            require File::Copy;
            File::Copy::copy( $credentials_file, $backup_file )
              or die "Failed to backup credentials file: $!";

            # Set permissions on backup file to 0600
            chmod( 0600, $backup_file )
              or die "Failed to set permissions on backup file: $!";

            $logger->debug("Backup created successfully with permissions 0600");
        }

        # Write new credentials to file
        require Cpanel::FileUtils::Write;
        Cpanel::FileUtils::Write::overwrite( $credentials_file, $credentials_json, 0600 );

        $logger->debug("Successfully wrote credentials file with permissions 0600");
    };

    if ($@) {
        my $error_msg = "Failed to upload credentials file: $@";
        $logger->error($error_msg);
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, $error_msg );
        return { uploaded => 0 };
    }

    $logger->debug("Successfully uploaded credentials file with permissions 0600");

    # Check if disaster_recovery mode is requested
    my $disaster_recovery = $decoded_args->{disaster_recovery} || 0;

    if ($disaster_recovery) {
        $logger->debug("Disaster recovery mode enabled - double forking recovery process");

        # Double fork to properly daemonize and prevent parent from waiting
        my $pid = fork();

        if ( !defined $pid ) {

            # Fork failed - log error but don't fail the credential upload
            $logger->error("Failed to fork disaster recovery process: $!");
        }
        elsif ( $pid == 0 ) {

            # First child process - fork again and exit immediately
            my $pid2 = fork();

            if ( !defined $pid2 ) {

                # Second fork failed - exit child
                exit(1);
            }
            elsif ( $pid2 == 0 ) {

                # Second child (grandchild) - this will be orphaned and adopted by init

                # Become session leader FIRST
                require POSIX;
                POSIX::setsid();

                # Close ALL inherited file descriptors (0-255 range covers most cases)
                # This ensures we don't inherit any database handles, file locks, or sockets
                for my $fd ( 0 .. 255 ) {
                    POSIX::close($fd);
                }

                # Reopen standard handles to /dev/null
                open( STDIN,  '<', '/dev/null' );
                open( STDOUT, '>', '/dev/null' );
                open( STDERR, '>', '/dev/null' );

                # Change to root directory to avoid holding directory handles
                chdir('/');

                # Clear environment variables that might hold connections
                delete $ENV{DBI_DRIVER};
                delete $ENV{DBIC_TRACE};

                # Set our name so we are easily identifiable
                $0 = 'comet-disaster-recovery';

                # Run disaster recovery and exit
                _run_disaster_recovery();
                exit(0);
            }
            else {
                # First child - exit immediately so parent can continue
                exit(0);
            }
        }
        else {
            # Parent process - wait for first child to exit (which happens immediately)
            # This prevents zombie processes
            waitpid( $pid, 0 );
            $logger->info("Disaster recovery process daemonized successfully");
        }
    }

    Whostmgr::API::1::Utils::set_metadata_ok($metadata);

    my $response = {
        uploaded => 1,
        file     => $credentials_file,
    };

    $response->{backup_file}               = $backup_file if $backup_file;
    $response->{disaster_recovery_started} = 1            if $disaster_recovery;

    return $response;
}

=head2 _run_disaster_recovery

Internal method that performs disaster recovery restore operation. This runs in a daemonized
process to restore the recovery_data Protected Item containing /var/cpanel/comet/ data.

This is called automatically when upload_credentials is invoked with disaster_recovery=1.

=head3 Parameters

None - reads configuration from server state.

=head3 Returns

Does not return - this runs as a daemon and logs to /var/cpanel/comet/recovery.log

=cut

sub _run_disaster_recovery {

    # This runs in the daemonized grandchild process
    my $recovery_flag = '/var/cpanel/comet/in_recovery';
    my $recovery_log  = '/var/cpanel/comet/recovery.log';

    eval {
        require Cpanel::FileUtils::TouchFile;
        require Whostmgr::CometBackup::CometBackupAPI;
        require Whostmgr::CometBackup::CometBackupAPI::ProtectedItemBatch;
        require Whostmgr::CometBackup::CometBackupAPI::ProtectedItemRestore;
        require Whostmgr::CometBackup::Constants;
        require Whostmgr::CometBackup::RestoreRunLogs;
        require Data::UUID;

        # Set recovery flag with PID before starting
        my $pid = $$;
        $logger->debug("Setting disaster recovery flag: $recovery_flag (PID: $pid)");

        # Write PID to recovery flag file so we can track the process
        require Cpanel::FileUtils::Write;
        Cpanel::FileUtils::Write::overwrite( $recovery_flag, "$pid", 0644 );

        # Get server config
        my $server_config = Whostmgr::CometBackup::CometBackupAPI::get_server_config();
        my $username      = $server_config->{username};

        unless ($username) {
            die "Unable to determine Comet username from server config";
        }

        $logger->info("Starting disaster recovery for user: $username");

        # Get user profile to find snapshots for recovery_data
        my $profile_result = Whostmgr::CometBackup::CometBackupAPI::make_server_request("user/web/get-user-profile-and-hash");

        unless ( $profile_result && $profile_result->{Status} == 200 && $profile_result->{Profile} ) {
            die "Failed to retrieve user profile from Comet server";
        }

        my $profile      = $profile_result->{Profile};
        my $destinations = $profile->{Destinations} || {};
        my $sources      = $profile->{Sources}      || {};

        # Find the recovery_data Protected Item by searching for one that backs up the recovery path
        # This is more reliable than using a generated UID since we might be restoring to a new server
        my $recovery_item_uid;
        my $recovery_path = $Whostmgr::CometBackup::Constants::RECOVERY_DATA_PATH;

        for my $source_id ( keys %$sources ) {
            my $source      = $sources->{$source_id};
            my $description = $source->{Description} || '';

            # Check if this is a recovery_data item by description or path
            if (
                $description =~ /recovery[_\s]data/i
                || (   $source->{PreExec}
                    && ref $source->{PreExec} eq 'ARRAY'
                    && grep { /recovery[_\s]?data/i } @{ $source->{PreExec} } )
            ) {
                $recovery_item_uid = $source_id;
                $logger->debug("Found recovery_data Protected Item: $recovery_item_uid (Description: $description)");
                last;
            }
        }

        unless ($recovery_item_uid) {

            # Try the deterministic UID as fallback
            require Whostmgr::CometBackup::CometBackupAPI::ProtectedItemBatch;
            my $batch         = Whostmgr::CometBackup::CometBackupAPI::ProtectedItemBatch->new( target_user => $username );
            my $generated_uid = $batch->_generate_recovery_data_uid();

            if ( $sources->{$generated_uid} ) {
                $recovery_item_uid = $generated_uid;
                $logger->debug("Found recovery_data Protected Item using generated UID: $recovery_item_uid");
            }
            else {
                die "recovery_data Protected Item not found on Comet server. Available sources: " . join( ", ", keys %$sources );
            }
        }

        $logger->debug("Found recovery_data Protected Item on Comet server");

        # Get active connection ID for restore
        my $connection_id = Whostmgr::CometBackup::CometBackupAPI::_get_user_connection_id();
        unless ($connection_id) {
            die "No live-connected device available for restore operation";
        }

        $logger->debug("Using connection ID: $connection_id");

        # Select the best destination for disaster recovery
        # Priority: cloud storage over local storage (for disaster recovery scenarios)
        my $destination_id;

        # Get the actual destinations configured for this source from the profile
        my $recovery_source       = $sources->{$recovery_item_uid};
        my $override_destinations = $recovery_source->{OverrideDestinationRetention} || {};

        # Build list of candidate destinations
        my @candidate_dest_ids;
        if ( keys %$override_destinations ) {

            # Use destinations configured for this source
            @candidate_dest_ids = keys %$override_destinations;
            $logger->debug( "Found " . scalar(@candidate_dest_ids) . " destinations configured for recovery_data source" );
        }
        else {
            # Fallback to all available destinations
            @candidate_dest_ids = keys %$destinations;
            $logger->debug( "Using all " . scalar(@candidate_dest_ids) . " available destinations as candidates" );
        }

        unless (@candidate_dest_ids) {
            die "No destinations available for recovery_data restore";
        }

        # Prioritize cloud storage (DestinationType == 1000) over local (DestinationType == 1002)
        # This is critical for disaster recovery - we want to restore from off-site backups
        my $cloud_dest_id;
        my $local_dest_id;

        for my $dest_id (@candidate_dest_ids) {
            my $dest = $destinations->{$dest_id};
            next unless $dest;

            my $dest_type = $dest->{DestinationType} || 0;

            if ( $dest_type == 1000 && !$cloud_dest_id ) {

                # Comet Cloud Storage - highest priority
                $cloud_dest_id = $dest_id;
                $logger->debug("Found cloud storage destination: $dest_id");
            }
            elsif ( $dest_type == 1002 && !$local_dest_id ) {

                # Local Copy - fallback option
                $local_dest_id = $dest_id;
                $logger->debug("Found local storage destination: $dest_id");
            }
        }

        # Select destination: cloud first, then local, then any available
        if ($cloud_dest_id) {
            $destination_id = $cloud_dest_id;
            $logger->debug("Selected CLOUD storage for disaster recovery: $destination_id");
        }
        elsif ($local_dest_id) {
            $destination_id = $local_dest_id;
            $logger->debug("Using LOCAL storage for disaster recovery (cloud not available): $destination_id");
        }
        else {
            # Fallback to first available destination (might be S3, B2, etc.)
            $destination_id = $candidate_dest_ids[0];
            $logger->debug("Using first available destination: $destination_id");
        }

        $logger->debug("Will restore latest available snapshot from destination: $destination_id");

        # Generate run UUID for this restore job
        my $ug       = Data::UUID->new();
        my $run_uuid = lc( $ug->create_str() );

        # Track the restore job in database
        my $restore_logs = Whostmgr::CometBackup::RestoreRunLogs->new();
        $restore_logs->create_restore_job(
            restore_type => 'disaster_recovery',
            users        => [],                    # No user accounts involved in recovery_data restore
            initiated_by => 'system',
            run_uuid     => $run_uuid,
        );

        # Create restore manager
        my $restore_manager = Whostmgr::CometBackup::CometBackupAPI::ProtectedItemRestore->new( target_device => $connection_id );

        # Open log file for restore monitoring
        my $log_file = Whostmgr::CometBackup::Constants::get_text_log_file_path( $run_uuid, $run_uuid );
        my $log_dir  = $log_file;
        $log_dir =~ s{/[^/]+$}{};

        unless ( -d $log_dir ) {
            require File::Path;
            File::Path::make_path($log_dir);
        }

        open( my $log_fh, '>', $log_file ) or die "Cannot open log file $log_file: $!";
        my $orig_fh = select($log_fh);
        $| = 1;    # Autoflush
        select($orig_fh);

        my $log_msg = sub {
            my ($msg) = @_;
            my $timestamp = scalar localtime();
            print $log_fh "[$timestamp] $msg\n";
            $logger->info($msg);
        };

        $log_msg->("Disaster recovery restore started");
        $log_msg->("Run UUID: $run_uuid");
        $log_msg->("Recovery data source: $recovery_item_uid");
        $log_msg->("Destination: $destination_id");
        $log_msg->("Connection ID: $connection_id");
        $log_msg->("Using latest available snapshot (Comet will select automatically)");

        # Restore recovery_data to original location
        # recovery_data backs up /var/cpanel/comet/ - restore to original location
        my $restore_path = $Whostmgr::CometBackup::Constants::RECOVERY_DATA_PATH;

        $log_msg->("Starting restore to original location (files will be restored to: $restore_path)");

        # For file-based backups like recovery_data, use standard restore_protected_item
        # which restores to the original backed-up paths
        # Note: snapshot_id is omitted - Comet will automatically use the latest snapshot
        my $job_id = $restore_manager->restore_protected_item(
            source_id      => $recovery_item_uid,
            destination_id => $destination_id,

            # No username needed for system-level recovery_data restore
        );

        unless ($job_id) {
            die "Failed to initiate recovery_data restore";
        }

        $log_msg->("Restore job initiated successfully");
        $log_msg->("Comet Job ID: $job_id");

        # Link the Comet job ID to our internal run UUID
        $restore_logs->add_comet_job_id( $run_uuid, $job_id );

        $log_msg->("Monitoring restore progress via Comet server job status...");

        # Monitor the restore job by polling get_recent_jobs
        # We need to keep the in_recovery flag until the restore actually completes
        my $max_wait_time     = 3600;    # 1 hour maximum
        my $poll_interval     = 5;       # Check every 5 seconds
        my $elapsed_time      = 0;
        my $restore_completed = 0;
        my $restore_error;

        while ( $elapsed_time < $max_wait_time ) {
            sleep($poll_interval);
            $elapsed_time += $poll_interval;

            # Get recent jobs from the last 5 minutes (0.1 hours)
            my $recent_jobs = eval {
                Whostmgr::CometBackup::CometBackupAPI::get_recent_jobs(
                    target_user => $username,
                    hours       => 0.1,
                );
            };

            if ($@) {
                $log_msg->("Warning: Failed to check job status: $@");
                next;
            }

            unless ( $recent_jobs && ref($recent_jobs) eq 'HASH' ) {
                $log_msg->("Warning: No recent jobs found, continuing to wait...");
                next;
            }

            # Look for a restore job for our recovery_data Protected Item
            for my $job_id ( keys %$recent_jobs ) {
                my $job = $recent_jobs->{$job_id};

                # Skip if not a restore job or not for our recovery_data source
                # Classification 4001 = restore with original paths, 4002 = restore to custom location
                next unless $job->{Classification} && ( $job->{Classification} == 4001 || $job->{Classification} == 4002 );
                next unless $job->{SourceGUID}     && $job->{SourceGUID} eq $recovery_item_uid;

                # Check job status
                # Status codes: 5000 = success, 5001 = warning, 5002 = error, 5003 = cancellation
                my $status = $job->{Status};

                if ( $status == 5000 ) {
                    $log_msg->("Restore completed successfully (Job ID: $job_id)");
                    $restore_completed = 1;
                    last;
                }
                elsif ( $status == 5001 ) {
                    $log_msg->("Restore completed with warnings (Job ID: $job_id)");
                    $restore_completed = 1;
                    last;
                }
                elsif ( $status == 5002 ) {
                    $restore_error = "Restore failed with error (Job ID: $job_id)";
                    $log_msg->($restore_error);
                    last;
                }
                elsif ( $status == 5003 ) {
                    $restore_error = "Restore was cancelled (Job ID: $job_id)";
                    $log_msg->($restore_error);
                    last;
                }
                else {
                    # Job still running or in unknown state
                    $log_msg->("Restore in progress (Job ID: $job_id, Status: $status, Elapsed: ${elapsed_time}s)");
                }
            }

            last if $restore_completed || $restore_error;
        }

        if ( !$restore_completed && !$restore_error ) {
            $restore_error = "Restore monitoring timed out after ${max_wait_time}s - check Comet server for status";
            $log_msg->($restore_error);
        }

        # Update final status
        if ($restore_completed) {
            $restore_logs->update_restore_job_status( $run_uuid, 'completed', 'Disaster recovery restore completed successfully' );
        }
        elsif ($restore_error) {
            $restore_logs->update_restore_job_status( $run_uuid, 'failed', $restore_error );
            die $restore_error;
        }

        close($log_fh);

        # Remove recovery flag only after restore completes successfully
        if ( $restore_completed && -f $recovery_flag ) {
            unlink($recovery_flag) or $logger->warn("Failed to remove recovery flag: $!");
        }

        $logger->info("Disaster recovery completed successfully");
    };

    if ($@) {
        my $error = $@;
        $logger->error("Disaster recovery failed: $error");

        # Write error to recovery log
        eval {
            require Cpanel::FileUtils::Write;
            my $log_content = "Disaster Recovery Failed";
            $log_content .= "=" x 50 . "";
            $log_content .= "Time: " . scalar( localtime() ) . "";
            $log_content .= "Error: $error";

            # If we have a log file from the restore attempt, append it
            my $run_uuid = eval {
                require Data::UUID;
                my $ug = Data::UUID->new();
                lc( $ug->create_str() );
            };

            if ($run_uuid) {
                my $log_file = Whostmgr::CometBackup::Constants::get_text_log_file_path( $run_uuid, $run_uuid );
                if ( -f $log_file ) {
                    require Cpanel::LoadFile;
                    my $restore_log = Cpanel::LoadFile::load($log_file);
                    $log_content .= "Restore Log: " . $restore_log;
                }
            }

            Cpanel::FileUtils::Write::overwrite( $recovery_log, $log_content, 0600 );
        };

        # Remove recovery flag on failure
        if ( -f $recovery_flag ) {
            unlink($recovery_flag) or $logger->warn("Failed to remove recovery flag: $!");
        }
    }
}

=head2 webpros_auth_status

Check if a valid WebPros authentication token exists.

=head3 Returns

Hashref with:
  - valid: boolean indicating if token is valid
  - expires_in: seconds until expiration (if valid)
  - email: user email from token (if available)
  - reason: reason for invalidity (if not valid)

=cut

sub webpros_auth_status {
    my ( $args, $metadata, $api_info_hr ) = @_;

    my $result;
    eval {
        require Whostmgr::CometBackup::WebProsAuth;

        $result = Whostmgr::CometBackup::WebProsAuth::get_auth_status();

        Whostmgr::API::1::Utils::set_metadata_ok($metadata);
    };

    if ($@) {
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "Failed to check WebPros auth status: $@" );
        $result = { valid => 0, reason => 'error', error => "$@" };
    }

    return $result;
}

=head2 webpros_auth_start

Start the WebPros SSO authentication flow for Comet Backup.

=head3 Returns

Hashref with:
  - auth_url: URL to redirect to for authentication
  - verifier: PKCE verifier (store in session)

=cut

sub webpros_auth_start {
    my ( $args, $metadata, $api_info_hr ) = @_;

    my $result;
    eval {
        require Whostmgr::CometBackup::WebProsAuth;

        my $state = {
            user_agent => $ENV{HTTP_USER_AGENT} || 'unknown',
            ip_address => $ENV{REMOTE_ADDR}     || 'unknown',
        };

        # Extract session ID from the request path if available
        my $session_id;
        if ( $ENV{REQUEST_URI} && $ENV{REQUEST_URI} =~ m{/cpsess(\d+)/} ) {
            $session_id = $1;
        }

        $result = Whostmgr::CometBackup::WebProsAuth::start_auth_flow( $state, $session_id );

        Whostmgr::API::1::Utils::set_metadata_ok($metadata);
    };

    if ($@) {
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "Failed to start WebPros auth: $@" );
        $result = { error => "$@" };
    }

    return $result;
}

=head2 webpros_auth_callback

Handle the OAuth callback from WebPros SSO.

=head3 Parameters

=over 4

=item C<code> - Authorization code from OAuth provider

=item C<verifier> - PKCE verifier from initial auth request

=item C<state> - State data passed through OAuth flow

=back

=head3 Returns

Hashref with:
  - success: boolean indicating if authentication succeeded
  - error: error message (if failed)

=cut

sub webpros_auth_callback {
    my ( $args, $metadata, $api_info_hr ) = @_;

    my $result = { success => 0 };    # Initialize with default failure
    eval {
        require Whostmgr::CometBackup::WebProsAuth;

        # Decode JSON arguments
        my ( $status, $decoded_args ) = _decode_json_args( $metadata, $args );
        if ( $status < 1 ) {
            $result = { success => 0, error => 'Failed to decode JSON arguments' };
            return;
        }

        my $code     = $decoded_args->{code};
        my $verifier = $decoded_args->{verifier};
        my $state    = $decoded_args->{state};

        unless ($code) {
            $result = { success => 0, error => 'Missing required parameter: code' };
            Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, $result->{error} );
            return;
        }
        unless ($verifier) {
            $result = { success => 0, error => 'Missing required parameter: verifier' };
            Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, $result->{error} );
            return;
        }

        $result = Whostmgr::CometBackup::WebProsAuth::handle_callback(
            $code,
            $verifier,
            $state
        );

        if ( $result->{success} ) {
            Whostmgr::API::1::Utils::set_metadata_ok($metadata);
        }
        else {
            Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, $result->{error} || 'Authentication failed' );
        }
    };

    if ($@) {
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "Failed to handle WebPros auth callback: $@" );
        $result = { success => 0, error => "$@" };
    }

    return $result;
}

=head2 webpros_auth_clear

Clear the stored WebPros authentication token.

=head3 Returns

Hashref with:
  - success: boolean indicating if token was cleared

=cut

sub webpros_auth_clear {
    my ( $args, $metadata, $api_info_hr ) = @_;

    my $result;
    eval {
        require Whostmgr::CometBackup::WebProsAuth;

        my $success = Whostmgr::CometBackup::WebProsAuth::clear_token();

        $result = { success => $success };

        Whostmgr::API::1::Utils::set_metadata_ok($metadata);
    };

    if ($@) {
        Whostmgr::API::1::Utils::set_metadata_not_ok( $metadata, "Failed to clear WebPros auth: $@" );
        $result = { success => 0, error => "$@" };
    }

    return $result;
}

1;
