package Whostmgr::CometBackup::CometBackupAPI::ProtectedItemBatch;

#                                      Copyright 2025 WebPros International, LLC
#                                                           All rights reserved.
# copyright@cpanel.net                                         http://cpanel.net
# This code is subject to the cPanel license. Unauthorized copying is prohibited.

# This module handles batch-oriented Protected Item management for Comet servers

use cPstrict;

use Cpanel::JSON                  ();
use Cpanel::PwCache               ();
use Data::UUID                    ();
use Whostmgr::CometBackup::Logger ();

=head1 NAME

Whostmgr::CometBackup::CometBackupAPI::ProtectedItemBatch - Batch-oriented Protected Item management for Comet servers

=head1 SYNOPSIS

    use Whostmgr::CometBackup::CometBackupAPI::ProtectedItemBatch;

    # Create a new batch manager for a user
    my $batch = Whostmgr::CometBackup::CometBackupAPI::ProtectedItemBatch->new(
        target_user => 'cpanel-backup-user'
    );

    # Add multiple protected items to the batch (UUIDs auto-generated)
    my $homedir_uid = $batch->add_protected_item(
        item_type    => 'homedir',
        system_user  => 'tau',
        configuration => {
            exclude_regex => ['.*/tmp/.*', '.*/cache/.*']
        }
    );

    my $db_uid = $batch->add_protected_item(
        item_type    => 'databases_mysql',
        system_user  => 'tau',
        configuration => {
            mysql_opts => '--single-transaction --routines --triggers'
        }
    );

    # Submit all items to Comet server at once
    my $success = $batch->submit();

=head1 DESCRIPTION

This module provides batch-oriented Protected Item management for Comet servers,
designed to efficiently handle multiple Protected Items per user profile.

=head2 new

    my $batch = Whostmgr::CometBackup::CometBackupAPI::ProtectedItemBatch->new(
        target_user => 'cpanel-backup-user'
    );

Creates a new Protected Item batch manager for a specific Comet user.

Parameters:
- target_user: The Comet server username (required)

=cut

sub new ( $class, %args ) {
    my $target_user = $args{target_user} or die "target_user parameter required";

    return bless {
        target_user     => $target_user,
        protected_items => [],
        user_profile    => undef,
        _logger         => Whostmgr::CometBackup::Logger->new( log_level => $ENV{DEBUG} ? 'debug' : 'info' ),
    }, $class;
}

=head2 add_protected_item

    my $item_uid = $batch->add_protected_item(
        item_type    => 'homedir',
        system_user  => 'tau',
        configuration => { ... }
    );

Adds a Protected Item to the batch. The item will be submitted to the Comet server
when submit() is called. Automatically generates and returns a UUID for the item.

Parameters:
- item_type: Type of protected item (required)
- system_user: cPanel system username (required)
- configuration: Hash reference with item-specific configuration (optional)

Returns: The generated UUID for the Protected Item

=cut

sub add_protected_item ( $self, %args ) {
    my $item_type     = $args{item_type}   or die "item_type parameter required";
    my $system_user   = $args{system_user} or die "system_user parameter required";
    my $configuration = $args{configuration} || {};

    # Use provided UUID if available, otherwise generate a new one
    my $item_uid = $args{item_uid};
    if ( !$item_uid ) {
        $item_uid = Data::UUID->new()->create_str();
        $self->{_logger}->debug(
            "DEBUG: Generated NEW UUID for Protected Item",
            {
                generated_uuid => $item_uid,
                item_type      => $item_type,
                system_user    => $system_user
            }
        );
    }
    else {
        $self->{_logger}->debug(
            "DEBUG: Using PROVIDED UUID for Protected Item",
            {
                provided_uuid => $item_uid,
                item_type     => $item_type,
                system_user   => $system_user
            }
        );
    }

    # Create Protected Item structure based on item type
    my $protected_item = $self->_create_protected_item_structure( $item_uid, $item_type, $system_user, $configuration );

    push @{ $self->{protected_items} }, {
        item_uid       => $item_uid,
        item_type      => $item_type,
        system_user    => $system_user,
        configuration  => $configuration,
        protected_item => $protected_item,
    };

    $self->{_logger}->debug(
        "Added Protected Item to batch",
        {
            item_uid    => $item_uid,
            item_type   => $item_type,
            system_user => $system_user,
            batch_size  => scalar( @{ $self->{protected_items} } )
        }
    );

    return $item_uid;
}

=head2 sync_protected_items

    my $success = Whostmgr::CometBackup::CometBackupAPI::ProtectedItemBatch->sync_protected_items(
        target_user => 'cpanel-backup-user'  # optional, defaults to server config
    );

Class method that reads all protected items from the database and syncs them to the
Comet server. This is the "source of truth" sync function that ensures the Comet
server's protected items match what's in the local database.

Parameters:
- target_user: The Comet server username (optional, defaults to username from server config)

Returns: 1 on success, 0 on failure

=cut

sub sync_protected_items ( $class, %args ) {
    my $logger = Whostmgr::CometBackup::Logger->new( log_level => $ENV{DEBUG} ? 'debug' : 'info' );

    # Get target_user from args or server config
    my $target_user = $args{target_user};
    unless ($target_user) {
        require Whostmgr::CometBackup::CometBackupAPI;
        my $server_config = Whostmgr::CometBackup::CometBackupAPI::get_server_config();
        $target_user = $server_config->{username} or die "Unable to determine target_user from server config";
    }

    $logger->debug(
        "Starting protected items sync from database",
        {
            target_user => $target_user,
            database    => $Whostmgr::CometBackup::Constants::PROTECTED_ITEMS_DB
        }
    );

    # Create a batch instance
    my $self = $class->new( target_user => $target_user );

    # Load all protected items from the database
    require Whostmgr::CometBackup::ProtectedItemMap;
    my $map = Whostmgr::CometBackup::ProtectedItemMap->new();

    # Get all active protected items across all users
    my $dbh = $map->_get_database_handle();
    my $sth = $dbh->prepare(
        q{
        SELECT protected_item_uid, job_uuid, system_user, cpanel_user_uuid,
               item_type, item_configuration, status
        FROM protected_items
        WHERE status IN ('pending', 'active')
        ORDER BY system_user, item_type
    }
    );

    $sth->execute();

    my $item_count = 0;
    my %items_by_user;

    # Collect all items and group by user
    while ( my $row = $sth->fetchrow_hashref() ) {
        my $system_user = $row->{system_user};
        my $item_type   = $row->{item_type};
        my $item_uid    = $row->{protected_item_uid};

        # Decode configuration
        my $configuration = {};
        if ( $row->{item_configuration} ) {
            eval { $configuration = Cpanel::JSON::Load( $row->{item_configuration} ); };
            if ($@) {
                $logger->warn(
                    "Failed to decode configuration for protected item",
                    {
                        item_uid    => $item_uid,
                        system_user => $system_user,
                        error       => $@
                    }
                );
                $configuration = {};
            }
        }

        # Add to batch using internal structure
        my $protected_item = $self->_create_protected_item_structure(
            $item_uid,
            $item_type,
            $system_user,
            $configuration
        );

        push @{ $self->{protected_items} }, {
            item_uid       => $item_uid,
            item_type      => $item_type,
            system_user    => $system_user,
            configuration  => $configuration,
            protected_item => $protected_item,
        };

        $items_by_user{$system_user}++;
        $item_count++;

        $logger->debug(
            "Loaded protected item from database",
            {
                item_uid    => $item_uid,
                item_type   => $item_type,
                system_user => $system_user
            }
        );
    }

    $logger->debug(
        "Loaded protected items from database",
        {
            total_items => $item_count,
            total_users => scalar( keys %items_by_user ),
            target_user => $target_user
        }
    );

    # Submit all items to Comet server
    if ( $item_count > 0 ) {
        my $success = $self->submit();

        if ($success) {

            # Mark all items as synced
            my @item_uids = map { $_->{item_uid} } @{ $self->{protected_items} };
            $map->mark_items_synced_to_comet( \@item_uids );

            # Clear the batch after marking items as synced
            $self->{protected_items} = [];

            $logger->debug(
                "Successfully synced protected items to Comet server",
                {
                    synced_count => scalar(@item_uids),
                    target_user  => $target_user
                }
            );
        }
        else {
            $logger->error(
                "Failed to sync protected items to Comet server",
                {
                    item_count  => $item_count,
                    target_user => $target_user
                }
            );
        }

        return $success;
    }
    else {
        $logger->debug("No protected items to sync");
        return 1;    # No items is considered success
    }
}

=head2 ensure_recovery_data_protected_item

    my $success = $batch->ensure_recovery_data_protected_item();

Creates or updates the special recovery_data Protected Item with scheduling and retention.
This item backs up critical recovery data from /var/cpanel/comet/ and ensures all
available destinations are configured.

Returns: 1 on success, 0 on failure

=cut

=head2 ensure_recovery_data_protected_item

Ensures that the special "recovery_data" Protected Item exists and is configured correctly.
This item backs up /var/cpanel/comet/ every 5 minutes for disaster recovery purposes.

This is a special system-level Protected Item that is the same across all servers and
uses Comet's native scheduling to provide automatic backup of critical server configuration.

=head3 Returns

Returns 1 on success, 0 on failure.

=head3 Example

    my $batch = Whostmgr::CometBackup::CometBackupAPI::ProtectedItemBatch->new(
        target_user => 'backup-user'
    );

    my $success = $batch->ensure_recovery_data_protected_item();

=cut

sub ensure_recovery_data_protected_item ($self) {
    my $logger = $self->{_logger};

    require Whostmgr::CometBackup::Constants;
    my $item_name = $Whostmgr::CometBackup::Constants::RECOVERY_DATA_ITEM_NAME;

    $logger->debug("Ensuring recovery_data Protected Item exists and is up-to-date");

    # Generate a consistent UUID for recovery_data using the name
    # This ensures we always use the same ID for this special item
    my $item_uid = $self->_generate_recovery_data_uid();

    # Create the recovery_data Protected Item structure
    my $protected_item = $self->_create_protected_item_structure(
        $item_uid,
        'recovery_data',
        'root',    # Use root as the system_user for this system-level item
        {}
    );

    # Get current user profile to check if it already exists
    my $profile_response = $self->_get_user_profile();
    unless ( $profile_response && $profile_response->{Profile} ) {
        $logger->error("Unable to retrieve user profile for recovery_data setup");
        return 0;
    }

    my $profile = $profile_response->{Profile};

    # Decode if it's a JSON string
    if ( !ref($profile) ) {
        eval { $profile = Cpanel::JSON::Load($profile); };
        if ($@) {
            $logger->error( "Failed to decode profile", { error => $@ } );
            return 0;
        }
    }

    my $sources      = $profile->{Sources}     ||= {};
    my $backup_rules = $profile->{BackupRules} ||= {};
    my $profile_hash = $profile_response->{ProfileHash};
    my $needs_update = 0;
    my $action       = 'none';

    # Get current destinations
    my $current_destinations = $protected_item->{OverrideDestinationRetention} || {};
    my @current_dest_ids     = sort keys %$current_destinations;

    # Check if recovery_data item already exists
    if ( exists $sources->{$item_uid} ) {
        $logger->debug("recovery_data Protected Item already exists, checking if update needed");

        # Check if destinations match
        my $existing_item         = $sources->{$item_uid};
        my $existing_destinations = $existing_item->{OverrideDestinationRetention} || {};
        my @existing_dest_ids     = sort keys %$existing_destinations;

        if ( join( ',', @existing_dest_ids ) ne join( ',', @current_dest_ids ) ) {
            $logger->debug(
                "Destinations have changed, updating recovery_data Protected Item",
                {
                    old_destinations => \@existing_dest_ids,
                    new_destinations => \@current_dest_ids
                }
            );
            $needs_update = 1;
            $action       = 'update';
        }

        # Check if BackupRules exist for all destinations
        my %existing_rules;
        for my $rule_id ( keys %$backup_rules ) {
            my $rule = $backup_rules->{$rule_id};
            if ( $rule->{Source} && $rule->{Source} eq $item_uid ) {
                $existing_rules{ $rule->{Destination} } = $rule_id;
            }
        }

        for my $dest_id (@current_dest_ids) {
            unless ( exists $existing_rules{$dest_id} ) {
                $logger->debug(
                    "Missing BackupRule for destination, update needed",
                    { destination_id => $dest_id }
                );
                $needs_update = 1;
                $action       = 'update';
                last;
            }
        }
    }
    else {
        $logger->debug("recovery_data Protected Item does not exist, creating");
        $needs_update = 1;
        $action       = 'create';
    }

    if ($needs_update) {

        # Add/update the Protected Item
        $sources->{$item_uid} = $protected_item;

        # Create or update BackupRules for each destination
        # Remove old rules that are no longer needed
        my %current_dest_map = map { $_ => 1 } @current_dest_ids;
        my @rules_to_remove;

        for my $rule_id ( keys %$backup_rules ) {
            my $rule = $backup_rules->{$rule_id};
            if ( $rule->{Source} && $rule->{Source} eq $item_uid ) {

                # This is a rule for our recovery_data source
                unless ( exists $current_dest_map{ $rule->{Destination} } ) {

                    # Destination no longer exists, remove this rule
                    push @rules_to_remove, $rule_id;
                }
            }
        }

        # Remove obsolete rules
        delete $backup_rules->{$_} for @rules_to_remove;

        # Create BackupRules for each destination
        for my $dest_id (@current_dest_ids) {

            # Check if a rule already exists for this source+destination combo
            my $existing_rule_id;
            for my $rule_id ( keys %$backup_rules ) {
                my $rule = $backup_rules->{$rule_id};
                if ( $rule->{Source} && $rule->{Source} eq $item_uid && $rule->{Destination} eq $dest_id ) {
                    $existing_rule_id = $rule_id;
                    last;
                }
            }

            # Create new rule if it doesn't exist
            unless ($existing_rule_id) {
                my $rule_uid = Data::UUID->new()->create_str();

                $backup_rules->{$rule_uid} = {
                    Description => "Recovery Backup Schedule",
                    CreateTime  => int( time() ),
                    ModifyTime  => int( time() ),
                    PreExec     => [],
                    ThawExec    => [],
                    PostExec    => [],
                    Source      => $item_uid,
                    Destination => $dest_id,
                    Schedules   => [
                        {
                            FrequencyType   => 8015,    # Periodic (seconds-based)
                            SecondsPast     => 300,     # Every 5 minutes (300 seconds)
                            RestrictRuntime => \0,      # JSON false
                            FromTime        => {
                                Hour    => 0,
                                Minutes => 0,
                            },
                            ToTime => {
                                Hour    => 0,
                                Minutes => 0,
                            },
                            RestrictDays => \0,         # JSON false
                            DaysSelect   => {
                                Sunday    => \1,        # JSON true
                                Monday    => \1,
                                Tuesday   => \1,
                                Wednesday => \1,
                                Thursday  => \1,
                                Friday    => \1,
                                Saturday  => \1,
                            },
                            SelectedMonth => 0,
                            SelectedDay   => 0,
                        }
                    ],
                    EventTriggers => {
                        OnPCBootIfLastJobMissed => \1,    # JSON true
                        LastJobFailDoRetryCount => 1,
                        LastJobFailDoRetryTime  => 30,
                    },
                    SkipAlreadyRunning    => \0,    # JSON false
                    StopAfter             => 0,
                    LimitVaultSpeedBps    => 0,
                    ReduceDiskConcurrency => \0,    # JSON false
                    UseOnDiskIndexes      => \0,    # JSON false
                    AllowZeroFilesSuccess => \0,    # JSON false
                    AutoRetentionLevel    => 0,
                    ConcurrencyCount      => 0,
                    LogLevel              => "",
                };

                $logger->debug(
                    "Created BackupRule for recovery_data",
                    {
                        rule_id        => $rule_uid,
                        source_id      => $item_uid,
                        destination_id => $dest_id,
                    }
                );
            }
        }

        # Update the profile on the Comet server
        require Whostmgr::CometBackup::CometBackupAPI;

        my $profile_json = Cpanel::JSON::Dump($profile);
        my $request_data = {
            ProfileData => $profile_json,
            ProfileHash => $profile_hash,
        };

        my $result = Whostmgr::CometBackup::CometBackupAPI::make_server_request(
            "user/web/set-profile-hash",
            $request_data
        );

        if ( $result && $result->{Status} == 200 ) {
            $logger->debug(
                "Successfully ${action}d recovery_data Protected Item and BackupRules",
                {
                    item_uid     => $item_uid,
                    destinations => \@current_dest_ids,
                    backup_rules => scalar(@current_dest_ids),
                    schedule     => 'every 5 minutes'
                }
            );

            # Cache the latest hash
            $self->_cache_latest_hash($result);
            return 1;
        }
        else {
            $logger->error(
                "Failed to ${action} recovery_data Protected Item",
                {
                    status  => $result->{Status}  // 'unknown',
                    message => $result->{Message} // 'unknown error'
                }
            );
            return 0;
        }
    }
    else {
        $logger->debug("recovery_data Protected Item is already up-to-date");
        return 1;
    }
}

=head2 _generate_recovery_data_uid

Generates a consistent UUID for the recovery_data Protected Item.
This ensures we always use the same ID for this special system item.

=cut

sub _generate_recovery_data_uid ($self) {

    # Generate a deterministic UUID based on the constant name
    # Using a namespace UUID (v5) approach with a fixed namespace
    require Whostmgr::CometBackup::Constants;
    my $item_name = $Whostmgr::CometBackup::Constants::RECOVERY_DATA_ITEM_NAME;

    # Create a simple hash-based UUID from the name
    # This ensures it's always the same for recovery_data
    require Digest::SHA;
    my $hash = Digest::SHA::sha256_hex($item_name);

    # Format as UUID
    my $uuid = substr( $hash, 0, 8 ) . '-' . substr( $hash, 8, 4 ) . '-' . '4' . substr( $hash, 13, 3 ) . '-' .    # Version 4
      substr( $hash, 16, 4 ) . '-' . substr( $hash, 20, 12 );

    return lc($uuid);
}

=head2 verify_recovery_data_is_backing_up

    my $status = Whostmgr::CometBackup::CometBackupAPI::ProtectedItemBatch->verify_recovery_data_is_backing_up();

Class method that verifies the recovery_data Protected Item is actively being backed up.
Checks if a backup job has run in the last 10 minutes (since schedule is every 5 minutes).

Returns: hashref with status information
  {
    is_backing_up    => 1 or 0,
    last_backup_time => Unix timestamp or undef,
    error            => error message if check failed
  }

=cut

sub verify_recovery_data_is_backing_up ($class) {
    my $logger = Whostmgr::CometBackup::Logger->new( log_level => $ENV{DEBUG} ? 'debug' : 'info' );

    require Whostmgr::CometBackup::Constants;
    require Whostmgr::CometBackup::CometBackupAPI;

    my $result = {
        is_backing_up    => 0,
        last_backup_time => undef,
        error            => undef,
    };

    eval {

        # Get server config
        my $server_config = Whostmgr::CometBackup::CometBackupAPI::get_server_config();
        my $username      = $server_config->{username};
        my $password      = $server_config->{password};

        unless ( $username && $password ) {
            $result->{error} = "Server credentials not configured";
            return;
        }

        # Build the query to search for recent backup jobs
        # Query structure matches the format shown in the user's curl example
        my $query_obj = {
            ClauseType     => "and",
            ClauseChildren => [
                {
                    RuleField    => "BackupJobDetail.TimeSinceStarted",
                    RuleOperator => "int_lt",
                    RuleValue    => "600",                                # Last 10 minutes (600 seconds)
                }
            ]
        };

        my $query_json = Cpanel::JSON::Dump($query_obj);

        # Make API request to search for recent backup jobs
        my $api_result = Whostmgr::CometBackup::CometBackupAPI::make_server_request(
            "user/web/get-jobs-for-custom-search",
            {
                Query => $query_json,
            }
        );

        if ( $api_result && $api_result->{Status} == 200 ) {
            my $jobs = $api_result->{Jobs} || [];

            $logger->debug(
                "Recovery data backup verification",
                {
                    jobs_found  => scalar(@$jobs),
                    time_window => "last 10 minutes"
                }
            );

            # Check if any jobs match our recovery_data source
            my $recovery_uid = $class->new( target_user => $username )->_generate_recovery_data_uid();

            for my $job (@$jobs) {

                # Check if this job is for our recovery_data Protected Item
                if ( $job->{SourceGUID} && $job->{SourceGUID} eq $recovery_uid ) {
                    $result->{is_backing_up}    = 1;
                    $result->{last_backup_time} = $job->{StartTime} || time();

                    $logger->debug(
                        "recovery_data is actively backing up",
                        {
                            source_id  => $recovery_uid,
                            start_time => $result->{last_backup_time},
                            job_status => $job->{Status} || 'unknown'
                        }
                    );

                    last;
                }
            }

            if ( !$result->{is_backing_up} ) {
                $logger->warn(
                    "No recent backup jobs found for recovery_data",
                    {
                        expected_source_id => $recovery_uid,
                        total_jobs_checked => scalar(@$jobs),
                        time_window        => "last 10 minutes"
                    }
                );
            }
        }
        else {
            $result->{error} = "Failed to query backup jobs: " . ( $api_result->{Message} || "Unknown error" );
            $logger->error(
                "Failed to verify recovery_data backup status",
                {
                    status => $api_result->{Status} // 'unknown',
                    error  => $result->{error}
                }
            );
        }
    };

    if ($@) {
        $result->{error} = "Exception during verification: $@";
        $logger->error( "Exception verifying recovery_data backup", { error => $@ } );
    }

    return $result;
}

=head2 submit

    my $success = $batch->submit();

Submits all queued Protected Items to the Comet server by updating the user profile.
This method handles retrieving the current user profile, adding all Protected Items
as Sources, and updating the profile on the server.

Returns: 1 on success, 0 on failure

=cut

sub submit ($self) {
    my $logger = $self->{_logger};

    unless ( @{ $self->{protected_items} } ) {
        $logger->warn("No Protected Items to submit");
        return 1;    # Nothing to do is considered success
    }

    $logger->debug(
        "Submitting Protected Items batch",
        {
            target_user => $self->{target_user},
            item_count  => scalar( @{ $self->{protected_items} } )
        }
    );

    # Retry logic for hash mismatch errors
    my $max_retries = 3;
    for my $attempt ( 1 .. $max_retries ) {
        $logger->debug("DEBUG: Submit attempt $attempt of $max_retries for target_user: $self->{target_user}");

        # Get current user profile (always fetch fresh on retry attempts)
        $logger->debug("DEBUG: Getting user profile for attempt $attempt");
        my $profile_response = $self->_get_user_profile();
        unless ($profile_response) {
            $logger->error("Failed to retrieve user profile for batch submission");
            return 0;
        }

        my $profile_hash = $profile_response->{ProfileHash} || 'UNDEFINED';
        $logger->debug("DEBUG: Retrieved user profile with hash: $profile_hash");

        # Extract the actual profile from the response
        # The Profile field may be a JSON string that needs to be decoded
        my $user_profile = $profile_response->{Profile};
        unless ($user_profile) {
            $logger->error("Profile data not found in API response");
            return 0;
        }

        # If Profile is a JSON string, decode it
        if ( !ref($user_profile) ) {
            $logger->debug("DEBUG: Profile is a string, decoding JSON");
            eval { $user_profile = Cpanel::JSON::Load($user_profile); };
            if ($@) {
                $logger->error( "Failed to decode Profile JSON", { error => $@ } );
                return 0;
            }
        }

        $logger->debug( "DEBUG: Profile is now a " . ref($user_profile) . " reference" );

        # Debug: Check username consistency
        my $current_profile_username = $user_profile->{Username} || 'UNDEFINED';
        $logger->debug("DEBUG: Profile contains Username: '$current_profile_username', target_user is: '$self->{target_user}'");

        if ( $current_profile_username ne $self->{target_user} ) {
            $logger->debug("DEBUG: WARNING - Username mismatch! Profile username '$current_profile_username' != target_user '$self->{target_user}'");

            # Fix the username in the profile to match target_user
            $user_profile->{Username} = $self->{target_user};
            $logger->debug("DEBUG: Updated profile Username to: '$self->{target_user}'");
        }

        # Add all Protected Items as Sources
        my $sources = $user_profile->{Sources} ||= {};
        my $sources_before_count = scalar( keys %$sources );
        $logger->debug("DEBUG: User profile has $sources_before_count existing sources");

        # Log what Source IDs we're about to send
        my @source_ids_being_added;
        for my $item ( @{ $self->{protected_items} } ) {
            my $source_id = $item->{item_uid};
            push @source_ids_being_added, $source_id;
            $sources->{$source_id} = $item->{protected_item};

            $logger->debug(
                "Added Protected Item as Source",
                {
                    item_uid    => $item->{item_uid},
                    source_id   => $source_id,
                    item_type   => $item->{item_type},
                    system_user => $item->{system_user}
                }
            );
        }

        my $sources_after_count = scalar( keys %$sources );
        $logger->debug("DEBUG: After adding items, user profile has $sources_after_count sources");
        $logger->debug( "DEBUG: Source IDs being submitted to Comet server", { source_ids => \@source_ids_being_added } );
        $logger->debug("DEBUG: About to submit profile with hash: $profile_hash");

        # Update user profile on server
        require Whostmgr::CometBackup::CometBackupAPI;

        # Serialize the profile for debugging
        my $profile_json         = Cpanel::JSON::Dump($user_profile);
        my $profile_json_length  = length($profile_json);
        my $profile_json_preview = substr( $profile_json, 0, 500 );

        $logger->debug("DEBUG: Serialized profile JSON length: $profile_json_length");
        $logger->debug("DEBUG: Profile JSON preview (first 500 chars): $profile_json_preview");

        # Pass ProfileData as JSON string and ProfileHash as string
        # The Comet API expects ProfileData to be a JSON-encoded string
        my $request_data = {
            ProfileData => $profile_json,
            ProfileHash => $profile_hash,
        };

        $logger->debug("DEBUG: Submitting to user/web/set-profile-hash with ProfileHash: $profile_hash");
        my $result = Whostmgr::CometBackup::CometBackupAPI::make_server_request(
            "user/web/set-profile-hash",
            $request_data
        );

        $logger->debug( "DEBUG: set-profile-hash API returned: " . ( $result ? "Status: $result->{Status}, Message: " . ( $result->{Message} || 'NO MESSAGE' ) : 'NO RESULT' ) );

        if ( $result && $result->{Status} == 200 ) {
            $logger->debug(
                "Successfully submitted Protected Items batch",
                {
                    target_user => $self->{target_user},
                    item_count  => scalar( @{ $self->{protected_items} } ),
                    attempt     => $attempt
                }
            );

            # Verify what was actually created by fetching the profile again
            $logger->debug("DEBUG: Verifying submitted Source IDs by fetching profile...");
            eval {
                my $verify_profile = $self->_get_user_profile();
                if ( $verify_profile && $verify_profile->{Profile} ) {
                    my $verify_profile_data = $verify_profile->{Profile};
                    if ( !ref($verify_profile_data) ) {
                        $verify_profile_data = Cpanel::JSON::Load($verify_profile_data);
                    }
                    my $verify_sources    = $verify_profile_data->{Sources} || {};
                    my @actual_source_ids = keys %$verify_sources;

                    $logger->debug(
                        "DEBUG: Server now has Source IDs",
                        {
                            count      => scalar(@actual_source_ids),
                            source_ids => \@actual_source_ids
                        }
                    );

                    # Check if our submitted IDs match what's on the server
                    my %submitted = map { $_ => 1 } @source_ids_being_added;
                    my @missing;
                    my @unexpected;

                    for my $submitted_id (@source_ids_being_added) {
                        unless ( exists $verify_sources->{$submitted_id} ) {
                            push @missing, $submitted_id;
                        }
                    }

                    if (@missing) {
                        $logger->warn( "DEBUG: Some submitted Source IDs are missing from server!", { missing_ids => \@missing } );
                    }
                    else {
                        $logger->debug("DEBUG: All submitted Source IDs were successfully created on server");
                    }
                }
            };
            if ($@) {
                $logger->warn( "DEBUG: Failed to verify Source IDs", { error => $@ } );
            }

            # Cache the latest hash after successful update
            $self->_cache_latest_hash($result);

            # Return success (don't clear the batch - caller needs to access protected_items for marking as synced)
            return 1;
        }

        # Check if this is a hash mismatch error (409)
        if ( $result && $result->{Status} == 409 && $result->{Message} =~ /Hash Mismatch/i ) {
            $logger->warn(
                "Hash mismatch detected, will retry",
                {
                    target_user => $self->{target_user},
                    attempt     => $attempt,
                    max_retries => $max_retries,
                    error       => $result->{Message}
                }
            );

            # Don't retry on the last attempt
            if ( $attempt < $max_retries ) {

                # Brief delay before retry to allow other operations to complete
                sleep(1);
                next;
            }
        }

        # For any other error or final attempt, log and fail
        $logger->error(
            "Failed to submit Protected Items batch",
            {
                target_user   => $self->{target_user},
                status        => $result->{Status}  // 'unknown',
                message       => $result->{Message} // 'unknown error',
                attempt       => $attempt,
                final_attempt => $attempt == $max_retries ? 1 : 0
            }
        );

        # Don't retry for non-hash-mismatch errors
        last unless ( $result && $result->{Status} == 409 );
    }

    return 0;
}

=head2 _create_protected_item_structure

Creates the appropriate Comet Protected Item (Source) structure based on the item type.

=cut

sub _create_protected_item_structure ( $self, $item_uid, $item_type, $system_user, $config ) {
    my $display_name = "$system_user - " . ucfirst($item_type);

    # Get the device ID from the user profile
    my $device_id = $self->_get_device_id();

    # Base structure for all Protected Items (Comet Source format)
    # Must match the structure of Sources created through Comet UI
    my $base_source = {
        Description        => "cPanel $item_type backup for user $system_user",
        OwnerDevice        => $device_id,
        CreateTime         => int( time() ),
        ModifyTime         => int( time() ),
        PreExec            => [],
        ThawExec           => [],
        PostExec           => [],
        EngineProps        => {},
        PolicySourceID     => "",
        ExistingUserUpdate => \0,                                                 # JSON false as boolean reference
        Statistics         => {
            LastStartTime => 0,
            LastBackupJob => {
                GUID             => "",
                Username         => "",
                Classification   => 0,
                Status           => 0,
                StartTime        => 0,
                EndTime          => 0,
                RetryCount       => 0,
                SourceGUID       => "",
                DestinationGUID  => "",
                DeviceID         => "",
                ClientVersion    => "",
                TotalDirectories => 0,
                TotalFiles       => 0,
                TotalSize        => 0,
                TotalChunks      => 0,
                UploadSize       => 0,
                DownloadSize     => 0,
            },
            LastSuccessfulBackupJob => {
                GUID             => "",
                Username         => "",
                Classification   => 0,
                Status           => 0,
                StartTime        => 0,
                EndTime          => 0,
                RetryCount       => 0,
                SourceGUID       => "",
                DestinationGUID  => "",
                DeviceID         => "",
                ClientVersion    => "",
                TotalDirectories => 0,
                TotalFiles       => 0,
                TotalSize        => 0,
                TotalChunks      => 0,
                UploadSize       => 0,
                DownloadSize     => 0,
            },
        },
    };

    if ( $item_type eq 'homedir_files_folders' ) {

        # Files and Folders PI for homedir with include/exclude patterns for email content filtering
        # Get the actual home directory for the user
        my $homedir = Cpanel::PwCache::gethomedir($system_user);

        my $source = {
            %$base_source,
            Description => "$system_user - Home Directory",
            Engine      => "engine1/file",
            EngineProps => {
                USE_WIN_VSS      => "1",
                EXTRA_ATTRIBUTES => "1",
            }
        };

        # Always include homedir as literal path
        my $include_key = "INCLUDE-" . $self->_generate_uuid();
        $source->{EngineProps}{$include_key} = $homedir;

        # Add standard exclusions
        my @exclude_patterns = (
            "$homedir/.trash",
            "$homedir/tmp",
            "$homedir/logs"    # May want to keep logs ?
        );

        # Add email filtering based on configuration
        if ( $config->{exclude_junk} ) {
            push @exclude_patterns,
              "$homedir/mail/[^/]*/\\.INBOX\\.INBOX\\/.*",    # Spam folders
              "$homedir/mail/[^/]*/cur/.*:.*,.*S.*",          # Spam-flagged messages
              "$homedir/mail/.Junk",                          # Junk folders
              "$homedir/mail/.*/.Junk"                        # Email account level junk such as /home/sysuser/mail/domain.tld/luser/.Junk/
        }

        # Add custom exclude patterns if provided
        # These are converted to regex by ProtectedItemMap::_glob_to_regex()
        # The patterns are designed to match against full paths (including homedir)
        # e.g., "*.log" becomes ".*\.log$" which matches "/home/user/file.log"
        # e.g., "**/cache/*" becomes "(?:^|.*/)cache/.*" which matches "/home/user/app/cache/file"
        if ( $config->{exclude_regex} && ref $config->{exclude_regex} eq 'ARRAY' ) {
            push @exclude_patterns, @{ $config->{exclude_regex} };
        }

        # Add exclude patterns with UUID keys
        for my $pattern (@exclude_patterns) {
            my $exclude_key = "REXCLUDE-" . $self->_generate_uuid();
            $source->{EngineProps}{$exclude_key} = $pattern;
        }

        return $source;
    }
    elsif ( $item_type eq 'databases_program_output' ) {

        my $homedir = Cpanel::PwCache::gethomedir($system_user);
        my $command = "/usr/local/cpanel/scripts/pkgacct $system_user";

        # Options is everything needed to ensure only databases are part of this particular cpmove archive
        my $options = '--stdout-archive --skipuserdata --skipssl --skipdomainkeys --skipshell --skipresellerconfig --skipquota --skippublichtml --skippgsql --skippasswd --skipmailman --skipmailconfig --skiplogs --skiplocale --skipintegrationlinks --skiphomedir --skipftpusers --skipdomains --skipdnszones --skipbwdata --skiplinkednodes --skipauthnlinks --skipapitokens --dbbackup=all';
        $command .= " $options";

        # For right now we are just going to back up all DBs together
        # if ( $config->{mysql} ) {
        #     $command .= " --include-mysql=yes";
        # }
        # if ( $config->{postgresql} ) {
        #     $command .= " --include-postgresql=yes";
        # }

        return {
            %$base_source,
            Description => "$system_user - Databases",
            Engine      => "engine1/stdout",
            EngineProps => {
                COMMAND => $command,
                SAVEAS  => "$system_user-databases.tar.gz",
                WORKDIR => $homedir,
            }
        };
    }
    elsif ( $item_type eq 'pkgacct_program_output' ) {

        # Program Output PI for pkgacct with flags to get everything except databases and homedir
        my $homedir = Cpanel::PwCache::gethomedir($system_user);
        my $command = '/usr/local/cpanel/scripts/pkgacct ' . $system_user . ' --skiphomedir --skipacctdb --stdout-archive';

        return {
            %$base_source,
            Description => "$system_user - pkgacct",
            Engine      => "engine1/stdout",
            EngineProps => {
                COMMAND => $command,
                SAVEAS  => "$system_user-pkgacct.tar.gz",
                WORKDIR => $homedir,
            }
        };
    }
    elsif ( $item_type eq 'recovery_data' ) {

        # Special recovery_data Protected Item - files and folders type with schedule
        # This backs up critical recovery metadata from /var/cpanel/comet/
        require Whostmgr::CometBackup::Constants;
        my $recovery_path = $Whostmgr::CometBackup::Constants::RECOVERY_DATA_PATH;

        my $source = {
            %$base_source,
            Description => "recovery_data",    # Must match exactly as specified
            Engine      => "engine1/file",
            EngineProps => {
                USE_WIN_VSS => "1",
            }
        };

        # Add the recovery data path
        my $include_key = "INCLUDE-" . $self->_generate_uuid();
        $source->{EngineProps}{$include_key} = $recovery_path;

        # Get all available destinations and configure retention for each
        my $destinations = $self->_get_available_destinations();
        if ( $destinations && ref($destinations) eq 'HASH' && %$destinations ) {
            $source->{OverrideDestinationRetention} = {};

            for my $dest_id ( keys %$destinations ) {

                # Configure retention: keep last 10 jobs, and last 30 jobs total
                $source->{OverrideDestinationRetention}{$dest_id} = {
                    Mode   => 802,    # Custom retention mode
                    Ranges => [
                        {
                            Type        => 900,    # Jobs-based retention
                            Timestamp   => 0,
                            Jobs        => 10,     # Keep last 10 jobs
                            Days        => 0,
                            Weeks       => 0,
                            Months      => 0,
                            Years       => 0,
                            WeekOffset  => 0,
                            MonthOffset => 1,
                            YearOffset  => 1,
                        },
                        {
                            Type        => 907,    # Additional jobs retention
                            Timestamp   => 0,
                            Jobs        => 30,     # Keep total of 30 jobs
                            Days        => 0,
                            Weeks       => 0,
                            Months      => 0,
                            Years       => 0,
                            WeekOffset  => 0,
                            MonthOffset => 1,
                            YearOffset  => 1,
                        }
                    ]
                };
            }
        }

        return $source;
    }
    else {
        # Fallback for unknown types
        die "Unknown Protected Item type: $item_type. Expected one of: homedir_files_folders, databases_program_output, pkgacct_program_output, recovery_data";
    }
}

=head2 _get_user_profile

Retrieves the current user profile from the Comet server with hash caching support.
Implements fallback logic to refresh hash in case of hash mismatch errors.

=cut

sub _get_user_profile ($self) {
    require Whostmgr::CometBackup::CometBackupAPI;

    my $logger = $self->{_logger};

    $logger->debug("DEBUG: About to fetch user profile and hash via get-user-profile-and-hash API");

    # Always fetch fresh profile to ensure we have the latest hash
    my $result = Whostmgr::CometBackup::CometBackupAPI::make_server_request("user/web/get-user-profile-and-hash");

    if ( $result && $result->{Status} == 200 ) {
        my $hash_value = $result->{ProfileHash} || 'UNDEFINED';
        $logger->debug("DEBUG: Successfully retrieved user profile with hash: $hash_value");

        # DEBUG: Check the type of Profile field
        my $profile_ref_type = ref( $result->{Profile} ) || 'SCALAR';
        $logger->debug("DEBUG: Profile field ref type: $profile_ref_type");

        if ( !ref( $result->{Profile} ) ) {
            my $profile_length  = length( $result->{Profile} || '' );
            my $profile_preview = substr( $result->{Profile} || '', 0, 200 );
            $logger->debug("DEBUG: Profile is a STRING with length $profile_length, preview: $profile_preview");
        }
        else {
            $logger->debug("DEBUG: Profile is already a reference type: $profile_ref_type");

            # Show some keys if it's a hash
            if ( $profile_ref_type eq 'HASH' ) {
                my @keys = sort keys %{ $result->{Profile} };
                $logger->debug( "DEBUG: Profile hash has keys: " . join( ', ', @keys[ 0 .. 9 ] ) . ( scalar(@keys) > 10 ? '...' : '' ) );
            }
        }

        $logger->debug(
            "Successfully retrieved user profile",
            {
                target_user => $self->{target_user},
                has_hash    => defined( $result->{ProfileHash} ) ? 1 : 0,
                hash_value  => $hash_value
            }
        );

        # Cache the hash from this fresh fetch
        if ( $result->{ProfileHash} ) {
            $logger->debug("DEBUG: Caching hash value: $result->{ProfileHash}");
            $self->_cache_latest_hash($result);
        }
        else {
            $logger->debug("DEBUG: No ProfileHash in response to cache");
        }

        return $result;
    }

    # If we get here, the request failed
    my $error_msg = $result ? "Status: $result->{Status}, Message: $result->{Message}" : "No response from server";
    $logger->debug("DEBUG: Failed to get user profile: $error_msg");
    $logger->warn(
        "Failed to retrieve user profile",
        {
            target_user => $self->{target_user},
            status      => $result->{Status}  // 'unknown',
            error       => $result->{Message} // 'unknown error'
        }
    );

    return undef;
}

=head2 _cache_latest_hash

Caches the latest hash from a successful API response to the server credentials file.
This helps avoid hash mismatch errors on subsequent API calls.

=cut

sub _cache_latest_hash ( $self, $api_response ) {
    my $logger = $self->{_logger};

    # Extract hash from the response if available
    my $latest_hash = $api_response->{ProfileHash} || $api_response->{Hash};
    unless ($latest_hash) {
        $logger->debug("No hash found in API response to cache");
        return;
    }

    require Whostmgr::CometBackup::Constants;
    my $credentials_file = $Whostmgr::CometBackup::Constants::CREDENTIALS_FILE;

    eval {
        # Read current credentials
        my $credentials = {};
        if ( -f $credentials_file ) {
            open my $fh, '<', $credentials_file or die "Cannot read credentials file: $!";
            my $content = do { local $/; <$fh> };
            close $fh;
            $credentials = Cpanel::JSON::Load($content) if $content;
        }

        # Update the cached hash
        $credentials->{cached_profile_hash}  = $latest_hash;
        $credentials->{hash_cache_timestamp} = time();

        # Write back to file
        require File::Basename;
        require File::Path;
        File::Path::make_path( File::Basename::dirname($credentials_file) );

        open my $fh, '>', $credentials_file or die "Cannot write credentials file: $!";
        print $fh Cpanel::JSON::Dump($credentials);
        close $fh;

        $logger->debug(
            "Cached latest profile hash",
            {
                hash      => substr( $latest_hash, 0, 8 ) . '...',    # Log only first 8 chars for security
                timestamp => $credentials->{hash_cache_timestamp}
            }
        );
    };

    if ($@) {
        $logger->warn(
            "Failed to cache profile hash",
            {
                error => $@,
                file  => $credentials_file
            }
        );
    }
}

=head2 _generate_uuid

Generates a UUID for use in EngineProps keys (e.g., INCLUDE-uuid, REXCLUDE-uuid).

=cut

sub _generate_uuid ($self) {

    # Simple UUID v4 generation
    my @chars = ( '0' .. '9', 'a' .. 'f' );
    my $uuid  = '';

    for my $i ( 0 .. 31 ) {
        if ( $i == 8 || $i == 12 || $i == 16 || $i == 20 ) {
            $uuid .= '-';
        }
        $uuid .= $chars[ rand @chars ];
    }

    # Set version (4) and variant bits
    substr( $uuid, 14, 1 ) = '4';
    substr( $uuid, 19, 1 ) = $chars[ 8 + rand(4) ];    # 8, 9, a, or b

    return $uuid;
}

=head2 _get_device_id

Gets the device ID from the cached user profile, or fetches it if not cached.

=cut

sub _get_device_id ($self) {
    my $logger = $self->{_logger};

    # Return cached device ID if we have it
    if ( $self->{_cached_device_id} ) {
        return $self->{_cached_device_id};
    }

    # If we already have the user profile cached, extract device ID from it
    if ( $self->{user_profile} && $self->{user_profile}->{Devices} ) {
        my @device_ids = keys %{ $self->{user_profile}->{Devices} };
        if (@device_ids) {
            $self->{_cached_device_id} = $device_ids[0];    # Cache it
            return $device_ids[0];
        }
    }

    # Otherwise fetch the profile to get the device ID
    my $profile_response = $self->_get_user_profile();
    if ( $profile_response && $profile_response->{Profile} ) {
        my $profile = $profile_response->{Profile};

        # Decode if it's a JSON string
        if ( !ref($profile) ) {
            eval { $profile = Cpanel::JSON::Load($profile); };
            if ($@) {
                $logger->error( "Failed to decode profile to get device ID", { error => $@ } );
                return "";
            }
        }

        # Cache the decoded profile for future use
        $self->{user_profile} = $profile;

        # Extract device ID
        if ( $profile->{Devices} ) {
            my @device_ids = keys %{ $profile->{Devices} };
            if (@device_ids) {
                $self->{_cached_device_id} = $device_ids[0];    # Cache it
                return $device_ids[0];
            }
        }
    }

    $logger->warn("Unable to determine device ID, returning empty string");
    return "";
}

=head2 _get_current_user_hash

Debug helper method to get the current hash from the user profile.
Used for debugging hash mismatch issues.

=cut

sub _get_current_user_hash ($self) {
    my $logger = $self->{_logger};

    $logger->debug("DEBUG: _get_current_user_hash called");

    require Whostmgr::CometBackup::CometBackupAPI;
    my $result = Whostmgr::CometBackup::CometBackupAPI::make_server_request("user/web/get-user-profile-and-hash");

    if ( $result && $result->{Status} == 200 ) {
        my $hash = $result->{ProfileHash};
        $logger->debug( "DEBUG: _get_current_user_hash returning hash: " . ( $hash || 'UNDEFINED' ) );
        return $hash;
    }

    $logger->debug( "DEBUG: _get_current_user_hash failed to get profile: " . ( $result ? "Status $result->{Status}" : "No result" ) );
    return undef;
}

=head2 _get_available_destinations

Gets all available destinations from the user profile for backup configuration.
Returns a hashref with destination IDs as keys.

=cut

sub _get_available_destinations ($self) {
    my $logger = $self->{_logger};

    # Get the user profile to extract destinations
    my $profile_response = $self->_get_user_profile();
    unless ( $profile_response && $profile_response->{Profile} ) {
        $logger->warn("Unable to retrieve user profile for destinations");
        return {};
    }

    my $profile = $profile_response->{Profile};

    # Decode if it's a JSON string
    if ( !ref($profile) ) {
        eval { $profile = Cpanel::JSON::Load($profile); };
        if ($@) {
            $logger->error( "Failed to decode profile for destinations", { error => $@ } );
            return {};
        }
    }

    # Extract destinations
    my $destinations = $profile->{Destinations} || {};

    $logger->debug(
        "Retrieved available destinations",
        {
            destination_count => scalar( keys %$destinations ),
            destination_ids   => [ keys %$destinations ]
        }
    );

    return $destinations;
}

1;
