package Cpanel::Sitejet::Publish;

# cpanel - Cpanel/Sitejet/Publish.pm               Copyright 2023 cPanel, L.L.C.
#                                                           All rights reserved.
# copyright@cpanel.net                                         http://cpanel.net
# This code is subject to the cPanel license. Unauthorized copying is prohibited

use cPstrict;

use Cpanel::AccessIds;
use Cpanel::Autodie;
use Cpanel::Autodie::Unlink;
use Cpanel::Exception;
use Cpanel::Logger;
use Cpanel::SafeRun::Errors;
use Cpanel::Tar;
use Cpanel::WebDisk();
use Cpanel::AccessIds::ReducedPrivileges();
use Cpanel::Chdir();
use Cpanel::DomainLookup          ();
use Cpanel::DomainLookup::DocRoot ();
use Cpanel::Sitejet::Connector    ();
use Cpanel::WebVhosts             ();
use Cpanel::XMLParser             ();
use File::Basename                ();
use File::MimeInfo                ();
use File::Path                    ();
use LWP::UserAgent;
use URI::Split    qw(uri_split);
use URI::XSEscape ();

my @CDN_HOSTS = (
    'https://inter-cdn.com',
    'https://cdn1.site-media.eu',
    'https://cdn2.site-media.eu',
    'https://cdn3.site-media.eu',
    'https://cdn4.site-media.eu',
    'https://cdn5.site-media.eu',
    'https://cdn6.site-media.eu',
    'https://cdn7.site-media.eu'
);

my $PATH_SITEMAP = '/sitemap.xml';

# Regex to parse all URLs referring to assets
my $REGEX_ASSETS = qr/\/(?:images|webcard|g|js|css)\/[^"\') ]+/;

# Regex to parse JS chunks in bundled JS file
my $REGEX_JS_CHUNKS = qr/\((\d+)\)\.then/;

# Regex to detect whether a filename has an extension
my $REGEX_FILE_EXT = qr/(.*)\.[a-zA-Z0-9]{2,5}$/;

our $start;
our $origin;

my $retry_attempts      = 0;
my $retry_sitemap_limit = 0;

our $logger;
our $document_root;

=encoding utf-8

=head1 NAME

Cpanel::Sitejet::Publish

=head1 DESCRIPTION

Module to publish (clone) a complete website from Sitejet's site to a cPanel
user's server.  Any files residing external to Sitejet's site such as a CDN
server will be copied to the cPanel server so that everything for that website
is local to the cPanel server.

This module matches Plesk's SitejetPublish.php in the sitejet-plesk repo. The
names of subroutines and variables in Perl match as much as possible, so that
future changes can be easily replicated.

The Plesk PHP naming convention for subroutines and variables is camelCase, so
their counterparts in this cPanel module match.

The cPanel standard naming convention is snake_case, except for module names
which are camelCase. Any subroutine or variable that is named using snake_case
does NOT have a corresponding PHP equivalent in the Plesk source code.

Subroutines whose names begin with "cp_" have no corresponding subroutine in
the original Plesk PHP source code. "cp_" just indicates the subroutine
originated with cPanel.

Publishing a website puts that website in the document root directory for the
appropriate domain or sub-domain.  If it is the first time a Sitejet publish is
done for that document root and if any files or directories already exist in
that document root directory then a compressed backup copy of those files and
directories is made in the cPanel user's home directory before they are
removed.  Subsequent Sitejet publish actions will overwrite everything that is
in that document root directory.

=cut

Cpanel::Sitejet::Publish::publish(shift) unless caller;

=head1 METHODS

=over

=item * publish -- Publishes Sitejet server-based website on cPanel server.

    ARGUMENTS
        websiteId (string) -- Unique website ID
        logfile (string) -- Full pathname of log file which is linked for UI streaming

    RETURNS
        1 if no errors.

    ERRORS
        All failures are fatal.
        Fails if cannot contact Sitejet server.

    Note:  Will not fail if some files cannot be downloaded, but resulting
    cloned site will appear to be incomplete.  Log will contain list of files
    that failed to download.

    EXAMPLE
        my $result = Cpanel::Sitejet::Publish::publish( $websiteId, $logfile );

=back

=cut

sub publish {
    my ( $domain, $logfile ) = @_;
    $logger = Cpanel::Logger->new( { 'alternate_logfile' => $logfile } );
    $start  = time;

    # Must write something to logger immediately so the streamer can find it
    log_message("Starting sync for $domain at $start.");

    my $connector = Cpanel::Sitejet::Connector->new();
    $document_root = get_document_root($domain);
    my ( $urls, $etag ) = cp_process_sitemap( $connector, $domain );

    my @finishedUrls  = ();
    my $trackProgress = 1;

    # backs up all, and removes select public_html files
    my $user_metadata = $connector->cp_load_sitejet_metadata($domain);
    cleanup() if !$user_metadata->{publish_status};

    my @iterations = ();
    my @retryUrls  = ();
    my $iteration  = 0;

    # get the latest sitemap urls before we parse assets
    my %processed_sitemap;
    _ensure_sitemap_etags_are_updated( $urls, $etag, \%processed_sitemap );

    # Recursively parse assets and download
    do {
        # Do not download already existing static files
        my @urls_filtered;

        for my $url (@$urls) {
            if ( isStatic($url) ) {
                if ( $url =~ /^(\S*)\s/ ) {
                    $url = $1;
                }
                my $this_file = createFilename($url);
                next if -e $this_file;
            }

            # skip /webcard/static/ folder
            next if $url =~ m{^/webcard/static/$};
            push @urls_filtered, $url;
        }
        $urls = \@urls_filtered;

        @retryUrls = ();

        # Execute all requests
        my @responses;
        foreach my $url (@$urls) {
            $url = _add_url_origin($url);

            # skip executing request for sitemap_urls
            if ( exists $processed_sitemap{$url} ) {
                push @responses, $processed_sitemap{$url};
                next;
            }
            my $response = executeRequests($url);
            push @responses, $response;
        }

        $iteration++;

        # Process responses
        my $index = 0;
        foreach my $response (@responses) {
            my $url = $urls->[$index];

            eval {
                my $processedResponse = processResponse( $response, $url, $etag, 1 );

                #if ( $processedResponse && !isBinary($processedResponse) ) {

                #parseAndInstallLanguages( $processedResponse ) if $iteration == 1 && $index == 0;
                #}
            };

            if ($@) {

                # Retry 3 times, then keep what we got
                if ( ( grep { $_ eq $url } @finishedUrls ) <= 3 ) {
                    push @retryUrls, $url;
                }
                else {
                    log_message("Reached max retry limit: Processing $url as it is.");
                    my $processedResponse = processResponse( $response, $url, $etag, 0 );
                    push @finishedUrls, $url;

                    #if ( $processedResponse && !isBinary($processedResponse) ) {

                    #parseAndInstallLanguages(  $processedResponse ) if $iteration == 1 && $index == 0;
                    #}
                }
            }

            push @finishedUrls, $url;
            $index++;
        }

        # Parse assets and JS chunks
        $urls = parseAssets( \@responses, \@finishedUrls );
        push @$urls, @retryUrls;

        # Wait a second before retrying URLs
        sleep(1) if @retryUrls;
    } while @$urls;

    my %publish_data = ( 'publish_status' => 1, 'document_root' => $document_root, 'latest_publish_date' => time );
    $connector->cp_update_sitejet_metadata( $domain, \%publish_data );

    return 1;
}

=over

=item * fetchUrlsFromSitemap -- downloads sitemap file.

    The starting point for all publishing is the sitemap XML file.  This file
    contains a list of top-level files to be downloaded.  Those files will
    refer internally to other files to be downloaded, which in turn may refer
    to other files to be downloaded, and so on.  There is no depth limit.
    All assets for a complete website (html, images, scripts, styles, etc.)
    will be either be listed in this sitemap file or referred to indirectly.

    This method gets this initial list of files as URLs from the sitemap XML
    file.

    ARGUMENTS
        etag (string) -- Unique identifier

    RETURNS
        Reference to array that contains URL strings.

    ERRORS
        All failures are fatal.
        Fails if cannot contact Sitejet server.

    EXAMPLE
        my $urls = Cpanel::Sitejet::Publish::fetchUrlsFromSitemap($etag);

=back

=cut

sub fetchUrlsFromSitemap {
    my $orig_etag = shift;
    log_message( "Download sitemap xml:" . $origin . $PATH_SITEMAP );
    my ( $ua, $uri, $xml, $response, $current_etag );
    $ua = LWP::UserAgent->new();
    $ua->ssl_opts( verify_hostname => 1 );
    $uri = URI->new( $origin . $PATH_SITEMAP );
    $uri->query_form( { 'etag' => 'true' } );

    $response = $ua->get( $uri->as_string );
    $retry_attempts++;
    $current_etag = _get_etag($response);
    if ( _etag_mismatch( $orig_etag, $current_etag ) ) {
        log_message("fetchUrlsFromSitemap etag mismatch found. current etag : $current_etag expected_etag : $orig_etag ");
        if ( $retry_attempts >= 5 ) {
            die Cpanel::Exception::create( 'AdminError', 'The publish process did not update all required files. Please try the publish process again.' );
        }
        _sleep();
        return fetchUrlsFromSitemap($orig_etag);
    }

    if ( !$response->is_success ) {
        return;
    }
    $xml = Cpanel::XMLParser::XMLin( $response->content );
    my @urls = ();
    foreach my $url ( @{ $xml->{url} } ) {
        push @urls, $url->{loc};
    }

    push @urls, '/robots.txt';

    #        log_message("Store sitemap");
    #        storeContent( $PATH_SITEMAP, $sitemap, 'application/xml' ); # Not sure why storing.

    return \@urls;
}

sub _sleep {
    sleep(5);
}

=over

=item * cleanup -- Backups/removes files that would be affected by new website.

    Only called the first time Sitejet publishes in a particular document
    root.

    ARGUMENTS
        none

    RETURNS
        1 if no errors.

    ERRORS
        All failures are fatal.
        Fails if cannot identify document root.

    EXAMPLE
        my $ok = Cpanel::Sitejet::Publish::cleanup();

=back

=cut

sub cleanup {
    my $homedir = $Cpanel::homedir;

    my ($target_folder) = $document_root =~ m{([^/]+)$};
    my $target_dir_name = $target_folder . '_' . time . '.gz';
    my $target_dir      = "$homedir/$target_dir_name";

    # get the first immediate folder name after doc root
    my @sub_docroots = map { m<$document_root/([^/\n]+)>; } keys %{ Cpanel::DomainLookup::getdocrootlist($Cpanel::user) };

    my @webdisks = Cpanel::WebDisk::api2_listwebdisks( home_dir => $homedir, );

    my @webdiskusers_docroots = map {
        $_->{homedir} =~ m<^$document_root/([^/]+)>;    # this one gives $1
    } @webdisks;
    my @dont_delete = ( @webdiskusers_docroots, @sub_docroots );
    opendir my $dh, $document_root or do {
        warn("Cannot open directory: '$document_root'.");
        return;
    };

    # create .gz backup file in user home dir
    Cpanel::SafeRun::Errors::saferunallerrors( Cpanel::Tar::load_tarcfg()->{'bin'}, '-c', '-v', '-z', '-f', $target_dir, $document_root );

    # collect files to delete
    my @contents = grep { !/^\.\.?$/ } readdir($dh);

    # delete the files/dirs
    require Cpanel::SafeDir::RM;

    foreach my $content (@contents) {

        # exclude any immediate folders for webdisk & sub_docroots
        next if grep { $_ eq $content } @dont_delete;
        my $file = "$document_root/$content";
        -d $file ? Cpanel::SafeDir::RM::safermdir($file) : Cpanel::Autodie::unlink_if_exists($file);
    }

    return 1;
}

=over

=item * createFilename -- Creates local filenames

    Given a URL or relative filename returns a complete path filename for the
    local cPanel server.  For example, for cPanel user 'jim',
    'http://xyz.com/img.png' will be changed to '/home/jim/public_html/img.png'
    and 'smile.html' will be changed to '/home/jim/public_html/smile.html'.

    ARGUMENTS
        uri (string) -- Full URI or relative pathname for a file or directory
        mimeType (string, optional) -- Standard mime type of the file, something like 'text/html'

    RETURNS
        Full new local pathname (string)

    ERRORS
        All failures are fatal.
        URI is invalid.
        mime type is invalid.

    EXAMPLE
        my $fullname = Cpanel::Sitejet::Publish::createFilename($uri);

=back

=cut

sub createFilename {
    my ( $uri, $mimeType ) = @_;
    $uri = ( uri_split($uri) )[2];
    $uri = $uri =~ s{g/fonts/css}{g/fonts.css}gr;
    $uri =~ s{^/*(.*)/*$}{$1};    # trim '/'
    my $filename = File::Basename::basename($uri);
    if ( $filename eq './' ) {
        $filename = '';
    }

    my $path = $document_root . '/' . File::Basename::dirname($uri);
    $path     =~ s{/\.$}{};       # look for . directories
    $filename =~ s/\?.*$//;       # Strip off cgi params.

    if ( !$filename || $filename eq './' ) {
        $filename = 'index';
    }
    if ( $filename !~ $REGEX_FILE_EXT ) {
        my $extension = File::MimeInfo::extensions( $mimeType // 'text/html' );
        $extension = 'html' if $extension eq 'htm';
        if ( $extension eq 'html' && $filename ne 'index' ) {
            $path .= '/' . $filename;
            $filename = 'index';
        }

        $filename .= '.' . $extension;
    }

    return $path . '/' . $filename;
}

=over

=item * isBinary -- Is the string likely to be from a binary file

    This is intended to determine if a file (contained in $value as a string)
    is a binary file, such as an image.

    CURRENTLY THIS SUBROUTINE IS NOT USED ANYWHERE.  It was used only
    internally in conjunction with parseAndInstallLanguages(), but that is not
    used either yet.

    ARGUMENTS
        value (string) -- Any string

    RETURNS
        1 or empty string

    ERRORS
        Only fails if $value is undef.

    EXAMPLE
        my $bin = Cpanel::Sitejet::Publish::isBinary($value);

=back

=cut

sub isBinary {
    my ($value) = @_;
    return $value !~ /[\x00-\x08\x0B-\x0C\x0E-\x1F]/;
}

=over

=item * isStatic -- Identify files in certain paths that are considered static

    ARGUMENTS
        url (string) -- Path for a file or directory
        extended (boolean, optional) -- For some paths containing 'webcard', 'js', or 'css'

    RETURNS
        1 or empty string

    ERRORS
        $url is undef.

    EXAMPLE
        my $static = Cpanel::Sitejet::Publish::isStatic($url);
        my $ok     = Cpanel::Sitejet::Publish::isStatic( $url, $extended );

=back

=cut

sub isStatic {
    my ( $url, $extended ) = @_;
    my $ret =
         $url =~ m{^/(images|g|uploads)/[^"\') ]+}
      || ( $extended && $url =~ m{/images/} )
      || ( $extended && $url =~ m{/webcard/} )
      || ( $extended && $url =~ m{^$origin/(js/custom\.js|css/custom.css)} );

    return $ret // '';
}

=over

=item * log_message -- Send message to log file

    ARGUMENTS
        message (string) -- String to be logged

    RETURNS
        1 or empty string

    ERRORS
        $message is undef.
        $start global is undef.

    EXAMPLE
       Cpanel::Sitejet::Publish::log_message($message);

=back

=cut

sub log_message {
    my ($message) = @_;
    my $elapsed = time - $start;

    # Cpanel::Logger does not fail on disk quota exceeded.
    # The return status is the only way of knowing if the logger
    # failed to write.
    $logger->info("[$elapsed s] $message") or die Cpanel::Exception::create( 'IO::FileWriteError', [ 'error' => 'Cpanel::Logger failed to write the log.' ] );
}

=over

=item * parseAssets -- Search downloaded text assets for more assets

    Given a reference to an array of HTTP::Response assets, parse the content
    of each, looking for additional assets referred to.  Skips parsing non-text
    assets such as images.

    ARGUMENTS
        responses (array reference) -- HTTP::Response object for downloaded assets
        finishedUrls (array reference) -- Array contains URLs that have already been downloaded

    RETURNS
        Reference to array of strings, each containing a newly found asset url

    ERRORS
        $response is not an array of HTTP::Response object.
        $finishedUrls is not an array reference.

    EXAMPLE
        my $new_urls = Cpanel::Sitejet::Publish::parseAssets($responses, $finishedUrls);

=back

=cut

sub parseAssets {
    my ( $responses, $finishedUrls ) = @_;
    log_message('Parse assets from all responses that are not known binary formats');

    my $all_responses = join "\n", map { $_->headers->{'content-type'} !~ m{^(image|video|audio)/}n ? $_->content : '' } @$responses;
    $all_responses =~ s/&quot;/"/g;
    my @hits        = $all_responses =~ /$REGEX_ASSETS/g;
    my %seen        = ();
    my @unique_urls = grep { !$seen{$_}++ } @hits;

    my @urls;

    # Do not process already processed urls
    foreach my $url (@unique_urls) {
        if ( !grep /^\Q$origin$url\E$/, @$finishedUrls ) {
            push @urls, $url;
        }
    }

    my @more_urls = @urls;

    # detect images that are downloaded as originals
    # for which thumb urls might be generated dynamically
    foreach my $url (@urls) {

        # /images/0/2437851/pexels-photo-1139793.jpg
        if ( $url =~ m{/0/} ) {
            push @more_urls, $url =~ s{/0/}{/1920/}r;
            push @more_urls, $url =~ s{/0/}{/1024/}r;
        }

        # /images/0%2C2249x1416%2B0%2B83/2437401/pexels-photo-356378.jpg
        if ( $url =~ m{/0,} ) {
            push @more_urls, $url =~ s{/0, }{/1920,}r;
            push @more_urls, $url =~ s{/0, }{/1024,}r;
            push @more_urls, $url =~ s{/0,[^/]*}{}r =~ s/0,/1920/r;
            push @more_urls, $url =~ s{/0,[^/]*}{}r =~ s/0,/1024/r;
        }

        # /images/0%2CNaNx2973%2BNaN%2B0/2471538/bed-bedroom-blanket-545012.jpg
        if ( $url =~ m{/0%2} ) {
            push @more_urls, $url =~ s{/0_}{/1920_}r;
            push @more_urls, $url =~ s{/0_}{/1024_}r;

            # URL encoded + , x NaN
            push @more_urls, $url =~ s{/0[%BCxNa0-9]+/}{/1920/}r;
            push @more_urls, $url =~ s{/0[%BCxNa0-9]+/}{/1024/}r;
        }
    }

    my @js_chks = $all_responses =~ /$REGEX_JS_CHUNKS/gm;
    push @more_urls, grep { length } map { $_ = '/webcard/static/' . $_ . '.js' if !m{/?webcard/static/}; } @js_chks;

    my %unique_urls = map { $_ => 1 } @more_urls;
    @more_urls = keys %unique_urls;

    return \@more_urls;
}

# Original Plesk subroutine.  We're not using this.
# sub parseAndInstallLanguages {}

=over

=item * executeRequests -- Fetch asset

    Downloads an asset described by URL.

    ARGUMENTS
        url (string) -- Fully qualified URL

    RETURNS
        Reference to HTTP::Response object

    ERRORS
        $url is invalid

    EXAMPLE
        my $response = Cpanel::Sitejet::Publish::executeRequests($url);

=back

=cut

sub executeRequests {
    my $url = shift;
    log_message("Prepare request for $url");
    my %headers = (
        'User-Agent' => 'Sitejet API Client/1.0.0 (Perl ' . $^V . ')',
        'Connection' => 'Close',
        'Accept'     => 'application/json'
    );
    my $ua = LWP::UserAgent->new(
        timeout  => 30,
        ssl_opts => { verify_hostname => 1 },
    );

    my $response = $ua->get( $url, %headers );

    log_message( "URI: '$url', " . $response->code );

    return $response;
}

=over

=item * processResponse -- process HTTP::Response for asset

    Process includes validating and saving as a local file.

    ARGUMENTS
        response (HTTP::Response object) -- Result of fetching asset using executeRequest()
        uri (string) -- The same URI used when the asset was fetched
        etag (string) -- From HTTP::Header etag field
        check etag (boolean) -- 0 or 1

    RETURNS
        Content string from HTTP::Response object

    ERRORS
        $etag mismatch
        Saving local copy of file error--bad filename, no disc space, no permission, etc.

    EXAMPLE
        my $file_contents = Cpanel::Sitejet::Publish::processResponse( $response, $uri, $etag, 0 );

=back

=cut

sub processResponse {
    my ( $response, $uri, $etag, $checkEtag ) = @_;

    log_message("Process response of $uri");

    if ( !$response->is_success ) {
        return '';
    }

    my $content = $response->content;
    my $headers = $response->headers->as_string;

    # Validate ETag for non-static pages
    if ( $checkEtag && !isStatic( $uri, 1 ) ) {
        my $parsedEtag = _get_etag($response);
        if ( _etag_mismatch( $etag, $parsedEtag ) ) {
            log_message("processResponse: etag mismatch found.  uri : $uri current etag: $parsedEtag  expected etag : $etag ");
            die Cpanel::Exception::create( 'InvalidParameter', 'Incorrect tag: “[_1]”.', [$parsedEtag] );

            # Had PHP: Modules_PleskSitejet_Task_WrongETagException->new($uri);
        }
    }

    my $body        = $content;
    my $contentType = $response->header('Content-Type');
    my ($mimeType)  = $contentType =~ /^([^;]*);?/;

    foreach my $host (@CDN_HOSTS) {
        $body =~ s/\Q$host\E//g;
    }

    storeContent( $uri, $body, $mimeType );

    return $body;
}

=over

=item * storeContent -- save asset as a file

    Given a URI, the contents of the file, and the mime type, determines the
    filename, path, and extension to save as.  And saves it.

    ARGUMENTS
        uri (string) -- The same URI used when the asset was fetched
        body (string) -- contents of the file, obtained through HTTP::Response content field
        mimeType (string) -- Something like 'text/html' or 'image/png'

    RETURNS
        body (contents of file as a string)

    ERRORS
        Saving local copy of file error--bad filename, no disc space, no permission, etc.

    EXAMPLE
        my $body = Cpanel::Sitejet::Publish::storeContent( $uri, $body, $mimeType );

=back

=cut

sub storeContent {
    my ( $uri, $body, $mimeType ) = @_;

    log_message("Store content for $uri");

    my $filename = createFilename( $uri, $mimeType );
    my $path     = File::Basename::dirname($filename);

    if ( !-e $path ) {
        log_message("Create directory $path");
        File::Path::make_path( $path, { mode => 0755 } );
    }

    my $bodyToStore = $body;
    $bodyToStore =~ s{g/fonts/css}{g/fonts.css}g;

    log_message("Save file $filename");
    open my $fh, '>', $filename or Cpanel::Exception::create( "IO::FileOpenError", [ path => $filename, error => $! ] );
    print $fh $bodyToStore;
    close $fh;

    # make an extra dir and save file with both encoded and unencoded path
    # no harm to unescape everything, even though we only want to unescape first depth of images dir,
    # e.g. /images/800%2C600x600%2B0%2B100/6613015/pexels-alleksana-40513971.jpg
    $filename = URI::XSEscape::uri_unescape($filename);
    $path     = File::Basename::dirname($filename);

    if ( !-e $path ) {
        log_message("Create directory $path");
        File::Path::make_path( $path, { mode => 0755 } );
    }

    log_message("Save file $filename");
    open $fh, '>', $filename or Cpanel::Exception::create( "IO::FileOpenError", [ path => $filename, error => $! ] );
    print $fh $bodyToStore;
    close $fh;

    return $body;
}

=over

=item * cp_process_sitemap -- Build URL for sitemap.XML, download it and return URLs from it.

    Given a URI, the contents of the file, and the mime type, determines the
    filename, path, and extension to save as.  And saves it.

    Please note that the global, $document_root, will need to be given a value
    *before* calling this subroutine.

    ARGUMENTS
        connector -- Cpanel::Sitejet::Connector object
        domain (string) -- Something like 'xyz.com'

    RETURNS
        Reference to array of URL strings from sitemap XML file
        etag (string) for site

    ERRORS
        All errors are fatal.
        File system error due to no permissions, no space on disc, etc.
        Unable to acquire sitemap XML file
        Domain invalid
        Cannot obtain site API key
        Cannot obtain website preview URL

    EXAMPLE
        $document_root = _get_document_root($domain);
        my $connector = Cpanel::Sitejet::Connector->new();
        my ( $urls, $etag ) = Cpanel::Sitejet::Publish::cp_process_sitemap( $connector, $domain );

=back

=cut

sub cp_process_sitemap {
    my ( $connector, $domain ) = @_;

    my $path = $document_root;
    if ( !-e $path ) {
        Cpanel::Autodie::mkdir_if_not_exists( $path, 0755 );
    }
    my $chdir       = Cpanel::Chdir->new($path);
    my $token       = $connector->loadApiKey();
    my $previewData = $connector->getPreviewDataForWebsite( $domain, $token );
    my $sitemap_url = $previewData->{url};

    if ( $sitemap_url !~ /^http/i ) {
        $sitemap_url = 'https://' . $sitemap_url;
    }

    $origin = $sitemap_url;
    my $etag = $previewData->{etag};

    my $urls = fetchUrlsFromSitemap($etag);
    if ( !$urls ) {
        die Cpanel::Exception::create( 'InvalidParameter', 'Unable to obtain URLs from sitemap at “[_1]”.', [$sitemap_url] );
    }

    return ( $urls, $etag );
}

sub _ensure_sitemap_etags_are_updated {
    my ( $sitemap_urls, $etag, $processed_sitemap ) = @_;
    my @retry_sitemap_url;
    $retry_sitemap_limit++;
    foreach my $sitemap_url (@$sitemap_urls) {
        $sitemap_url = _add_url_origin($sitemap_url);
        my $response   = executeRequests($sitemap_url);
        my $parsedEtag = _get_etag($response);

        if ( _etag_mismatch( $etag, $parsedEtag ) ) {
            log_message("_ensure_sitemap_etags_are_updated: Wrong etag found: $parsedEtag Expected etag : $etag for uri : $sitemap_url");
            push @retry_sitemap_url, $sitemap_url;
        }
        elsif ( defined $parsedEtag ) {
            $processed_sitemap->{$sitemap_url} = $response;
        }
    }
    if (@retry_sitemap_url) {
        if ( $retry_sitemap_limit >= 5 ) {
            die Cpanel::Exception::create( 'AdminError', 'The publish process did not update all required files. Please try the publish process again.' );
        }
        _sleep();
        _ensure_sitemap_etags_are_updated( \@retry_sitemap_url, $etag, $processed_sitemap );
        return;
    }
    return;
}

sub _add_url_origin {
    my $url = shift;
    if ( isStatic($url) ) {
        $url = 'https://inter-cdn.com/' . $url;
    }
    else {
        if ( $url !~ /^http/i ) {
            $url = $origin . '/' . $url;
        }
    }
    $url =~ s{(?<!http:|https:)//}{/}g;
    return $url;
}

sub _etag_mismatch {
    my ( $source_etag, $current_etag ) = @_;
    return ( $current_etag && ( $source_etag ne $current_etag ) ) ? 1 : 0;
}

sub _get_etag {
    my $response   = shift;
    my $parsedEtag = $response->header('etag');
    $parsedEtag =~ s/\"//g if $parsedEtag;
    return $parsedEtag;
}

sub get_document_root {
    my $domain   = shift;
    my $docroots = Cpanel::DomainLookup::DocRoot::getdocroots($Cpanel::user);
    if ( exists $docroots->{$domain} ) {
        return $docroots->{$domain};
    }
    return;
}

1;
