Filename: ///scripts/import_exim_data

#!/usr/local/cpanel/3rdparty/bin/perl

# cpanel - scripts/import_exim_data                Copyright 2022 cPanel, L.L.C.
#                                                           All rights reserved.
# copyright@cpanel.net                                         http://cpanel.net
# This code is subject to the cPanel license. Unauthorized copying is prohibited

package scripts::import_exim_data;

=encoding utf-8

=head1 NAME

import_exim_data

=head1 SYNOPSIS

    import_exim_data ( --help | <logfile1> [ <logfile2> .. ] )

=head1 DESCRIPTION

This command re-populates the email statistics database based on the contents
of the log files at the given paths.

The format of the data in each exim log file should conform to the data format
of F</var/log/exim_mainlog>. Any of the files may be in gzip format, in which
case the filename must end with C<.gz>.

=cut

use strict;
use warnings;

use parent qw( Cpanel::HelpfulScript );

use Cpanel::Config::LoadCpConf          ();
use Cpanel::EximStats::ImportInProgress ();
use Cpanel::Finally                     ();
use Cpanel::IONice                      ();
use Cpanel::IP::Loopback                ();
use Cpanel::Locale                      ();
use Cpanel::PIDFile                     ();
use Cpanel::Sys::Load                   ();
use Cpanel::TailWatch                   ();    # PPI USE OK - inline below
use Cpanel::TailWatch::Eximstats        ();
use Cpanel::TimeHiRes                   ();
use Cpanel::IO                          ();
use Cpanel::Timezones                   ();

use Try::Tiny;

use constant _OPTIONS => ();

use constant _ACCEPT_UNNAMED => 1;

{

    package FakeTailWatch;    # Because Cpanel::TailWatch::Eximstats::process_line()
                              # thinks it needs a tailwatch obj

    use parent 'Cpanel::TailWatch';    # PPI USE OK - needed to mock this properly

    sub new {
        my ($class) = @_;
        return bless {}, $class;
    }

    sub register_module        { }
    sub register_reload_module { }
    sub register_action_module { }
    *_is_loopback = sub {
        my $self = shift;

        goto \&Cpanel::IP::Loopback::is_loopback;
    };
    sub log              { shift @_; printf STDERR "@_\n" }
    sub log_sql          { shift @_; printf STDERR "@_\n" }
    sub debug            { shift @_; printf STDERR "@_\n" }
    sub error            { shift @_; printf STDERR "@_\n" }
    sub clear_data_cache { }
    {
        no warnings 'once';
        no warnings 'redefine';
        *Cpanel::TailWatch::Eximstats::_send_limit_exceeded_notification = sub { return 1; };
        *Cpanel::TailWatch::Eximstats::_validate_email_limits_data       = sub { return 1; };
    }
}

$Cpanel::TailWatch::Eximstats::LOG_DUPLICATES = 0;

# Our for tests
our $LOOP_SIZE = 10000;

our $MAX_SLEEP_TIME = 120;    # seconds

__PACKAGE__->new(@ARGV)->run() unless caller();

sub run {
    my ($self) = @_;

    Cpanel::PIDFile->do(
        $Cpanel::EximStats::ImportInProgress::PATH,
        sub { $self->script( $self->getopt_unnamed() ) },
    );

    return;
}

sub script {
    my ( $self, @infiles ) = @_;

    local $ENV{'TZ'} = Cpanel::Timezones::calculate_TZ_env();

    if ( !@infiles ) {
        die $self->help( Cpanel::Locale->get_handle()->maketext('Give at least one filesystem path.') );
    }

    my $tailwatch_obj = FakeTailWatch->new;
    $tailwatch_obj->init_global_share();
    my $logfile = '/dev/null';
    my $now     = time;

    Cpanel::TailWatch::Eximstats->init($tailwatch_obj);

    my $eximstats = Cpanel::TailWatch::Eximstats->new( $tailwatch_obj, 'buffered_sql' => 1, 'import' => 1 );

    my $cpconf_ref = Cpanel::Config::LoadCpConf::loadcpconf();

    if ( Cpanel::IONice::ionice( 'best-effort', exists $cpconf_ref->{'ionice_import_exim_data'} ? $cpconf_ref->{'ionice_import_exim_data'} : 6 ) ) {
        print "[import_exim_data] Setting I/O priority to reduce system load: " . Cpanel::IONice::get_ionice() . "\n";
    }

    setpriority( 0, 0, 19 );

    $eximstats->_ensure_dbh();

    $eximstats->{'dbh'}->do('PRAGMA synchronous = OFF;');
    my $finally = Cpanel::Finally->new( sub { $eximstats->{'dbh'}->do('PRAGMA synchronous = ON;'); } );

  INFILE: for my $infile (@infiles) {
        print "$0: Processing input infile: $infile\n";
        local $0 = "$0 - processing input file: $infile";

        if ( !-f $infile ) {
            warn "Skipping input file $infile, file does not exist or is not a regular file";
            next INFILE;
        }

        my $fh;

        if ( $infile =~ /[.]gz$/ ) {
            my @cmd = ( qw[ gunzip -c -f ], $infile );
            if ( !open $fh, '-|', @cmd ) {
                warn "Skipping $infile: Cannot open pipe to read stdout from command '@{ [ join ' ', @cmd ] }' : $!";
                next INFILE;
            }
        }
        else {
            if ( !open $fh, '<', $infile ) {
                warn "Skipping $infile: Cannot open $infile for reading: $!";
                next INFILE;
            }
        }

        my $loop_count = 0;
        my $lasttime   = $now = [ Cpanel::TimeHiRes::gettimeofday() ];
        my ( $load_one, $load_five, $load_fifteen ) = Cpanel::Sys::Load::getloadavg($Cpanel::Sys::Load::ForceFloat);

        $load_one = sprintf( "%.01f", $load_one );    # Reduce precision on load_one to avoid divide by zero below (FPU fun)

        my ( $post_load_one, $post_load_five, $post_load_fifteen );

        my $process_line_regex = $eximstats->{'process_line_regex'}->{'/var/log/exim/mainlog'};

        while ( my $block = Cpanel::IO::read_bytes_to_end_of_line( $fh, 65_535 ) ) {
            foreach my $line ( grep { m{$process_line_regex}o } split( m{\n}, $block ) ) {
                if ( ++$loop_count % $LOOP_SIZE == 0 ) {
                    $eximstats->commit_buffer();

                    ( $post_load_one, $post_load_five, $post_load_fifteen ) = Cpanel::Sys::Load::getloadavg($Cpanel::Sys::Load::ForceFloat);
                    $lasttime = $now;
                    $now      = [ Cpanel::TimeHiRes::gettimeofday() ];
                    print(
                        "($loop_count) Processed " . sprintf(
                            '%0.2f',
                            (
                                $LOOP_SIZE / (
                                    (

                                        ( $now->[0] - $lasttime->[0] ) + ( ( $now->[1] - $lasttime->[1] ) / 1000000 )
                                    )
                                      || 1
                                )
                            )
                          )
                          . " records per second\n"
                    );
                    if ( $loop_count > 75000 ) {
                        $eximstats->{'quote_cache'} = {};
                    }
                    if ( $post_load_one > 1 && $post_load_one > ( $load_one * 1.1 ) ) {
                        my $sleep_time = calculate_sleep_time( $load_one, $post_load_one );

                        print "Load before import: $load_one, Load after import: $post_load_one.   Waiting ${sleep_time}s for load to decrease.\n";

                        sleep($sleep_time);

                        # We want to sleep a bit for disk i/o to catch up.   The idea here is to
                        # prevent this process for hogging the disk and create a large wait backlog.

                        # The goal is to still complete the optimization, however prevent the user
                        # from having the desire to kill off import_exim_data

                        # If we have driven up the load a bit we sleep in the end as well in order
                        # to give the system time to return to normal before upcp proceeds to
                        # prevent normal operations from being slowed down too much.
                        ( $load_one, $load_five, $load_fifteen ) = Cpanel::Sys::Load::getloadavg($Cpanel::Sys::Load::ForceFloat);
                    }

                }

                #next if $line !~ m{$process_line_regex}o;

                $eximstats->process_line( $line, $tailwatch_obj, $logfile, $now->[0] );
            }
        }

        $eximstats->commit_buffer();
        close $fh;

        print "Data has been imported to the database from the exim log.\n";
    }

    return 1;
}

sub calculate_sleep_time {
    my ( $load_one, $post_load_one ) = @_;

    # adding .00001 to avoid dividing by 0.0
    my $sleep = $load_one ? int( 60 * ( $post_load_one / ( $load_one + 0.00001 ) ) ) : 60;
    return $sleep > $MAX_SLEEP_TIME ? $MAX_SLEEP_TIME : $sleep;
}

1;

./Ninja\.