#!/usr/local/cpanel/3rdparty/bin/perl
# cpanel - SOURCES/summarize-api1-logs Copyright 2022 cPanel, L.L.C.
# All rights reserved.
# copyright@cpanel.net http://cpanel.net
# This code is subject to the cPanel license. Unauthorized copying is prohibited
package scripts::summarize_api1_logs;
=head1 MODULE
C<scripts::summarize_api1_logs>
=head1 DESCRIPTION
C<scripts::summarize_api1_logs> generates summary files from the API1 logs.
=head1 FUNCTIONS
=cut
use strict;
use warnings;
use lib '/var/cpanel/perl';
use Getopt::Long ();
use Cpanel::Alarm ();
use Cpanel::Analytics::Config ();
use Cpanel::Analytics::KnownPages::Api1 ();
use Cpanel::Autodie ();
use Cpanel::PIDFile ();
use JSON::XS ();
use constant SAFE_SHUTDOWN_WINDOW => 15; # 15 seconds.
use constant DEFAULT_TIMEOUT => 2 * 60 * 60; # 2 hours in seconds
use constant PID_FILE => '/var/cpanel/analytics/summarize_api1_logs.pid';
=head2 only_one(ARGS)
Make sure the program cannot be started multiple time simultaneously.
=cut
sub only_one {
my @args = @_;
Cpanel::PIDFile->do(
PID_FILE,
sub { main(@args) },
);
return 0;
}
=head2 main(ARGS)
Main program entry point.
=cut
sub main { ##no critic qw(ProhibitExcessComplexity)
my @argv = @_;
my $verbose = 0;
my $timeout = DEFAULT_TIMEOUT;
my $help = 0;
if (
!Getopt::Long::GetOptionsFromArray(
\@argv,
'verbose!' => \$verbose,
'timeout=i' => \$timeout,
'help|?' => \$help,
)
) {
usage(1);
}
if ($help) {
usage(0);
}
my $alarm;
my $log_dir = Cpanel::Analytics::Config::ANALYTICS_DATA_DIR();
# If the log directory does not exist, we do not
# have anything to do. So return.
return unless -e $log_dir;
Cpanel::Autodie::opendir( my $dh, $log_dir );
my $shutdown = 0;
my $processing = 1;
my $signal;
# We want to leave a little time to finish processing
# the current log. So we split the timeout into the
# initial period and the safe shutdown window. When the
# initial period expires, we tell the loop to shut down
# and set a second timer to go off in SAFE_SHUTDOWN_WINDOW
# additional seconds. If the loop can finish an iteration,
# it will shut down, otherwise we force a shutdown after
# the SAFE_SHUTDOWN_WINDOW expires.
$timeout = $timeout - SAFE_SHUTDOWN_WINDOW;
$timeout = SAFE_SHUTDOWN_WINDOW if $timeout <= 0;
$alarm = Cpanel::Alarm->new(
$timeout,
sub {
if ( !$shutdown && $processing ) {
$signal = 'ALARM';
print STDERR "\nPreparing to stop generating summaries!\n";
# Tell the processing loop to shut down
# on next iteration. This way the current
# log will finish processing, but it won't
# pick up any more logs.
$shutdown = 1;
# Set another alarm for a little bit more to let the loop try to finish
$alarm->set(SAFE_SHUTDOWN_WINDOW);
}
else {
handle_signal( $signal, $timeout );
}
}
);
my $terminate_fn = sub {
print STDERR "\nPreparing to stop generating summaries!\n";
$shutdown = 1;
# Set an alarm to limit how log the program will run after the interrupt.
$alarm->set(SAFE_SHUTDOWN_WINDOW);
};
local $SIG{INT} = sub { $signal = 'INT'; $terminate_fn->() };
local $SIG{TERM} = sub { $signal = 'TERM'; $terminate_fn->() };
my ( $year, $month, $day ) = ( localtime(time) )[ 5, 4, 3 ];
my $today = sprintf "%4d-%02d-%02d", $year + 1900, $month + 1, $day;
my $entry_checker = Cpanel::Analytics::KnownPages::Api1->new();
while ( my $file = readdir($dh) ) {
$processing = 1;
if ($shutdown) {
handle_signal( $signal, $timeout );
}
my $path = "$log_dir/$file";
next if !-f $path || !-r $path; # Skip if it is not a file or not readable.
if ( $file eq 'api1.log' ) {
# Clean up the legacy logging
Cpanel::Autodie::unlink($path);
next;
}
next unless ( $file =~ /^api1\.\d\d\d\d-\d\d-\d\d\.log$/ ); # Skip anything that is not api1.####-##-##.log
next if $file =~ /^\Qapi1.$today.log\E$/; # Skip today's log file since it presumably isn't finished writing yet
print STDOUT "Processing $path: " if $verbose;
open( my $fh, "<", $path ) or do {
print STDERR "Failed to open $path: $!\n";
next;
};
my $summary = {};
my $line_num = 0;
my $decode_result;
while (<$fh>) {
$line_num++;
next unless /\S/;
my $line;
eval { $line = JSON::XS::decode_json($_); };
$decode_result = $@;
if ($decode_result) {
my $filepath = "$log_dir/$file";
my $renamed_file = "$filepath.corrupted";
print STDERR "decode_json() failed\n\n$@\n";
if ( rename $filepath, $renamed_file ) {
print STDERR "$filepath is corrupt!\nRenamed it to $renamed_file\n";
}
else {
print STDERR "$filepath is corrupt!\nFailed to rename it to $renamed_file: $!\n";
}
last;
}
next if $entry_checker->is_cpanel_builtin( $line->{call}, $line->{page} );
next if $entry_checker->ignore_known_special_cases( $line->{call}, $line->{page} );
# special handling for the index page.
if ( defined $line->{page} && $line->{page} eq '/usr/local/cpanel/base/frontend/paper_lantern/index.html' && $line->{call} =~ m/^Installatron::/ ) {
$summary->{api1_pages}{ $line->{page} . ' (' . $line->{call} . ')' }++ if exists $line->{page};
}
else {
$summary->{api1_pages}{ $line->{page} }++ if exists $line->{page};
}
$summary->{api1_calls}{ $line->{call} }++;
if ( $verbose && $line_num % 10000 == 0 ) {
print STDOUT '.';
flush STDOUT;
}
}
close($fh);
unless ($decode_result) {
print STDOUT "\nWriting summary: $path.summary\n" if $verbose;
Cpanel::Autodie::open( my $out_fh, ">", "$path.summary" );
Cpanel::Autodie::print( $out_fh, JSON::XS::encode_json($summary) );
Cpanel::Autodie::close($out_fh);
Cpanel::Autodie::unlink($path);
}
}
$processing = 0;
return 0;
}
=head2 handle_signal(SIGNAL, TIMEOUT)
Process the signal received earlier.
=head3 ARGUMENTS
=over
=item SIGNAL - string
One of ALARM, INT, TERM.
=item TIMEOUT - number
Number of seconds before the process times out.
=back
=cut
sub handle_signal {
my ( $signal, $timeout ) = @_;
for ($signal) {
if (/ALARM/) { timeout($timeout) }
elsif (/INT/) { interrupt() }
elsif (/TERM/) { terminate() }
else { die 'Unknown signal' }
}
return;
}
=head2 timeout(TIMEOUT)
Reports the timeout message.
=head3 ARGUMENTS
=over
=item TIMEOUT - number
Number of seconds for the timeout. The actual runtime may be a bit
more since the timeout is only processed once the processing loop hits a clean stopping point.
=back
=cut
sub timeout {
my $timeout = shift;
end_early("The process took longer than $timeout seconds.");
return;
}
=head2 interrupt()
Reports the process was interrupted by the caller. This message is sent after a graceful shutdown attempt;
=cut
sub interrupt {
end_early("The process was interrupted.");
return;
}
=head2 terminate()
Reports the process was terminated by the caller. This message is sent after a graceful shutdown attempt;
=cut
sub terminate {
end_early("The process was terminated.");
return;
}
=head2 end_eary(MESSAGE)
End the process with a message.
=cut
sub end_early {
my $message = shift;
print STDERR "Shutdown $0.\n$message\nAdditional logs will be processed during the next run.\n";
exit 1; ## no critic qw(NoExitsFromSubroutines)
}
=head2 usage(EXIT)
Print the usage for the script.
=head3 ARGUMENTS
=over
=item EXIT - number
Optional, the exit code you want the process to exit with.
=back
=cut
sub usage {
my $stderr_flag = shift || 0;
my $oh = $stderr_flag ? \*STDERR : \*STDOUT;
print $oh <<USAGE;
Usage: $0 <options>
This script creates summary files from the logs generated by deprecated
access to the cPanel API1 calls.
Options:
--timeout=# Allows the caller to adjust the timeout in seconds. Defaults to 2 hours.
--verbose Generates more detailed output from the script.
--help Brief help message
USAGE
exit $stderr_flag; ## no critic qw(NoExitsFromSubroutines)
}
unless ( caller() ) {
exit only_one(@ARGV);
}
1;
Mini Shell