#!/usr/bin/env perl
#------------------------------------------------------------------------------
#
# pgbadger_tools - Tools based on pgBadger binary files
#
# This program is open source, licensed under the PostgreSQL Licence.
# For license terms, see the LICENSE file.
#------------------------------------------------------------------------------
#
# This program is first used to demonstrate how to deal with pgBadger binary
# files to build your own tool.
#
# See README.tools for more explanations.
#
#------------------------------------------------------------------------------
use vars qw($VERSION);

use strict;

use Getopt::Long qw(:config no_ignore_case bundling);
use File::Spec qw/ tmpdir /;
use File::Temp qw/ tempfile /;
use IO::File;
use IO::Handle;
use Storable qw(store_fd fd_retrieve);

$VERSION = '2.0';

my $PSQL_BIN     = 'psql';
my $TMP_DIR      = File::Spec->tmpdir() || '/tmp';

my @SQL_ACTION   = ('SELECT', 'INSERT', 'UPDATE', 'DELETE', 'COPY FROM', 'COPY TO', 'CTE', 'DDL', 'TCL');

# Where statistics are stored in pgbadger binary files
my %overall_stat        = ();
my %pgb_overall_stat    = ();
my %overall_checkpoint  = ();
my %top_slowest         = ();
my %normalyzed_info     = ();
my %error_info          = ();
my %pgb_error_info      = ();
my %logs_type           = ();
my %per_minute_info     = ();
my %pgb_per_minute_info = ();
my %pgb_pool_info       = ();
my %lock_info           = ();
my %tempfile_info       = ();
my %connection_info     = ();
my %pgb_connection_info = ();
my %database_info       = ();
my %application_info    = ();
my %user_info           = ();
my %host_info           = ();
my %session_info        = ();
my %pgb_session_info    = ();
my %conn_received       = ();
my %checkpoint_info     = ();
my %autovacuum_info     = ();
my %autoanalyze_info    = ();
my %cur_info            = ();
my %cur_temp_info       = ();
my %cur_lock_info       = ();
my %tsung_session       = ();
my %errors_code          = ();
my @top_locked_info     = ();
my @top_tempfile_info   = ();
my @log_files           = ();
my $nlines              = 0;
my $sample              = 3;
my $top                 = 20;
my $anonymize           = 0;

# Let's say --explode was not used to generate the binary files
# in this case all data are stored under the postgres database.
my $def_db = 'postgres';

##################################################################
# Get the command line parameters
# Add your own option
##################################################################
# General options
my $help   = 0;
my $quiet  = 0;
my $debug  = 0;
my $pghost = '';
my $pgport = '';
my $pguser = '';
my $pgdb   = '';

# Tools related option
my $explain_slowest = 0;
my $analyze         = 0;
my $max_duration    = 0;
my $file_per_query  = 0;
my $format_query    = 0;
my $explain_time_consuming = 0;
my $explain_normalized = 0;
my $csv_time_consuming = 0;
my $csv_slowest        = 0;
my $csv_normalized     = 0;
my $csv_filename       = 'out.csv';

my $result = GetOptions(
	'h|host=s'         => \$pghost,
	'p|port=i'         => \$pgport,
	'U|username=s'     => \$pguser,
	'd|dbname=s'       => \$pgdb,
	'q|quiet!'         => \$quiet,
	'v|verbose!'       => \$debug,
        'help!'            => \$help,
	# Explain slowest query tool
	'explain-slowest!' => \$explain_slowest,
	'explain-time-consuming!' => \$explain_time_consuming,
	'explain-normalized!' => \$explain_normalized,
	'analyze!'         => \$analyze,
	'max-duration=i'   => \$max_duration,
	'file-per-query!'  => \$file_per_query,
	'format-query!'    => \$format_query,
	'top=i'            => $top,
	# CSV output of top queries tool
	'csv-time-consuming!'      => \$csv_time_consuming,
	'csv-slowest!'      => \$csv_slowest,
	'csv-normalized!'      => \$csv_normalized,
	'csv-filename=s'   => \$csv_filename,
);

# Show help an exit
if ($help) {
	&usage();
}

# Lookup for binary files to load from command line
my @file_list = ();
foreach my $f (@ARGV) {
	push(@file_list, $f) if (-e $f && ($f =~ /\.bin$/));
}

die "FATAL: no binary file found, see usage (--help).\n" if ($#file_list == -1);

# Load all data gathered by all the different processes. This function
# is responsible of loading pgbadger statistics from binary files
foreach my $f (@file_list) {
	next if (-z "$f");
	my $fht = new IO::File;
	$fht->open("< $f") or die "FATAL: can't open temp file $f, $!\n";
	&load_stats($fht);
	$fht->close();
}

# Set the psql command follwing the option
my $psql_cmd = '';
$psql_cmd   .= " -h $pghost" if ($pghost);
$psql_cmd   .= " -p $pgport" if ($pgport);
$psql_cmd   .= " -U $pguser" if ($pguser);
$psql_cmd   .= " -d $pgdb"   if ($pgdb  );
if ($psql_cmd) {
	$psql_cmd = $PSQL_BIN . ' ' . $psql_cmd;
}

####################################################################
# 1srt tool: Dump top slowest queries inside explain statement. Will
# be executed when option --explain-slowest is set at command line
# Add your own condition. See also TOOL FUNCTIONS bellow 
##################################################################
if ($explain_slowest) {
    &logmsg('LOG', "Output each slowest queries within explain statement.");
    &dump_slowest_queries($def_db);
}
# when option --explain-time-consuming is set at command line
if ($explain_time_consuming) {
    &logmsg('LOG', "Output each most times consuming queries within explain statement.");
    &dump_times_consuming_queries($def_db);
}
# when option --explain-normalized is set at command line
if ($explain_normalized) {
    &logmsg('LOG', "Output each slowest normalized queries within explain statement.");
    &dump_normalized_slowest_queries($def_db);
}

####################################################################
# 2nd tool: Dump top queries into CSV file. Will be executed when
# option --csv-time_consuming or --csv-slowest or --csv-normalized
# are enabled.
##################################################################
if ($csv_time_consuming || $csv_slowest || $csv_normalized) {
	# Check that Text::CSV is available
	if (eval {require Text::CSV;1;} ne 1) {
		die("Can not save output in csv format, please install Perl module Text::CSV first.\n");
	} else {
		Text::CSV->import();
	}
	if ($csv_time_consuming) {
		&csv_times_consuming_queries($def_db);
	} elsif ($csv_slowest) {
		&csv_slowest_queries($def_db);
	} elsif ($csv_normalized) {
		&csv_normalized_queries($def_db);
	}
}


##################################################################
# Add your own bellow
##################################################################

exit 0;

##################################################################
# Display pgbadger_tools usage with the list of options
# Add your own in the Tools section with a sample.
##################################################################
sub usage
{

	print qq{
Usage: pgbadger_tools [options] [options tools] BINARY_FILE

Options:

    -d | --dbname DBNAME : same as in psql command, see psql --help
    -h | --host HOST     : same as in psql command, see psql --help
    -p | --port PORT     : same as in psql command, see psql --help
    -q | --quiet         : do not print any information
    -U | --username NAME : same as in psql command, see psql --help
    -v | --verbose       : show debug information
    --help               : Show this message

Note: option -d, -h, -p and -U are passed directly to the psql command.
The psql command must be in the PATH environment variable. If you have
authentication for the connection, use .pgpass. This allow to execute
queries to a PostgreSQL backend and get the output.

Options Tools:

  Generate EXPLAIN statements
  ---------------------------

  This tool allow to generate EXPLAIN statements with the top slowest queries
  reported by pgBadger. Here are the supported options, only the first one is
  mandatory:

    --explain-slowest : generate explain statements of slowest queries

	./pgbadger_tools --explain-slowest out.bin

    --explain-time-consuming : generate explain statements of time consuming queries

	./pgbadger_tools --explain-time-consuming out.bin

    --explain-normalized : generate explain statements of normalized slowest queries

	./pgbadger_tools --explain-normalized out.bin

    --analyze         : generate explain analyze statements of slowest queries

	./pgbadger_tools --explain-slowest --analyze out.bin

    --max-duration MS : set the number of milliseconds above which queries
			will not be reported. Use it if you want to auto
			execute explain statements.

    --file-per-query  : will dump each query into a separate file named qryXXX.sql

    --format-query    : do simple query formating

  To automatically execute those EXPLAIN statements and get the results with
  the queries, you just have to set at least one of the -d, -h, -p or -U
  command. For example, if the PostgreSQL instance is local and use peer as
  authent method for the postgres user and listen on default port:

	./pgbadger_tools --explain-slowest --analyze -d postgres out.bin

  Default is to generate EXPLAIN statement for the top 20 of queries, you can
  change this value using --top N.

  Options --explain-slowest, --explain-time-consuming and --explain-normalized
  can be use together in the same pgbadger_tools command.

  Generate CSV file with top queries
  ----------------------------------

  This tool allow to dump most time consuming statements reported by pgBadger.
  Here are the supported options, only one of the following is mandatory:

    --csv-time-consuming : generate a CSV file with top time consuming queries
    --csv-slowest        : generate a CSV file with top slowest queries
    --csv-normalized     : generate a CSV file with top normalized queries

	./pgbadger_tools --csv-time-consuming out.bin

  Those options can not be used together. By default the output file is named
  out.csv, you can use the --csv-filename option to renamed this file. Ex:

	./pgbadger_tools --csv-slowest --csv-filename slowest.csv out.bin

  Option to limit top queries to minimum duration:

    --max-duration MS : set the number of milliseconds above which queries
			will not be reported. Use it if you want to auto
			execute explain statements.


};
	exit 0;
}

##################################################################
# Internal method, do not edit unless you know what you are doing
##################################################################

####
# Display message following the log level
####
sub logmsg
{
	my ($level, $str) = @_;

	return if ($quiet && ($level ne 'FATAL'));
	return if (!$debug && ($level eq 'DEBUG'));

	if ($level =~ /(\d+)/) {
		print STDERR "\t" x $1;
	}

	print STDERR "$level: $str\n";
}

####
# Stores top N error sample queries, taken from pgbadger script
####
sub set_top_error_sample
{
	my ($curdb, $q, $date, $real_error, $detail, $context, $statement, $hint, $db, $user, $app, $remote, $sqlstate) = @_;

	$errors_code{$curdb}{$sqlstate}++ if ($sqlstate);

	# Stop when we have our number of samples
	if (!exists $error_info{$curdb}{$q}{date} || ($#{$error_info{$curdb}{$q}{date}}+1 < $sample)) {
		if ( ($q =~ /deadlock detected/) || ($real_error && !grep(/^\Q$real_error\E$/, @{$error_info{$curdb}{$q}{error}})) ) {
			if ($anonymize) {
				$context = &anonymize_query($context);
				$statement = &anonymize_query($statement);
				$detail = &anonymize_query($detail);
			}

			push(@{$error_info{$curdb}{$q}{date}},      $date);
			push(@{$error_info{$curdb}{$q}{detail}},    $detail);
			push(@{$error_info{$curdb}{$q}{context}},   $context);
			push(@{$error_info{$curdb}{$q}{statement}}, $statement);
			push(@{$error_info{$curdb}{$q}{hint}},      $hint);
			push(@{$error_info{$curdb}{$q}{error}},     $real_error);
			push(@{$error_info{$curdb}{$q}{db}},    $db);
			push(@{$error_info{$curdb}{$q}{user}},      $user);
			push(@{$error_info{$curdb}{$q}{app}},       $app);
			push(@{$error_info{$curdb}{$q}{remote}},    $remote);
			push(@{$error_info{$curdb}{$q}{sqlstate}},  $sqlstate);
		}
	}
}

####
# Load statistics from binary file into memory, the function is copied from pgbadger script
####
sub load_stats
{

	my $fd = shift;

	no strict;

	my %stats = %{ fd_retrieve($fd) };
	my %_overall_stat = %{$stats{overall_stat}};
	my %_pgb_overall_stat = %{$stats{pgb_overall_stat}};
	my %_overall_checkpoint = %{$stats{overall_checkpoint}};
	my %_normalyzed_info = %{$stats{normalyzed_info}};
	my %_error_info = %{$stats{error_info}};
	my %_pgb_error_info = %{$stats{pgb_error_info}};
	my %_pgb_pool_info = %{$stats{pgb_pool_info}};
	my %_connection_info = %{$stats{connection_info}};
	my %_pgb_connection_info = %{$stats{pgb_connection_info}};
	my %_database_info = %{$stats{database_info}};
	my %_application_info = %{$stats{application_info}};
	my %_user_info = %{$stats{user_info}};
	my %_host_info = %{$stats{host_info}};
	my %_checkpoint_info = %{$stats{checkpoint_info}};
	my %_session_info = %{$stats{session_info}};
	my %_pgb_session_info = %{$stats{pgb_session_info}};
	my %_tempfile_info = %{$stats{tempfile_info}};
	my %_cancelled_info = %{$stats{cancelled_info}};
	my %_logs_type = %{$stats{logs_type}};
	my %_errors_code = %{$stats{errors_code}};
	my %_lock_info = %{$stats{lock_info}};
	my %_per_minute_info = %{$stats{per_minute_info}};
	my %_pgb_per_minute_info = %{$stats{pgb_per_minute_info}};
	my %_top_slowest = %{$stats{top_slowest}};
	my $_nlines = $stats{nlines};
	my $_first_log_timestamp = $stats{first_log_timestamp};
	my $_last_log_timestamp = $stats{last_log_timestamp};
	my @_log_files = @{$stats{log_files}};
	my %_autovacuum_info = %{$stats{autovacuum_info}};
	my %_autoanalyze_info = %{$stats{autoanalyze_info}};
	my %_top_locked_info = %{$stats{top_locked_info}};
	my %_top_tempfile_info = %{$stats{top_tempfile_info}};
	my %_top_cancelled_info = %{$stats{top_cancelled_info}};

	### overall_stat ###

	# Those a used to update the progress bar and are not related to any database
	$overall_stat{'queries_number'} += $_overall_stat{'queries_number'};
	$overall_stat{'errors_number'} += $_overall_stat{'errors_number'};
	$pgb_overall_stat{'queries_number'} += $_pgb_overall_stat{'queries_number'};
	$pgb_overall_stat{'errors_number'} += $_pgb_overall_stat{'errors_number'};
	foreach my $d (keys %{ $_overall_stat{nlines} })
	{
		$overall_stat{'nlines'}{$d} += $_overall_stat{'nlines'}{$d};
	}

	foreach my $curdb (keys %_overall_stat)
	{
		next if (grep(/^$curdb$/, 'queries_number', 'errors_number', 'queries_number', 'errors_number', 'nlines'));
		if ($_overall_stat{$curdb}{'first_log_ts'}) {
			$overall_stat{$curdb}{'first_log_ts'} = $_overall_stat{$curdb}{'first_log_ts'}
				if (!$overall_stat{$curdb}{'first_log_ts'} ||
					($overall_stat{$curdb}{'first_log_ts'} gt $_overall_stat{$curdb}{'first_log_ts'}));
		}

		$overall_stat{$curdb}{'last_log_ts'} = $_overall_stat{$curdb}{'last_log_ts'}
			if not $overall_stat{$curdb}{'last_log_ts'}
				or $overall_stat{$curdb}{'last_log_ts'} lt $_overall_stat{$curdb}{'last_log_ts'};

		$overall_stat{$curdb}{'queries_number'} += $_overall_stat{$curdb}{'queries_number'};
		$overall_stat{$curdb}{'errors_number'} += $_overall_stat{$curdb}{'errors_number'};
		$pgb_overall_stat{$curdb}{'queries_number'} += $_pgb_overall_stat{$curdb}{'queries_number'};
		$pgb_overall_stat{$curdb}{'errors_number'} += $_pgb_overall_stat{$curdb}{'errors_number'};

		if ($_overall_stat{$curdb}{'first_query_ts'}) {
			$overall_stat{$curdb}{'first_query_ts'} = $_overall_stat{$curdb}{'first_query_ts'}
				if (!$overall_stat{$curdb}{'first_query_ts'} ||
					($overall_stat{$curdb}{'first_query_ts'} gt $_overall_stat{$curdb}{'first_query_ts'}));
		}

		$overall_stat{$curdb}{'last_query_ts'} = $_overall_stat{$curdb}{'last_query_ts'}
			if not $overall_stat{$curdb}{'last_query_ts'}
				or $overall_stat{$curdb}{'last_query_ts'} lt $_overall_stat{$curdb}{'last_query_ts'};

		$overall_stat{$curdb}{queries_duration} += $_overall_stat{$curdb}{queries_duration};

		foreach my $a (@SQL_ACTION) {
			$overall_stat{$curdb}{$a} += $_overall_stat{$curdb}{$a}
			if exists $_overall_stat{$curdb}{$a};
		}

		foreach my $k (keys %{$_overall_stat{$curdb}{peak}}) {
			$overall_stat{$curdb}{peak}{$k}{query} += $_overall_stat{$curdb}{peak}{$k}{query};
			$overall_stat{$curdb}{peak}{$k}{select} += $_overall_stat{$curdb}{peak}{$k}{select};
			$overall_stat{$curdb}{peak}{$k}{write} += $_overall_stat{$curdb}{peak}{$k}{write};
			$overall_stat{$curdb}{peak}{$k}{connection} += $_overall_stat{$curdb}{peak}{$k}{connection};
			$overall_stat{$curdb}{peak}{$k}{session} += $_overall_stat{$curdb}{peak}{$k}{session};
			$overall_stat{$curdb}{peak}{$k}{tempfile_size} += $_overall_stat{$curdb}{peak}{$k}{tempfile_size};
			$overall_stat{$curdb}{peak}{$k}{tempfile_count} += $_overall_stat{$curdb}{peak}{$k}{tempfile_count};
			$overall_stat{$curdb}{peak}{$k}{cancelled_size} += $_overall_stat{$curdb}{peak}{$k}{cancelled_size};
			$overall_stat{$curdb}{peak}{$k}{cancelled_count} += $_overall_stat{$curdb}{peak}{$k}{cancelled_count};
		}

		foreach my $k (keys %{$_overall_stat{$curdb}{histogram}{query_time}}) {
			$overall_stat{$curdb}{histogram}{query_time}{$k} += $_overall_stat{$curdb}{histogram}{query_time}{$k};
		}
		$overall_stat{$curdb}{histogram}{query_total} += $_overall_stat{$curdb}{histogram}{total};
		$overall_stat{$curdb}{histogram}{query_total} += $_overall_stat{$curdb}{histogram}{query_total};
		foreach my $k (keys %{$_overall_stat{$curdb}{histogram}{session_time}}) {
			$overall_stat{$curdb}{histogram}{session_time}{$k} += $_overall_stat{$curdb}{histogram}{session_time}{$k};
		}
		$overall_stat{$curdb}{histogram}{session_total} += $_overall_stat{$curdb}{histogram}{session_total};

		foreach my $k ('prepare', 'bind','execute') {
			$overall_stat{$curdb}{$k} += $_overall_stat{$curdb}{$k};
		}
	}

	$overall_checkpoint{checkpoint_warning} += $_overall_checkpoint{checkpoint_warning};
	$overall_checkpoint{checkpoint_write} = $_overall_checkpoint{checkpoint_write}
		if ($_overall_checkpoint{checkpoint_write} > $overall_checkpoint{checkpoint_write});
	$overall_checkpoint{checkpoint_sync} = $_overall_checkpoint{checkpoint_sync}
		if ($_overall_checkpoint{checkpoint_sync} > $overall_checkpoint{checkpoint_sync});
	foreach my $k (keys %{$_overall_checkpoint{peak}}) {
		$overall_checkpoint{peak}{$k}{checkpoint_wbuffer} += $_overall_checkpoint{peak}{$k}{checkpoint_wbuffer};
		$overall_checkpoint{peak}{$k}{walfile_usage} += $_overall_checkpoint{peak}{$k}{walfile_usage};
		$overall_checkpoint{peak}{$k}{distance} += $_overall_checkpoint{peak}{$k}{distance};
		$overall_checkpoint{peak}{$k}{estimate} += $_overall_checkpoint{peak}{$k}{estimate};
	}

	### pgbouncer related overall stats ###
	foreach my $k (keys %{$_pgb_overall_stat{peak}}) {
		$pgb_overall_stat{peak}{$k}{connection} += $_pgb_overall_stat{peak}{$k}{connection};
		$pgb_overall_stat{peak}{$k}{session} += $_pgb_overall_stat{peak}{$k}{session};
		$pgb_overall_stat{peak}{$k}{t_req} += $_pgb_overall_stat{peak}{$k}{t_req};
		$pgb_overall_stat{peak}{$k}{t_inbytes} += $_pgb_overall_stat{peak}{$k}{t_inbytes};
		$pgb_overall_stat{peak}{$k}{t_outbytes} += $_pgb_overall_stat{peak}{$k}{t_outbytes};
		$pgb_overall_stat{peak}{$k}{t_avgduration} += $_pgb_overall_stat{peak}{$k}{t_avgduration};
	}

	foreach my $k (keys %{$_pgb_overall_stat{histogram}{session_time}}) {
		$pgb_overall_stat{histogram}{session_time}{$k} += $_pgb_overall_stat{histogram}{session_time}{$k};
	}
	$pgb_overall_stat{histogram}{session_total} += $_pgb_overall_stat{histogram}{session_total};

	### Logs level ###

	foreach my $l (qw(LOG WARNING ERROR FATAL PANIC DETAIL HINT STATEMENT CONTEXT))
	{
		foreach my $curdb (keys %_logs_type)
		{
			$logs_type{$curdb}{$l} += $_logs_type{$curdb}{$l} if exists $_logs_type{$curdb}{$l};
		}
	}

	### Errors code ###

	foreach my $curdb (keys %_errors_code)
	{
		foreach my $c (keys %{$_errors_code{$curdb}}) {
			$errors_code{$curdb}{$c} += $_errors_code{$curdb}{$c};
		}
	}

	### database_info ###

	foreach my $curdb (keys %_database_info)
	{
		foreach my $db (keys %{$_database_info{$curdb}}) {
			foreach my $k (keys %{ $_database_info{$curdb}{$db} }) {
				$database_info{$curdb}{$db}{$k} += $_database_info{$curdb}{$db}{$k};
			}
		}
	}

	### application_info ###

	foreach my $curdb (keys %_application_info)
	{
		foreach my $app (keys %{$_application_info{$curdb}}) {
			foreach my $k (keys %{ $_application_info{$curdb}{$app} }) {
				$application_info{$curdb}{$app}{$k} += $_application_info{$curdb}{$app}{$k};
			}
		}
	}

	### user_info ###

	foreach my $curdb (keys %_user_info)
	{
		foreach my $u (keys %{$_user_info{$curdb}}) {
			foreach my $k (keys %{ $_user_info{$curdb}{$u} }) {
				$user_info{$curdb}{$u}{$k} += $_user_info{$curdb}{$u}{$k};
			}
		}
	}

	### host_info ###

	foreach my $curdb (keys %_host_info)
	{
		foreach my $h (keys %{$_host_info{$curdb}}) {
			foreach my $k (keys %{ $_host_info{$curdb}{$h} }) {
				$host_info{$curdb}{$h}{$k} += $_host_info{$curdb}{$h}{$k};
			}
		}
	}


	### connection_info ###

	foreach my $curdb (keys %_connection_info)
	{
		foreach my $db (keys %{ $_connection_info{$curdb}{database} }) {
			$connection_info{$curdb}{database}{$db} += $_connection_info{$curdb}{database}{$db};
		}

		foreach my $db (keys %{ $_connection_info{$curdb}{database_user} }) {
			foreach my $user (keys %{ $_connection_info{$curdb}{database_user}{$db} }) {
				$connection_info{$curdb}{database_user}{$db}{$user} += $_connection_info{$curdb}{database_user}{$db}{$user};
			}
		}

		foreach my $db (keys %{ $_connection_info{$curdb}{database_host} }) {
			foreach my $user (keys %{ $_connection_info{$curdb}{database_host}{$db} }) {
				$connection_info{$curdb}{database_host}{$db}{$user} += $_connection_info{$curdb}{database_host}{$db}{$user};
			}
		}

		foreach my $user (keys %{ $_connection_info{$curdb}{user} }) {
			$connection_info{$curdb}{user}{$user} += $_connection_info{$curdb}{user}{$user};
		}

		foreach my $host (keys %{ $_connection_info{$curdb}{host} }) {
			$connection_info{$curdb}{host}{$host} += $_connection_info{$curdb}{host}{$host};
		}

		$connection_info{$curdb}{count} += $_connection_info{$curdb}{count};

		foreach my $day (keys %{ $_connection_info{$curdb}{chronos} })
		{
			foreach my $hour (keys %{ $_connection_info{$curdb}{chronos}{$day} }) {

				$connection_info{$curdb}{chronos}{$day}{$hour}{count} += $_connection_info{$curdb}{chronos}{$day}{$hour}{count};
			}
		}
	}

	### pgbouncer connection_info ###

	foreach my $db (keys %{ $_pgb_connection_info{database} }) {
		$pgb_connection_info{database}{$db} += $_pgb_connection_info{database}{$db};
	}

	foreach my $db (keys %{ $_pgb_connection_info{database_user} }) {
		foreach my $user (keys %{ $_pgb_connection_info{database_user}{$db} }) {
			$pgb_connection_info{database_user}{$db}{$user} += $_pgb_connection_info{database_user}{$db}{$user};
		}
	}

	foreach my $db (keys %{ $_pgb_connection_info{database_host} }) {
		foreach my $user (keys %{ $_pgb_connection_info{database_host}{$db} }) {
			$pgb_connection_info{database_host}{$db}{$user} += $_pgb_connection_info{database_host}{$db}{$user};
		}
	}

	foreach my $user (keys %{ $_pgb_connection_info{user} }) {
		$pgb_connection_info{user}{$user} += $_pgb_connection_info{user}{$user};
	}

	foreach my $host (keys %{ $_pgb_connection_info{host} }) {
		$pgb_connection_info{host}{$host} += $_pgb_connection_info{host}{$host};
	}

	$pgb_connection_info{count} += $_pgb_connection_info{count};

	foreach my $day (keys %{ $_pgb_connection_info{chronos} }) {
		foreach my $hour (keys %{ $_pgb_connection_info{chronos}{$day} }) {
			$pgb_connection_info{chronos}{$day}{$hour}{count} += $_pgb_connection_info{chronos}{$day}{$hour}{count}
		}
	}

	### log_files ###

	foreach my $f (@_log_files) {
		push(@log_files, $f) if (!grep(m#^$f$#, @_log_files));
	}

	### error_info ###

	foreach my $curdb (keys %_error_info)
	{
		foreach my $q (keys %{$_error_info{$curdb}}) {
			$error_info{$curdb}{$q}{count} += $_error_info{$curdb}{$q}{count};
			foreach my $day (keys %{ $_error_info{$curdb}{$q}{chronos} }) {
				foreach my $hour (keys %{$_error_info{$curdb}{$q}{chronos}{$day}}) {
					$error_info{$curdb}{$q}{chronos}{$day}{$hour}{count} += $_error_info{$curdb}{$q}{chronos}{$day}{$hour}{count};
					foreach my $min (keys %{$_error_info{$curdb}{$q}{chronos}{$day}{$hour}{min}}) {
						$error_info{$curdb}{$q}{chronos}{$day}{$hour}{min}{$min} += $_error_info{$curdb}{$q}{chronos}{$day}{$hour}{min}{$min};
					}
				}
			}
			for (my $i = 0; $i <= $#{$_error_info{$curdb}{$q}{date}}; $i++) {
				&set_top_error_sample( $curdb, $q,
										$_error_info{$curdb}{$q}{date}[$i],
										$_error_info{$curdb}{$q}{error}[$i],
										$_error_info{$curdb}{$q}{detail}[$i],
										$_error_info{$curdb}{$q}{context}[$i],
										$_error_info{$curdb}{$q}{statement}[$i],
										$_error_info{$curdb}{$q}{hint}[$i],
										$_error_info{$curdb}{$q}{db}[$i],
										$_error_info{$curdb}{$q}{user}[$i],
										$_error_info{$curdb}{$q}{app}[$i],
										$_error_info{$curdb}{$q}{remote}[$i],
										$_error_info{$curdb}{$q}{sqlstate}[$i]
										);
			}
		}
	}

	### pgbouncer error_info ###

	foreach my $q (keys %_pgb_error_info)
	{
		$pgb_error_info{$q}{count} += $_pgb_error_info{$q}{count};
		foreach my $day (keys %{ $_pgb_error_info{$q}{chronos} }) {
			foreach my $hour (keys %{$_pgb_error_info{$q}{chronos}{$day}}) {
				$pgb_error_info{$q}{chronos}{$day}{$hour}{count} += $_pgb_error_info{$q}{chronos}{$day}{$hour}{count};
				foreach my $min (keys %{$_pgb_error_info{$q}{chronos}{$day}{$hour}{min}}) {
					$pgb_error_info{$q}{chronos}{$day}{$hour}{min}{$min} += $_pgb_error_info{$q}{chronos}{$day}{$hour}{min}{$min};
				}
			}
		}
		for (my $i = 0; $i <= $#{$_pgb_error_info{$q}{date}}; $i++) {
			&pgb_set_top_error_sample($q,
									$_pgb_error_info{$q}{date}[$i],
									$_pgb_error_info{$q}{error}[$i],
									$_pgb_error_info{$q}{db}[$i],
									$_pgb_error_info{$q}{user}[$i],
									$_pgb_error_info{$q}{remote}[$i]
									);
		}
	}

	### pgbouncer pool_info ###

	foreach my $q (keys %_pgb_pool_info) {
		$pgb_pool_info{$q}{count} += $_pgb_pool_info{$q}{count};
		foreach my $day (keys %{ $_pgb_pool_info{$q}{chronos} }) {
			foreach my $hour (keys %{$_pgb_pool_info{$q}{chronos}{$day}}) {
				$pgb_pool_info{$q}{chronos}{$day}{$hour}{count} += $_pgb_pool_info{$q}{chronos}{$day}{$hour}{count};
				foreach my $min (keys %{$_pgb_pool_info{$q}{chronos}{$day}{$hour}{min}}) {
					$pgb_pool_info{$q}{chronos}{$day}{$hour}{min}{$min} += $_pgb_pool_info{$q}{chronos}{$day}{$hour}{min}{$min};
				}
			}
		}
	}

	### per_minute_info ###

	foreach my $curdb (keys %_per_minute_info) {
		foreach my $day (keys %{$_per_minute_info{$curdb}}) {
			foreach my $hour (keys %{ $_per_minute_info{$curdb}{$day} }) {
				foreach my $min (keys %{ $_per_minute_info{$curdb}{$day}{$hour} }) {
					$per_minute_info{$curdb}{$day}{$hour}{$min}{connection}{count} +=
						($_per_minute_info{$curdb}{$day}{$hour}{$min}{connection}{count} || 0);
					$per_minute_info{$curdb}{$day}{$hour}{$min}{session}{count} +=
						($_per_minute_info{$curdb}{$day}{$hour}{$min}{session}{count} || 0);
					$per_minute_info{$curdb}{$day}{$hour}{$min}{query}{count} +=
						($_per_minute_info{$curdb}{$day}{$hour}{$min}{query}{count} || 0);
					$per_minute_info{$curdb}{$day}{$hour}{$min}{query}{duration} += $_per_minute_info{$curdb}{$day}{$hour}{$min}{query}{duration};
					$per_minute_info{$curdb}{$day}{$hour}{$min}{query}{min} = $_per_minute_info{$curdb}{$day}{$hour}{$min}{query}{min} if (!exists $per_minute_info{$curdb}{$day}{$hour}{$min}{query}{min} || ($per_minute_info{$curdb}{$day}{$hour}{$min}{query}{min} > $_per_minute_info{$curdb}{$day}{$hour}{$min}{query}{min}));
					$per_minute_info{$curdb}{$day}{$hour}{$min}{query}{max} = $_per_minute_info{$curdb}{$day}{$hour}{$min}{query}{max} if (!exists $per_minute_info{$curdb}{$day}{$hour}{$min}{query}{max} || ($per_minute_info{$curdb}{$day}{$hour}{$min}{query}{max} < $_per_minute_info{$curdb}{$day}{$hour}{$min}{query}{max}));

					foreach my $sec (keys %{ $_per_minute_info{$curdb}{$day}{$hour}{$min}{connection}{second} }) {
						$per_minute_info{$curdb}{$day}{$hour}{$min}{connection}{second}{$sec} +=
							($_per_minute_info{$curdb}{$day}{$hour}{$min}{connection}{second}{$sec} || 0);
					}
					foreach my $sec (keys %{ $_per_minute_info{$curdb}{$day}{$hour}{$min}{session}{second} }) {
						$per_minute_info{$curdb}{$day}{$hour}{$min}{session}{second}{$sec} +=
							($_per_minute_info{$curdb}{$day}{$hour}{$min}{session}{second}{$sec} || 0);
					}
					foreach my $sec (keys %{ $_per_minute_info{$curdb}{$day}{$hour}{$min}{query}{second} }) {
						$per_minute_info{$curdb}{$day}{$hour}{$min}{query}{second}{$sec} +=
							($_per_minute_info{$curdb}{$day}{$hour}{$min}{query}{second}{$sec} || 0);
					}
					foreach my $action (@SQL_ACTION) {
						next if (!exists $_per_minute_info{$curdb}{$day}{$hour}{$min}{$action});
						$per_minute_info{$curdb}{$day}{$hour}{$min}{$action}{count} += ($_per_minute_info{$curdb}{$day}{$hour}{$min}{$action}{count} || 0);
						$per_minute_info{$curdb}{$day}{$hour}{$min}{$action}{duration} += ($_per_minute_info{$curdb}{$day}{$hour}{$min}{$action}{duration} || 0);
						foreach my $sec (keys %{ $_per_minute_info{$curdb}{$day}{$hour}{$min}{$action}{second} }) {
							$per_minute_info{$curdb}{$day}{$hour}{$min}{$action}{second}{$sec} +=
								($_per_minute_info{$curdb}{$day}{$hour}{$min}{$action}{second}{$sec} || 0);
						}
					}
					foreach my $k ('prepare', 'bind','execute') {
						if (exists $_per_minute_info{$curdb}{$day}{$hour}{$min}{$k}) {
							$per_minute_info{$curdb}{$day}{$hour}{$min}{$k} += $_per_minute_info{$curdb}{$day}{$hour}{$min}{$k};
						}
					}
					foreach my $log (keys %{ $_per_minute_info{$curdb}{$day}{$hour}{$min}{log_level} }) {
						$per_minute_info{$curdb}{$day}{$hour}{$min}{log_level}{$log} +=
							($_per_minute_info{$curdb}{$day}{$hour}{$min}{log_level}{$log} || 0);
					}

					$per_minute_info{$curdb}{$day}{$hour}{$min}{cancelled}{count} += $_per_minute_info{$curdb}{$day}{$hour}{$min}{cancelled}{count}
							if defined $_per_minute_info{$curdb}{$day}{$hour}{$min}{cancelled}{count};
					$per_minute_info{$curdb}{$day}{$hour}{$min}{'tempfile'}{count} += $_per_minute_info{$curdb}{$day}{$hour}{$min}{'tempfile'}{count}
							if defined $_per_minute_info{$curdb}{$day}{$hour}{$min}{'tempfile'}{count};
					$per_minute_info{$curdb}{$day}{$hour}{$min}{'tempfile'}{size} += $_per_minute_info{$curdb}{$day}{$hour}{$min}{'tempfile'}{size}
							if defined $_per_minute_info{$curdb}{$day}{$hour}{$min}{'tempfile'}{size};

					$per_minute_info{$curdb}{$day}{$hour}{$min}{checkpoint}{file_removed} += $_per_minute_info{$curdb}{$day}{$hour}{$min}{checkpoint}{file_removed};
					$per_minute_info{$curdb}{$day}{$hour}{$min}{checkpoint}{sync} += $_per_minute_info{$curdb}{$day}{$hour}{$min}{checkpoint}{sync};
					$per_minute_info{$curdb}{$day}{$hour}{$min}{checkpoint}{wbuffer} += $_per_minute_info{$curdb}{$day}{$hour}{$min}{checkpoint}{wbuffer};
					$per_minute_info{$curdb}{$day}{$hour}{$min}{checkpoint}{file_recycled} += $_per_minute_info{$curdb}{$day}{$hour}{$min}{checkpoint}{file_recycled};
					$per_minute_info{$curdb}{$day}{$hour}{$min}{checkpoint}{total} += $_per_minute_info{$curdb}{$day}{$hour}{$min}{checkpoint}{total};
					$per_minute_info{$curdb}{$day}{$hour}{$min}{checkpoint}{file_added} += $_per_minute_info{$curdb}{$day}{$hour}{$min}{checkpoint}{file_added};
					$per_minute_info{$curdb}{$day}{$hour}{$min}{checkpoint}{write} += $_per_minute_info{$curdb}{$day}{$hour}{$min}{checkpoint}{write};
					$per_minute_info{$curdb}{$day}{$hour}{$min}{autovacuum}{count} += $_per_minute_info{$curdb}{$day}{$hour}{$min}{autovacuum}{count};
					$per_minute_info{$curdb}{$day}{$hour}{$min}{autoanalyze}{count} += $_per_minute_info{$curdb}{$day}{$hour}{$min}{autoanalyze}{count};

					$per_minute_info{$curdb}{$day}{$hour}{$min}{checkpoint}{sync_files} += $_per_minute_info{$curdb}{$day}{$hour}{$min}{checkpoint}{sync_files};
					$per_minute_info{$curdb}{$day}{$hour}{$min}{checkpoint}{sync_avg} += $_per_minute_info{$curdb}{$day}{$hour}{$min}{checkpoint}{sync_avg};
					$per_minute_info{$curdb}{$day}{$hour}{$min}{checkpoint}{sync_longest} = $_per_minute_info{$curdb}{$day}{$hour}{$min}{checkpoint}{sync_longest}
						if ($_per_minute_info{$curdb}{$day}{$hour}{$min}{checkpoint}{sync_longest} > $per_minute_info{$curdb}{$day}{$hour}{$min}{checkpoint}{sync_longest});
					$per_minute_info{$curdb}{$day}{$hour}{$min}{checkpoint}{distance} = $_per_minute_info{$curdb}{$day}{$hour}{$min}{checkpoint}{distance};
					$per_minute_info{$curdb}{$day}{$hour}{$min}{checkpoint}{distance_count} = $_per_minute_info{$curdb}{$day}{$hour}{$min}{checkpoint}{distance_count};
					$per_minute_info{$curdb}{$day}{$hour}{$min}{checkpoint}{estimate} = $_per_minute_info{$curdb}{$day}{$hour}{$min}{checkpoint}{estimate};
					$per_minute_info{$curdb}{$day}{$hour}{$min}{checkpoint}{warnning} = $_per_minute_info{$curdb}{$day}{$hour}{$min}{checkpoint}{warning};
					$per_minute_info{$curdb}{$day}{$hour}{$min}{checkpoint}{warning_seconds} = $_per_minute_info{$curdb}{$day}{$hour}{$min}{checkpoint}{warning_seconds};
				}
			}
		}
	}

	### pgbouncer per_minute_info ###

	foreach my $day (keys %_pgb_per_minute_info) {
		foreach my $hour (keys %{ $_pgb_per_minute_info{$day} }) {
			foreach my $min (keys %{ $_pgb_per_minute_info{$day}{$hour} }) {
				$pgb_per_minute_info{$day}{$hour}{$min}{connection}{count} +=
					($_pgb_per_minute_info{$day}{$hour}{$min}{connection}{count} || 0);
				$pgb_per_minute_info{$day}{$hour}{$min}{session}{count} +=
					($_pgb_per_minute_info{$day}{$hour}{$min}{session}{count} || 0);
				$pgb_per_minute_info{$day}{$hour}{$min}{t_req} +=
					($_pgb_per_minute_info{$day}{$hour}{$min}{t_req} || 0);
				$pgb_per_minute_info{$day}{$hour}{$min}{t_inbytes} +=
					($_pgb_per_minute_info{$day}{$hour}{$min}{t_inbytes} || 0);
				$pgb_per_minute_info{$day}{$hour}{$min}{t_outbytes} +=
					($_pgb_per_minute_info{$day}{$hour}{$min}{t_outbytes} || 0);
				$pgb_per_minute_info{$day}{$hour}{$min}{t_avgduration} +=
					($_pgb_per_minute_info{$day}{$hour}{$min}{t_avgduration} || 0);
			}
		}
	}

	### lock_info ###

	foreach my $curdb (keys %_lock_info)
	{
		foreach my $lock (keys %{$_lock_info{$curdb}}) {
			$lock_info{$curdb}{$lock}{count} += $_lock_info{$curdb}{$lock}{count};

			foreach my $day (keys %{ $_lock_info{$curdb}{chronos} }) {
				foreach my $hour (keys %{ $_lock_info{$curdb}{chronos}{$day} }) {
					$lock_info{$curdb}{chronos}{$day}{$hour}{count} += $_lock_info{$curdb}{chronos}{$day}{$hour}{count};
					$lock_info{$curdb}{chronos}{$day}{$hour}{duration} += $_lock_info{$curdb}{chronos}{$day}{$hour}{duration};
				}
			}

			$lock_info{$curdb}{$lock}{duration} += $_lock_info{$curdb}{$lock}{duration};

			foreach my $type (keys %{$_lock_info{$curdb}{$lock}}) {
				next if $type =~ /^(count|chronos|duration)$/;

				$lock_info{$curdb}{$lock}{$type}{count} += $_lock_info{$curdb}{$lock}{$type}{count};
				$lock_info{$curdb}{$lock}{$type}{duration} += $_lock_info{$curdb}{$lock}{$type}{duration};
			}
		}
	}

	### nlines ###

	$nlines += $_nlines;

	### normalyzed_info ###

	foreach my $curdb (keys %_normalyzed_info) {
		foreach my $stmt (keys %{$_normalyzed_info{$curdb}}) {

			foreach my $dt (keys %{$_normalyzed_info{$curdb}{$stmt}{samples}} ) {
				foreach my $k (keys %{$_normalyzed_info{$curdb}{$stmt}{samples}{$dt}} ) {
					$normalyzed_info{$curdb}{$stmt}{samples}{$dt}{$k} = $_normalyzed_info{$curdb}{$stmt}{samples}{$dt}{$k};
				}
			}

			# Keep only the top N samples
			my $i = 1;
			foreach my $k (sort {$b <=> $a} keys %{$normalyzed_info{$curdb}{$stmt}{samples}}) {
				if ($i > $sample) {
					delete $normalyzed_info{$curdb}{$stmt}{samples}{$k};
				}
				$i++;
			}

			$normalyzed_info{$curdb}{$stmt}{count} += $_normalyzed_info{$curdb}{$stmt}{count};

			# Set min / max duration for this query
			if (!exists $normalyzed_info{$curdb}{$stmt}{min} || ($normalyzed_info{$curdb}{$stmt}{min} > $_normalyzed_info{$curdb}{$stmt}{min})) {
				$normalyzed_info{$curdb}{$stmt}{min} = $_normalyzed_info{$curdb}{$stmt}{min};
			}
			if (!exists $normalyzed_info{$curdb}{$stmt}{max} || ($normalyzed_info{$curdb}{$stmt}{max} < $_normalyzed_info{$curdb}{$stmt}{max})) {
				$normalyzed_info{$curdb}{$stmt}{max} = $_normalyzed_info{$curdb}{$stmt}{max};
			}

			foreach my $day (keys %{$_normalyzed_info{$curdb}{$stmt}{chronos}} ) {
				foreach my $hour (keys %{$_normalyzed_info{$curdb}{$stmt}{chronos}{$day}} ) {
					$normalyzed_info{$curdb}{$stmt}{chronos}{$day}{$hour}{count} +=
						$_normalyzed_info{$curdb}{$stmt}{chronos}{$day}{$hour}{count};
					$normalyzed_info{$curdb}{$stmt}{chronos}{$day}{$hour}{duration} +=
						$_normalyzed_info{$curdb}{$stmt}{chronos}{$day}{$hour}{duration};
					foreach my $min (keys %{$_normalyzed_info{$curdb}{$stmt}{chronos}{$day}{$hour}{min}} ) {
						$normalyzed_info{$curdb}{$stmt}{chronos}{$day}{$hour}{min}{$min} +=
							$_normalyzed_info{$curdb}{$stmt}{chronos}{$day}{$hour}{min}{$min};
					}
					foreach my $min (keys %{$_normalyzed_info{$curdb}{$stmt}{chronos}{$day}{$hour}{min_duration}} ) {
						$normalyzed_info{$curdb}{$stmt}{chronos}{$day}{$hour}{min_duration}{$min} +=
							$_normalyzed_info{$curdb}{$stmt}{chronos}{$day}{$hour}{min_duration}{$min};
					}
				}
			}

			$normalyzed_info{$curdb}{$stmt}{duration} += $_normalyzed_info{$curdb}{$stmt}{duration};

			if (exists $_normalyzed_info{$curdb}{$stmt}{locks}) {
				$normalyzed_info{$curdb}{$stmt}{locks}{count} += $_normalyzed_info{$curdb}{$stmt}{locks}{count};
				$normalyzed_info{$curdb}{$stmt}{locks}{wait} += $_normalyzed_info{$curdb}{$stmt}{locks}{wait};
				if (!exists $normalyzed_info{$curdb}{$stmt}{locks}{minwait} || ($normalyzed_info{$curdb}{$stmt}{locks}{minwait} > $_normalyzed_info{$stmt}{locks}{minwait})) {
					$normalyzed_info{$curdb}{$stmt}{locks}{minwait} = $_normalyzed_info{$curdb}{$stmt}{locks}{minwait};
				}
				if (!exists $normalyzed_info{$curdb}{$stmt}{locks}{maxwait} || ($normalyzed_info{$curdb}{$stmt}{locks}{maxwait} < $_normalyzed_info{$curdb}{$stmt}{locks}{maxwait})) {
					$normalyzed_info{$curdb}{$stmt}{locks}{maxwait} = $_normalyzed_info{$curdb}{$stmt}{locks}{maxwait};
				}
			}

			if (exists $_normalyzed_info{$curdb}{$stmt}{tempfiles}) {
				$normalyzed_info{$curdb}{$stmt}{tempfiles}{count} += $_normalyzed_info{$curdb}{$stmt}{tempfiles}{count};
				$normalyzed_info{$curdb}{$stmt}{tempfiles}{size} += $_normalyzed_info{$curdb}{$stmt}{tempfiles}{size};
				if (!exists $normalyzed_info{$curdb}{$stmt}{tempfiles}{minsize} || ($normalyzed_info{$curdb}{$stmt}{tempfiles}{minsize} > $_normalyzed_info{$curdb}{$stmt}{tempfiles}{minsize})) {
					$normalyzed_info{$curdb}{$stmt}{tempfiles}{minsize} = $_normalyzed_info{$curdb}{$stmt}{tempfiles}{minsize};
				}
				if (!exists $normalyzed_info{$curdb}{$stmt}{tempfiles}{maxsize} || ($normalyzed_info{$curdb}{$stmt}{tempfiles}{maxsize} < $_normalyzed_info{$curdb}{$stmt}{tempfiles}{maxsize})) {
					$normalyzed_info{$curdb}{$stmt}{tempfiles}{maxsize} = $_normalyzed_info{$curdb}{$stmt}{tempfiles}{maxsize};
				}
			}

			if (exists $_normalyzed_info{$curdb}{$stmt}{cancelled}) {
				$normalyzed_info{$curdb}{$stmt}{cancelled}{count} += $_normalyzed_info{$curdb}{$stmt}{cancelled}{count};
			}

			foreach my $u (keys %{$_normalyzed_info{$curdb}{$stmt}{users}} ) {
				foreach my $k (keys %{$_normalyzed_info{$curdb}{$stmt}{users}{$u}} ) {
					$normalyzed_info{$curdb}{$stmt}{users}{$u}{$k} += $_normalyzed_info{$curdb}{$stmt}{users}{$u}{$k};
				}
			}
			foreach my $u (keys %{$_normalyzed_info{$curdb}{$stmt}{apps}} ) {
				foreach my $k (keys %{$_normalyzed_info{$curdb}{$stmt}{apps}{$u}} ) {
					$normalyzed_info{$curdb}{$stmt}{apps}{$u}{$k} += $_normalyzed_info{$curdb}{$stmt}{apps}{$u}{$k};
				}
			}
		}
	}

	### session_info ###

	foreach my $curdb (keys %_session_info) {
		foreach my $db (keys %{ $_session_info{$curdb}{database}}) {
			$session_info{$curdb}{database}{$db}{count} += $_session_info{$curdb}{database}{$db}{count};
			$session_info{$curdb}{database}{$db}{duration} += $_session_info{$curdb}{database}{$db}{duration};
		}

		$session_info{$curdb}{count} += $_session_info{$curdb}{count};

		foreach my $day (keys %{ $_session_info{$curdb}{chronos}}) {
			foreach my $hour (keys %{ $_session_info{$curdb}{chronos}{$day}}) {
				$session_info{$curdb}{chronos}{$day}{$hour}{count} += $_session_info{$curdb}{chronos}{$day}{$hour}{count};
				$session_info{$curdb}{chronos}{$day}{$hour}{duration} += $_session_info{$curdb}{chronos}{$day}{$hour}{duration};
			}
		}

		foreach my $user (keys %{ $_session_info{$curdb}{user}}) {
			$session_info{$curdb}{user}{$user}{count}    += $_session_info{$curdb}{user}{$user}{count};
			$session_info{$curdb}{user}{$user}{duration} += $_session_info{$curdb}{user}{$user}{duration};
		}

		$session_info{$curdb}{duration} += $_session_info{$curdb}{duration};

		foreach my $host (keys %{ $_session_info{$curdb}{host}}) {
			$session_info{$curdb}{host}{$host}{count}    += $_session_info{$curdb}{host}{$host}{count};
			$session_info{$curdb}{host}{$host}{duration} += $_session_info{$curdb}{host}{$host}{duration};
		}

		foreach my $app (keys %{ $_session_info{$curdb}{app}}) {
			$session_info{$curdb}{app}{$app}{count}    += $_session_info{$curdb}{app}{$app}{count};
			$session_info{$curdb}{app}{$app}{duration} += $_session_info{$curdb}{app}{$app}{duration};
		}
	}

	### pgbouncer session_info ###

	foreach my $db (keys %{ $_pgb_session_info{database}}) {
		$pgb_session_info{database}{$db}{count} += $_pgb_session_info{database}{$db}{count};
		$pgb_session_info{database}{$db}{duration} += $_pgb_session_info{database}{$db}{duration};
	}

	$pgb_session_info{count} += $_pgb_session_info{count};

	foreach my $day (keys %{ $_pgb_session_info{chronos}}) {
		foreach my $hour (keys %{ $_pgb_session_info{chronos}{$day}}) {
			$pgb_session_info{chronos}{$day}{$hour}{count} += $_pgb_session_info{chronos}{$day}{$hour}{count};
			$pgb_session_info{chronos}{$day}{$hour}{duration} += $_pgb_session_info{chronos}{$day}{$hour}{duration};
		}
	}

	foreach my $user (keys %{ $_pgb_session_info{user}}) {
		$pgb_session_info{user}{$user}{count}    += $_pgb_session_info{user}{$user}{count};
		$pgb_session_info{user}{$user}{duration} += $_pgb_session_info{user}{$user}{duration};
	}

	$pgb_session_info{duration} += $_pgb_session_info{duration};

	foreach my $host (keys %{ $_pgb_session_info{host}}) {
		$pgb_session_info{host}{$host}{count}    += $_pgb_session_info{host}{$host}{count};
		$pgb_session_info{host}{$host}{duration} += $_pgb_session_info{host}{$host}{duration};
	}


	### tempfile_info ###

	foreach my $curdb (keys %_tempfile_info)
	{
		$tempfile_info{$curdb}{count} += $_tempfile_info{$curdb}{count}
			if defined $_tempfile_info{$curdb}{count};
		$tempfile_info{$curdb}{size} += $_tempfile_info{$curdb}{size}
			if defined $_tempfile_info{$curdb}{size};
		$tempfile_info{$curdb}{maxsize} = $_tempfile_info{$curdb}{maxsize}
			if defined $_tempfile_info{$curdb}{maxsize} and ( not defined $tempfile_info{$curdb}{maxsize}
				or $tempfile_info{$curdb}{maxsize} < $_tempfile_info{$curdb}{maxsize} );
	}

	### top_slowest ###
	foreach my $curdb (keys %_top_slowest)
	{
		my @tmp_top_slowest = sort {$b->[0] <=> $a->[0]} (@{$top_slowest{$curdb}}, @{$_top_slowest{$curdb}});
		@{$top_slowest{$curdb}} = ();
		for (my $i = 0; $i <= $#tmp_top_slowest; $i++) {
			push(@{$top_slowest{$curdb}}, $tmp_top_slowest[$i]);
			last if ($i == $end_top);
		}
	}

	### top_locked ###
	foreach my $curdb (keys %_top_locked_info)
	{
		my @tmp_top_locked_info = sort {$b->[0] <=> $a->[0]} (@{$top_locked_info{$curdb}}, @{$_top_locked_info{$curdb}});
		@{$top_locked_info{$curdb}} = ();
		for (my $i = 0; $i <= $#tmp_top_locked_info; $i++) {
			push(@{$top_locked_info{$curdb}}, $tmp_top_locked_info[$i]);
			last if ($i == $end_top);
		}
	}

	### top_tempfile ###
	foreach my $curdb (keys %_top_tempfile_info)
	{
		my @tmp_top_tempfile_info = sort {$b->[0] <=> $a->[0]} (@{$top_tempfile_info{$curdb}}, @{$_top_tempfile_info{$curdb}});
		@{$top_tempfile_info{$curdb}} = ();
		for (my $i = 0; $i <= $#tmp_top_tempfile_info; $i++) {
			push(@{$top_tempfile_info{$curdb}}, $tmp_top_tempfile_info[$i]);
			last if ($i == $end_top);
		}
	}

	### checkpoint_info ###
	$checkpoint_info{file_removed} += $_checkpoint_info{file_removed};
	$checkpoint_info{sync} += $_checkpoint_info{sync};
	$checkpoint_info{wbuffer} += $_checkpoint_info{wbuffer};
	$checkpoint_info{file_recycled} += $_checkpoint_info{file_recycled};
	$checkpoint_info{total} += $_checkpoint_info{total};
	$checkpoint_info{file_added} += $_checkpoint_info{file_added};
	$checkpoint_info{write} += $_checkpoint_info{write};
	$checkpoint_info{distance} += $_checkpoint_info{distance};
	$checkpoint_info{estimate} += $_checkpoint_info{estimate};
	$checkpoint_info{warning} += $_checkpoint_info{warning};
	$checkpoint_info{warning_seconds} += $_checkpoint_info{warning_seconds};

	#### Autovacuum info ####

	foreach my $curdb (keys %_autovacuum_info)
	{
		$autovacuum_info{$curdb}{count} += $_autovacuum_info{$curdb}{count};

		foreach my $day (keys %{ $_autovacuum_info{$curdb}{chronos} }) {
			foreach my $hour (keys %{ $_autovacuum_info{$curdb}{chronos}{$day} }) {
				$autovacuum_info{$curdb}{chronos}{$day}{$hour}{count} += $_autovacuum_info{$curdb}{chronos}{$day}{$hour}{count};
			}
		}
		foreach my $table (keys %{ $_autovacuum_info{$curdb}{tables} }) {
			$autovacuum_info{$curdb}{tables}{$table}{vacuums} += $_autovacuum_info{$curdb}{tables}{$table}{vacuums};
			$autovacuum_info{$curdb}{tables}{$table}{idxscans} += $_autovacuum_info{$curdb}{tables}{$table}{idxscans};
			$autovacuum_info{$curdb}{tables}{$table}{tuples}{removed} += $_autovacuum_info{$curdb}{tables}{$table}{tuples}{removed};
			$autovacuum_info{$curdb}{tables}{$table}{pages}{removed} += $_autovacuum_info{$curdb}{tables}{$table}{pages}{removed};
		}
		if ($_autovacuum_info{$curdb}{peak}{system_usage}{elapsed} > $autovacuum_info{$curdb}{peak}{system_usage}{elapsed}) {
			$autovacuum_info{$curdb}{peak}{system_usage}{elapsed} = $_autovacuum_info{$curdb}{peak}{system_usage}{elapsed};
			$autovacuum_info{$curdb}{peak}{system_usage}{table} = $_autovacuum_info{$curdb}{peak}{system_usage}{table};
			$autovacuum_info{$curdb}{peak}{system_usage}{date} = $_autovacuum_info{$curdb}{peak}{system_usage}{date};
		}
	}

	#### Autoanalyze info ####

	foreach my $curdb (keys %_autoanalyze_info)
	{
		$autoanalyze_info{$curdb}{count} += $_autoanalyze_info{$curdb}{count};

		foreach my $day (keys %{ $_autoanalyze_info{$curdb}{chronos} }) {
			foreach my $hour (keys %{ $_autoanalyze_info{$curdb}{chronos}{$day} }) {
				$autoanalyze_info{$curdb}{chronos}{$day}{$hour}{count} += $_autoanalyze_info{$curdb}{chronos}{$day}{$hour}{count};
			}
		}
		foreach my $table (keys %{ $_autoanalyze_info{$curdb}{tables} }) {
			$autoanalyze_info{$curdb}{tables}{$table}{analyzes} += $_autoanalyze_info{$curdb}{tables}{$table}{analyzes};
		}
		if ($_autoanalyze_info{$curdb}{peak}{system_usage}{elapsed} > $autoanalyze_info{$curdb}{peak}{system_usage}{elapsed}) {
			$autoanalyze_info{$curdb}{peak}{system_usage}{elapsed} = $_autoanalyze_info{$curdb}{peak}{system_usage}{elapsed};
			$autoanalyze_info{$curdb}{peak}{system_usage}{table} = $_autoanalyze_info{$curdb}{peak}{system_usage}{table};
			$autoanalyze_info{$curdb}{peak}{system_usage}{date} = $_autoanalyze_info{$curdb}{peak}{system_usage}{date};
		}
	}

	use strict;

	return;
}


# Do minimal query formatting.
# Add carriage return to be more human readable
sub fmtquery
{
	my $qry = shift;

	return $qry if (!$format_query);

	my @KEYWORDS = qw( FROM INNER WHERE AND OR ORDER RETURNING ); 

	foreach my $key (@KEYWORDS) {
		$qry =~ s/\s+($key\s+)/\n$1/ig;
	}
	return $qry;
}

###############################################################################
# TOOL FUNCTIONS
# Add your own below
###############################################################################

# Dump top slowest queries as EXPLAIN statement.
sub dump_slowest_queries
{
	my $curdb = shift;

	# Global information
	my $curdate    = localtime(time);
	my $logfile_str = join(',', @log_files);

	for (my $i = 0 ; $i <= $#{$top_slowest{$curdb}} ; $i++) {

		# Do not process request that are slower than $max_duration
		next if ( $max_duration && ($top_slowest{$curdb}[$i]->[0] > $max_duration) );

		my $head = qq{-- Generated on $curdate
-- Log file: $logfile_str
-- Log start from $overall_stat{'first_log_ts'} to $overall_stat{'last_log_ts'}
--
};
		$head .= "-- duration: $top_slowest{$curdb}[$i]->[0]\n" if ($top_slowest{$curdb}[$i]->[0]);
		$head .= "-- database: $top_slowest{$curdb}[$i]->[3]\n" if ($top_slowest{$curdb}[$i]->[3]);
		$head .= "-- user: $top_slowest{$curdb}[$i]->[4]\n" if ($top_slowest{$curdb}[$i]->[4]);
		$head .= "-- remote: $top_slowest{$curdb}[$i]->[5]\n" if ($top_slowest{$curdb}[$i]->[5]);
		$head .= "-- app: $top_slowest{$curdb}[$i]->[6]\n" if ($top_slowest{$curdb}[$i]->[6]);
		$head .= "--, bind query: yes\n" if ($top_slowest{$curdb}[$i]->[7]);
		$head =~ s/^, //;

		my $explain = "EXPLAIN\n";
		if ($analyze) {
			$explain = "BEGIN;\nEXPLAIN (ANALYZE, VERBOSE, BUFFERS)\n";
		}
		$explain .= "$top_slowest{$curdb}[$i]->[2]\n";
		$explain .= "ROLLBACK;\n" if ($analyze);
		if ($file_per_query) {
			my $fhqr = new IO::File;
			my $fname = sprintf("qry%03d.sql", $i);
			$fhqr->open("> $fname") or die "FATAL: can't open temp file $fname, $!\n";
			$fhqr->print($head);
			$fhqr->print(&fmtquery($explain));
			$fhqr->close();
		} else {
			print $head;
			print $explain;
		}
		if ($psql_cmd) {
			my @tmpfile = tempfile('tmp_pgbadger_tools_XXXX', SUFFIX => '.txt', DIR => $TMP_DIR, UNLINK => 1 );
			my $fht = new IO::File;
			$fht->open("> $tmpfile[1]") or die "FATAL: can't open temp file $tmpfile[1], $!\n";
			$fht->print("$explain");
			$fht->close();
			print `$psql_cmd -f $tmpfile[1]`;
			unlink($tmpfile[1]);
		}
	}
}

# Dump top time consuming queries as EXPLAIN statement.
sub dump_times_consuming_queries
{
	my $curdb = shift;

	# Global information
	my $curdate    = localtime(time);
	my $logfile_str = join(',', @log_files);

        my $rank = 1;
        foreach my $k (sort {$normalyzed_info{$curdb}{$b}{count} <=> $normalyzed_info{$curdb}{$a}{count}} keys %{$normalyzed_info{$curdb}}) {
                next if (!$normalyzed_info{$curdb}{$k}{count});
                foreach my $d (sort {$b <=> $a} keys %{$normalyzed_info{$curdb}{$k}{samples}}) {
			# Do not process request that are slower than $max_duration
			next if ( $max_duration && ( $d > $max_duration) );

			my $head = qq{-- Generated on $curdate
-- Log file: $logfile_str
-- Log start from $overall_stat{'first_log_ts'} to $overall_stat{'last_log_ts'}
--
};
			$head .= "-- duration: $d\n" if ($d);
			$head .= "-- database: $normalyzed_info{$curdb}{$k}{samples}{$d}{db}\n" if ($normalyzed_info{$curdb}{$k}{samples}{$d}{db});
			$head .= "-- user: $normalyzed_info{$curdb}{$k}{samples}{$d}{user}\n" if ($normalyzed_info{$curdb}{$k}{samples}{$d}{user});
			$head .= "-- remote: $normalyzed_info{$curdb}{$k}{samples}{$d}{remote}\n" if ($normalyzed_info{$curdb}{$k}{samples}{$d}{remote});
			$head .= "-- app: $normalyzed_info{$curdb}{$k}{samples}{$d}{app}\n" if ($normalyzed_info{$curdb}{$k}{samples}{$d}{app});
			$head .= "--, bind query: yes\n" if ($normalyzed_info{$curdb}{$k}{samples}{$d}{bind});
			$head =~ s/^, //;

			my $explain = "EXPLAIN\n";
			if ($analyze) {
				$explain = "BEGIN;\nEXPLAIN (ANALYZE, VERBOSE, BUFFERS)\n";
			}
			$explain .= "$normalyzed_info{$curdb}{$k}{samples}{$d}{query}\n";
			$explain .= "ROLLBACK;\n" if ($analyze);
			if ($file_per_query) {
				my $fhqr = new IO::File;
				my $fname = sprintf("time_consuming_query%03d.sql", $rank);
				$fhqr->open("> $fname") or die "FATAL: can't open temp file $fname, $!\n";
				$fhqr->print($head);
				$fhqr->print(&fmtquery($explain));
				$fhqr->close();
			} else {
				print $head;
				print $explain;
			}
			if ($psql_cmd) {
				my @tmpfile = tempfile('tmp_pgbadger_tools_XXXX', SUFFIX => '.txt', DIR => $TMP_DIR, UNLINK => 1 );
				my $fht = new IO::File;
				$fht->open("> $tmpfile[1]") or die "FATAL: can't open temp file $tmpfile[1], $!\n";
				$fht->print("$explain");
				$fht->close();
				print `$psql_cmd -f $tmpfile[1]`;
				unlink($tmpfile[1]);
			}
			last; # Just take one sample 
                }
                last if ($rank > $top);
                $rank++;
        }

}

# Dump top normalized slowest queries as EXPLAIN statement.
sub dump_normalized_slowest_queries
{
	my $curdb = shift;

	# Global information
	my $curdate    = localtime(time);
	my $logfile_str = join(',', @log_files);

        my $rank = 1;
	foreach my $k (sort {$normalyzed_info{$curdb}{$b}{average} <=> $normalyzed_info{$curdb}{$a}{average}} keys %{$normalyzed_info{$curdb}}) {
                next if (!$normalyzed_info{$curdb}{$k}{count});
                foreach my $d (sort {$b <=> $a} keys %{$normalyzed_info{$curdb}{$k}{samples}}) {
			# Do not process request that are slower than $max_duration
			next if ( $max_duration && ( $d > $max_duration) );

			my $head = qq{-- Generated on $curdate
-- Log file: $logfile_str
-- Log start from $overall_stat{'first_log_ts'} to $overall_stat{'last_log_ts'}
--
};
			$head .= "-- duration: $d\n" if ($d);
			$head .= "-- database: $normalyzed_info{$curdb}{$k}{samples}{$d}{db}\n" if ($normalyzed_info{$curdb}{$k}{samples}{$d}{db});
			$head .= "-- user: $normalyzed_info{$curdb}{$k}{samples}{$d}{user}\n" if ($normalyzed_info{$curdb}{$k}{samples}{$d}{user});
			$head .= "-- remote: $normalyzed_info{$curdb}{$k}{samples}{$d}{remote}\n" if ($normalyzed_info{$curdb}{$k}{samples}{$d}{remote});
			$head .= "-- app: $normalyzed_info{$curdb}{$k}{samples}{$d}{app}\n" if ($normalyzed_info{$curdb}{$k}{samples}{$d}{app});
			$head .= "--, bind query: yes\n" if ($normalyzed_info{$curdb}{$k}{samples}{$d}{bind});
			$head =~ s/^, //;

			my $explain = "EXPLAIN\n";
			if ($analyze) {
				$explain = "BEGIN;\nEXPLAIN (ANALYZE, VERBOSE, BUFFERS)\n";
			}
			$explain .= "$normalyzed_info{$curdb}{$k}{samples}{$d}{query}\n";
			$explain .= "ROLLBACK;\n" if ($analyze);
			if ($file_per_query) {
				my $fhqr = new IO::File;
				my $fname = sprintf("normalized_slowest_query%03d.sql", $rank);
				$fhqr->open("> $fname") or die "FATAL: can't open temp file $fname, $!\n";
				$fhqr->print($head);
				$fhqr->print(&fmtquery($explain));
				$fhqr->close();
			} else {
				print $head;
				print $explain;
			}
			if ($psql_cmd) {
				my @tmpfile = tempfile('tmp_pgbadger_tools_XXXX', SUFFIX => '.txt', DIR => $TMP_DIR, UNLINK => 1 );
				my $fht = new IO::File;
				$fht->open("> $tmpfile[1]") or die "FATAL: can't open temp file $tmpfile[1], $!\n";
				$fht->print("$explain");
				$fht->close();
				print `$psql_cmd -f $tmpfile[1]`;
				unlink($tmpfile[1]);
			}
			last; # Just take one sample 
                }
                last if ($rank > $top);
                $rank++;
        }

}

# Export time consuming queries as CSV
sub csv_times_consuming_queries
{
	my $curdb = shift;

	my $fhcsv = undef;

	my $csv = Text::CSV->new({
			 binary => 1,      # should set binary attribute.
			 eol    => $/,     # end of line character
		}) or die "Cannot use CSV: ".Text::CSV->error_diag ();
	open $fhcsv, ">", "$csv_filename" or die "FATAL: can not write to $csv_filename: $!\n";
	$csv->print($fhcsv, ['Database','Rank','Total_Duration','Times_Executed','Min','Max','Avg','Query']);

        my $rank = 1;
        foreach my $k (sort {$normalyzed_info{$curdb}{$b}{count} <=> $normalyzed_info{$curdb}{$a}{count}} keys %{$normalyzed_info{$curdb}}) {
                next if (!$normalyzed_info{$curdb}{$k}{count});
                foreach my $d (sort {$b <=> $a} keys %{$normalyzed_info{$curdb}{$k}{samples}}) {
			# Do not process request that are slower than $max_duration
			next if ( $max_duration && ( $d > $max_duration) );

			$normalyzed_info{$curdb}{$k}{average} = $normalyzed_info{$curdb}{$k}{duration} / $normalyzed_info{$curdb}{$k}{count};

			$csv->print ($fhcsv, [
				$normalyzed_info{$curdb}{$k}{samples}{$d}{db},
				$rank,
				$d,
				$normalyzed_info{$curdb}{$k}{count},
				$normalyzed_info{$curdb}{$k}{min},
				$normalyzed_info{$curdb}{$k}{max},
				$normalyzed_info{$curdb}{$k}{average},
				$normalyzed_info{$curdb}{$k}{samples}{$d}{query}
			]);
			# Report only the first sample
			last;

                }
                last if ($rank > $top);
                $rank++;
        }
	close $fhcsv if (defined $fhcsv);

}

# Export normalized slowest queries as CSV
sub csv_normalized_queries
{
	my $curdb = shift;

	my $fhcsv = undef;

	my $csv = Text::CSV->new({
			 binary => 1,      # should set binary attribute.
			 eol    => $/,     # end of line character
		}) or die "Cannot use CSV: ".Text::CSV->error_diag ();
	open $fhcsv, ">", "$csv_filename" or die "FATAL: can not write to $csv_filename: $!\n";
	$csv->print($fhcsv, ['Database','Rank','Total_Duration','Times_Executed','Min','Max','Avg','Query']);

        my $rank = 1;
        foreach my $k (sort {$normalyzed_info{$curdb}{$b}{average} <=> $normalyzed_info{$curdb}{$a}{average}} keys %{$normalyzed_info{$curdb}}) {
                next if (!$normalyzed_info{$curdb}{$k}{count});
                foreach my $d (sort {$b <=> $a} keys %{$normalyzed_info{$curdb}{$k}{samples}}) {
			# Do not process request that are slower than $max_duration
			next if ( $max_duration && ( $d > $max_duration) );

			$csv->print ($fhcsv, [
				$normalyzed_info{$curdb}{$k}{samples}{$d}{db},
				$rank,
				$d,
				$normalyzed_info{$curdb}{$k}{count},
				$normalyzed_info{$curdb}{$k}{min},
				$normalyzed_info{$curdb}{$k}{max},
				$normalyzed_info{$curdb}{$k}{average},
				$normalyzed_info{$curdb}{$k}{samples}{$d}{query}
			]);
			# Report only the first sample
			last;

                }
                last if ($rank > $top);
                $rank++;
        }
	close $fhcsv if (defined $fhcsv);

}

# Export slowest queries as CSV
sub csv_slowest_queries
{
	my $curdb = shift;

	my $fhcsv = undef;

	my $csv = Text::CSV->new({
			 binary => 1,      # should set binary attribute.
			 eol    => $/,     # end of line character
		}) or die "Cannot use CSV: ".Text::CSV->error_diag ();
	open $fhcsv, ">", "$csv_filename" or die "FATAL: can not write to $csv_filename: $!\n";
	$csv->print($fhcsv, ['Database','Rank','Duration','Query']);

	my $rank = 1;
	for (my $i = 0 ; $i <= $#{$top_slowest{$curdb}} ; $i++) {

		# Do not process request that are slower than $max_duration
		next if ( $max_duration && ($top_slowest{$curdb}[$i]->[0] > $max_duration) );

		$csv->print ($fhcsv, [
			$top_slowest{$curdb}[$i]->[3],
			$rank,
			$top_slowest{$curdb}[$i]->[0],
			$top_slowest{$curdb}[$i]->[2]
		]);
                last if ($rank > $top);
                $rank++;
        }
	close $fhcsv if (defined $fhcsv);
}


