#!/bin/ksh

#
# NetPerf - For Quick and Dirty charts.
# This script generates charts from several metrics including vmstat/iostat/netstat/pstat
#

export PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/pkg/bin
SQLITE=/usr/pkg/bin/sqlite3
WEBSERVER="thttpd" # ruby thttpd apache none
ROOTDIR="/home/performance/netperf"
PORT="2020"

if [[ X`whoami`X = X"root"X ]]
then
    echo "Please run this script as a regular user."
    exit 2;
fi

if [[ X`/usr/bin/uname`X != XNetBSDX ]]
then
    echo "NetPerf is only supported on NetBSD."
    exit 2;
fi

unset DISPLAY

if [[ ! -d ${ROOTDIR} ]]
then
    mkdir ${ROOTDIR}
fi

if [[ ! -f ${SQLITE} ]]
then
	echo "Could not find sqlite3."
	echo "please install pkgsrc/databases/sqlite3"
	exit 2
fi

if [[ ! -f /usr/pkg/bin/ruby && X${WEBSERVER}X = XrubyX ]]
then
	echo "Could not find ruby."
	echo "please install pkgsrc/land/ruby"
	exit 2
fi

if [[ ! -f /usr/pkg/bin/pl ]]
then
	echo "Could not find ploticus."
	echo "please install pkgsrc/graphics/ploticus"
	exit 2
fi

if [[ X`mount|grep nfs$`X != XX ]]
then
	NFSC="YES"
fi

if [[ X`grep -i ^nfs_server=YES /etc/rc.conf`X != XX ]]
then
	NFSS="YES"
fi


if [[ ! -x ${ROOTDIR}/webs.rb && X${WEBSERVER}X = XrubyX ]]
then

cat <<EOF> ${ROOTDIR}/webs.rb
#!/usr/pkg/bin/ruby -x
require 'webrick'
include WEBrick

s = HTTPServer.new(
  :Port            => ${PORT},
  :DocumentRoot    => "${ROOTDIR}"
)

trap("INT"){ s.shutdown }
s.start
EOF

chmod a+rx ${ROOTDIR}/webs.rb
fi

if [[ X${WEBSERVER}X = XthttpdX ]]
then
	/usr/pkg/sbin/thttpd -d ${ROOTDIR} -p ${PORT}
fi

export PLOTICUS_PREFABS=/usr/pkg/share/ploticus
DB=${ROOTDIR}/netperf.db
TF=${ROOTDIR}/$$.netperf
HOST=`/usr/bin/uname -n`

# Our Web server setup on port 2020 on localhost to allow charts to be seen.
# If it's already running it will exit harmlessly.
#cd ${ROOTDIR} && nohup ${ROOTDIR}/webs.rb &

# Create the tables. If they already exist then a harmless error will be shown.

${SQLITE} ${DB} "create table netstat(date date,time time,inpackets integer not null,outpackets integer not null,indrop integer not null,outdrop integer not null);" 2>/dev/null

${SQLITE} ${DB} "create table vmstat(date date, time time, r integer not null, b integer not null, w integer not null, avm integer not null, fre integer not null, flt integer not null, re integer not null, pi integer not null, po integer not null, fr integer not null, sr integer not null, fin integer not null, fsy integer not null, cs integer not null, us integer not null, csy integer not null, id integer not null );" 2>/dev/null

${SQLITE} ${DB} "create table vmstat_s(date date, time time,  bytes_per_page bigint not null, page_colors bigint not null, pages_managed bigint not null, pages_free bigint not null, pages_paging bigint not null, pages_wired bigint not null, zero_pages bigint not null, reserve_pagedaemon_pages bigint not null, reserve_kernel_pages bigint not null, anonymous_pages bigint not null, cached_file_pages bigint not null, cached_executable_pages bigint not null, minimum_free_pages bigint not null, target_free_pages bigint not null, maximum_wired_pages bigint not null, swap_devices bigint not null, swap_pages bigint not null, swap_pages_in_use bigint not null, swap_allocations bigint not null, total_faults_taken bigint not null, traps bigint not null, device_interrupts bigint not null, CPU_context_switches bigint not null, software_interrupts bigint not null, system_calls bigint not null, pagein_requests bigint not null, pageout_requests bigint not null, swap_ins bigint not null, swap_outs bigint not null, pages_swapped_in bigint not null, pages_swapped_out bigint not null, forks_total bigint not null, forks_blocked_parent bigint not null, forks_shared_address_space_with_parent bigint not null, pagealloc_zero_wanted_and_avail bigint not null, pagealloc_zero_wanted_and_not_avail bigint not null, aborts_of_idle_page_zeroing bigint not null, pagealloc_desired_color_avail bigint not null, pagealloc_desired_color_not_avail bigint not null, faults_with_no_memory bigint not null, faults_with_no_anons bigint not null, faults_had_to_wait_on_pages bigint not null, faults_found_released_page bigint not null, faults_relock bigint not null, anon_page_faults bigint not null, anon_retry_faults bigint not null, amap_copy_faults bigint not null, neighbour_anon_page_faults bigint not null, neighbour_object_page_faults bigint not null, locked_pager_get_faults bigint not null, unlocked_pager_get_faults bigint not null, anon_faults bigint not null, anon_copy_on_write_faults bigint not null, object_faults bigint not null, promote_copy_faults bigint not null, promote_zero_fill_faults bigint not null, times_daemon_wokeup bigint not null, revolutions_of_the_clock_hand bigint not null, times_daemon_attempted_swapout bigint not null, pages_freed_by_daemon bigint not null, pages_scanned_by_daemon bigint not null, anonymous_pages_scanned_by_daemon bigint not null, object_pages_scanned_by_daemon bigint not null, pages_reactivated bigint not null, pages_found_busy_by_daemon bigint not null, total_pending_pageouts bigint not null, pages_deactivated bigint not null, total_name_lookups bigint not null, good_hits bigint not null, negative_hits bigint not null, bad_hits bigint not null, false_hits bigint not null, miss bigint not null, too_long bigint not null, pass2_hits bigint not null, twopasses bigint not null );" 2>/dev/null

${SQLITE} ${DB} "create table pstat(date date, time time, filesopen integer not null, filestotal integer not null, vnodes integer not null, swapblocksused integer not null, swapblockstotal integer not null);" 2>/dev/null

${SQLITE} ${DB} "create table nfsc(date date, time time, getattr bigint not null, setattr bigint not null,  lookup bigint not null, read bigint not null, write bigint not null,  rename bigint not null,  access bigint not null , readdir bigint not null, others bigint not null);" 2>/dev/null

${SQLITE} ${DB} "create table nfss(date date, time time, getattr bigint not null, setattr bigint not null,  lookup bigint not null, read bigint not null, write bigint not null,  rename bigint not null,  access bigint not null , readdir bigint not null, others bigint not null);" 2>/dev/null

# Chart plotting functions.
plot1(){

    echo "date  ${1} ${2} ${3} ${4}" > ${TF}
    ${SQLITE} ${DB} "select date,time, ${1}, ${2}, ${3}, ${4} from netlite order by date,time"|grep ^2|/usr/bin/awk -F"|" '{OFS="\t"}{print $1"."$2,$3,$4,$5,$6}' >> ${TF}
    if [[ `/usr/bin/wc -l < ${TF}` -gt 300 ]]

    then
	pl -prefab chron  ygrid="color=blue"  "rectangle=1 1 8 5"  "yinc=1 1000000" "ystubfmt=%g0Mp" delim=tab data=${TF} header=yes x=date y=${1} y2=${2} y3=${3} y4=${4} xstubfmt=WWW xinc="1 day" datefmt=yyyy-mm-dd color=powderblue mode=line unittype=datetime legendfmt=singleline  nearest=day title="Netlite: ${1}-${2}-${3}-${4} for ${HOST}" -png -o ${ROOTDIR}/netlite_${1}-${2}-${3}-${4}.png
    else
	pl -prefab chron  ygrid="color=blue"  "rectangle=1 1 8 5"  "yinc=1 1000000" "ystubfmt=%g0Mp" delim=tab data=${TF} header=yes x=date y=${1} y2=${2} y3=${3} y4=${4} xstubfmt=hhA xinc="1 hour" datefmt=yyyy-mm-dd color=powderblue mode=line unittype=datetime legendfmt=singleline  nearest=hour title="Netlite: ${1}-${2}-${3}-${4} for ${HOST}" -png -o ${ROOTDIR}/netlite_${1}-${2}-${3}-${4}.png
    fi
}

plot4(){
# Plot 4 items.
    
    echo "date  ${1} ${2} ${3} ${4}" > ${TF}
    ${SQLITE} ${DB} "select date,time, ${1}, ${2}, ${3}, ${4} from netlite order by date,time"|grep ^2| /usr/bin/awk -F"|" '{OFS="\t"}{print $1"."$2,$3,$4,$5,$6}' >> ${TF}
    if [[ `/usr/bin/wc -l < ${TF}` -gt 300 ]]
    then
	pl -prefab chron  ygrid="color=blue"  "rectangle=1 1 8 5"  "yinc=1 1000000" "ystubfmt=%g0Mp" delim=tab data=${TF} header=yes x=date y=${1} y2=${2} y3=${3} y4=${4} xstubfmt=WWW xinc="1 day" datefmt=yyyy-mm-dd color=powderblue mode=line unittype=datetime legendfmt=singleline  nearest=day title="Netlite: ${1}-${2}-${3}-${4} for ${HOST}" -png -o ${ROOTDIR}/netlite_${1}-${2}-${3}-${4}.png
    else
	pl -prefab chron  ygrid="color=blue"  "rectangle=1 1 8 5"  "yinc=1 1000000" "ystubfmt=%g0Mp" delim=tab data=${TF} header=yes x=date y=${1} y2=${2} y3=${3} y4=${4} xstubfmt=hhA xinc="1 hour" datefmt=yyyy-mm-dd color=powderblue mode=line unittype=datetime legendfmt=singleline  nearest=hour title="Netlite: ${1}-${2}-${3}-${4} for ${HOST}" -png -o ${ROOTDIR}/netlite_${1}-${2}-${3}-${4}.png
    fi
}

plot_netstat(){
    echo "date	inpackets	outpackets" > ${TF}
    ${SQLITE} ${DB} "select date,time, inpackets, outpackets from netstat order by date,time"|grep ^2| /usr/bin/awk -F"|" '{OFS="\t"} ; { if ((FNR == 1) || ( PREVA > $3) || ( PREVB > $4)) {PREVA=$3; PREVB=$4} print $1"."$2,$3 - PREVA,$4 - PREVB; PREVA=$3; PREVB=$4;}' >> ${TF}
    if [[ `/usr/bin/wc -l < ${TF}` -gt 300 ]]
    then
	echo "plot A"
	pl -prefab chron  ygrid="color=blue"  "rectangle=1 1 8 5"  delim=tab data=${TF} header=yes ylbl="Packets/5 Minutes" x=date y=inpackets y2=outpackets xstubfmt=WWW xinc="1 day" datefmt=yyyy-mm-dd color=powderblue mode=line unittype=datetime legendfmt=singleline  nearest=day title="Netstat: for ${HOST}" -png -o ${ROOTDIR}/netstat.png
    else
	echo "plot B"
	pl -prefab chron  ygrid="color=blue"  "rectangle=1 1 8 5"  delim=tab data=${TF} header=yes x=date ylbl="Packets/5 Minutes" y=inpackets y2=outpackets stubfmt=hhA xinc="1 hour" datefmt=yyyy-mm-dd color=powderblue mode=line unittype=datetime legendfmt=singleline  nearest=hour title="Netstat: for ${HOST}" -png -o ${ROOTDIR}/netstat.png
    fi
	echo "done"
}


plot_nfs(){


if [[ -n ${NFSC} ]]
then

for metric in getattr setattr lookup read  write rename access readdir others
do
	echo "date	${metric}" > ${TF}

	${SQLITE} ${DB} "select date,time, ${metric} from nfsc order by date,time"|grep ^2| /usr/bin/awk -F"|" '{OFS="\t"} ; { if ((FNR == 1) || ( PREV > $3)) {PREV=$3;} print $1"."$2,$3 - PREV; PREV=$3; }' >> ${TF}
	if [[ `/usr/bin/wc -l < ${TF}` -gt 300 ]]
	then
		pl -prefab chron  ygrid="color=blue"  "rectangle=1 1 8 5"  delim=tab data=${TF} header=yes ylbl="${metric} RPCs/5 Minutes" x=date y=${metric} xstubfmt=WWW xinc="1 day" datefmt=yyyy-mm-dd color=powderblue mode=line unittype=datetime legendfmt=singleline  nearest=day title="Nfsstat: Client: ${metric} for ${HOST}" -png -o ${ROOTDIR}/nfsstat_client_${metric}.png
	else
		pl -prefab chron  ygrid="color=blue"  "rectangle=1 1 8 5"  delim=tab data=${TF} header=yes x=date ylbl="${metric} RPCs/5 Minutes" y=${metric} stubfmt=hhA xinc="1 hour" datefmt=yyyy-mm-dd color=powderblue mode=line unittype=datetime legendfmt=singleline  nearest=hour title="Nfsstat: Client: ${metric} for ${HOST}" -png -o ${ROOTDIR}/nfsstat_client_${metric}.png
	fi

done
fi # if client

if [[ -n ${NFSS} ]]
then

for metric in getattr setattr lookup read  write rename access readdir others
do
       echo "date      ${metric}" > ${TF}

	${SQLITE} ${DB} "select date,time, ${metric} from nfss order by date,time"|grep ^2| /usr/bin/awk -F"|" '{OFS="\t"} ; { if ((FNR == 1) || ( PREV > $3)) {PREV=$3;} print $1"."$2,$3 - PREV; PREV=$3; }' >> ${TF}
	if [[ `/usr/bin/wc -l < ${TF}` -gt 300 ]]
	then
		pl -prefab chron  ygrid="color=blue"  "rectangle=1 1 8 5"  delim=tab data=${TF} header=yes ylbl="${metric} RPCs/5 Minutes" x=date y=${metric} xstubfmt=WWW xinc="1 day" datefmt=yyyy-mm-dd color=powderblue mode=line unittype=datetime legendfmt=singleline  nearest=day title="Nfsstat: Server: ${metric} for ${HOST}" -png -o ${ROOTDIR}/nfsstat_server_${metric}.png
	else
		pl -prefab chron  ygrid="color=blue"  "rectangle=1 1 8 5"  delim=tab data=${TF} header=yes x=date ylbl="${metric} RPCs/5 Minutes" y=${metric} stubfmt=hhA xinc="1 hour" datefmt=yyyy-mm-dd color=powderblue mode=line unittype=datetime legendfmt=singleline  nearest=hour title="Nfsstat: Server: ${metric} for ${HOST}" -png -o ${ROOTDIR}/nfsstat_server_${metric}.png
	fi

done
fi # if server



}


plot_vmstat_procs(){
    echo "date	runq	blocked	swapped" > ${TF}
    ${SQLITE} ${DB} "select date,time, r,b,w from vmstat order by date,time"|grep ^2| /usr/bin/awk -F"|" '{OFS="\t"}{print $1"."$2,$3,$4,$5,$6}' >> ${TF}

    if [[ `/usr/bin/wc -l < ${TF}` -gt 300 ]]
    then
	pl -prefab chron  ygrid="color=blue"  "rectangle=1 1 8 5"  delim=tab data=${TF} header=yes x=date y=runq y2=blocked y3=swapped ylbl="VMSTAT: RunQ/Blocked/Swapped" xstubfmt=WWW xinc="1 day" datefmt=yyyy-mm-dd color=powderblue mode=line unittype=datetime legendfmt=singleline  nearest=day title="Vmstat: for ${HOST}" -png -o ${ROOTDIR}/vmstat_procs.png
    else
	pl -prefab chron  ygrid="color=blue"  "rectangle=1 1 8 5"  delim=tab data=${TF} header=yes x=date y=runq y2=blocked y3=swapped ylbl="VMSTAT: RunQ/Blocked/Swapped" xstubfmt=hhA xinc="1 hour" datefmt=yyyy-mm-dd color=powderblue mode=line unittype=datetime legendfmt=singleline  nearest=hour title="Vmstat: for ${HOST}" -png -o ${ROOTDIR}/vmstat_procs.png
    fi
}

plot_pstat(){
    echo "date	filesopen	filesmax" > ${TF}
    ${SQLITE} ${DB} "select date,time, filesopen, filestotal from pstat order by date,time"|grep ^2| /usr/bin/awk -F"|" '{OFS="\t"}{print $1"."$2,$3,$4}' >> ${TF}
    if [[ `/usr/bin/wc -l < ${TF}` -gt 300 ]]
    then
        pl -prefab chron  ygrid="color=blue"  "rectangle=1 1 8 5"  delim=tab data=${TF} header=yes x=date y=filesopen y2=filesmax xstubfmt=WWW xinc="1 day" datefmt=yyyy-mm-dd color=powderblue mode=line unittype=datetime legendfmt=singleline  nearest=day title="Pstat: FilesOpen for ${HOST}" -png -o ${ROOTDIR}/pstat_files.png
    else
        pl -prefab chron  ygrid="color=blue"  "rectangle=1 1 8 5"  delim=tab data=${TF} header=yes x=date y=filesopen y2=filesmax ylbl="Pstat: FilesOpen for ${HOST}" xstubfmt=hhA xinc="1 hour" datefmt=yyyy-mm-dd color=powderblue mode=line unittype=datetime legendfmt=singleline  nearest=hour title="Vmstat: for ${HOST}" -png -o ${ROOTDIR}/pstat_files.png
    fi
}

plot_vmstat_s(){

for metric in bytes_per_page page_colors pages_managed pages_free pages_paging pages_wired zero_pages reserve_pagedaemon_pages reserve_kernel_pages anonymous_pages cached_file_pages cached_executable_pages minimum_free_pages target_free_pages maximum_wired_pages swap_devices swap_pages swap_pages_in_use pagealloc_desired_color_not_avail 
do
	echo "plot ${metric}"

    echo "date  ${metric}" > ${TF}
    ${SQLITE} ${DB} "select date,time, ${metric} from vmstat_s order by date,time"|grep ^2| /usr/bin/awk -F"|" '{OFS="\t"}{print $1"."$2,$3}' >> ${TF}
    if [[ `/usr/bin/wc -l < ${TF}` -gt 300 ]]
    then
        pl -prefab chron  ygrid="color=blue"  "rectangle=1 1 8 5"  delim=tab data=${TF} header=yes x=date y=${metric} xstubfmt=WWW xinc="1 day" datefmt=yyyy-mm-dd color=powderblue mode=line unittype=datetime legendfmt=singleline  nearest=day title="Vmstat: ${metric} for ${HOST}" -png -o ${ROOTDIR}/vmstat_s_${metric}.png
    else
        pl -prefab chron  ygrid="color=blue"  "rectangle=1 1 8 5"  delim=tab data=${TF} header=yes x=date y=${metric} xstubfmt=hhA xinc="1 hour" datefmt=yyyy-mm-dd color=powderblue mode=line unittype=datetime legendfmt=singleline  nearest=hour title="Vmstat: ${metric} for ${HOST}" -png -o ${ROOTDIR}/vmstat_s_${metric}.png
    fi
done

# Metrics which are accumulative
for metric in CPU_context_switches amap_copy_faults anon_copy_on_write_faults total_faults_taken anon_page_faults faults_with_no_memory faults_with_no_anons faults_had_to_wait_on_pages faults_found_released_page faults_relock anon_retry_faults neighbour_anon_page_faults neighbour_object_page_faults locked_pager_get_faults unlocked_pager_get_faults anon_faults bad_hits false_hits device_interrupts forks_total forks_blocked_parent forks_shared_address_space_with_parent good_hits negative_hits object_faults pagealloc_desired_color_avail pagealloc_zero_wanted_and_avail pagealloc_zero_wanted_and_not_avail pass2_hits promote_copy_faults  promote_zero_fill_faults  software_interrupts system_calls  too_long total_name_lookups traps twopasses miss anonymous_pages_scanned_by_daemon object_pages_scanned_by_daemon pagealloc_desired_color_not_avail pagein_requests pageout_requests pages_deactivated pages_freed_by_daemon pages_scanned_by_daemon  pages_reactivated revolutions_of_the_clock_hand swap_allocations swap_ins swap_outs times_daemon_wokeup times_daemon_attempted_swapout total_pending_pageouts pages_swapped_out aborts_of_idle_page_zeroing pages_found_busy_by_daemon

do
	echo "plot ${metric}"
    echo "date  ${metric}" > ${TF}
    ${SQLITE} ${DB} "select date,time, ${metric} from vmstat_s order by date,time"|grep ^2| /usr/bin/awk -F"|" '{OFS="\t"}{ if((FNR == 1) || ( PREV > $3) ) {PREV=$3} print $1"."$2,$3 - PREV; PREV = $3;}' >> ${TF}
    if [[ `/usr/bin/wc -l < ${TF}` -gt 300 ]]
    then
        pl -prefab chron  ygrid="color=blue"  "rectangle=1 1 8 5"  delim=tab data=${TF} header=yes x=date y=${metric} xstubfmt=WWW xinc="1 day" datefmt=yyyy-mm-dd color=powderblue mode=line unittype=datetime legendfmt=singleline  nearest=day title="Vmstat: ${metric} for ${HOST}" -png -o ${ROOTDIR}/vmstat_s_${metric}.png
    else
        pl -prefab chron  ygrid="color=blue"  "rectangle=1 1 8 5"  delim=tab data=${TF} header=yes x=date y=${metric} xstubfmt=hhA xinc="1 hour" datefmt=yyyy-mm-dd color=powderblue mode=line unittype=datetime legendfmt=singleline  nearest=hour title="Vmstat: ${metric} for ${HOST}" -png -o ${ROOTDIR}/vmstat_s_${metric}.png
    fi
done

}

plot_vmstat_memory(){
    echo "date	active	freelist" > ${TF}
    ${SQLITE} ${DB} "select date,time, avm,fre from vmstat order by date,time"|grep ^2| /usr/bin/awk -F"|" '{OFS="\t"}{print $1"."$2,$3,$4,$5,$6}' >> ${TF}
    if [[ `/usr/bin/wc -l < ${TF}` -gt 300 ]]
    then
	pl -prefab chron  ygrid="color=blue"  "rectangle=1 1 8 5"  delim=tab data=${TF} header=yes x=date y=active y2=freelist ylbl="VMSTAT: R/B/W per 5 minutes" xstubfmt=WWW xinc="1 day" datefmt=yyyy-mm-dd color=powderblue mode=line unittype=datetime legendfmt=singleline  nearest=day title="Vmstat: Memory for ${HOST}" -png -o ${ROOTDIR}/vmstat_memory.png
    else
	pl -prefab chron  ygrid="color=blue"  "rectangle=1 1 8 5"  delim=tab data=${TF} header=yes x=date y=active y2=freelist ylbl="VMSTAT: R/B/W per 5 minutes" stubfmt=hhA xinc="1 hour" datefmt=yyyy-mm-dd color=powderblue mode=line unittype=datetime legendfmt=singleline  nearest=hour title="Vmstat: Memory for ${HOST}" -png -o ${ROOTDIR}/vmstat_memory.png
    fi
}

plot_vmstat_faults(){
    echo "date	interrupts	syscalls	ctsw" > ${TF}
    ${SQLITE} ${DB} "select date,time, fin, fsy, cs from vmstat order by date,time"|grep ^2| /usr/bin/awk -F"|" '{OFS="\t"}{print $1"."$2,$3,$4,$5,$6}' >> ${TF}
  
    if [[ `/usr/bin/wc -l < ${TF}` -gt 300 ]]
    then
	pl -prefab chron  ygrid="color=blue"  "rectangle=1 1 8 5"  delim=tab data=${TF} header=yes x=date y=interrupts y2=syscalls y3=ctsw ylbl="VMSTAT: Faults: Interrupts/Syscalls/Cxsw" xstubfmt=WWW xinc="1 day" datefmt=yyyy-mm-dd color=powderblue mode=line unittype=datetime legendfmt=singleline  nearest=day title="Vmstat: Memory for ${HOST}" -png -o ${ROOTDIR}/vmstat_faults.png
    else
	pl -prefab chron  ygrid="color=blue"  "rectangle=1 1 8 5"  delim=tab data=${TF} header=yes x=date y=interrupts y2=syscalls y3=ctsw ylbl="VMSTAT: Faults: Interrupts/Syscalls/Cxsw" stubfmt=hhA xinc="1 hour" datefmt=yyyy-mm-dd color=powderblue mode=line unittype=datetime legendfmt=singleline  nearest=hour title="Vmstat: Faults: Interrupts/Syscalls/Cxsw" -png -o ${ROOTDIR}/vmstat_faults.png
    fi
}

plot_vmstat_cpu(){
    echo "date	user	system	total" > ${TF}
    ${SQLITE} ${DB} "select date,time, us, csy from vmstat order by date,time"|grep ^2|/usr/bin/awk -F"|" '{OFS="\t"}{print $1"."$2,$3,$4,$3 + $4}' >> ${TF}
    if [[ `/usr/bin/wc -l < ${TF}` -gt 300 ]]
    then
	pl -prefab chron  ygrid="color=blue"  "rectangle=1 1 8 5"  yrange="0 100" delim=tab data=${TF} header=yes x=date y=user y2=system y3=total ylbl="VMSTAT: CPU: User/System" xstubfmt=WWW xinc="1 day" datefmt=yyyy-mm-dd color=powderblue mode=line unittype=datetime legendfmt=singleline  nearest=day title="Vmstat: CPU: User/System ${HOST}" "gapmissing=yes" -png -o ${ROOTDIR}/vmstat_cpu.png
    else
	pl -prefab chron  ygrid="color=blue"  "rectangle=1 1 8 5"  yrange="0 100" delim=tab data=${TF} header=yes x=date y=user y2=system y3=total ylbl="VMSTAT: CPU: Users/Systems" stubfmt=hhA xinc="1 hour" datefmt=yyyy-mm-dd color=powderblue mode=line unittype=datetime legendfmt=singleline  nearest=hour title="Vmstat: CPU: User/System on ${HOST}" "gapmissing=yes" -png -o ${ROOTDIR}/vmstat_cpu.png
    fi
}

#/usr/bin/vmstat 1 1|tail -n 1|/usr/bin/awk -v DATE=${DATE} -v TIME=${TIME} -v F="," -v FF="','" -v FFF="'" '{ print "insert into vmstat(date,time,r,b,w,avm,fre,flt,re,pi,po,fr,sr,f0,m0,fin,fsy,cs,us,csy,id) values("FFF DATE FF TIME FF $1 FF $2 FF $3 FF $4 FF $5 FF $6 FF $7 FF $8 FF $9 FF $10 FF $11 FF $12 FF $13 FF $14 FF $15 FF $16 FF $17 FF $18 FF $19 FFF");" }'|${SQLITE} ${DB}

while :
do
    
    DATE=`date +%Y-%m-%d`
    TIME=`date +%H:%M`
    
    #keep tmp cleaning scripts from nuking our work.
    #touch ${ROOTDIR}/*
    
#Run the data collectors

# Netstat 
/usr/bin/netstat -w 1|/usr/bin/head -n 3|/usr/bin/tail -n 1|/usr/bin/awk -v date=${DATE} -v time=${TIME} -v f="'" -v ff="','" '{ print "insert into netstat(date,time,inpackets,outpackets,indrop,outdrop) values(" f date ff time ff $6 ff $8 ff $7 ff $9 f ");"  }' |${SQLITE} ${DB}

# Vmstat output 
# We run it at 5 second intervals and twice to get a sample of the current load vs accumulated average that is the default.
/usr/bin/vmstat 5 2|tail -n 1|/usr/bin/awk -v DATE=${DATE} -v TIME=${TIME} -v FF="','" -v FFF="'" '{ print "insert into vmstat(date,time,r,b,w,avm,fre,flt,re,pi,po,fr,sr,fin,fsy,cs,us,csy,id) values("FFF DATE FF TIME FF $1 FF $2 FF $3 FF $4 FF $5 FF $6 FF $7 FF $8 FF $9 FF $10 FF $11 FF $(NF - 5) FF $(NF - 4) FF $(NF - 3) FF $(NF - 2) FF $(NF - 1) FF $NF FFF");" }'|${SQLITE} ${DB}

/usr/bin/vmstat -s|awk -v DATE=${DATE} -v TIME=${TIME} -v F="," -v FF="','" -v FFF="'" 'BEGIN{RS=""; FS="\n";} { split($1,a1," ");  split($2,a2," ");  split($3,a3," ");  split($4,a4," ");  split($5,a5," ");  split($6,a6," ");  split($7,a7," ");  split($8,a8," ");  split($9,a9," ");  split($10,a10," ");  split($11,a11," ");  split($12,a12," ");  split($13,a13," ");  split($14,a14," ");  split($15,a15," ");  split($16,a16," ");  split($17,a17," ");  split($18,a18," ");  split($19,a19," ");  split($20,a20," ");  split($21,a21," ");  split($22,a22," ");  split($23,a23," ");  split($24,a24," ");  split($25,a25," ");  split($26,a26," ");  split($27,a27," ");  split($28,a28," ");  split($29,a29," ");  split($30,a30," ");  split($31,a31," ");  split($32,a32," ");  split($33,a33," ");  split($34,a34," ");  split($35,a35," ");  split($36,a36," ");  split($37,a37," ");  split($38,a38," ");  split($39,a39," ");  split($40,a40," ");  split($41,a41," ");  split($42,a42," ");  split($43,a43," ");  split($44,a44," ");  split($45,a45," ");  split($46,a46," ");  split($47,a47," ");  split($48,a48," ");  split($49,a49," ");  split($50,a50," ");  split($51,a51," ");  split($52,a52," ");  split($53,a53," ");  split($54,a54," ");  split($55,a55," ");  split($56,a56," ");  split($57,a57," ");  split($58,a58," ");  split($59,a59," ");  split($60,a60," ");  split($61,a61," ");  split($62,a62," ");  split($63,a63," ");  split($64,a64," ");  split($65,a65," ");  split($66,a66," ");  split($67,a67," ");  split($68,a68," ");  split($69,a69," ");  split($70,a70," ");  split($71,a71," ");  split($72,a72," ");  split($73,a73," ");  split($74,a74," ");  split($75,a75," ");  split($76,a76," ");  printf "insert into vmstat_s(date,time, bytes_per_page, page_colors, pages_managed, pages_free, pages_paging, pages_wired, zero_pages, reserve_pagedaemon_pages, reserve_kernel_pages, anonymous_pages, cached_file_pages, cached_executable_pages, minimum_free_pages, target_free_pages, maximum_wired_pages, swap_devices, swap_pages, swap_pages_in_use, swap_allocations, total_faults_taken, traps, device_interrupts, CPU_context_switches, software_interrupts, system_calls, pagein_requests, pageout_requests, swap_ins, swap_outs, pages_swapped_in, pages_swapped_out, forks_total, forks_blocked_parent, forks_shared_address_space_with_parent, pagealloc_zero_wanted_and_avail, pagealloc_zero_wanted_and_not_avail, aborts_of_idle_page_zeroing, pagealloc_desired_color_avail, pagealloc_desired_color_not_avail, faults_with_no_memory, faults_with_no_anons, faults_had_to_wait_on_pages, faults_found_released_page, faults_relock, anon_page_faults, anon_retry_faults, amap_copy_faults, neighbour_anon_page_faults, neighbour_object_page_faults, locked_pager_get_faults, unlocked_pager_get_faults, anon_faults, anon_copy_on_write_faults, object_faults, promote_copy_faults, promote_zero_fill_faults, times_daemon_wokeup, revolutions_of_the_clock_hand, times_daemon_attempted_swapout, pages_freed_by_daemon, pages_scanned_by_daemon, anonymous_pages_scanned_by_daemon, object_pages_scanned_by_daemon, pages_reactivated, pages_found_busy_by_daemon, total_pending_pageouts, pages_deactivated, total_name_lookups, good_hits, negative_hits, bad_hits, false_hits, miss, too_long, pass2_hits, twopasses ) values( "  FFF DATE FF TIME FF a1[1]  FF a2[1]  FF a3[1]  FF a4[1]  FF a5[1]  FF a6[1]  FF a7[1]  FF a8[1]  FF a9[1]  FF a10[1]  FF a11[1]  FF a12[1]  FF a13[1]  FF a14[1]  FF a15[1]  FF a16[1]  FF a17[1]  FF a18[1]  FF a19[1]  FF a20[1]  FF a21[1]  FF a22[1]  FF a23[1]  FF a24[1]  FF a25[1]  FF a26[1]  FF a27[1]  FF a28[1]  FF a29[1]  FF a30[1]  FF a31[1]  FF a32[1]  FF a33[1]  FF a34[1]  FF a35[1]  FF a36[1]  FF a37[1]  FF a38[1]  FF a39[1]  FF a40[1]  FF a41[1]  FF a42[1]  FF a43[1]  FF a44[1]  FF a45[1]  FF a46[1]  FF a47[1]  FF a48[1]  FF a49[1]  FF a50[1]  FF a51[1]  FF a52[1]  FF a53[1]  FF a54[1]  FF a55[1]  FF a56[1]  FF a57[1]  FF a58[1]  FF a59[1]  FF a60[1]  FF a61[1]  FF a62[1]  FF a63[1]  FF a64[1]  FF a65[1]  FF a66[1]  FF a67[1]  FF a68[1]  FF a69[1]  FF a70[1]  FF a71[1]  FF a72[1]  FF a73[1]  FF a74[1]  FF a75[1]  FF a76[1]  FFF ");"}'|${SQLITE} ${DB}

# Collect Pstat -T output
pstat -T|awk -v DATE=${DATE} -v TIME=${TIME} -v F="'" -v FF="','" 'BEGIN{RS="";FS="\n"}{split($1,a,"/"); split(a[2],d," "); split($2,b," "); split($3,c,"/"); split(c[2],e," "); print "insert into pstat(date,time,filesopen,filestotal,vnodes,swapblocksused,swapblockstotal) values(" F DATE FF TIME FF a[1] FF d[1] FF b[1] FF c[1] FF e[1] F ");" }'|${SQLITE} ${DB}

# Collect NFS Client/Server info
if [[ -n ${NFSC} ]]
then
/usr/bin/nfsstat -c -w 1|head -n2|tail -n 1|awk -v DATE=${DATE} -v TIME=${TIME} -v F="'" -v FF="','" '{ print "insert into nfsc(date,time,getattr,setattr,lookup,read,write,rename,access,readdir,others) values(" F DATE FF TIME FF $2 FF $3 FF $4 FF $5 FF $6 FF $7 FF $8 FF $9 FF $10 F ");" }'|${SQLITE} ${DB} 
fi

if [[ -n ${NFSS} ]]
then
/usr/bin/nfsstat -s -w 1|head -n2|tail -n 1|awk -v DATE=${DATE} -v TIME=${TIME} -v F="'" -v FF="','" '{ print "insert into nfss(date,time,getattr,setattr,lookup,read,write,rename,access,readdir,others) values(" F DATE FF TIME FF $2 FF $3 FF $4 FF $5 FF $6 FF $7 FF $8 FF $9 FF $10 F ");" }'|${SQLITE} ${DB}
fi


#Getattr Setattr  Lookup    Read   Write  Rename  Access Readdir  Others
#Client: 39320238 1164914 3764207 5465660 5760531  192198 4337633  164839  578467
   
    
# Capture packets in/out drop in/out and plot delta
    
if [[ X`date +%M|cut -b2|grep -E "0|5"`X != XX ]]
then
    plot_netstat
    plot_vmstat_memory
    plot_vmstat_procs
    plot_vmstat_faults
    plot_vmstat_cpu
    plot_vmstat_s
    plot_pstat
    plot_nfs

fi
    /bin/ls ${ROOTDIR}/*png|/usr/bin/awk -F"/" '{split($NF,a,"."); print "<b>"a[1]"</b><br><img src=\"./" $NF "\"><br>"}' > ${ROOTDIR}/index.html
    sleep 60
    
done