268 lines
11 KiB
Bash
268 lines
11 KiB
Bash
#!/bin/bash
|
|
#if [ $# -eq 1 ]; then
|
|
# diconf_file=$1
|
|
#else
|
|
# echo "Missing monitor-config-file info!"
|
|
# exit 1
|
|
#fi
|
|
|
|
|
|
echo `date` >> /home/ceiec/nms/nmsclient/nc_logs/ttt.log
|
|
|
|
|
|
base_dir="/home/ceiec/nms/nmsclient"
|
|
conf=$base_dir"/conf/process.list"
|
|
pid_dir=$base_dir"/temp"
|
|
myselfpid=$pid_dir"/pidcheck.pid"
|
|
|
|
|
|
diconf_dir="/home/ceiec/nms/nmsdata/nc_config"
|
|
diconf_file=`echo $0 | awk -F '/' '{print $NF}' | awk -F '.' '{print $1}' `
|
|
file_name=$diconf_file
|
|
diconf_file=$diconf_dir"/"$diconf_file".cfg"
|
|
|
|
clog_conf=$base_dir"/conf/clog.list"
|
|
|
|
monitorstart=`stat $diconf_file | grep Modify | awk '{print $2" "$3}'`
|
|
mstimestamp=`date -d "$monitorstart" +%s%N`
|
|
remstimestamp=$((mstimestamp/1000000))
|
|
pubInfo=`cat $diconf_file | grep pubInfo | cut -d = -f 2`
|
|
|
|
mypid=$$
|
|
echo $$ > $myselfpid
|
|
|
|
#echo $conf
|
|
pdate=`date +'%Y%m%d%H%M%S'`
|
|
stat_file=$pid_dir"/$pdate"".rep"
|
|
incoming_file=$pid_dir"/$pdate"".csv"
|
|
temp_file=$pid_dir"/$pdate"".tmp"
|
|
|
|
timeStamp=`date +%s%N`
|
|
currentTimeStamp=$((timeStamp/1000000))
|
|
later15mTimeStamp=$((currentTimeStamp+checkGap*60*1000))
|
|
|
|
didata_dir=`cat $diconf_file | grep dataFileDir | cut -d = -f 2`
|
|
ditemp_dir=`cat $diconf_file | grep tempDataFileDir | cut -d = -f 2`
|
|
checkGap=`cat $diconf_file | grep checkGap | cut -d = -f 2`
|
|
dataFileDir=`cat $diconf_file | grep dataFileDir | cut -d = -f 2`
|
|
|
|
interstatus=""
|
|
perstatus=""
|
|
|
|
linenum=0
|
|
while read pline
|
|
do
|
|
if [ -n "$pline" ]; then
|
|
# echo line=$pline
|
|
num=0
|
|
isclog=0
|
|
firstc=`echo $pline | cut -c 1`
|
|
if [ "$firstc" != "#" ]; then
|
|
module=`echo $pline | cut -d = -f 1`
|
|
keyword=`echo $pline | cut -d = -f 2`
|
|
# echo $module $keyword
|
|
clog_dir=`cat $clog_conf | grep -v "#" | grep $module | cut -d = -f 2 | cut -d , -f 1`
|
|
clog_name=`cat $clog_conf | grep -v "#" | grep $module | cut -d = -f 2 | cut -d , -f 2`
|
|
# echo $module : $clog_dir , $clog_name
|
|
|
|
case $module in
|
|
"processcheck")
|
|
prostatus=`ps u -P "$mypid" | grep -v grep | grep -v vim | grep -v USER`;
|
|
num=`ps u -P "$mypid" | grep -v grep | grep -v vim | grep -v USER | wc -l`;
|
|
;;
|
|
"nmsweb")
|
|
prostatus=`ps aux | grep -E "$keyword" | grep "nms" | grep -v grep | grep -v galaxy`;
|
|
num=`ps aux | grep -E "$keyword" | grep "nms" | grep -v grep | grep -v galaxy | wc -l`;
|
|
;;
|
|
"nmsdc")
|
|
prostatus=`ps aux | grep -E "$keyword" | grep -v grep`;
|
|
num=`ps aux | grep -E "$keyword" | grep -v grep | wc -l`;
|
|
;;
|
|
"nmsclient")
|
|
prostatus=`ps aux | grep -E "$keyword" | grep -v shouhu | grep -v grep`;
|
|
num=`ps aux | grep -E "$keyword" | grep -v shouhu | grep -v grep | wc -l`;
|
|
;;
|
|
"ntcsweb")
|
|
prostatus=`ps aux | grep -E "$keyword" | grep -v grep | grep -v galaxy | grep -v nms`;
|
|
num=`ps aux | grep -E "$keyword" | grep -v grep | grep -v galaxy | grep -v nms | wc -l`;
|
|
;;
|
|
"mariadb")
|
|
prostatus=`ps aux | grep -E "$keyword" | grep -v grep | grep -v safe`;
|
|
num=`ps aux | grep -E "$keyword" | grep -v grep | grep -v safe | wc -l`;
|
|
;;
|
|
"galaxyService"|"galaxyFsync"|"galaxyPush"|"galaxyStat"|"galaxyTest"|"logComplete"|"logStat"|"logTopIP"|"logTopN"|"logUV")
|
|
prostatus=`ps aux | grep -E "$keyword" | grep -v grep | grep -v Daemon`;
|
|
num=`ps aux | grep -E "$keyword" | grep -v grep | grep -v Daemon | wc -l`;
|
|
;;
|
|
"pzffff"|"pzffhj"|"paffkeep")
|
|
prostatus=`ps aux | grep -E "$keyword" | grep -v grep | grep -v sh`;
|
|
num=`ps aux | grep -E "$keyword" | grep -v grep | grep -v sh | wc -l`;
|
|
;;
|
|
"dkAppBehavior"|"dkAppSSL"|"dkHdLdp"|"dkHdNtba"|"dkHdStrategy"|"dkQx"|"dkLogProcess")
|
|
prostatus=`ps aux | grep -E "$keyword" | grep -v grep | grep -v sh`;
|
|
num=`ps aux | grep -E "$keyword" | grep -v grep | grep -v sh | wc -l`;
|
|
;;
|
|
"Tensor"|"Redis"|"panguValve")
|
|
prostatus=`ps aux | grep -E "$keyword" | grep -v grep | grep -v sh`;
|
|
num=`ps aux | grep -E "$keyword" | grep -v grep | grep -v sh | wc -l`;
|
|
;;
|
|
"redis")
|
|
prostatus=`ps aux | grep -E "$keyword" | grep -v grep | grep -v sh`;
|
|
num=`ps aux | grep -E "$keyword" | grep -v grep | grep -v sh | wc -l`;
|
|
;;
|
|
"mrzcpd"|"Sapp"|"Oam")
|
|
prostatus=`ps aux | grep -E "$keyword" | grep -v grep | grep -v sh`;
|
|
num=`ps aux | grep -E "$keyword" | grep -v grep | grep -v sh | wc -l`;
|
|
;;
|
|
"maskey2.0")
|
|
prostatus=`ps aux | grep -E "$keyword" | grep -v grep | grep -v sh`;
|
|
num=`ps aux | grep -E "$keyword" | grep -v grep | grep -v sh | wc -l`;
|
|
;;
|
|
"rssbMaskey"|"fragIndexCluster"|"castleBlack"|"eastwatch"|"platformvideo"|"platformaudio"|"platformimage"|"platformvoip")
|
|
prostatus=`ps aux | grep -E "$keyword" | grep -v grep | grep -v sh`;
|
|
num=`ps aux | grep -E "$keyword" | grep -v grep | grep -v sh | wc -l`;
|
|
;;
|
|
"avanalyser"|"pic"|"voip"|"audio"|"video"|"fragcheck"|"blacklist")
|
|
prostatus=`ps aux | grep -E "$keyword" | grep -v grep | grep -v sh`;
|
|
num=`ps aux | grep -E "$keyword" | grep -v grep | grep -v sh | wc -l`;
|
|
;;
|
|
"dkLdp"|"dkZeusLogclct"|"dkSappClean"|"dkZeusNtbapre"|"dkSappNtba")
|
|
prostatus=`ps aux | grep -E "$keyword" | grep -v grep | grep -v sh`;
|
|
num=`ps aux | grep -E "$keyword" | grep -v grep | grep -v sh | wc -l`;
|
|
;;
|
|
"zookeeper"|"Kafka"|"Strom"|"clickhouse"|"Hades")
|
|
prostatus=`ps aux | grep -E "$keyword" | grep -v grep | grep -v sh`;
|
|
num=`ps aux | grep -E "$keyword" | grep -v grep | grep -v sh | wc -l`;
|
|
;;
|
|
"consul"|"telegraf"|"infludb"|"Grafana")
|
|
prostatus=`ps aux | grep -E "$keyword" | grep -v grep | grep -v sh`;
|
|
num=`ps aux | grep -E "$keyword" | grep -v grep | grep -v sh | wc -l`;
|
|
;;
|
|
"mrzcpd"|"Sapp"|"Torches")
|
|
prostatus=`ps aux | grep -E "$keyword" | grep -v grep | grep -v sh`;
|
|
num=`ps aux | grep -E "$keyword" | grep -v grep | grep -v sh | wc -l`;
|
|
;;
|
|
"irAccess"|"irFoward")
|
|
prostatus=`ps aux | grep -E "$keyword" | grep -v grep | grep -v sh`;
|
|
num=`ps aux | grep -E "$keyword" | grep -v grep | grep -v sh | wc -l`;
|
|
;;
|
|
"IPDiscovery")
|
|
prostatus=`ps aux | grep -E "$keyword" | grep -v grep | grep -v sh`;
|
|
num=`ps aux | grep -E "$keyword" | grep -v grep | grep -v sh | wc -l`;
|
|
;;
|
|
"Tfe"|"kni")
|
|
prostatus=`ps aux | grep -E "$keyword" | grep -v grep | grep -v sh`;
|
|
num=`ps aux | grep -E "$keyword" | grep -v grep | grep -v sh | wc -l`;
|
|
;;
|
|
"Minio"|"minioEventsPretreat"|"minioDeteteClient")
|
|
prostatus=`ps aux | grep -E "$keyword" | grep -v grep | grep -v sh`;
|
|
num=`ps aux | grep -E "$keyword" | grep -v grep | grep -v sh | wc -l`;
|
|
;;
|
|
"certStore")
|
|
prostatus=`ps aux | grep -E "$keyword" | grep -v grep | grep -v sh`;
|
|
num=`ps aux | grep -E "$keyword" | grep -v grep | grep -v sh | wc -l`;
|
|
;;
|
|
*)
|
|
;;
|
|
esac
|
|
pidfile=$pid_dir"/"$module".pid";
|
|
# echo HERE num=$num
|
|
# echo $prostatus
|
|
# echo "HERE? "
|
|
if [ $num = 1 ]; then
|
|
pid=`echo $prostatus | awk '{print $2}'` ;
|
|
echo $pid > $pidfile
|
|
pstatistics=`ps -eo pid,stat,priority,pcpu,pmem,lstart | grep $pid`
|
|
#pcpu=`echo $pstatistics | awk '{print $4}'`
|
|
pcpu=`top -c -b -p $pid -n 1 | tail -1 | grep $pid | awk '{print $9}'`
|
|
pmem=`echo $pstatistics | awk '{print $5}'`
|
|
prostatus=`echo $pstatistics | awk '{print $2}'`
|
|
propri=`echo $pstatistics | awk '{print $3}'`
|
|
lstart=`echo $pstatistics | awk '{for (i=6 ;i<=NF;i++) printf $i " "; printf "\n" }'`
|
|
linenum=$((linenum+1))
|
|
# echo $module $clog_dir $clog_name
|
|
if [ -n "$clog_dir" ]; then
|
|
if [ -n "$clog_name" ]; then
|
|
if [ -d $clog_dir ]; then
|
|
# echo have filename $clog_dir $clog_name
|
|
case $module in
|
|
"pzffff")
|
|
if [ "$clog_name" = "cc_recv" ]; then
|
|
isclog=`find $clog_dir -maxdepth 3 -mmin -$checkGap -name "$clog_name*" | wc -l`
|
|
if [ "$isclog" = "0" ]; then
|
|
isclog="--!!! $clog_name LOG FILES MISSED IN $checkGap MINUTE ! PLEASE CHECK ON SERVER!!!--"
|
|
interstatus=$interstatus"Bad finding on $module $clog_name behavior. | "
|
|
else
|
|
isclog="OK"
|
|
fi
|
|
fi
|
|
;;
|
|
"pzffhj")
|
|
isclog=`find $clog_dir -maxdepth 3 -mmin -$checkGap -name "$clog_name*" | wc -l`
|
|
if [ "$isclog" = "0" ]; then
|
|
isclog="OK"
|
|
else
|
|
isclog="--!!! $module $clog_name IN $checkGap MINUTE ! PLEASE CHECK ON SERVER!!!--"
|
|
interstatus=$interstatus"Bad finding on $module $clog_name behavior. | "
|
|
fi
|
|
;;
|
|
"pzffkeep")
|
|
isclog="null"
|
|
;;
|
|
*)
|
|
isclog=`find $clog_dir -mmin -$checkGap -name "$clog_name*" | wc -l`
|
|
if [ $isclog -gt 0 ]; then
|
|
interstatus=$interstatus"Bad finding on $module $clog_name behavior. | "
|
|
fi
|
|
isclog="$isclog critical log files are modified in $checkGap minutes."
|
|
;;
|
|
esac
|
|
else
|
|
# echo $module log not found
|
|
isclog="Log path not found: $clog_dir"
|
|
interstatus=$interstatus"Log path wrong. | "
|
|
fi
|
|
else
|
|
if [ -d $clog_dir ]; then
|
|
# echo no filename $clog_dir
|
|
isclog=`find $clog_dir -maxdepth 1 -mmin -$checkGap | wc -l`
|
|
isclog="$isclog critical log files are modified in $checkGap minutes."
|
|
fi
|
|
fi
|
|
else
|
|
isclog="null"
|
|
fi
|
|
echo -e "$module,$pid,$pcpu,$pmem,$prostatus,$propri,$lstart,$isclog," >> $stat_file
|
|
else
|
|
if [ "$module" = "pzffc2res" ]; then
|
|
isclog=`find $clog_dir -mmin -$checkGap -name "$clog_name*" | wc -l`
|
|
if [ $isclog -gt 0 ]; then
|
|
pzffffline=`grep pzffff $stat_file`
|
|
if [ -n "$pzffffline" ]; then
|
|
sed -i '/pzffff/d' $stat_file
|
|
echo $pzffffline" | RESTART log is modified in $checkGap minites." >> $stat_file
|
|
fi
|
|
fi
|
|
else
|
|
pid=""; pstatistics="--"; pcpu=""; pmem=""; prostatus="NotFound"; propri=""; lstart=""; linenum=$((linenum+1));
|
|
isclog="null"
|
|
interstatus=$interstatus"$module miss something this interval. | "
|
|
echo -e "$module,$pid,$pcpu,$pmem,$prostatus,$propri,$lstart,$isclog," >> $stat_file
|
|
if [ -f $pidfile ]; then
|
|
rm $pidfile
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
done < $conf
|
|
# echo "$pubInfo,$remstimestamp,0,$currentTimeStamp,1,$later15mTimeStamp,1,\"$interstatus\",\"$perstatus\"" > $incoming_file
|
|
# echo "details,$linenum" >> $incoming_file
|
|
# cat $stat_file >> $incoming_file
|
|
|
|
echo "0,1,\"$interstatus\"" > $temp_file
|
|
echo "details,$linenum" >> $temp_file
|
|
cat $stat_file >> $temp_file
|
|
mv $temp_file $ditemp_dir
|
|
rm -rf $stat_file
|