一个支持正则表达式的日志关键字统计的shell工具

Usage: pafa_mon/statistics_pafa.sh -i|--instance [-fn|--filename ] [-ip|--instancepath] [-kw|--keywords] [-bt|--begintime] [-et|--endtime] [-t|--type]
instance name: 实例英文名称
instance path: 实例所在路径，默认为/wls/applogs/rtlog/
file name: 文件名称，默认为pafa.log
key words: 匹配关键字,如有多个关键字请用冒号隔开，传参不支持带空格的关键字
  从文件读取支持带空格关键字，多个关键字关键字用换行符分隔，支持正则表达式
  默认从文件"/instance path/instance name/pafa.key"读取，如：/wls/applogs/rtlog/TOA-CORESF6802/pafa.key
begin time: 开始时间，默认从文件开头匹配
end time: 结束时间，默认匹配到文件末尾
progress type: 操作类型: 默认为1. 
  1，指定文件范围内扫描关键字，打印内容；（最后100w行）
  2，指定文件范围内扫描关键字，统计次数；（最后100w行）
  3，指定时间范围内扫描关键字，打印内容；
  4，指定时间范围内扫描关键字，统计次数；
for example :
 ./statistics_pafa.sh -i TOA-CORESF6801
 ./statistics_pafa.sh -i TOA-CORESF6801 -t 2 
 ./statistics_pafa.sh -i TOA-CORESF6801 -t 2  -kw ERROR
 ./statistics_pafa.sh -i TOA-CORESF6801 -kw ERROR
 ./statistics_pafa.sh -i TOA-CORESF6801 -fn TOA-CORESF6801.acc -t 2  -kw ERROR
 ./statistics_pafa.sh -i TOA-CORESF6801 -t 2  -kw ERROR -ip /tmp/ -bt 08:00:00
 ./statistics_pafa.sh -i TOA-CORESF6801 -t 2  -kw ERROR -bt 08:00:00  -et 18:00:00
 ./statistics_pafa.sh -i TOA-CORESF6801 -ip /wls/applogs/rtlog/ -fn pafa.log -kw ERROR:WARN -bt 08:00:00  -et 18:00:00 --type 2


#!/bin/bash
#
# File Name: statistics.sh
#
# Function: statistics the keywords from the specified log-file.
#
# Written by: stone
#
# Written on: 2013-01-16
#
#######################################################################

## default values for the variables below ##

HOST_NAME=$(hostname)
HOST_IP=$(hostname -i)
INSTANCE_NAME=''
INSTANCE_PATH="/wls/applogs/rtlog/"
FILE_NAME='pafa.log'
KEY_WORDS=''
MONTH=$(date +%m)
DAY=$(date +%e)
BEGIN_TIME=''
END_TIME=''
PROGRESS_TYPE=1

########################################################################

#### function for usage of the shell script ############
function Usage(){
  echo "Usage: $0 -i|--instance [-fn|--filename ] [-ip|--instancepath] [-kw|--keywords] [-bt|--begintime] [-et|--endtime] [-t|--type] "
  echo "instance name: 实例英文名称"
  echo "instance path: 实例所在路径，默认为/wls/applogs/rtlog/"
  echo "file name: 文件名称，默认为pafa.log "
  echo 'key words: 匹配关键字,如有多个关键字请用冒号隔开，传参不支持带空格的关键字'
  echo '  从文件读取支持带空格关键字，多个关键字关键字用换行符分隔'
  echo '  默认从文件"/instance path/instance name/pafa.key"读取，如：/wls/applogs/rtlog/TOA-CORESF6802/pafa.key'
  echo "begin time: 开始时间，默认从文件开头匹配 "
  echo "end time: 结束时间，默认匹配到文件末尾 "
  echo 'progress type: 操作类型: 默认为1.  '
  echo '  1，指定文件范围内扫描关键字，打印内容；（最后100w行）'
  echo '  2，指定文件范围内扫描关键字，统计次数；（最后100w行）'
  echo '  3，指定时间范围内扫描关键字，打印内容；'
  echo '  4，指定时间范围内扫描关键字，统计次数；'
  echo 'for example :'
  echo ' ./statistics_pafa.sh -i TOA-CORESF6801 '
  echo ' ./statistics_pafa.sh -i TOA-CORESF6801 -t 2  '
  echo ' ./statistics_pafa.sh -i TOA-CORESF6801 -t 2  -kw ERROR'
  echo ' ./statistics_pafa.sh -i TOA-CORESF6801 -kw ERROR'
  echo ' ./statistics_pafa.sh -i TOA-CORESF6801 -fn TOA-CORESF6801.acc -t 2  -kw ERROR'
  echo ' ./statistics_pafa.sh -i TOA-CORESF6801 -t 2  -kw ERROR -ip /tmp/ -bt 08:00:00'
  echo ' ./statistics_pafa.sh -i TOA-CORESF6801 -t 2  -kw ERROR -bt 08:00:00  -et 18:00:00'
  echo ' ./statistics_pafa.sh -i TOA-CORESF6801 -ip /wls/applogs/rtlog/ -fn pafa.log -kw ERROR:WARN -bt 08:00:00  -et 18:00:00 --type 2'
  exit 1
}

#### function for setting variables for the shell script #############
function Set_variables(){
    if [[ $# == 0 ]]; then
         if [[ -n $INSTANCE_NAME ]];then
              FILE_PATH="${INSTANCE_PATH}/${INSTANCE_NAME}/"
              Choose
              exit 0
         else
              Usage
         fi
    fi
   
    case "$1" in
      "-i")  shift
      INSTANCE_NAME=$1 
      shift
      Set_variables $@
        ;;
      "--instance")  shift
      INSTANCE_NAME=$1 
      shift
      Set_variables $@
        ;;
      "-fn")  shift
                    FILE_NAME=$1
      shift
      Set_variables $@
        ;;
      "--filename")  shift
                    FILE_NAME=$1
      shift
      Set_variables $@
        ;;
      "-ip")  shift
                    INSTANCE_PATH=$1
      shift
      Set_variables $@
        ;;
      "--instancepath")  shift
                    INSTANCE_PATH=$1
      shift
      Set_variables $@
        ;;
      "-kw")  shift
                    KEY_WORDS=$1
      shift
      Set_variables $@
        ;;
      "--keywords")  shift
                    KEY_WORDS=$1
      shift
      Set_variables $@
        ;;
      "-bt")  shift
                    BEGIN_TIME=$1
      shift
      Set_variables $@
        ;;
      "--begintime")  shift
                    BEGIN_TIME=$1
      shift
      Set_variables $@
        ;;
      "-et")  shift
                    END_TIME=$1
      shift
      Set_variables $@
        ;;
      "--endtime")  shift
                    END_TIME=$1
      shift
      Set_variables $@
        ;;
      "-t")  shift
                    PROGRESS_TYPE=$1
      shift
      Set_variables $@
        ;;
      "--type")  shift
                    PROGRESS_TYPE=$1
      shift
      Set_variables $@
        ;;
      *)     Usage
    esac
}

#### function for show the value of the shell script's variables ############
function Show_variables(){
    echo HOST_NAME=$HOST_NAME
    echo HOST_IP=$HOST_IP
    echo INSTANCE_NAME=$INSTANCE_NAME
    echo FILE_PATH=$FILE_PATH
    echo INSTANCE_PATH=$INSTANCE_PATH
    echo FILE_NAME=$FILE_NAME
    echo KEY_WORDS=$KEY_WORDS
    echo MONTH=$MONTH
    echo DAY=$DAY
    echo BEGIN_TIME=$BEGIN_TIME
    echo END_TIME=$END_TIME 
   }

#### function for statistic the key-words in the specified file from the last 1 million lines ############
function Statistic_contents(){
    tail -1000000 ${FILE_PATH}${FILE_NAME} |awk '/^'${MONTH}'\/'${DAY}'/ && /'"${KEY}"'/ {OFS=""; print "KeyWord=","'"${KEY}:"'",$0}  ' | tail -n 10 
}

#### function for statistic the key-words arise times in the specified file from the last 1 million lines ############
function Statistic_times(){
    tail -1000000 ${FILE_PATH}${FILE_NAME} |awk '/^'${MONTH}'\/'${DAY}'/ && /'"${KEY}"'/ {a++} END{OFS=""; print "KeyWord=","'"${KEY}:"'",a}  '  
}

#### function for statistic the key-words in the specified file by time ############
function Statistic_contents_bytime(){
    if [[ -z $BEGIN_TIME ]];then
        if [[ -z $END_TIME ]];then
            awk '/^'${MONTH}'\/'${DAY}'/ && /'"${KEY}"'/ {OFS=""; print "KeyWord=","'"${KEY}:"'",$0}  ' ${FILE_PATH}${FILE_NAME} | tail -n 10  
        else
            awk '/^'${MONTH}'\/'${DAY}'/ && $2<"'$END_TIME'" && /'"${KEY}"'/ {OFS=""; print "KeyWord=","'"${KEY}:"'",$0}' ${FILE_PATH}${FILE_NAME} | tail -n 10  
        fi   
    else
        if [[ -z $END_TIME ]];then
            awk '/^'${MONTH}'\/'${DAY}'/ && $2>"'$BEGIN_TIME'" && /'"${KEY}"'/ {OFS=""; print "KeyWord=","'"${KEY}:"'",$0}' ${FILE_PATH}${FILE_NAME} | tail -n 10  
        else
            awk '/^'${MONTH}'\/'${DAY}'/ && $2>"'$BEGIN_TIME'" && $2<"'$END_TIME'" && /'"${KEY}"'/ {OFS=""; print "KeyWord=","'"${KEY}:"'",$0}' ${FILE_PATH}${FILE_NAME} | tail -n 10 
        fi   
    fi
}

#### function for statistic the key-words arise times in the specified file by time ############
function Statistic_times_bytime(){
    if [[ -z $BEGIN_TIME ]];then
        if [[ -z $END_TIME ]];then
            awk '/^'${MONTH}'\/'${DAY}'/ && /'"${KEY}"'/ {a++}  END{OFS=""; print "KeyWord=","'"${KEY}:"'",a}'   ${FILE_PATH}${FILE_NAME}
        else
            awk '/^'${MONTH}'\/'${DAY}'/ && $2<"'$END_TIME'" && /'"${KEY}"'/ {a++} END{OFS=""; print "KeyWord=","'"${KEY}:"'",a}'    ${FILE_PATH}${FILE_NAME}
        fi   
    else
        if [[ -z $END_TIME ]];then
            awk '/^'${MONTH}'\/'${DAY}'/ && $2>"'$BEGIN_TIME'" && /'"${KEY}"'/ {a++}  END{OFS=""; print "KeyWord=","'"${KEY}:"'",a}'   ${FILE_PATH}${FILE_NAME}
         else
            awk '/^'${MONTH}'\/'${DAY}'/ && $2>"'$BEGIN_TIME'" && $2<"'$END_TIME'" && /'"${KEY}"'/  {a++}  END{OFS=""; print "KeyWord=","'"${KEY}:"'",a}'   ${FILE_PATH}${FILE_NAME}  
        fi   
    fi
}

#### function for progressing for statistic ##########################
function Progress(){
    if [[ -z $KEY_WORDS ]];then
        if [[ -f ${FILE_PATH}/pafa.key ]];then
              grep -v "^#" ${FILE_PATH}/pafa.key | while read KEY
              do 
                   $1 
              done  
         else
              echo "Attention please !"
              echo "You should create the key-word file named pafa.key in the directory ${FILE_PATH} at first !"
              exit 1
         fi
    else
         Key_Words="$(echo ${KEY_WORDS} |sed s/':'/' '/g )"
         for KEY in $Key_Words
         do 
              $1 
         done
    fi
}

#### function for choose progress type for statistic ###############
function Choose(){
    case "$PROGRESS_TYPE" in
      "1")  Progress "Statistic_contents"
      ;;   
      "2")  Progress "Statistic_times"
      ;;   
      "3")  Progress "Statistic_contents_bytime"
      ;;   
      "4")  Progress "Statistic_times_bytime"
      ;;   
      *)  Usage
      ;;   
    esac
}

#### function for the start of the shell script #################
function Main(){
    Set_variables $@
}

Main  $@