实时 日志收集 - 实时统计 的起点

使用 ：
测试 代码中的 一些 sleep 都去除 就可以用了 .

#encoding: utf-8
import sys,time,os
import threading
import pyinotify

# 全局变量
# 一个 job ，只能 有一个 TAIL_JOB 线程
GLOBAL_MEM = '' # buff
TAIL_JOB = None

class ThreadTail(threading.Thread):

    def __init__(self,log,sep='\n',read_seek=-1,buff_size=12,blocking=-1,sleep=2):
        threading.Thread.__init__(self)
        self.log = log  # 日志文件名
        self.sep = sep  # 读取文件分割名称
        self.read_seek = read_seek  # 开始读取 数据位置,默认 文件开始
        self.buff_size = buff_size  # 读取 缓冲区
        self.blocking = blocking    # 文件 读取到最后 是否  拥塞/-1 ; 或等待 n/num 个sleep 时间
        self.sleep = sleep          # 文件读取到最后 sleep 单元时间
        self._isStop =  False

    def isStop(self,isstop):
        self._isStop = isstop

    def run(self):
        global GLOBAL_MEM
        if self.read_seek == -1 :
            self.read_seek = os.stat(self.log)[6]
        try :
            log_file=open(self.log,'r')
            print "read ~ %s " % self.log
        except : return
        log_file.seek(self.read_seek)
        blocking_num = 0
        while self._isStop == False :
            time.sleep(2)
            block = log_file.read(self.buff_size)
            if block :
                if block[-1] == self.sep : block = block[:-1]
                self.read_seek += len(block)

                buffs = block.split(self.sep)
                if len(buffs)!=1:
                    GLOBAL_MEM += buffs[0]
                    send(GLOBAL_MEM)

                    for t_buff in buffs[1:-1] :
                        send(t_buff)

                    GLOBAL_MEM = buffs[-1]
                else :
                    GLOBAL_MEM += buffs[0]
            else :
                time.sleep(self.sleep)
                log_file.seek(self.read_seek )
                blocking_num += 1
                if self.blocking != -1  and blocking_num > self.blocking :
                    send(GLOBAL_MEM)
                    print 'end ~ %s' % self.log
                    break



def send(row):
    print row


class EventHandler(pyinotify.ProcessEvent):

    def __init__(self,log_file):
        pyinotify.ProcessEvent.__init__(self)
        self.log_file = log_file
        self.dir,self.filename = os.path.split(log_file)
        self.isMoveLog = False


    def process_IN_MOVED_FROM(self, event):
        global TAIL_JOB
        if event.name == self.filename :
            print "process_IN_MOVED_FROM %s " % event.name
            TAIL_JOB.isStop(True)
            self.isMoveLog = True

    def process_IN_MOVED_TO(self, event):
        global TAIL_JOB
        if self.isMoveLog :
            print "process_IN_MOVED_TO %s " % event.name
            if TAIL_JOB :
                TAIL_JOB.isStop(True)
            ttail = ThreadTail(self.dir+'/'+event.name,read_seek= TAIL_JOB.read_seek-1,blocking=1)
            ttail.start()

            TAIL_JOB = ThreadTail(self.log_file,read_seek=0)
            TAIL_JOB.start()

    def process_IN_CREATE(self, event):
        global TAIL_JOB
        if self.filename == event.name :
            print "process_IN_CREATE %s " % event.name
            if TAIL_JOB  : TAIL_JOB.isStop(True)
            TAIL_JOB = ThreadTail(log_file,read_seek=0)
            TAIL_JOB.start()



#   /opt/sohumc/bin/python  read_log.py /tmp/t.txt
#   动态 收集 日志
#   支持即使 有   mv /tmp/t.txt /tmp/x.txt 的变动,一样能收集
#     如有需要 只需 重写 下 send 方法
if __name__ == '__main__':
    log_file = sys.argv[1]
    dir,filename = os.path.split(log_file)

    handler = EventHandler(log_file)
    wm = pyinotify.WatchManager()
    mask = pyinotify.IN_MOVED_FROM | pyinotify.IN_MOVED_TO | pyinotify.IN_CREATE
    wdd = wm.add_watch( dir , mask, auto_add=True )
    notifier = pyinotify.Notifier(wm, handler)

    TAIL_JOB = ThreadTail(log_file,read_seek=0)
    try :
        TAIL_JOB.start()
    except :
        print "not open %s " % log_file

    print 'test~'

    notifier.loop()