PPPOE路由日志分析系统源码［原创］

采集端：

#-*-coding:utf-8-*-
"""
功能：采集日志的脚本
"""
import os,sys
import ConfigParser
import time
path = os.path.abspath(os.path.dirname(sys.argv[0]))
N = 1000
#1.输出当前系统时间 文件名：20100630
tmplog = time.strftime("%Y%m%d", time.localtime())
fullpath = path + os.sep + tmplog
if not os.path.isdir( fullpath ):os.mkdir(fullpath)
#2.开始分析日志文件提取前N行日志到指定的文件里面去
currhour = time.strftime("%d", time.localtime())
file_object = open(fullpath + os.sep + currhour + '.txt', 'a')

#读取配置文件提取当前的起点下标如果为1则表示读从1 --- N条
cf = ConfigParser.ConfigParser()
cf.read("test.conf")
s = cf.sections()
o = cf.options("db")
v = cf.items("db")
position = cf.getint("db", "position")
#读取配置文件提取当前的起点下标如果为1则表示读从1 --- N条 完成

#读取当前临时日志文件的行数 开始
#logfileobject = open('tmplog.log', 'rU')
#count = -1
#for count, line in enumerate(logfileobject):pass
#count += 1
count = 0
logfileobject = open('tmplog.log', 'rb')
while True:
    buffer = logfileobject.read(81920*1024)
    if not buffer:break
    count += buffer.count('\n')
#读取当前临时日志文件的行数 结束

#判断INI里面的值与当前全部的记录值
if position > count:
    tmpcount = -1
    #表示当前这个日志为新创建的日志文件了,读取顺序从1开始读N条
    for current_line_number,line in enumerate(logfileobject):
        if current_line_number < N:
            file_object.write(line)
            tmpcount += 1
        else:break
    #将这个值设置到INI文件里面去
    cf.set("db", "position", tmpcount)
    cf.write(open("test.conf", "w"))
else:
    #表示从POSITION开始读
    tmpcount = -1
    countnum = 0
    for current_line_number,line in enumerate(open('tmplog.log','r+')):
        tmpcount += 1
        if tmpcount >= position:
            file_object.write(line)
            countnum += 1
            if countnum < N:continue
            else:break
        else:pass
    newposition = position + countnum
    cf.set("db", "position", newposition)
    cf.write(open("test.conf", "w"))
#解析读取完毕
logfileobject.close()
file_object.close()
file_object = None
cf = None


入库：

#-*-coding:utf-8-*-
"""
功能：解析日志入库
"""
import os,sys,time,re
import MySQLdb
regex= "(\|\d+\.\d+\.\d+\.\d+:\d+|\d+\.\d+\.\d+\.\d+|\d+-\w.*from)"
p = re.compile(regex)
conn = MySQLdb.connect("localhost","root","123","syslog",port=3306,connect_timeout=10,compress=True,charset='utf8',use_unicode=True)
cursor=conn.cursor()
path = os.path.abspath(os.path.dirname(sys.argv[0]))
tmplog = time.strftime("%Y%m%d", time.localtime())
currhour = time.strftime("%d", time.localtime())
fullpath = path + os.sep + tmplog
file_object = open(fullpath + os.sep + currhour + '.txt', 'r')
for current_line_number,line in enumerate(file_object):
    results = p.findall(line)
    try:
        pppoe = results[0].replace("<","").replace(">","").replace("pppoe-","").strip()
        sip = results[1].strip()
        vip = results[2].strip()
        months = results[3].replace("from","").split("-")[1].split(" ")[0]
        days = results[3].replace("from","").split("-")[0]
        times = results[3].replace("from","").split("-")[1].split(" ")[1]
        routeip = results[4]
    except Exception,e:
        pppoe = ''
        sip = results[0].strip()
        vip = results[1].strip()
        months = results[2].replace("from","").split("-")[1].split(" ")[0]
        days = results[2].replace("from","").split("-")[0]
        times = results[2].replace("from","").split("-")[1].split(" ")[1]
        routeip = results[3]
    sql = "insert into syslog(pppoe,sip,vip,months,days,times,routeip) values('%s','%s','%s','%s','%s','%s','%s')"%(pppoe,sip,vip,months,days,times,routeip)
    cursor.execute(sql)
    conn.commit()
cursor = None
conn.close()
conn = None

数据表结构：

CREATE TABLE `syslog` (
  `id` bigint(20) NOT NULL auto_increment,
  `pppoe` char(50) default '',
  `sip` char(25) default NULL,
  `vip` char(25) default NULL,
  `months` char(20) default NULL,
  `days` char(4) default NULL,
  `times` char(25) default NULL,
  `routeip` char(25) default NULL,
  PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8