nginx 日志分析

我使用 系统自带的轮转日志  logrotate, 配置文件如下 /etc/logrotate.d/nginx

/usr/local/nginx/logs/*.log {
daily
rotate 5
notifempty
missingok
sharedscripts
copytruncate

postrotate
if [ -f /usr/local/nginx/logs/nginx.pid ]; then
kill -USR1 `cat /usr/local/nginx/logs/nginx.pid`
fi
/root/tj_script/test.sh
endscript
}

配置文件中 轮转完成后 对刚刚进行轮转的日志进行导入数据库操作,/root/tj_script/test.sh 这个脚本如下

#!/bin/sh

pid_file="/root/tj_script/test.pid";

if [ -f ${pid_file} ]
then
pid=`cat ${pid_file}`
isRunning=`ps -p ${pid} |grep ${pid} |wc -l`
else
isRunning="0"
fi


if [ ${isRunning} = '1' ] ; then
echo "script is running now"
exit 0;
fi

echo $$ > ${pid_file}

logFileName="/usr/local/nginx/logs/1234.com_access.log.1"

if [ -f $logFileName ]; then
echo "job begin"
php /root/tj_script/access.php $logFileName > /root/tj_script/test.log
echo "job end"
fi

echo "Running is done."

调用了一个php 程序， access.php 文件内容如下

if($_SERVER['argc'] < 2){
die(0);
}

define('DEBUG' ,true);
define('NEWLINE',"\n");

define('ROW_LIMIT',50);

define('CUR_DIR',dirname(__FILE__));

require CUR_DIR.'/mysql.class.php';

function wlog($v = '',$stop = false){
if(defined('DEBUG')){
print_r($v);
echo NEWLINE;
}

if(true == $stop){
exit(0);
}
}


/**
* 批量插入
*/
function batch_insert($p){
if(empty($p) || strlen($p[0]) < 1){
wlog($p);
return false;
}

global $db;
$tmp = '';

$insertSQL['head'] = "INSERT INTO `site_access_log` VALUES ";
$insertSQL['tail'] = array();

foreach($p as $k => $v){
if(false == preg_match("/^(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}).*?\[(.*?)\s+.*?\]\s+\"(\w{3,6})\s+(.*?)\s+.*\s+(\d+)\s+(\d+)\s+\"(.*?)\"\s+\"(.*?)\"/",$v,$match)){
wlog($v);
continue;
}

$now = time();
$ip = explode('.',$match[1]);

//29/Dec/2011:09:22:06
//var_dump(strtotime("02 Dec 2011 14:14:03"));

$req_hm = substr($match[2],strpos($match[2],':') + 1,5);
$match[2] = str_replace('/',' ',$match[2]);
$match[2]{strpos($match[2],':')} = ' ';


$req_time = date("Y-m-d H:i:s",strtotime($match[2]));


$browser = "";
if(strpos($match[8],"MSIE") !== false){
$browser = "MSIE";
}else if(strpos($match[8],"Firefox") !== false){
$browser = "Firefox";
}else if(strpos($match[8],"Safari") !== false){
$browser = "Safari";
}else if(strpos($match[8],"Konqueror") !== false){
$browser = "Konqueror";
}else{
$browser = "Other";
}

$os = '';
if(strpos($match[8],"Windows") !== false){
$os = "windows";
}else if(strpos($match[8],"Linux") !== false){
$os = "Linux";
}else if(strpos($match[8],"Mac") !== false){
$os = "Mac OS";
}else{
$os = "Other";
}

$match[3] = mysql_escape_string($match[3]);
$match[7] = mysql_escape_string($match[7]);
$match[4] = mysql_escape_string($match[4]);

$insertSQL['tail'][] = "(NULL,'{$req_time}','{$req_hm}',{$ip[0]},{$ip[1]},{$ip[2]},{$ip[3]},'{$os}','{$browser}','{$match[3]}','{$match[7]}','{$match[4]}',{$match[6]},{$match[5]},1,{$now})";
}

try {
$db->query($insertSQL['head'].implode(',',$insertSQL['tail']));

if($db->affectedRows() < 1){
wlog("batch insert rows=".count($insertSQL['tail'])."=".$db->affectedRows());
wlog($insertSQL);
}
}catch(Exception $e){
wlog("Query Error");
wlog($insertSQL);
return false;
}

return true;
}

if(defined('DEBUG')){
error_reporting(E_ERROR|E_WARNING|E_PARSE);
ini_set("display_errors",true);
}else{
error_reporting(0);
ini_set("display_errors",false);
}

$fileName = $_SERVER['argv'][1];

if(substr($fileName,0,1) != '/'){
$fileName = "/usr/local/nginx/logs/{$fileName}";
}

wlog($fileName);

$dbConf = array(
'host' => '192.168.202.151',
'username' => 'root',
'password' => '123456',
'dbname' => 'm_log',

);

$db = new Mysql($dbConf);

if(file_exists($fileName) && is_file($fileName)){

$fp = fopen($fileName,"r");

if(!$fp){
wlog("File {$fileName} cant not open",true);
}


$line = array();
try{

$db->connect();
while(!feof($fp)){
$line[] = fgets($fp,2048);

if(count($line) >= ROW_LIMIT){
if(!batch_insert($line)){
wlog($line);
}else{
$line = array();
}
}
}

}catch(Exception $e){
//db error, todo send message to administrator
}

if($fp){
fclose($fp);
}

if(!batch_insert($line)){
wlog($line);
}else{
unset($line);
}
}else{
wlog("{$fileName} not found");

}

wlog('END of '. $_SERVER['argv'][0]);

输入库表结构如下

CREATE TABLE `site_access_log` (
  `rid` int(10) unsigned NOT NULL AUTO_INCREMENT COMMENT '自增',
  `req_time` datetime DEFAULT NULL COMMENT '日期',
  `req_hm` varchar(5) DEFAULT '00:00' COMMENT '24小时制中 小时和分钟 如12:30',
  `ip1` tinyint(4) unsigned DEFAULT NULL COMMENT 'IP 地址第一个域',
  `ip2` tinyint(4) unsigned DEFAULT NULL,
  `ip3` tinyint(4) unsigned DEFAULT NULL,
  `ip4` tinyint(4) unsigned DEFAULT NULL,
  `os` varchar(30) DEFAULT '' COMMENT '操作系统',
  `browser` varchar(30) DEFAULT '' COMMENT '浏览器',
  `method` varchar(8) DEFAULT NULL COMMENT 'http方法',
  `referer` varchar(500) DEFAULT '',
  `url` varchar(500) DEFAULT '' COMMENT '请求地址',
  `body_len` int(11) unsigned DEFAULT '0' COMMENT '数据包长度',
  `resp_code` int(11) unsigned DEFAULT NULL COMMENT 'http 响应码',
  `web_site` int(11) unsigned DEFAULT NULL COMMENT '那个站点',
  `gmt_create` int(11) unsigned DEFAULT NULL,
  PRIMARY KEY (`rid`)
) ENGINE=MyISAM;


还有一个潜在的问题，如果日志文件特别大以至于在一个轮转周期内还没有处理完毕就会出现问题, 目前一天一个轮转还不会出现跑了一天 脚本还没有结束的情况, 等量大了以后再来修复

定时任务
0 16 * * * logrotate /etc/logrotate.d/nginx