#fields: ["beat.hostname", "beat.name", "beat.version", "input_type", "beat"]
#fields: ["beat.hostname", "beat.name", "beat.version", "input_type", "beat"]
#
数据从Redis写入到ES中
192.168.10.8中的
cd /home/haoren/data/logstash-5.0.2
[haoren@IM-SJ01-Server18 logstash-5.0.2]$ ll
总用量 276
drwxrwxr-x 2 haoren haoren 4096 11月 30 15:34 bin
-rw-rw-r-- 1 haoren haoren 111731 11月 24 18:22 CHANGELOG.md
drwxrwxr-x 2 haoren haoren 4096 12月 1 22:25 config
-rw-rw-r-- 1 haoren haoren 2249 11月 24 18:22 CONTRIBUTORS
drwxrwxr-x 2 haoren haoren 4096 11月 24 18:22 data
-rwxrwxr-x 1 haoren haoren 1125 4月 7 18:06 Del-logstash-log.sh
-rw-r--r-- 1 haoren haoren 7385 1月 12 11:57 filebeat.20170112
-rw-rw-r-- 1 haoren haoren 2636 1月 12 11:47 filebeat20170112.txt
-rw-r--r-- 1 haoren haoren 2534 12月 9 21:06 filebeat.bak
-rw-r--r-- 1 haoren haoren 9011 2月 8 20:16 filebeat.conf
-rw-r--r-- 1 haoren haoren 5130 1月 11 16:20 filebeat.conf.20170111.bak
-rw-r--r-- 1 haoren haoren 6163 1月 11 16:20 filebeat.conf.20170112.bak
-rw-r--r-- 1 haoren haoren 7853 2月 8 20:16 filebeat.conf.20170208.bak
-rw-r--r-- 1 haoren haoren 7390 1月 18 10:31 filebeat.conf.2070118.bak
-rw-r--r-- 1 haoren haoren 3457 12月 20 16:28 filebeat.conf.bak
-rw-rw-r-- 1 haoren haoren 2636 1月 12 11:47 filebeat.json
-rwxrwxr-x 1 haoren haoren 78 12月 6 18:33 filebeat.sh
-rw-r--r-- 1 haoren haoren 292 1月 11 21:31 filebeat.txt
-rw-rw-r-- 1 haoren haoren 3686 11月 24 18:26 Gemfile
-rw-rw-r-- 1 haoren haoren 20837 11月 24 18:22 Gemfile.jruby-1.9.lock
drwxrwxr-x 5 haoren haoren 4096 11月 30 15:34 lib
-rw-rw-r-- 1 haoren haoren 589 11月 24 18:22 LICENSE
drwxrwxr-x 2 haoren haoren 4096 4月 20 08:47 logs
-rw-rw-r-- 1 haoren haoren 555 12月 20 16:27 logstash.conf
drwxrwxr-x 5 haoren haoren 4096 11月 30 15:34 logstash-core
drwxrwxr-x 3 haoren haoren 4096 11月 30 15:34 logstash-core-event-java
drwxrwxr-x 3 haoren haoren 4096 11月 30 15:34 logstash-core-plugin-api
-rwxrwxr-x 1 haoren haoren 78 11月 30 17:40 logstash.sh
-rw-rw-r-- 1 haoren haoren 149 11月 24 18:22 NOTICE.TXT
drwxrwxr-x 4 haoren haoren 4096 11月 30 15:34 vendor
[haoren@IM-SJ01-Server18 logstash-5.0.2]$ cat logstash.sh
#!/bin/bash
/home/haoren/data/logstash-5.0.2/bin/logstash -f logstash.conf &
[haoren@IM-SJ01-Server18 logstash-5.0.2]$ cat logstash.conf
input {
redis {
host => ["192.168.10.8"]
port => 6379
password => "A8841c09BAD52E63067C4DA"
data_type => "list"
key => "logstash"
codec => json {
charset => "UTF-8"
}
}
}
filter {
ruby {
code=>"event.set('daytag',event.timestamp.time.localtime.strftime('%Y.%m.%d'))"
}
}
output {
elasticsearch {
hosts => ["127.0.0.1:19200"]
index => "%{type}-%{daytag}"
#index => "%{type}-%{+yyyy.MM.dd}"
user => "logstashserver"
password => "A950C0FB2D833E42C1AC59210CD5CDF8"
}
}
[haoren@IM-SJ01-Server18 logstash-5.0.2]$ cat filebeat.sh
#!/bin/bash
/home/haoren/data/logstash-5.0.2/bin/logstash -f filebeat.conf &
[haoren@IM-SJ01-Server18 logstash-5.0.2]$ cat filebeat.conf
input {
redis {
host => ["192.168.10.8"]
port => 6379
password => "A8841c09BAD52E63067C4DA"
data_type => "list"
key => "filebeat"
codec => json {
charset => "UTF-8"
}
}
#file {
# path => "/home/haoren/data/logstash-5.0.2/filebeat.txt"
# codec => plain {
# charset => "GBK"
# }
# start_position => "beginning"
# sincedb_path => "/dev/null"
# type => "pchannelserver"
#}
#file {
# path => "/home/haoren/data/logstash-5.0.2/filebeat.json"
# codec => json {
# charset => "UTF-8"
# }
# start_position => "beginning"
# sincedb_path => "/dev/null"
# type => "pchannelserver"
#}
}
filter {
if( [type] == "sessionserver" ){
if( [message] =~ "登陆统计"){
mutate {replace => { "type" => "userlogin" }}
grok {
#161206-16:00:00 SS[4306] TRACE: [登陆统计]收到角色(82559870)登陆(PC)IP(124.239.95.209)MAC(52229449286)机器ID(454070640)渠道(0)gameid(6)端口(3889)登陆类型(1)
match => [ "message", "(?(?>\d{6}-\d\d:\d\d:\d\d)) SS\[\d+\].*?TRACE: \[登陆统计\]收到角色\(%{NUMBER:userid:int}\)登陆\(%{DATA:sclient}\)IP\(%{DATA:sip}\)MAC\(%{DATA:smac}\)机器ID\(%{NUMBER:mid:int}\)渠道\(%{NUMBER:apkid:int}\)gameid\(%{NUMBER:gameid:int}\)端口\(%{NUMBER:port:int}\)登陆类型\(%{NUMBER:logintype:int}\)"]
}
}
else if ([message] =~ "注册统计"){
mutate {replace => { "type" => "userreg" }}
grok{
#161205-15:33:22 SS[4306] TRACE: [注册统计]用户(87475178)注册(Android)渠道(8)账号(_wx_omj_avq_nz8obsk6yy5dsmfrlfmk)
match => [ "message", "(?(?>\d{6}-\d\d:\d\d:\d\d)) SS\[\d+\].*?TRACE: \[注册统计\]用户\(%{NUMBER:userid:int}\)注册\(%{DATA:sclient}\)渠道\(%{NUMBER:apkid:int}\)账号\(%{DATA:saccount}\)" ]
}
}
#else{
# drop{}
#}
}
else if( [type] == "activityserver" ){
if( [message] =~ "用户注册渠道奖励统计"){
mutate {replace => { "type" => "actvityregreward" }}
grok {
#161212-10:17:29 ActivityServer[17702] INFO: [UserRegisterReward.cpp:90] [用户注册渠道奖励统计]用户(87582819)客户端(0)注册渠道(1001)机器码(476931654)奖励包裹(77)个数(100)当日奖励(100)上限(1000000)
match => [ "message", "(?(?>\d{6}-\d\d:\d\d:\d\d)) ActivityServer\[\d+\].*?INFO: .*?\[用户注册渠道奖励统计\]用户\(%{NUMBER:userid:int}\)客户端\(%{NUMBER:client:int}\)注册渠道\(%{NUMBER:apkid:int}\)机器码\(%{NUMBER:mid:int}\)奖励包裹\(%{NUMBER:packid:int}\)个数\(%{NUMBER:num:int}\)当日奖励\(%{NUMBER:todaynum:int}\)上限\(%{NUMBER:maxnum:int}\)\)"]
}
}
else if( [message] =~ "充值礼包统计"){
mutate {replace => { "type" => "activityrechargebag" }}
grok {
#161220-17:02:53 ActivityServer[17700] INFO: [ActivityRechargeBag.cpp:559] [充值礼包统计]用户(60163778)获得(8800)礼包类型(2)用户获得( 1029:1)消费者获得( 1062:688)
match => [ "message", "(?(?>\d{6}-\d\d:\d\d:\d\d)) ActivityServer\[\d+\].*?INFO: \[.*?\] \[充值礼包统计\]用户\(%{NUMBER:userid:int}\)获得\(%{NUMBER:pid:int}\)礼包类型\(%{NUMBER:ptype:int}\)用户获得\(%{DATA:userget}\)消费者获得\(%{DATA:singerget}\)"]
}
}
else if( [message] =~ "16年度盛典消费统计"){
mutate {replace => { "type" => "activitymodules" }}
grok {
#161220-17:05:31 ActivityServer[17702] INFO: [ActivityModules.cpp:837][2016购物活动] [16年度盛典消费统计]用户(84795176)购物平台(65780103)消费者(65780103)个数(20)礼物(92)价值(0)原分数(385399)增加分数(20)现分数(385419)淘汰(0)新旧(0)
match => [ "message", "(?(?>\d{6}-\d\d:\d\d:\d\d)) ActivityServer\[\d+\] INFO: \[.*?\]\[2016购物活动\] \[16年度盛典消费统计\]用户\(%{NUMBER:userid:int}\)购物平台\(%{NUMBER:channelid:int}\)消费者\(%{NUMBER:singerid:int}\)个数\(%{NUMBER:num:int}\)礼物\(%{NUMBER:itemid:int}\)价值\(%{NUMBER:coin:int}\)原分数\(%{NUMBER:oldscore:int}\)增加分数\(%{NUMBER:addscore:int}\)现分数\(%{NUMBER:nowscore:int}\)淘汰\(%{NUMBER:out:int}\)新旧\(%{NUMBER:isnew:int}\)"]
}
}
}
else if( [type] == "vchannelserver" ){
if( [message] =~ "进出购物平台统计"){
mutate {replace => { "type" => "vchannelin" }}
grok {
#170109-12:59:39 VChannelServer[15000] INFO: [Channel.cpp:414] [进出购物平台统计]用户(22016998)(进入)购物平台(3998186)端(0)渠道(0)IMState(6)机器ID(246418457)消费者(87261227)游客(0)
#170109-12:59:40 VChannelServer[15000] INFO: [Channel.cpp:414] [进出购物平台统计]用户(83735196)(离开)购物平台(4029779)端(3)渠道(0)IMState(13)机器ID(0)消费者(0)游客(0)
#170118-09:59:58 VChannelServer[15001] INFO: [Channel.cpp:417] [进出购物平台统计]用户(87455625)(进入)购物平台(3830989)端(3)渠道(2017)IMState(13)机器ID(0)消费者(65256549)游客(0)游戏id(0)
#170118-09:59:59 VChannelServer[15001] INFO: [Channel.cpp:417] [进出购物平台统计]用户(3015482583)(离开)购物平台(3830989)端(0)渠道(0)IMState(13)机器ID(0)消费者(65256549)游客(1)游戏id(0)
match => [ "message", "(?(?>\d{6}-\d\d:\d\d:\d\d)) VChannelServer\[\d+\] INFO: \[.*?\] \[进出购物平台统计\]用户\(%{NUMBER:userid:int}\)\(%{DATA:sop}\)购物平台\(%{NUMBER:channelid:int}\)端\(%{NUMBER:client:int}\)渠道\(%{NUMBER:apkid:int}\)IMState\(%{NUMBER:imstate:int}\)机器ID\(%{NUMBER:mid:int}\)消费者\(%{NUMBER:singerid:int}\)游客\(%{NUMBER:istemp:int}\)游戏id\(%{NUMBER:gameid:int}\)"]
}
}
}
else if( [type] == "pchannelserver" ){
if( [message] =~ "进出购物平台统计"){
mutate {replace => { "type" => "pchannelin" }}
grok {
#170111-00:00:17 PChannelServer[18701] INFO: [进出购物平台统计]用户(88464974)(进入)购物平台(86972527)端(0)渠道(0)IMState(6)机器ID(0)消费者(86972527)游客(0)
#170111-00:00:43 PChannelServer[18701] INFO: [进出购物平台统计]用户(88464904)(退出)购物平台(86972527)端(0)渠道(0)IMState(6)机器ID(0)消费者(86972527)游客(0)
match => [ "message", "(?(?>\d{6}-\d\d:\d\d:\d\d)) PChannelServer\[\d+\] INFO: \[进出购物平台统计\]用户\(%{NUMBER:userid:int}\)\(%{DATA:sop}\)购物平台\(%{NUMBER:channelid:int}\)端\(%{NUMBER:client:int}\)渠道\(%{NUMBER:apkid:int}\)IMState\(%{NUMBER:imstate:int}\)机器ID\(%{NUMBER:mid:int}\)消费者\(%{NUMBER:singerid:int}\)游客\(%{NUMBER:istemp:int}\)"]
}
}
}
else if( [type] == "billserver" ){
if( [message] =~ "人民币统计"){
mutate {replace => { "type" => "dubijiesuan" }}
grok {
#170208-10:00:28 Bill[40268] INFO: [人民币结算]时间(1486519228),用户ID(30581009),原来人民币(12567),现在人民币(16567),人民币操作(1),增加(4000),操作类型(19),操作详情(2),操作数量(0).描述:通用人民币操作
#170208-10:00:01 Bill[40268] INFO: [人民币结算]时间(1486519201),用户ID(22327945),原来人民币(2572),现在人民币(2532),人民币操作(2),扣除(40),操作类型(19),操作详情(15),操作数量(0).描述:通用人民币操作
match => [ "message", "(?(?>\d{6}-\d\d:\d\d:\d\d)) Bill\[\d+\] INFO: \[人民币结算\]时间\(%{NUMBER:time:int}\),用户ID\(%{NUMBER:userid:int}\),原来人民币\(%{NUMBER:oldcoin:int}\),现在人民币\(%{NUMBER:currentcoin:int}\),人民币操作\(%{NUMBER:coinop:int}\),%{DATA:sop}\(%{NUMBER:coinnum:int}\),操作类型\(%{NUMBER:optype:int}\),操作详情\(%{NUMBER:opdetail:int}\),操作数量\(%{NUMBER:opnum:int}\).描述:%{DATA:sinfo}"]
}
}
}
else{
drop{}
}
date {
match => ["datetime", "yyMMdd-HH:mm:ss"]
}
ruby {
code => "event.timestamp.time.localtime"
}
ruby {
code => "event.set('daytag',event.timestamp.time.localtime.strftime('%Y.%m.%d'))"
remove_field => ["tags"]
}
}
output {
#stdout {
# codec => plain {
# charset => "UTF-8"
# #charset => "GBK"
# }
#}
#file {
# path => "/tmp/logstash.log"
# codec => json {
# charset => "UTF-8"
# }
#}
elasticsearch {
hosts => ["127.0.0.1:9200"]
index => "%{type}-%{daytag}"
#index => "%{type}-%{+yyyy.MM.dd}"
user => "logstashserver"
password => "33E42C1AC59210CD5CDF8"
}
}
cat /home/haoren/data/logstash-5.0.2/billserver.conf
input {
file {
path => "/log/abcbillserver.log"
codec => plain {
charset => "GBK"
}
#start_position => "beginning"
#sincedb_path => "/dev/null"
type => "billserver"
}
}
filter {
if ([message] =~ "人民币统计"){
mutate {replace => { "type" => "dubijiesuan" }}
grok {
#170208-10:00:28 Bill[40268] INFO: [人民币结算]时间(1486519228),用户ID(30581009),原来人民币(12567),现在人民币(16567),人民币操作(1),增加(4000),操作类型(19),操作详情(2),操作数量(0).描述:通用人民币操作
#170208-10:00:01 Bill[40268] INFO: [人民币结算]时间(1486519201),用户ID(22327945),原来人民币(2572),现在人民币(2532),人民币操作(2),扣除(40),操作类型(19),操作详情(15),操作数量(0).描述:通用人民币操作
match => [ "message", "(?(?>\d{6}-\d\d:\d\d:\d\d)) Bill\[\d+\] INFO: \[人民币结算\]时间\(%{NUMBER:time:int}\),用户ID\(%{NUMBER:userid:int}\),原来人民币\(%{NUMBER:oldcoin:int}\),现在人民币\(%{NUMBER:currentcoin:int}\),人民币操作\(%{NUMBER:coinop:int}\),%{DATA:sop}\(%{NUMBER:coinnum:int}\),操作类型\(%{NUMBER:optype:int}\),操作详情\(%{NUMBER:opdetail:int}\),操作数量\(%{NUMBER:opnum:int}\).描述:%{DATA:sinfo}"]
}
}
else{
drop{}
}
date {
match => ["datetime", "yyMMdd-HH:mm:ss"]
#timezone => ["Asia/Hong_Kong"]
#remove_field => ["time"]
}
ruby {
code => "event.timestamp.time.localtime"
}
mutate {
#some pc no host
replace => { "host" => "192.168.10.7" }
}
}
output {
#stdout {
# codec => plain {
# charset => "UTF-8"
# #charset => "GBK"
# }
#}
#file {
# path => "/tmp/logstash.log"
# codec => json {
# charset => "UTF-8"
# }
#}
redis {
host => ["192.168.10.18"]
port => 6379
data_type => "list"
key => "logstash"
password => "A8841c09BAD52E63067C4DA"
codec => json {
charset => "UTF-8"
}
}
}
cat /home/haoren/data/logstash-5.0.2/activityserver.conf
input {
file {
path => "/log/aactivityserver.log"
codec => plain {
charset => "GBK"
}
#start_position => "beginning"
#sincedb_path => "/dev/null"
type => "aactivityserver"
}
}
filter {
if ([message] =~ "运统计"){
mutate {replace => { "type" => "activityescort" }}
grok {
#161201-13:12:28 ActivityServer[17701] INFO: [Escort.cpp:595] [统计]序号(53)用户(23619530)(攻)值(4360)暴击率(4)使用道具(57)本次花费(0)本总花费(0)车原始量(1706792)剩余量(1702432)总值(4360)
match => [ "message", "(?(?>\d{6}-\d\d:\d\d:\d\d)) ActivityServer\[\d+\].*?INFO: \[.*?\] \[运镖统计\]序号\(%{NUMBER:carid:int}\)用户\(%{NUMBER:userid:int}\)\(%{DATA:sop}\).*?\(%{NUMBER:itemvalue:int}\)暴击率\(%{NUMBER:baoji:int}\)使用道具\(%{NUMBER:itemid:int}\)本次花费\(%{NUMBER:coin:int}\)本总花费\(%{NUMBER:allusercoin:int}\)车原始量\(%{NUMBER:oldblood:int}\)剩余量\(%{NUMBER:blood:int}\)总值\(%{NUMBER:allvalue:int}\)"]
}
}
else if ([message] =~ "运通用奖励"){
mutate {replace => { "type" => "activityescort" }}
grok {
#161201-13:15:16 ActivityServer[17701] INFO: [运]序号(54), [运通用奖励] 用户(21772699)奖励方式(0)奖励类型(34)购物车ID(61)数量(16)
match => [ "message", "(?(?>\d{6}-\d\d:\d\d:\d\d)) ActivityServer\[\d+\] INFO: \[运\]序号\(%{NUMBER:carid:int}\), \[运通用奖励\] 用户\(%{NUMBER:userid:int}\)奖励方式\(%{NUMBER:method:int}\)奖励类型\(%{NUMBER:addtype:int}\)包裹ID\(%{NUMBER:itemid:int}\)数量\(%{NUMBER:num:int}\)"]
}
}
else{
drop{}
}
date {
match => ["datetime", "yyMMdd-HH:mm:ss"]
#timezone => ["Asia/Hong_Kong"]
#remove_field => ["time"]
}
ruby {
code => "event.timestamp.time.localtime"
}
mutate {
#some pc no host
replace => { "host" => "192.168.10.13" }
}
}
output {
#stdout {
# codec => plain {
# charset => "UTF-8"
# #charset => "GBK"
# }
#}
#file {
# path => "/tmp/logstash.log"
# codec => json {
# charset => "UTF-8"
# }
#}
redis {
host => ["192.168.10.8"]
port => 6379
data_type => "list"
key => "logstash"
password => "A8841c09BAD52E63067C4DA"
codec => json {
charset => "UTF-8"
}
}
}
cat /home/haoren/data/filebeat-5.0.2-linux-x86_64/filebeat.yml
filebeat.prospectors:
- input_type: log
paths:
- /log/billserver.log
#- /home/haoren/data/filebeat-5.0.2-linux-x86_64/test.log
encoding: gbk
symlinks: true
include_lines: ['\[.*?统计\]','\[.*?结算\]']
document_type: billserver
fields_under_root: true
fields:
host: 192.168.10.7
processors:
- drop_fields:
#fields: ["beat.hostname", "beat.name", "beat.version", "input_type", "beat"]
fields: ["input_type", "beat", "offset", "source"]
output.redis:
enabled: true
hosts: ["192.168.10.18"]
password: "A8841c09BAD52E63067C4DA"
port: 6379
datatype: list
key: "filebeat"
db: 0
output.file:
enabled: false
path: "/tmp/filebeat"
output.console:
enabled: false
