• 博客访问: 592432
  • 博文数量: 195
  • 博客积分: 4939
  • 博客等级: 上校
  • 技术积分: 2085
  • 用 户 组: 普通用户
  • 注册时间: 2010-04-08 17:15
文章分类

全部博文(195)

文章存档

2017年(3)

2016年(11)

2015年(43)

2014年(24)

2013年(11)

2012年(19)

2011年(47)

2010年(37)

微信关注

IT168企业级官微



微信号:IT168qiye



系统架构师大会



微信号:SACC2013

订阅
热词专题
Ruby 杂记 加代码练习 2016-08-13 19:16:19

分类: Python/Ruby

前言,基础略过,基本上跟python等差不多,看基础就可以了。

1. json嵌套list的数据提取
  1. result = { "api.xxx.cn": [
  2. {
  3. "address": "10.0.16.19:9022",
  4. "ttl": 0
  5. }
  6. ]
  7. }

  8. route = JSON.parse(result)

  9. temp_route = {}
  10. route.each do |host, ip_metas|
  11. temp_route[host] = ip_metas.map { |ip_meta| ip_meta["address"]}
  12. end
  13. ########
  14. map, _ = build_map  #放弃第二个返回参数

2. 根据条件检索ElasticSearch,找过message里面proxy error的IP数量,然后打印出结果,后续可以写到zabbix等监控工具里。

  1. # Name: RouterLogRetriver 

  2. #!/usr/bin/env ruby
  3. require 'net/http'
  4. require 'uri'
  5. require 'openssl'
  6. require 'json'
  7. require 'date'
  8. require 'yaml'


  9. class RouterLogRetriver
  10.   class << self                 # 无需生成一个对象然后再调用方法,见 114行。
  11.     attr_accessor :host, :user, :password, :time_span_minute    # 和Python不同,Ruby的类变量不允许被类方法直接访问,只能通过accessor存取器这种。
  12.     def configure(host, user, password, time_span_minute)
  13.       @host = host
  14.       @user = user
  15.       @password = password
  16.       @time_span_minute = time_span_minute
  17.     end

  18.     def is_configured?
  19.       !(@host.nil? || @user.nil? || @password.nil? || @time_span_minute.nil?)
  20.     end

  21.     def construct_uri
  22.       date = DateTime.now.new_offset(0).strftime('%C%y.%m.%d')
  23.       return URI.parse("http://#{@host}/logstash-#{date}/_search")
  24.     end

  25.     def construct_payload
  26.        timestamp_e = DateTime.now.strftime('%Q')
  27.        timestamp_s = (timestamp_e.to_i - 1000*60*@time_span_minute).to_s
  28.        payload = '        # ES可以通过JSON方式查询,下边是构造一个查询体。
  29. {
  30.   "query": {
  31.     "filtered": {
  32.       "query": {
  33.         "bool": {
  34.           "should": [
  35.             {
  36.               "query_string": {
  37.                 "query": "\"proxy error:\"-\"EOF\""
  38.               }
  39.             }
  40.           ]
  41.         }
  42.       },
  43.       "filter": {
  44.         "bool": {
  45.           "must": [
  46.             {
  47.               "range": {
  48.                 "@timestamp": {
  49.                   "from": ' + timestamp_s + ',
  50.                  "to": ' + timestamp_e + '
  51.                 }
  52.               }
  53.             }
  54.           ]
  55.         }
  56.       }
  57.     }
  58.   },
  59.  "size": 500,
  60.   "sort": [
  61.     {
  62.       "msg_timestamp": {
  63.         "order": "desc"
  64.       }
  65.     }
  66.   ]
  67. }
  68.        '
  69.        payload    # 等价于 return payload。
  70.     end

  71.     def search_result
  72.       return nil unless is_configured?
  73.       uri = construct_uri
  74.       http = Net::HTTP.new(uri.host, uri.port)
  75.       req = Net::HTTP::Get.new(uri.path)
  76.       req.basic_auth @user, @password
  77.       req["Content-Type"]="application/json"
  78.       req.body = construct_payload

  79.       errors = {}

  80.       begin
  81.         response = http.request(req)
  82.         result = response.body

  83.         logs = JSON.parse(result)
  84.         logs['hits']['hits'].each do |l|
  85.           error = l['_source']['@message']
  86.           next if error.nil?
  87.           match_set = error.match(/.*proxy error: \w* tcp (\d+\.\d+.\d+.\d+:\d+):.*/)
  88.           if match_set
  89.             backend = match_set[1]
  90.             errors[backend] = 0 unless errors.has_key?(backend)
  91.             errors[backend] += 1
  92.           end
  93.         end
  94.       rescue Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, EOFError,
  95.         Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError => e
  96.         puts "[ERR] CAN NOT GET Logstash logs, #{e.message}"
  97.       end

  98.       errors
  99.     end
  100.   end
  101. end
  102. # log_url = ENV['LOGSEARCH_HOST'] || "es.xxx.com" 可以从环境变量取值
  103. RouterLogRetriver.configure("es.xxx.com","elk","xxx",20)
  104. print RouterLogRetriver.search_result
3. 一般来说要写个项目,把class、module放./lib下,执行文件放bin下。然后bin下加载

  1. libdir = File.expand_path(File.join(File.dirname(__FILE__), "../lib"))
  2. $LOAD_PATH.unshift(libdir) unless $LOAD_PATH.include?(libdir)

4. rspec一例,测试代码

  1. # spec_helper.rb

  2. require 'logger'
  3. LOGGER=Logger.new('/dev/null')
  4. require 'router_log_retriver.rb'


  5. $:.unshift(File.expand_path("../lib", File.dirname(__FILE__)))

  6. SPEC_ROOT = File.expand_path(File.dirname(__FILE__))

  7. RSpec.configure do |config|
  8.   config.color = true
  9.   config.run_all_when_everything_filtered = true
  10.   config.filter_run :focus
  11. end

  12. ## routerlogretriver.rb    主测试
  13. require File.expand_path '../spec_helper.rb', __FILE__

    describe RouterLogRetriver do

      before :each do
        described_class.configure('logsearch.run.xxx.com', 'fake_log_user', 'fake_log_pass', 5)
      end

      it 'load configure and populate variables' do
        expect(described_class.host).to eq('logsearch.run.xxx.com')
        expect(described_class.user).to eq('fake_log_user')
        expect(described_class.password).to eq('fake_log_pass')
        expect(described_class.time_span_minute).to eq(5)
      end

      it 'construct the logstash uri base on time stamp' do
        allow(DateTime).to receive(:now).and_return(DateTime.new(2014,10,10))
        uri = described_class.construct_uri
        expect(uri).to eq(URI.parse("https://es.run.xxx.com/logstash-2014.10.10/_search"))
      end

      it 'construct the payload base on the time stamp' do
        allow(DateTime).to receive(:now).and_return(DateTime.new(2014,10,10))
        payload = described_class.construct_payload
        pl = JSON.parse(payload)
        expect(pl['query']['filtered']['filter']['bool']['must'][0]['range']['@timestamp']['to']).to eq(1412899200000)
        expect(pl['query']['filtered']['filter']['bool']['must'][0]['range']['@timestamp']['from']).to eq(1412899200000-5*60*1000)
      end

      context 'search result' do

        before :each do
          @http = double(Net::HTTP)
          allow(@http).to receive(:verify_mode=).with(0)
          @req = double(Net::HTTP::Get)
          @response = double(Net::HTTPResponse)

          allow(DateTime).to receive(:now).and_return(DateTime.new(2014,10,10))
          allow(Net::HTTP).to receive(:new).with('es.run.xxx.com',443).and_return(@http)
          allow(Net::HTTP::Get).to receive(:new).with('/logstash-2014.10.10/_search').and_return(@req)
          allow(@http).to receive(:use_ssl=).with(true)
          allow(@req).to receive(:basic_auth).with('fake_log_user','fake_log_pass')
          allow(@req).to receive(:[]=).with('Content-Type','application/json')

          allow(@req).to receive(:body=)

        end

        it 'raise errors when have Net::Http exceptions' do
          allow(@http).to receive(:request).with(@req).and_raise(Timeout::Error)
          allow(STDOUT).to receive(:puts).with("[ERR] CAN NOT GET Logstash logs, Timeout::Error")
          result = described_class.search_result
          expect(result).to eq({})
        end

        it 'grab search result and return a hash with error instance ip:port' do

          simulate_response_body = '
        {
          "took" : 19,
          "timed_out" : false,
          "_shards" : {
            "total" : 6,
            "successful" : 6,
            "failed" : 0
          },
          "hits" : {
            "total" : 1,
            "max_score" : null,
            "hits" : [ {
              "_index" : "logstash-2014.10.21",
              "_type" : "syslog",
              "_id" : "UyuLFsXGSriM-gVfom3-hw",
              "_score" : null,
              "_source" : {"@message":"<11>2014-10-21T07:58:43.092792+00:00 10.10.80.16 vcap.gorouter_ctl.stderr [job=router_z2 index=1]  2014/10/21 07:58:43 http: proxy error: read tcp 10.10.17.37:63426: i/o timeout\n","@version":"1","@timestamp":"2014-10-21T07:59:05.425Z","type":"syslog","host":"10.10.66.17","tags":["_grokparsefailure"],"priority":13,"severity":5,"facility":1,"facility_label":"user-level","severity_label":"Notice","job":"router_z2","index":"1","msg_timestamp":"2014-10-21T07:58:43.092792+00:00","origin":"10.10.80.16"},
              "sort" : [ 1413878323092 ]
            },
            {
              "_index" : "logstash-2014.10.21",
              "_type" : "syslog",
              "_id" : "UyuLFsXGSriM-gVfom3-hw",
              "_score" : null,
              "_source" : {"@message":"<11>2014-10-21T07:58:43.092792+00:00 10.10.80.16 vcap.gorouter_ctl.stderr [job=router_z2 index=1]  2014/10/21 07:58:43 http: proxy error: read empty string to test","@version":"1","@timestamp":"2014-10-21T07:59:05.425Z","type":"syslog","host":"10.10.66.17","tags":["_grokparsefailure"],"priority":13,"severity":5,"facility":1,"facility_label":"user-level","severity_label":"Notice","job":"router_z2","index":"1","msg_timestamp":"2014-10-21T07:58:43.092792+00:00","origin":"10.10.80.16"},
              "sort" : [ 1413878323092 ]
              }
            ]
          }
        }'

        allow(@http).to receive(:request).with(@req).and_return(@response)
        allow(@response).to receive(:body).and_return(simulate_response_body)

        result = described_class.search_result
        expect(result).to eq({"10.10.17.37:63426"=>1})
        end

        it 'returns the search result when message field is missing' do

          simulate_response_body = '
        {
          "took" : 19,
          "timed_out" : false,
          "_shards" : {
            "total" : 6,
            "successful" : 6,
            "failed" : 0
          },
          "hits" : {
            "total" : 1,
            "max_score" : null,
            "hits" : [ {
              "_index" : "logstash-2014.10.21",
              "_type" : "syslog",
              "_id" : "UyuLFsXGSriM-gVfom3-hw",
              "_score" : null,
              "_source" : {"@message":"<11>2014-10-21T07:58:43.092792+00:00 10.10.80.16 vcap.gorouter_ctl.stderr [job=router_z2 index=1]  2014/10/21 07:58:43 http: proxy error: read tcp 10.10.17.37:63426: i/o timeout\n","@version":"1","@timestamp":"2014-10-21T07:59:05.425Z","type":"syslog","host":"10.10.66.17","tags":["_grokparsefailure"],"priority":13,"severity":5,"facility":1,"facility_label":"user-level","severity_label":"Notice","job":"router_z2","index":"1","msg_timestamp":"2014-10-21T07:58:43.092792+00:00","origin":"10.10.80.16"},
              "sort" : [ 1413878323092 ]
            },
            {
              "_index" : "logstash-2014.10.21",
              "_type" : "syslog",
              "_id" : "UyuLFsXGSriM-gVfom3-hw",
              "_score" : null, "_source" : {},
              "sort" : [ 1413878323092 ]
              }
            ]
          }
        }'

        allow(@http).to receive(:request).with(@req).and_return(@response)
        allow(@response).to receive(:body).and_return(simulate_response_body)

        result = described_class.search_result
        expect(result).to eq({"10.10.17.37:63426"=>1})
        end
      end

    end


## assert/config_spec.yml

log_search:
url: fake_log_uri
login: fake_log_user
pass: fake_log_pass
timespan: 5


5. 简单rakefile

  1. require 'yaml'
  2. require 'erb'


  3. ENV['RUBY_ENVIRONMENT'] ||= 'staging'

  4. OUTPUT_FILE='manifest.yml'
  5. TEMPLATE_FILE=OUTPUT_FILE+ '.erb'


  6. def get_template
  7.   File.read(TEMPLATE_FILE)
  8. end


  9. RSpec::Core::RakeTask.new(:spec)
  10. task default: [:spec]


  11. desc "push to xxx.com"
  12. task :push do
  13.   Rake::Task["login"].invoke
  14.   puts %x[cf push --no-start]
  15.   Rake::Task["set-env"].invoke
  16.   puts %x[cf start app-instance-monitor]
  17.   Rake::Task["logout"].invoke
  18. end


  19. desc "set env of app"
  20. task :'set-env' do
  21.   puts system("COMMAND RUBY_ENVIRONMENT #{ENV['RUBY_ENVIRONMENT']}")
  22. end


  23. desc "login to xxx.com"
  24. task :login do
  25.   conf = YAML::load(File.open('config/config.yml'))[ENV['RUBY_ENVIRONMENT']]
  26.   puts %x[COMMAND #{conf['xxxx']['api']} --skip-ssl-validation]
  27.   puts %x[cf login -u #{conf['xxxx']['username']} -p #{conf['user']['password']} -o system -s monitor]
  28. end


  29. desc "logout of xxx.com"
  30. task :logout do
  31.   puts %x[cf logout]
  32. end

 6.  关于 Gemfile
source 'https://ruby.taobao.org'   #国外的就是rubygems.org,但国内用这个太慢了。
gem 'rake'  '~> xxx'   指定gem 的版本

7.    .ruby-version
2.2.4

8. 获取AWS的使用情况,代码练习。

  1. #!/usr/bin/env ruby

  2. require 'optparse'
  3. require 'aws'

  4. class AwsInfoHandler
  5.   def self.acquire_account_info(account)
  6.     name=account.fetch('name')
  7.     id=account.fetch('account_id')


  8.     ec2 = AWS::EC2.new(
  9.       :access_key_id => account.fetch('aws_key'),
  10.       :secret_access_key => account.fetch('aws_sec'),
  11.       :region => account.fetch('region')
  12.     )

  13.     ec2.instances.inject({}) do |m, i|
  14.       if m[i.availability_zone].nil?
  15.         m[i.availability_zone] = {}
  16.       end

  17.       if m[i.availability_zone][i.instance_type].nil?
  18.         m[i.availability_zone][i.instance_type]=1
  19.       else
  20.         m[i.availability_zone][i.instance_type]+=1
  21.       end
  22.       m
  23.     end
  24.   end


  25. #### reservation, for S3 ###


  26.   def self.publish_s3_report(s3_input_config)
  27.     account = s3_input_config[:account]
  28.     s3 = AWS::S3.new(
  29.       :access_key_id => account.fetch('aws_key'),
  30.       :secret_access_key => account.fetch('aws_sec')
  31.     )
  32.     bucket = s3.buckets[s3_input_config[:target_bucket]]
  33.     object = bucket.objects[s3_input_config[:target_filename]]
  34.     object.write(s3_input_config[:contents])
  35.   end
  36. end


  37. ######################


  38. class ReportGenerator
  39.   def initialize(accounts)
  40.     @accounts = accounts
  41.   end


  42.   def generate_report
  43.     report=[]


  44.     @accounts.each do |ac|
  45.       info = AwsInfoHandler.acquire_account_info(ac)
  46.       puts "[INFO] Generating account info for #{ac.fetch('name')}"
  47.       info.each do |k,v|
  48.         v.each do |type, count|
  49.           rp = {
  50.             'account_name' => ac.fetch('name'),
  51.             'account_id' => ac.fetch('account_id'),
  52.             'AZ' => k,
  53.             'flavor' => type,
  54.             'count' => count
  55.           }
  56.           report << rp
  57.         end
  58.       end
  59.     end
  60.     report
  61.   end
  62. end


  63. class EmailAgent
  64.   def self.generate_and_email(report,report_to=nil)
  65.     puts "[DEBUG] report for #{Time.now.strftime("%m/%d/%Y")}"
  66.     puts "=========================="
  67.     puts report
  68.     puts "=========================="
  69.     puts "[DEBUG] End of report"


  70. =begin
  71. MAIL_CONF={"mail-provider":[
  72.    {
  73.            "credentials": {
  74.            "password": "qdWC4sqfLoD9Zg",
  75.            "username": "cloudfoundry"
  76.           }
  77.     ]
  78.    }
  79. }
  80. =end
  81.     credentials = host = username = password = ''
  82.     if !ENV['MAIL_CONF'].to_s.empty?
  83.       JSON.parse(ENV['MAIL_CONF']).each do |k,v|
  84.         if !k.scan("mail-provider").to_s.empty?
  85.           credentials = v.first.select {|k1,v1| k1 == "credentials"}["credentials"]
  86.           host = credentials["hostname"]
  87.           username = credentials["username"]
  88.           password = credentials["password"]
  89.         end
  90.         end
  91.     end


  92. ### SMTP setings ###
  93. # ......


  94. # attachement


  95.       report_file = File.new('/tmp/report.csv', 'w+')
  96.       report_file.puts(report[0].keys.join(','))


  97.       report.each do |r|
  98.         report_file.puts(r.values.join(','))
  99.       end


  100.       report_file.close
  101. ### send out ###
  102. ## ......
  103.   end
  104. end

  105. ###  -c 来指定配置文件,实现代码和配置分离。
  106. options = {}
  107. optparse = OptionParser.new do |opts|
  108.   opts.on('-h', '--help', 'Help Messages') do
  109.     puts opts
  110.     exit
  111.   end
  112.   opts.on('-c', '--config File', 'The Config file') do |f|
  113.     options[:file] = f
  114.   end
  115. end
  116. optparse.
  117. unless options[:file]
  118.   puts 'no config file specified, exit'
  119.   exit
  120. end


  121. ruby_env = ENV['PLATFORM'] || 'production'
  122. conf = YAML::load(File.open(options[:file]))[ruby_env]


  123. accounts = conf.fetch('accounts')
  124. mail_to = conf.fetch('to')
  125. report = ReportGenerator.new(accounts).generate_report
  126. EmailAgent.generate_and_email(report, mail_to)
# config.yml

  1. ---
    production:
      accounts:
      - name: web
        account_id: xxxxxxxx7842
        aws_key: xxxx
        aws_sec: xxxx
      - name: db
        account_id: xxxxxxxx6344
        aws_key: xxx
        aws_sec: xxxx

      reports_email_to:
      - xxxt@yyy.com

阅读(261) | 评论(0) | 转发(0) |
给主人留下些什么吧!~~
评论热议
请登录后评论。

登录 注册