Ruby 杂记 加代码练习

前言，基础略过，基本上跟python等差不多，看基础就可以了。

1. json嵌套list的数据提取

result = {  "api.xxx.cn": [
{
"address": "10.0.16.19:9022",
"ttl": 0
}
]
}

route = JSON.parse(result)

temp_route = {}
route.each do |host, ip_metas|
temp_route[host] = ip_metas.map { |ip_meta| ip_meta["address"]}
end
########
map, _ = build_map  #放弃第二个返回参数

2. 根据条件检索ElasticSearch，找过message里面proxy error的IP数量，然后打印出结果，后续可以写到zabbix等监控工具里。

# Name: RouterLogRetriver

#!/usr/bin/env ruby
require 'net/http'
require 'uri'
require 'openssl'
require 'json'
require 'date'
require 'yaml'


class RouterLogRetriver
  class << self                 # 无需生成一个对象然后再调用方法，见 114行。
    attr_accessor :host, :user, :password, :time_span_minute    # 和Python不同，Ruby的类变量不允许被类方法直接访问，只能通过accessor存取器这种。
    def configure(host, user, password, time_span_minute)
      @host = host
      @user = user
      @password = password
      @time_span_minute = time_span_minute
    end

    def is_configured?
      !(@host.nil? || @user.nil? || @password.nil? || @time_span_minute.nil?)
    end

    def construct_uri
      date = DateTime.now.new_offset(0).strftime('%C%y.%m.%d')
      return URI.parse("{@host}/logstash-#{date}/_search")
    end

    def construct_payload
       timestamp_e = DateTime.now.strftime('%Q')
       timestamp_s = (timestamp_e.to_i - 1000*60*@time_span_minute).to_s
       payload = '        # ES可以通过JSON方式查询，下边是构造一个查询体。
{
  "query": {
    "filtered": {
      "query": {
        "bool": {
          "should": [
            {
              "query_string": {
                "query": "\"proxy error:\"-\"EOF\""
              }
            }
          ]
        }
      },
      "filter": {
        "bool": {
          "must": [
            {
              "range": {
                "@timestamp": {
                  "from": ' + timestamp_s + ',
                 "to": ' + timestamp_e + '
                }
              }
            }
          ]
        }
      }
    }
  },
 "size": 500,
  "sort": [
    {
      "msg_timestamp": {
        "order": "desc"
      }
    }
  ]
}
       '
       payload    # 等价于 return payload。
    end

    def search_result
      return nil unless is_configured?
      uri = construct_uri
      http = Net::HTTP.new(uri.host, uri.port)
      req = Net::HTTP::Get.new(uri.path)
      req.basic_auth @user, @password
      req["Content-Type"]="application/json"
      req.body = construct_payload

      errors = {}

      begin
        response = http.request(req)
        result = response.body

        logs = JSON.parse(result)
        logs['hits']['hits'].each do |l|
          error = l['_source']['@message']
          next if error.nil?
          match_set = error.match(/.*proxy error: \w* tcp (\d+\.\d+.\d+.\d+:\d+):.*/)
          if match_set
            backend = match_set[1]
            errors[backend] = 0 unless errors.has_key?(backend)
            errors[backend] += 1
          end
        end
      rescue Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, EOFError,
        Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError => e
        puts "[ERR] CAN NOT GET Logstash logs, #{e.message}"
      end

      errors
    end
  end
end
# log_url = ENV['LOGSEARCH_HOST'] || "es.xxx.com" 可以从环境变量取值
RouterLogRetriver.configure("es.xxx.com","elk","xxx",20)
print RouterLogRetriver.search_result

3. 一般来说要写个项目，把class、module放./lib下，执行文件放bin下。然后bin下加载

libdir = File.expand_path(File.join(File.dirname(__FILE__), "../lib"))
$LOAD_PATH.unshift(libdir) unless $LOAD_PATH.include?(libdir)

4. rspec一例，测试代码

# spec_helper.rb

require 'logger'
LOGGER=Logger.new('/dev/null')
require 'router_log_retriver.rb'


$:.unshift(File.expand_path("../lib", File.dirname(__FILE__)))

SPEC_ROOT = File.expand_path(File.dirname(__FILE__))

RSpec.configure do |config|
  config.color = true
  config.run_all_when_everything_filtered = true
  config.filter_run :focus
end

## routerlogretriver.rb    主测试
require File.expand_path '../spec_helper.rb', __FILE__

describe RouterLogRetriver do

  before :each do
    described_class.configure('logsearch.run.xxx.com', 'fake_log_user', 'fake_log_pass', 5)
  end

  it 'load configure and populate variables' do
    expect(described_class.host).to eq('logsearch.run.xxx.com')
    expect(described_class.user).to eq('fake_log_user')
    expect(described_class.password).to eq('fake_log_pass')
    expect(described_class.time_span_minute).to eq(5)
  end

  it 'construct the logstash uri base on time stamp' do
    allow(DateTime).to receive(:now).and_return(DateTime.new(2014,10,10))
    uri = described_class.construct_uri
    expect(uri).to eq(URI.parse(""))
  end

  it 'construct the payload base on the time stamp' do
    allow(DateTime).to receive(:now).and_return(DateTime.new(2014,10,10))
    payload = described_class.construct_payload
    pl = JSON.parse(payload)
    expect(pl['query']['filtered']['filter']['bool']['must'][0]['range']['@timestamp']['to']).to eq(1412899200000)
    expect(pl['query']['filtered']['filter']['bool']['must'][0]['range']['@timestamp']['from']).to eq(1412899200000-5*60*1000)
  end

  context 'search result' do

    before :each do
      @http = double(Net::HTTP)
      allow(@http).to receive(:verify_mode=).with(0)
      @req = double(Net::HTTP::Get)
      @response = double(Net::HTTPResponse)

      allow(DateTime).to receive(:now).and_return(DateTime.new(2014,10,10))
      allow(Net::HTTP).to receive(:new).with('es.run.xxx.com',443).and_return(@http)
      allow(Net::HTTP::Get).to receive(:new).with('/logstash-2014.10.10/_search').and_return(@req)
      allow(@http).to receive(:use_ssl=).with(true)
      allow(@req).to receive(:basic_auth).with('fake_log_user','fake_log_pass')
      allow(@req).to receive(:[]=).with('Content-Type','application/json')

      allow(@req).to receive(:body=)

    end

    it 'raise errors when have Net::Http exceptions' do
      allow(@http).to receive(:request).with(@req).and_raise(Timeout::Error)
      allow(STDOUT).to receive(:puts).with("[ERR] CAN NOT GET Logstash logs, Timeout::Error")
      result = described_class.search_result
      expect(result).to eq({})
    end

    it 'grab search result and return a hash with error instance ip:port' do

      simulate_response_body = '
    {
      "took" : 19,
      "timed_out" : false,
      "_shards" : {
        "total" : 6,
        "successful" : 6,
        "failed" : 0
      },
      "hits" : {
        "total" : 1,
        "max_score" : null,
        "hits" : [ {
          "_index" : "logstash-2014.10.21",
          "_type" : "syslog",
          "_id" : "UyuLFsXGSriM-gVfom3-hw",
          "_score" : null,
          "_source" : {"@message":"<11>2014-10-21T07:58:43.092792+00:00 10.10.80.16 vcap.gorouter_ctl.stderr [job=router_z2 index=1]  2014/10/21 07:58:43 http: proxy error: read tcp 10.10.17.37:63426: i/o timeout\n","@version":"1","@timestamp":"2014-10-21T07:59:05.425Z","type":"syslog","host":"10.10.66.17","tags":["_grokparsefailure"],"priority":13,"severity":5,"facility":1,"facility_label":"user-level","severity_label":"Notice","job":"router_z2","index":"1","msg_timestamp":"2014-10-21T07:58:43.092792+00:00","origin":"10.10.80.16"},
          "sort" : [ 1413878323092 ]
        },
        {
          "_index" : "logstash-2014.10.21",
          "_type" : "syslog",
          "_id" : "UyuLFsXGSriM-gVfom3-hw",
          "_score" : null,
          "_source" : {"@message":"<11>2014-10-21T07:58:43.092792+00:00 10.10.80.16 vcap.gorouter_ctl.stderr [job=router_z2 index=1]  2014/10/21 07:58:43 http: proxy error: read empty string to test","@version":"1","@timestamp":"2014-10-21T07:59:05.425Z","type":"syslog","host":"10.10.66.17","tags":["_grokparsefailure"],"priority":13,"severity":5,"facility":1,"facility_label":"user-level","severity_label":"Notice","job":"router_z2","index":"1","msg_timestamp":"2014-10-21T07:58:43.092792+00:00","origin":"10.10.80.16"},
          "sort" : [ 1413878323092 ]
          }
        ]
      }
    }'

    allow(@http).to receive(:request).with(@req).and_return(@response)
    allow(@response).to receive(:body).and_return(simulate_response_body)

    result = described_class.search_result
    expect(result).to eq({"10.10.17.37:63426"=>1})
    end

    it 'returns the search result when message field is missing' do

      simulate_response_body = '
    {
      "took" : 19,
      "timed_out" : false,
      "_shards" : {
        "total" : 6,
        "successful" : 6,
        "failed" : 0
      },
      "hits" : {
        "total" : 1,
        "max_score" : null,
        "hits" : [ {
          "_index" : "logstash-2014.10.21",
          "_type" : "syslog",
          "_id" : "UyuLFsXGSriM-gVfom3-hw",
          "_score" : null,
          "_source" : {"@message":"<11>2014-10-21T07:58:43.092792+00:00 10.10.80.16 vcap.gorouter_ctl.stderr [job=router_z2 index=1]  2014/10/21 07:58:43 http: proxy error: read tcp 10.10.17.37:63426: i/o timeout\n","@version":"1","@timestamp":"2014-10-21T07:59:05.425Z","type":"syslog","host":"10.10.66.17","tags":["_grokparsefailure"],"priority":13,"severity":5,"facility":1,"facility_label":"user-level","severity_label":"Notice","job":"router_z2","index":"1","msg_timestamp":"2014-10-21T07:58:43.092792+00:00","origin":"10.10.80.16"},
          "sort" : [ 1413878323092 ]
        },
        {
          "_index" : "logstash-2014.10.21",
          "_type" : "syslog",
          "_id" : "UyuLFsXGSriM-gVfom3-hw",
          "_score" : null, "_source" : {},
          "sort" : [ 1413878323092 ]
          }
        ]
      }
    }'

    allow(@http).to receive(:request).with(@req).and_return(@response)
    allow(@response).to receive(:body).and_return(simulate_response_body)

    result = described_class.search_result
    expect(result).to eq({"10.10.17.37:63426"=>1})
    end
  end

end

## assert/config_spec.yml

log_search:

url: fake_log_uri

login: fake_log_user

pass: fake_log_pass

timespan: 5

5. 简单rakefile

require 'yaml'
require 'erb'


ENV['RUBY_ENVIRONMENT'] ||= 'staging'

OUTPUT_FILE='manifest.yml'
TEMPLATE_FILE=OUTPUT_FILE+ '.erb'


def get_template
  File.read(TEMPLATE_FILE)
end


RSpec::Core::RakeTask.new(:spec)
task default: [:spec]


desc "push to xxx.com"
task :push do
  Rake::Task["login"].invoke
  puts %x[cf push --no-start]
  Rake::Task["set-env"].invoke
  puts %x[cf start app-instance-monitor]
  Rake::Task["logout"].invoke
end


desc "set env of app"
task :'set-env' do
  puts system("COMMAND RUBY_ENVIRONMENT  #{ENV['RUBY_ENVIRONMENT']}")
end


desc "login to xxx.com"
task :login do
  conf = YAML::load(File.open('config/config.yml'))[ENV['RUBY_ENVIRONMENT']]
  puts %x[COMMAND #{conf['xxxx']['api']} --skip-ssl-validation]
  puts %x[cf login -u #{conf['xxxx']['username']} -p #{conf['user']['password']} -o system -s monitor]
end


desc "logout of xxx.com"
task :logout do
  puts %x[cf logout]
end

 6.  关于 Gemfile
source ''   #国外的就是rubygems.org，但国内用这个太慢了。
gem 'rake'  '~> xxx'   指定gem 的版本

7.    .ruby-version
2.2.4

8. 获取AWS的使用情况，代码练习。

#!/usr/bin/env ruby

require 'optparse'
require 'aws'

class AwsInfoHandler
  def self.acquire_account_info(account)
    name=account.fetch('name')
    id=account.fetch('account_id')


    ec2 = AWS::EC2.new(
      :access_key_id => account.fetch('aws_key'),
      :secret_access_key => account.fetch('aws_sec'),
      :region => account.fetch('region')
    )

    ec2.instances.inject({}) do |m, i|
      if m[i.availability_zone].nil?
        m[i.availability_zone] = {}
      end

      if m[i.availability_zone][i.instance_type].nil?
        m[i.availability_zone][i.instance_type]=1
      else
        m[i.availability_zone][i.instance_type]+=1
      end
      m
    end
  end


#### reservation, for S3 ###


  def self.publish_s3_report(s3_input_config)
    account = s3_input_config[:account]
    s3 = AWS::S3.new(
      :access_key_id => account.fetch('aws_key'),
      :secret_access_key => account.fetch('aws_sec')
    )
    bucket = s3.buckets[s3_input_config[:target_bucket]]
    object = bucket.objects[s3_input_config[:target_filename]]
    object.write(s3_input_config[:contents])
  end
end


######################


class ReportGenerator
  def initialize(accounts)
    @accounts = accounts
  end


  def generate_report
    report=[]


    @accounts.each do |ac|
      info = AwsInfoHandler.acquire_account_info(ac)
      puts "[INFO] Generating account info for #{ac.fetch('name')}"
      info.each do |k,v|
        v.each do |type, count|
          rp = {
            'account_name' => ac.fetch('name'),
            'account_id' => ac.fetch('account_id'),
            'AZ' => k,
            'flavor' => type,
            'count' => count
          }
          report << rp
        end
      end
    end
    report
  end
end


class EmailAgent
  def self.generate_and_email(report,report_to=nil)
    puts "[DEBUG] report for #{Time.now.strftime("%m/%d/%Y")}"
    puts "=========================="
    puts report
    puts "=========================="
    puts "[DEBUG] End of report"


=begin
MAIL_CONF={"mail-provider":[
   {
           "credentials": {
           "password": "qdWC4sqfLoD9Zg",
           "username": "cloudfoundry"
          }
    ]
   }
}
=end
    credentials = host = username = password = ''
    if !ENV['MAIL_CONF'].to_s.empty?
      JSON.parse(ENV['MAIL_CONF']).each do |k,v|
        if !k.scan("mail-provider").to_s.empty?
          credentials = v.first.select {|k1,v1| k1 == "credentials"}["credentials"]
          host = credentials["hostname"]
          username = credentials["username"]
          password = credentials["password"]
        end
        end
    end


### SMTP setings ###
# ......


# attachement


      report_file = File.new('/tmp/report.csv', 'w+')
      report_file.puts(report[0].keys.join(','))


      report.each do |r|
        report_file.puts(r.values.join(','))
      end


      report_file.close
### send out ###
## ......
  end
end

###  -c 来指定配置文件，实现代码和配置分离。
options = {}
optparse = OptionParser.new do |opts|
  opts.on('-h', '--help', 'Help Messages') do
    puts opts
    exit
  end
  opts.on('-c', '--config File', 'The Config file') do |f|
    options[:file] = f
  end
end
optparse.
unless options[:file]
  puts 'no config file specified, exit'
  exit
end


ruby_env = ENV['PLATFORM'] || 'production'
conf = YAML::load(File.open(options[:file]))[ruby_env]


accounts = conf.fetch('accounts')
mail_to = conf.fetch('to')
report = ReportGenerator.new(accounts).generate_report
EmailAgent.generate_and_email(report, mail_to)

# config.yml

---
production:
  accounts:
  - name: web
    account_id: xxxxxxxx7842
    aws_key: xxxx
    aws_sec: xxxx
  - name: db
    account_id: xxxxxxxx6344
    aws_key: xxx
    aws_sec: xxxx

  reports_email_to:
  - [e-mail]