gluster增加xlator操作（一）

查看更多，请猛点向glusterfs中增加xlator（一）
前面已经介绍过，glusterfs中主线为xlator，几乎每一个大的功能点，或者性能，都可以以xlator形式参与进glusterfs中，那么xlator的开发就必不可少，所以，这里介绍一下简单的向glusterfs中增加xlator的方法，首先创建一个卷testvol

[root@CM addxlator]# mkdir /opt/test
[root@CM addxlator]# gluster volume create testvol 192.168.0.194:/opt/test force
volume create: testvol: success: please start the volume to access data
[root@CM addxlator]# gluster volume info all
 
Volume Name: testvol
Type: Distribute
Volume ID: f0ef2a09-09c0-4a84-aeb6-3f2e0aebf4c2
Status: Created
Number of Bricks: 1
Transport-type: tcp
Bricks:
Brick1: 192.168.0.194:/opt/test
[root@CM addxlator]# 
[root@CM addxlator]# gluster volume start testvol force
volume start: testvol: success
[root@CM addxlator]# 

创建成功后，可以查看volume file中的内容

 cat /var/lib/glusterd/vols/testvol/testvol-fuse.vol 

volume testvol-client-0
    type protocol/client
    option transport-type tcp
    option remote-subvolume /opt/test
    option remote-host 192.168.0.194
end-volume

volume testvol-dht
    type cluster/distribute
    subvolumes testvol-client-0
end-volume

volume testvol-write-behind
    type performance/write-behind
    subvolumes testvol-dht
end-volume

volume testvol-read-ahead
    type performance/read-ahead
    subvolumes testvol-write-behind
end-volume

volume testvol-io-cache
    type performance/io-cache
    subvolumes testvol-read-ahead
end-volume

volume testvol-quick-read
    type performance/quick-read
    subvolumes testvol-io-cache
end-volume

volume testvol-open-behind
    type performance/open-behind
    subvolumes testvol-quick-read
end-volume

volume testvol-md-cache
    type performance/md-cache
    subvolumes testvol-open-behind
end-volume

volume testvol
    type debug/io-stats
    option count-fop-hits off
    option latency-measurement off
    subvolumes testvol-md-cache
end-volume

这个配置为默认的配置，接下来需要将自己的xlator增加入该配置中，根据之前所提到过的xlator_dyload的过程，需要修改这个volume file，来增加xlator，修改配置如下：

volume testvol-client-0
    type protocol/client
    option transport-type tcp
    option remote-subvolume /opt/test
    option remote-host 192.168.0.194
end-volume

volume testvol-dht
    type cluster/distribute
    subvolumes testvol-client-0
end-volume

volume testvol-write-behind
    type performance/write-behind
    subvolumes testvol-dht
end-volume

volume testvol-read-ahead
    type performance/read-ahead
    subvolumes testvol-write-behind
end-volume

volume testvol-io-cache
    type performance/io-cache
    subvolumes testvol-read-ahead
end-volume

volume testvol-quick-read
    type performance/quick-read
    subvolumes testvol-io-cache
end-volume

volume testvol-open-behind
    type performance/open-behind
    subvolumes testvol-quick-read
end-volume

volume testvol-md-cache
    type performance/md-cache
    subvolumes testvol-open-behind
end-volume  volume testvol-test
    type debug/test
    subvolumes testvol-md-cache
end-volume volume testvol
    type debug/io-stats
    option count-fop-hits off
    option latency-measurement off
    subvolumes testvol-test
end-volume

可以看到，配置中多了个testvol-test这么一层，接下来将glusterfs挂载至挂载点查看输出log

[root@CM addxlator]# 
[root@CM addxlator]# mount -t glusterfs 192.168.0.194:testvol /media
[root@CM addxlator]# 

查看gluster挂载到的挂载点对应的log输出信息
cat /usr/local/var/log/glusterfs/media.log

[2013-06-10 11:00:29.899440] I [glusterfsd.c:1878:main] 0-/usr/local/sbin/glusterfs: Started running /usr/local/sbin/glusterfs version 3git (/usr/local/sbin/glusterfs --volfile-id=testvol --volfile-server=192.168.0.194 /media)
[2013-06-10 11:00:29.903627] I [socket.c:3515:socket_init] 0-glusterfs: SSL support is NOT enabled
[2013-06-10 11:00:29.903823] I [socket.c:3530:socket_init] 0-glusterfs: using system polling thread
[2013-06-10 11:00:29.909423] E [test.c:54:init] 0-testvol-test: test translator loaded
[2013-06-10 11:00:29.909718] I [dht-shared.c:311:dht_init_regex] 0-testvol-dht: using regex rsync-hash-regex = ^\.(.+)\.[^.]+$
[2013-06-10 11:00:29.911047] I [socket.c:3515:socket_init] 0-testvol-client-0: SSL support is NOT enabled
[2013-06-10 11:00:29.911173] I [socket.c:3530:socket_init] 0-testvol-client-0: using system polling thread
[2013-06-10 11:00:29.911218] I [client.c:2236:notify] 0-testvol-client-0: parent translators are ready, attempting connect on transport
Final graph:
+------------------------------------------------------------------------------+
  1: volume testvol-client-0
  2:     type protocol/client
  3:     option remote-host 192.168.0.194
  4:     option remote-subvolume /opt/test
  5:     option transport-type socket
  6: end-volume
  7: 
  8: volume testvol-dht
  9:     type cluster/distribute
 10:     subvolumes testvol-client-0
 11: end-volume
 12: 
 13: volume testvol-write-behind
 14:     type performance/write-behind
 15:     subvolumes testvol-dht
 16: end-volume
 17: 
 18: volume testvol-read-ahead
 19:     type performance/read-ahead
 20:     subvolumes testvol-write-behind
 21: end-volume
 22: 
 23: volume testvol-io-cache
 24:     type performance/io-cache
 25:     subvolumes testvol-read-ahead
 26: end-volume
 27: 
 28: volume testvol-quick-read
 29:     type performance/quick-read
 30:     subvolumes testvol-io-cache
 31: end-volume
 32: 
 33: volume testvol-open-behind
 34:     type performance/open-behind
 35:     subvolumes testvol-quick-read
 36: end-volume
 37: 
 38: volume testvol-md-cache
 39:     type performance/md-cache
 40:     subvolumes testvol-open-behind
 41: end-volume
 42: 
 43: volume testvol-test
 44:     type debug/test
 45:     subvolumes testvol-md-cache
 46: end-volume
 47: 
 48: volume testvol
 49:     type debug/io-stats
 50:     option latency-measurement off
 51:     option count-fop-hits off
 52:     subvolumes testvol-test
 53: end-volume
 54: 
+------------------------------------------------------------------------------+
[2013-06-10 11:00:29.914847] I [rpc-clnt.c:1675:rpc_clnt_reconfig] 0-testvol-client-0: changing port to 49152 (from 0)
[2013-06-10 11:00:29.918227] I [client-handshake.c:1658:select_server_supported_programs] 0-testvol-client-0: Using Program GlusterFS 3.3, Num (1298437), Version (330)
[2013-06-10 11:00:29.918482] I [client-handshake.c:1456:client_setvolume_cbk] 0-testvol-client-0: Connected to 192.168.0.194:49152, attached to remote volume '/opt/test'.
[2013-06-10 11:00:29.918548] I [client-handshake.c:1468:client_setvolume_cbk] 0-testvol-client-0: Server and Client lk-version numbers are not same, reopening the fds
[2013-06-10 11:00:29.923683] I [fuse-bridge.c:4767:fuse_graph_setup] 0-fuse: switched to graph 0
[2013-06-10 11:00:29.923986] I [client-handshake.c:450:client_set_lk_version_cbk] 0-testvol-client-0: Server lk version = 1
[2013-06-10 11:00:29.924164] I [fuse-bridge.c:3724:fuse_init] 0-glusterfs-fuse: FUSE inited with protocol versions: glusterfs 7.13 kernel 7.13
[2013-06-10 11:00:29.924225] E [test.c:28:test_lookup] 0-testvol-test: in test translator lookup
[2013-06-10 11:00:29.925153] E [test.c:28:test_lookup] 0-testvol-test: in test translator lookup

经过查看log，可以看到test模块中的打印输出， in test translator lookup
接下来查看描述一下外挂xlator的方式
首先现在glusterfs的源代码，编译以后，在本地也就存在了glusterfs的开发环境，为了开发调试方便，可暂不将xlator存于glusterfs工程中，这时可以编写一个Makefile
首先建立一个目录

mkdir addxlator_sample
cd  addxlator_sample

Makefile中内容可如下：

#Add translator into glusterfs volume file
#Author Steven Liu
#E-mail [e-mail]
#Blog: http://blog.fs-linux.org

TARGET	= test.so
OBJECTS	= test.o

GLUSTERFS_SRC	= /opt/glusterfs
GLUSTERFS_LIB	= /usr/local/lib
HOST_OS	= HF_LINUX_HOST_OS

CFLAGS	= -fPIC -Wall -O0 -g \
		  -DHAVE_CONFIG_H -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -D$(HOST_OS) \
		  -I$(GLUSTERFS_SRC) -I$(GLUSTERFS_SRC)/libglusterfs/src \
		  -I$(GLUSTERFS_SRC)/contrib/uuid

LDFLAGS	= -shared -nostartfiles -L$(GLUSTERFS_LIB) -lglusterfs -lpthread

$(TARGET): $(OBJECTS)
	$(CC) $(OBJECTS) $(LDFLAGS) -o $(TARGET)


clean:
	rm -rf $(TARGET) $(OBJECTS)

Makefile编写完成之后，需要OBJECTS，这时需要编写test.c来保证编译通过，当然，test这个xlator也需要在这里生成

#ifndef _CONFIG_H
#define _CONFIG_H
#include "config.h"
#include "xlator.h"
#endif

#include 
#include 
#include "glusterfs.h"
#include "xlator.h"
#include 
#include "defaults.h"
#include "logging.h"

int test_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno,
                     inode_t *inode, struct iatt *buf,
                     dict_t *xdata, struct iatt *postparent)
{
        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf, xdata,
                             postparent);
        return 0;
}


static int test_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
{
	gf_log(this->name, GF_LOG_ERROR, "in test translator lookup");
	STACK_WIND (frame, test_lookup_cbk, 
			FIRST_CHILD(this), FIRST_CHILD(this)->fops->lookup, 
			loc, xdata);
	return 0;
}

static int test_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
{
	gf_log(this->name, GF_LOG_ERROR, "in test translator stat");
	return 0;
}

int
reconfigure (xlator_t *this, dict_t *options)
{
        return 0;
}


int
init (xlator_t *this)
{
        struct ios_conf    *conf = NULL;
        int                 ret = -1;

	gf_log (this->name, GF_LOG_ERROR, "test translator loaded");
        if (!this)
                return -1;

        if (!this->children) {
                gf_log (this->name, GF_LOG_ERROR,
                        "test translator requires atleast one subvolume");
                return -1;
        }

        if (!this->parents) {
              gf_log (this->name, GF_LOG_ERROR, "dangling volume. check volfile ");
        }

		conf = this->private;

        this->private = conf;
        ret = 0;
        return ret;
}


void
fini (xlator_t *this)
{
        struct ios_conf *conf = NULL;

        if (!this)
                return;

        conf = this->private;

        if (!conf)
                return;
        this->private = NULL;

        GF_FREE(conf);
        gf_log (this->name, GF_LOG_ERROR, "test translator unloaded");
        return;
}

int
notify (xlator_t *this, int32_t event, void *data, ...)
{
	default_notify (this, event, data);
        return 0;
}

struct xlator_fops fops = {
        .stat        = test_stat,
        .lookup      = test_lookup,
};

struct xlator_cbks cbks = {
};

struct volume_options options[] = {
 };

目前仅用于测试的xlator而已，所以自己定义了lookup与stat，还有lookup_cbk，其他的并未定义，全部走default，这个原理在之前已经提到过。
然后执行make既可生成对应的xlator文件

Makefile  test.c  test.o  test.so

将test.so复制至/usr/local/lib/glusterfs/3git/xlator/debug中
然后执行glusterfs的客户端，mount至挂载节点，如果成功，可以查看log，整个过程成功矣~~！