Posted on 2006/04/07 21:11
Filed Under 클러스터란/파일시스템_CFS 조회수: view 5098

* Redhat ES4 환경에서의 GFS 와 GNBD 구성하기
                               *** 작성일 :  2005년 9 월15일
                               *** 작성자 :  서진우 (alang@sysmng.com)

- 기본 환경 구축 하기

3대의 서버에 Redhat ES4 를 설치 한다.

* hostname 정보

n001        192.168.123.111                gfs client1
n002        192.168.123.112                gfs client2
n003        192.168.123.113                gfs,gnbd server

먼저 3대 서버에 2.6.9-11 대 kernel 을 설치 한다.

3대 서버에 GFS, GNBD 관련 패키지를 설치 한다.

* 기본 Cluster 관련 패키지 설치

[root@n003 gfs+rhcs]# rpm -Uvh magma-1.0.0-0.i686.rpm
[root@n003 gfs+rhcs]# rpm -Uvh magma-devel-1.0.0-0.i686.rpm
[root@n003 gfs+rhcs]# rpm -Uvh ccs-1.0.0-0.i686.rpm
[root@n003 gfs+rhcs]# rpm -Uvh ccs-devel-1.0.0-0.i686.rpm
[root@n003 gfs+rhcs]# rpm -Uvh gulm-1.0.0-0.i686.rpm
[root@n003 gfs+rhcs]# rpm -Uvh gulm-devel-1.0.0-0.i686.rpm
[root@n003 gfs+rhcs]# rpm -Uvh magma-plugins-1.0.0-0.i386.rpm
[root@n003 gfs+rhcs]# rpm -Uvh cman-kernel-2.6.9-36.0.i686.rpm
[root@n003 gfs+rhcs]# rpm -Uvh cman-1.0.0-0.i686.rpm
[root@n003 gfs+rhcs]# rpm -Uvh dlm-kernel-2.6.9-34.0.i686.rpm
[root@n003 gfs+rhcs]# rpm -Uvh dlm-1.0.0-0.i686.rpm
[root@n003 gfs+rhcs]# rpm -Uvh dlm-devel-1.0.0-0.i686.rpm
[root@n003 gfs+rhcs]# rpm -Uvh perl-Net-Telnet-3.03-3.noarch.rpm
[root@n003 gfs+rhcs]# rpm -Uvh fence-1.32.1-0.i686.rpm
[root@n003 gfs]# rpm -Uvh gnbd-kernel-2.6.9-8.27.i686.rpm
[root@n003 gfs]# rpm -Uvh gnbd-1.0.0-0.i686.rpm

* Cluster File System 관련 패키지 설치

[root@n003 gfs]# rpm -Uvh GFS-kernel-2.6.9-35.5.i686.rpm
[root@n003 gfs]# rpm -Uvh GFS-6.1.0-0.i386.rpm
[root@n003 rhn-packages]# rpm -Uvh device-mapper-1.01.01-1.RHEL4.i386.rpm
[root@n003 rhn-packages]# rpm -Uvh lvm2-2.01.08-1.0.RHEL4.i386.rpm
[root@n003 gfs]# rpm -Uvh lvm2-cluster-2.01.09-5.0.RHEL4.i386.rpm
[root@n003 gfs+rhcs]# rpm -Uvh iddev-2.0.0-0.i686.rpm
[root@n003 gfs+rhcs]# rpm -Uvh iddev-devel-2.0.0-0.i686.rpm

기본적으로 GFS, GNBD 를 이용하기 위한 기본 패키지는 위와 같다. Redhat 의 Cluster Suit 과
같이 사용하기 위해서는 몇가지 패키지가 더 지원된다.

rgmanager-1.9.34-1.i386.rpm   ;; cluster resource 관련 프로그램
system-config-cluster-1.0.12-1.0.noarch.rpm ;; CCS 설정 프로그램


일단 기본 cluster.conf 파일을 설정한다.

[root@n003 gfs+rhcs]# cd /etc
[root@n003 etc]# ensh mkdir cluster
### executing in n001
### executing in n002
### executing in n003
[root@n003 cluster]# vi cluster.conf
--------------------------------------------------------------------------------------
<?xml version="1.0"?>
<cluster name="cluster" config_version="1">

<cman>
</cman>

<clusternodes>
<clusternode name="n001" votes="1">
        <fence>
                <method name="single">
                        <device name="human" ipaddr="n001"/>
                </method>
        </fence>
</clusternode>

<clusternode name="n002" votes="1">
        <fence>
                <method name="single">
                        <device name="human" ipaddr="n002"/>
                </method>
        </fence>
</clusternode>

<clusternode name="n003" votes="1">
        <fence>
                <method name="single">
                        <device name="human" ipaddr="nd03"/>
                </method>
        </fence>
</clusternode>
</clusternodes>

<fence_devices>
        <device name="human" agent="fence_manual"/>
</fence_devices>

</cluster>
--------------------------------------------------------------------------------------

[root@n003 cluster]# ensync cluster.conf

모듈을 탑재한다.

[root@n003 cluster]# depmod -a
[root@n003 cluster]# modprobe dm-mod
[root@n003 cluster]# modprobe gfs
[root@n003 cluster]# modprobe lock_dlm

[root@n003 cluster]# ccsd
[root@n003 cluster]# cman_tool join              
[root@n003 cluster]# fence_tool join
[root@n003 cluster]# clvmd
[root@n003 cluster]# vgchange -aly

이걸로 GFS 를 사용할 수 있는 준비 되었다.

정상적으로 동작하는지 몇가지 테스트를 해 보자 ..


[root@n003 cluster]# ccs_test connect
Connect successful.
Connection descriptor = 1

[root@n003 cluster]# ccs_test get node '//cluster/@name'
Get successful.
Value = <cluster>

[root@n003 cluster]# cat /proc/cluster/nodes
Node  Votes Exp Sts  Name
   1    1    3   M   n003
   2    1    3   M   n001
   3    1    3   M   n002

[root@n003 cluster]# cat /proc/cluster/dlm_stats
DLM stats (HZ=1000)

Lock operations:          0
Unlock operations:        0
Convert operations:       0
Completion ASTs:          0
Blocking ASTs:            0

Lockqueue        num  waittime   ave

[root@n003 cluster]# cat /proc/cluster/services
Service          Name                              GID LID State     Code
Fence Domain:    "default"                           1   2 run       -
[1 2 3]

DLM Lock Space:  "clvmd"                             2   3 run       -
[1 2 3]

[root@n003 cluster]# cman_tool status
Protocol version: 5.0.1
Config version: 1
Cluster name: cluster
Cluster ID: 13364
Cluster Member: Yes
Membership state: Cluster-Member
Nodes: 3
Expected_votes: 3
Total_votes: 3
Quorum: 2  
Active subsystems: 3
Node name: n003
Node addresses: 192.168.123.113  

이제 정상적으로 작동하는 것을 확인했다.

만일 서버를 Shutdown 시키기 위해서는 아래와 같은 절차가 필요하다.

[root@n003 cluster]# umount /mnt        
[root@n003 cluster]# vgchange -aln    
[root@n003 cluster]# killall clvmd    
[root@n003 cluster]# fence_tool leave
[root@n003 cluster]# cman_tool leave  
[root@n003 cluster]# killall ccsd    

Redhat RPM 으로 Install 시 chkconfig 에 자동 등록이 되어 Redhat 방식의 설정과 충돌이
발생 할 수 있으니 해당 deamon 을 모두 제거해 준다.

# chkconfig ccsd off
# chkconfig clvmd off
# chkconfig cman off
# chkconfig fenced off
# chkconfig gfs off
# chkconfig lock_gulmd off


GNBD 환경 준비 하기

gnbd export 시킬 hdd device 를 준비한다. fdisk 를 이용하여 partition 을 지정하도록 한다.

Gnbd 서버 셋팅

[root@n003 cluster]# modprobe gnbd
[root@n003 cluster]# /sbin/gnbd_serv -v
[root@n003 cluster]# gnbd_export -v -e export1 -d /dev/hda6

정상적으로 등록 되었는지 확인 한다.

[root@n003 cluster]# gnbd_export -v -l
Server[1] : export1
--------------------------
      file : /dev/hda6
   sectors : 19551042
  readonly : no
    cached : no
   timeout : 60

Gnbd 클라이언트 셋팅

[root@n001 ~]# modprobe gnbd
[root@n001 ~]# gnbd_import -v -i n003

정상적으로 셋팅 되었는지 확인 한다.
[root@n001 ~]# gnbd_import -v -l
Device name : export1
----------------------
    Minor # : 0
  Proc name : /dev/gnbd0
     Server : n003
       Port : 14567
      State : Close Connected Clear
   Readonly : No
    Sectors : 19551042

[root@n001 ~]# ll /dev/gnbd/export1
brw-r--r--  1 root root 251, 0 Oct  7 13:46 /dev/gnbd/export1


이제 /dev/gnbd/export1 device 를 local device 처럼 사용이 가능하다.

이제 Gnbd 서버로 가서 gfs file system 을 생성한다.

[root@n003 cluster]# gfs_mkfs -p lock_dlm -t cluster:export1 -j 3 /dev/hda6
This will destroy any data on /dev/hda6.

Are you sure you want to proceed? [y/n] y

이제 각 서버와 클라이어트에서 Mount 를 시킨다.

[root@n003 cluster]# mount -t gfs /dev/hda6 /mnt

[root@n001 ~]# mount -t gfs /dev/gnbd/export1 /mnt
[root@n002 ~]# mount -t gfs /dev/gnbd/export1 /mnt

[root@n001 ~]# df -Th | grep /mnt
/dev/gnbd/export1    gfs  9.0G   20K  9.0G   1% /mnt
[root@n002 ~]# df -Th | grep /mnt
/dev/gnbd/export1    gfs  9.0G   20K  9.0G   1% /mnt
[root@n003 cluster]# df -Th | grep /mnt
/dev/hda6            gfs 9.0G   20K  9.0G   1% /mnt

[root@n003 cluster]# cat /proc/cluster/services
Service          Name                              GID LID State     Code
Fence Domain:    "default"                           1   2 run       -
[1 2 3]

DLM Lock Space:  "clvmd"                             2   3 run       -
[1 2 3]

DLM Lock Space:  "export1"                           4   5 run       -
[1 2 3]

GFS Mount Group: "export1"                           5   6 run       -
[1 2 3]

User:            "gnbd"                              3   4 run       -
[1]



이제 LVM 을 이용하여 GFS+GNBD 클러스터 파일 시스템 구성을 해보자..
LVM 을 이용하면 GNBD 스토리지의 확장이 매우 편리하다.

먼저 GNBD Server 에 LVM 포맷으로 파티션을 두개정도 생성한다.

[root@n003 ~]# fdisk /dev/hda

   Device Boot      Start         End      Blocks   Id  System
/dev/hda1   *           1          13      104391   83  Linux
/dev/hda2              14         395     3068415   83  Linux
/dev/hda3             396        1670    10241437+  83  Linux
/dev/hda4            1671        9729    64733917+   5  Extended
/dev/hda5            1671        1801     1052226   82  Linux swap
/dev/hda6            1802        3018     9775521   83  Linux
/dev/hda7            3019        4235     9775521   8e  Linux LVM
/dev/hda8            4236        5452     9775521   8e  Linux LVM

[root@n003 ~]# partprobe

[root@n003 ~]# pvcreate /dev/hda7 /dev/hda8
  Physical volume "/dev/hda7" successfully created
  Physical volume "/dev/hda8" successfully created
[root@n003 ~]# vgcreate vg0 /dev/hda7 /dev/hda8
  Volume group "vg0" successfully created
[root@n003 ~]# lvcreate -L 15G -n lvmdata vg0
[root@n003 ~]# lvcreate -L 10G -n lvm0 vg0
[root@n003 ~]# gnbd_export -v -e export2 -d /dev/vg0/lvm0
[root@n002 data]# gnbd_import -i n003
gnbd_import: created gnbd device export2
gnbd_recvd: gnbd_recvd started

[root@n002 data]# ll /dev/gnbd/export
export1  export2  
[root@n002 data]# gnbd_import -l
Device name : export1
----------------------
    Minor # : 0
  Proc name : /dev/gnbd0
     Server : n003
       Port : 14567
      State : Open Connected Clear
   Readonly : No
    Sectors : 19551042

Device name : export2
----------------------
    Minor # : 1
  Proc name : /dev/gnbd1
     Server : n003
       Port : 14567
      State : Close Connected Clear
   Readonly : No
    Sectors : 20971520

gnbd_export: created GNBD export2 serving file /dev/vg0/lvm0

[root@n003 ~]# gfs_mkfs -p lock_dlm -t cluster:export2 -j 3 /dev/vg0/lvm0

[root@n003 ~]# mount -t gfs /dev/vg0/lvm0 /lvmdata
[root@n002 ~]# mount -t gfs /dev/gnbd/export2 /lvmdata/
[root@n001 ~]# mount -t gfs /dev/gnbd/export2 /lvmdata/

[root@n003 ~]# df -Th
Filesystem           Type    Size  Used Avail Use% Mounted on
/dev/hda2            ext3    2.9G  231M  2.6G   9% /
/dev/hda1            ext3     99M   11M   83M  12% /boot
none                 tmpfs    249M     0  249M   0% /dev/shm
/dev/hda3            ext3    9.7G  2.3G  6.9G  26% /usr
/dev/hda6             gfs    9.0G   81M  8.9G   1% /data
/dev/mapper/vg0-lvm0  gfs    9.7G   20K  9.7G   1% /lvmdata



##################################################################


* gfs start scripts *

modprobe dm-mod
modprobe gfs
modprobe lock_dlm
ccsd
sleep 2
cman_tool join
sleep 2
fence_tool join
sleep 2
clvmd
modprobe gnbd
gnbd_import -v -i n003
mount -t gfs /dev/gnbd/export1 /data



* gfs stop scripts *

umount /data
sleep 2
gnbd_import -r export1
killall clvmd
fence_tool leave
sleep 2
cman_tool leave
killall ccsd
rmmod gnbd
rmmod lock_dlm
rmmod gfs
rmmod dm-mod


### gfs initsripts ########################################################################


#!/bin/bash
#
#
#
# description: mount/unmount gfs filesystems configured in /etc/fstab
#
#              
### BEGIN INIT INFO
# Provides:
### END INIT INFO

. /etc/init.d/functions
[ -f /etc/sysconfig/cluster ] && . /etc/sysconfig/cluster

#
# This script's behavior is modeled closely after the netfs script.  
#
GFSFSTAB=`cat /etc/fstab | grep gfs | awk '{print $2}'`
GFSMTAB=$(LC_ALL=C awk '!/^#/ && $3 ~ /^gfs/ && $2 != "/" { print $2 }' /proc/mounts)

# See how we were called.
case "$1" in
  start)
        if [ -n "$GFSFSTAB" ]
        then
                #load_module lock_harness MODULE_LOCK_HARNESS
                #load_module crc32        MODULE_CRC32
                modprobe lock_dlm
                modprobe gfs

                action $"Mounting GFS filesystems: " mount -a -t gfs
        fi
        touch /var/lock/subsys/gfs
        ;;

  stop)
        if [ -n "$GFSMTAB" ]
        then
                sig=
                retry=6
                remaining=`LC_ALL=C awk '!/^#/ && $3 ~ /^gfs/ && $2 != "/" {print $2}' /proc/mounts`
                while [ -n "$remaining" -a "$retry" -gt 0 ]
                do
                        action $"Unmounting GFS filesystems: " umount -a -t gfs

                        if [ $retry -eq 0 ]
                        then
                                action $"Unmounting GFS filesystems (lazy): " umount -l -a -t gfs
                                break
                        fi

                        sleep 2
                        remaining=`LC_ALL=C awk '!/^#/ && $3 ~ /^gfs/ && $2 != "/" {print $2}' /proc/mounts`
                        [ -z "$remaining" ] && break
                        /sbin/fuser -k -m $sig $remaining >/dev/null
                        sleep 10
                        retry=$(($retry - 1))
                        sig=-9
                done
        fi

        modprobe -r gfs
        modprobe -r lock_dlm
        rm -f /var/lock/subsys/gfs
        ;;

  status)
        if [ -f /proc/mounts ]
        then
                [ -n "$GFSFSTAB" ] && {
                     echo $"Configured GFS mountpoints: "
                     for fs in $GFSFSTAB; do echo $fs ; done
                }
                [ -n "$GFSMTAB" ] && {
                      echo $"Active GFS mountpoints: "
                      for fs in $GFSMTAB; do echo $fs ; done
                }
        else
                echo "/proc filesystem unavailable"
        fi
        ;;

  restart)
        $0 stop
        $0 start
        ;;

  reload)
        $0 start
        ;;
  *)
        echo $"Usage: $0 {start|stop|restart|reload|status}"
        exit 1
esac

exit 0
-----------------------------------------------------------------------------------------

#### /etc/fstab modify ######################################################################


/dev/sda1               /gfs                    gfs     defaults        0 0


#### nfs + ha + gfs #########################################################################

- nfs server

/etc/exports
----------------------------------------------------------------------------------------------
/gfs   *(rw,sync,no_root_squash,fsid=1)

/etc/ha.d/ha.cf
----------------------------------------------------------------------------------------------
debugfile /var/log/ha-debug
logfile /var/log/ha-log
logfacility     local0
keepalive 2
deadtime 5
hopfudge 1
udpport 1001
udp     eth0
auto_failback off
node    xeon01
node    xeon02
ping 192.168.123.254
#respawn hacluster /usr/lib/heartbeat/ccm
respawn hacluster /usr/lib/heartbeat/ipfail
apiauth ipfail uid=hacluster


/etc/ha.d/haresources
--------------------------------------------------------------------------------------------------
xeon01 192.168.123.118 nfslock nfs
xeon02 192.168.123.119 nfslock nfs


Writer profile
author image
-아랑 -
2006/04/07 21:11 2006/04/07 21:11

트랙백 주소 : 이 글에는 트랙백을 보낼 수 없습니다

About

by 서진우
Twitter :@muchunalang

Counter

• Total
: 4164447
• Today
: 385
• Yesterday
: 1075