用 FC FOR nexenta 山寨SAN儲存

edwardking888發表於2012-12-11

最近公司到了一臺R510滿配伺服器，就裝了2張光纖卡，用來山寨SAN儲存。

作業系統選擇的是 nexenta （www.nexenta.com下載，社群版本，其實nexenta 是基於opensolaris系統的，所以讀效能還不錯，比OPENFILER好很多！要是可以裝上SSD硬碟，做讀寫的1級cache，效能會有大幅度提高）

nexenta 每1TB資料需要4GB記憶體做後設資料快取。

nexenta 安裝就不介紹了，網上一大堆，

nexenta 社群版本，容量有限制，必須小於18TB，並且圖形介面不能做 FC FOR nexenta，但是可以使用命令列實現。

言歸正傳，安裝好，nexenta後，進行如下設定：

1，ssh登入伺服器

2.升級和安裝必要的軟體包

setup appliance upgrade -v

setup appliance upgrade sunwlibc

3.進入expert mode模式：

$ option expert_mode=1

$ !bash

4. 檢視並設定光纖卡

root@myhost:/volumes# svcadm enable stmf

root@myhost:/volumes# svcs -v stmf

STATE NSTATE STIME CTID FMRI

online - 3:12:33 - svc:/system/stmf:default

root@myhost:/volumes# mdb -k

Loading modules: [ unix genunix specfs dtrace mac cpu.generic uppc pcplusmp scsi_vhci zfs sd sata mr_sas qlc sockfs ip hook

neti sctp arp usba uhci stmf_sbd stmf lofs idm cpc random crypto smbsrv nfs logindmux ptm nsctl sdbc ufs nsmb sv rdc ii ipc ]

> ::devbindings -q qlc

ffffff06f652da48 pciex1077,2432, instance #0 (driver name: qlc)

ffffff06f652d7c0 pciex1077,2432, instance #1 (driver name: qlc)

在/etc/driver_aliases檔案新增下面2行資訊：

vi /etc/driver_aliases:

qlt "pciex1077,2432"

qlc "pciex1077,2532"

重啟伺服器。

root@myhost:/volumes# fcinfo hba-port

HBA Port WWN: 21000024ff062e5c

Port Mode: Target

Port ID: 10400

OS Device Name: Not Applicable

Manufacturer: QLogic Corp.

Model: QLE2460

Firmware Version: 5.2.1

FCode/BIOS Version: N/A

Serial Number: not available

Driver Name: COMSTAR QLT

Driver Version: 20100505-1.05

Type: F-port

State: online

Supported Speeds: 1Gb 2Gb 4Gb

Current Speed: 4Gb

Node WWN: 20000024ff062e5c

HBA Port WWN: 2100001b32944fd3

Port Mode: Target

Port ID: 10500

OS Device Name: Not Applicable

Manufacturer: QLogic Corp.

Model: QLE2460

Firmware Version: 5.2.1

FCode/BIOS Version: N/A

Serial Number: not available

Driver Name: COMSTAR QLT

Driver Version: 20100505-1.05

Type: F-port

State: online

Supported Speeds: 1Gb 2Gb 4Gb

Current Speed: 4Gb

Node WWN: 2000001b32944fd3

root@myhost:/volumes# stmfadm list-target

Target: wwn.2100001B32944FD3

Target: wwn.21000024FF062E5C

root@myhost:/volumes# format

Searching for disks...done

AVAILABLE DISK SELECTIONS:

0. c0t0d0

/pci@0,0/pci8086,340a@3/pci1028,1f17@0/sd@0,0

1. c0t1d0

/pci@0,0/pci8086,340a@3/pci1028,1f17@0/sd@1,0

Specify disk (enter its number): ^C

/加引數 -f是強制建立

zpool create -f BIFC c0t1d0

root@myhost:/volumes# zpool status BIFC

pool: BIFC

state: ONLINE

scan: none requested

config:

NAME STATE READ WRITE CKSUM

BIFC ONLINE 0 0 0

c0t1d0 ONLINE 0 0 0

errors: No known data errors

root@myhost:/volumes# zfs list

NAME USED AVAIL REFER MOUNTPOINT

BIFC 95.5K 10.7T 31K /BIFC

syspool 19.1G 255G 34.5K legacy

syspool/dump 16.8G 255G 16.8G -

syspool/rootfs-nmu-000 1.28G 255G 860M legacy

syspool/rootfs-nmu-001 62.5K 255G 837M legacy

syspool/rootfs-nmu-002 61.5K 255G 840M legacy

syspool/rootfs-nmu-003 68.5K 255G 841M legacy

syspool/rootfs-nmu-004 75.5K 255G 841M legacy

syspool/rootfs-nmu-005 80.5K 255G 840M legacy

syspool/rootfs-nmu-006 87.5K 255G 840M legacy

syspool/rootfs-nmu-007 89.5K 255G 841M legacy

syspool/swap 1.03G 256G 16K -

root@myhost:/volumes# df -h

Filesystem size used avail capacity Mounted on

syspool/rootfs-nmu-000

274G 860M 255G 1% /

/devices 0K 0K 0K 0% /devices

/dev 0K 0K 0K 0% /dev

ctfs 0K 0K 0K 0% /system/contract

proc 0K 0K 0K 0% /proc

mnttab 0K 0K 0K 0% /etc/mnttab

swap 20G 304K 20G 1% /etc/svc/volatile

objfs 0K 0K 0K 0% /system/object

sharefs 0K 0K 0K 0% /etc/dfs/sharetab

/usr/lib/libc/libc_hwcap1.so.1

255G 860M 255G 1% /lib/libc.so.1

fd 0K 0K 0K 0% /dev/fd

swap 20G 152K 20G 1% /tmp

swap 20G 104K 20G 1% /var/run

BIFC 11T 31K 11T 1% /BIFC

After creating a data pool we can start carving out ZFS volumes (zVol):

zfs create [-ps] [-b blocksize] [-o property=value] ... -V size volume

root@myhost:/volumes# zfs create -V 7T BIFC/akebono-scratchvol

sbdadm – SCSI block device administration CLI

Albeit we have our zVol available we have to tell STMF that this is a volume that can be mapped as a LUN, this is why sbdadm

is here:

root@myhost:/volumes# sbdadm create-lu /dev/zvol/rdsk/BIFC/akebono-scratchvol

Created the following LU:

GUID DATA SIZE SOURCE

-------------------------------- ------------------- ----------------

600144f0383e8200000050c00a8e0004 7696581394432 /dev/zvol/rdsk/BIFC/akebono-scratchvol

root@myhost:/volumes# stmfadm add-view 600144f0383e8200000050c00a8e0004

root@myhost:/volumes# stmfadm list-view -l 600144f0383e8200000050c00a8e0004

View Entry: 0

Host group : All

Target group : All

LUN : 0

root@myhost:/volumes# stmfadm create-hg akebono

root@myhost:/volumes# stmfadm list-hg

Host Group: akebono

root@myhost:/volumes# stmfadm list-target -v

Target: wwn.2100001B32944FD3

Operational Status: Online

Provider Name : qlt

Alias : qlt1,0

Protocol : Fibre Channel

Sessions : 2

Initiator: wwn.2100001B3282A309

Alias: -

Logged in since: Wed Dec 5 17:27:51 2012

Initiator: wwn.2100001B3282270A

Alias: -

Logged in since: Wed Dec 5 17:27:51 2012

Target: wwn.21000024FF062E5C

Operational Status: Online

Provider Name : qlt

Alias : qlt0,0

Protocol : Fibre Channel

Sessions : 3

Initiator: wwn.2100001B32829B04

Alias: -

Logged in since: Wed Dec 5 17:28:00 2012

Initiator: wwn.2100001B3282EC05

Alias: -

Logged in since: Wed Dec 5 17:28:00 2012

Initiator: wwn.2100001B3282FE02

Alias: -

Logged in since: Wed Dec 5 17:28:00 2012

stmfadm add-hg-member -g akebono wwn.2100001B3282270A

stmfadm add-hg-member -g akebono wwn.2100001b32829b04

stmfadm add-hg-member -g akebono wwn.2100001b3282a309

stmfadm add-hg-member -g akebono wwn.2100001b3282fe02

root@myhost:/volumes# stmfadm list-hg -v

Host Group: akebono

Member: wwn.2100001B3282270A

Member: wwn.2100001B32829B04

Member: wwn.2100001B3282A309

Member: wwn.2100001B3282FE02

root@myhost:/volumes# stmfadm list-view -l 600144f0383e8200000050c00a8e0004

View Entry: 0

Host group : All

Target group : All

LUN : 0

root@myhost:/volumes# stmfadm remove-view -a -l 600144f0383e8200000050c00a8e0004

root@myhost:/dev/zvol/rdsk/BIFC# stmfadm add-view -h akebono 600144f0383e8200000050c00a8e0004

root@myhost:/dev/zvol/rdsk/BIFC# stmfadm list-view -l 600144f0383e8200000050c00a8e0004

View Entry: 0

Host group : akebono

Target group : All

LUN : 0

Target: wwn.2100001B32944FD3

Target: wwn.21000024FF062E5C

root@myhost:/dev/zvol/rdsk/BIFC# stmfadm create-tg fc-ports

root@myhost:/dev/zvol/rdsk/BIFC# stmfadm add-tg-member -g fc-ports wwn.2100001B32944FD3

stmfadm: STMF target must be offline

root@myhost:/dev/zvol/rdsk/BIFC# stmfadm offline-target wwn.2100001B32944FD3

root@myhost:/dev/zvol/rdsk/BIFC# stmfadm offline-target wwn.21000024FF062E5C

root@myhost:/dev/zvol/rdsk/BIFC# stmfadm add-tg-member -g fc-ports wwn.2100001B32944FD3

root@myhost:/dev/zvol/rdsk/BIFC# stmfadm add-tg-member -g fc-ports wwn.21000024FF062E5C

root@myhost:/dev/zvol/rdsk/BIFC# stmfadm online-target wwn.2100001B32944FD3

root@myhost:/dev/zvol/rdsk/BIFC# stmfadm online-target wwn.21000024FF062E5C

目標機（掛載儲存伺服器）設定：

root@prod # luxadm probe

No Network Array enclosures found in /dev/es

Found Fibre Channel device(s):

Node WWN:200200a0b8563d8e Device Type:Disk device

Logical Path:/dev/rdsk/c1t201300A0B8563D8Ed31s2

Logical Path:/dev/rdsk/c2t201200A0B8563D8Ed31s2

Node WWN:200200a0b8564f0e Device Type:Disk device

Logical Path:/dev/rdsk/c1t201300A0B8564F0Ed31s2

Logical Path:/dev/rdsk/c2t202200A0B8564F0Ed31s2

Node WWN:200200a0b8563d8e Device Type:Disk device

Logical Path:/dev/rdsk/c4t600A0B8000563D8E000002F64AA02509d0s2

Node WWN:200200a0b8563d8e Device Type:Disk device

Logical Path:/dev/rdsk/c4t600A0B8000563D8E000002F94AA02524d0s2

Node WWN:200200a0b8563d8e Device Type:Disk device

Logical Path:/dev/rdsk/c4t600A0B8000563D8E000002FB4AA02699d0s2

Node WWN:200200a0b8563d8e Device Type:Disk device

Logical Path:/dev/rdsk/c4t600A0B8000563DC6000005124BB1A243d0s2

Node WWN:200200a0b8564f0e Device Type:Disk device

Logical Path:/dev/rdsk/c4t600A0B8000564E18000006694BB30DB5d0s2

Node WWN:200200a0b8564f0e Device Type:Disk device

Logical Path:/dev/rdsk/c4t600A0B8000564F0E000006EE4BB30D1Fd0s2

Node WWN:2000001b32944fd3 Device Type:Disk device

Logical Path:/dev/rdsk/c4t600144F0383E8200000050C00A8E0004d0s2

root@prod # luxadm -e port

/devices/pci@400/pci@0/pci@8/pci@0/pci@9/SUNW,qlc@0/fp@0,0:devctl CONNECTED

/devices/pci@500/pci@0/pci@d/SUNW,qlc@0/fp@0,0:devctl CONNECTED

從中可以看到有2塊光纖卡連線到儲存裝置。

選擇已經連線的HBA 卡,檢視其WWN 號：

格式: # luxadm -e dump_map 裝置port。其中port 值可從步驟2 得到。如下所示:

Current Speed: 4Gb

Node WWN: 2000001b3282fe02

root@prod # luxadm -e dump_map /devices/pci@500/pci@0/pci@d/SUNW,qlc@0/fp@0,0:devctl

Pos Port_ID Hard_Addr Port WWN Node WWN Type

0 10100 0 2100001b32829b04 2000001b32829b04 0x1f (Unknown Type)

1 10200 0 201200a0b8563d8e 200200a0b8563d8e 0x0 (Disk device)

2 10300 0 2100001b3282ec05 2000001b3282ec05 0x1f (Unknown Type)

3 10400 0 21000024ff062e5c 20000024ff062e5c 0x0 (Disk device)

4 10500 0 202200a0b8564f0e 200200a0b8564f0e 0x0 (Disk device)

5 10000 0 2100001b3282fe02 2000001b3282fe02 0x1f (Unknown Type,Host Bus Adapter)

root@prod # luxadm -e dump_map /devices/pci@400/pci@0/pci@8/pci@0/pci@9/SUNW,qlc@0/fp@0,0:devctl

Pos Port_ID Hard_Addr Port WWN Node WWN Type

0 10000 0 2100001b3282270a 2000001b3282270a 0x1f (Unknown Type)

1 10100 0 201300a0b8563d8e 200200a0b8563d8e 0x0 (Disk device)

2 10400 0 201300a0b8564f0e 200200a0b8564f0e 0x0 (Disk device)

3 10500 0 2100001b32944fd3 2000001b32944fd3 0x0 (Disk device)

4 10800 0 2100001b32822d0a 2000001b32822d0a 0x1f (Unknown Type)

5 10200 0 2100001b3282a309 2000001b3282a309 0x1f (Unknown Type,Host Bus Adapter)

從該命令我們輸出看到有5 個WWN，那麼究竟這5 個WWN 具體代表的含義是什麼呢？

Unknown Type,Host Bus Adapter，在此代表本地HBA 卡WWN。

Unknown Type，有時從上述命令輸出有該型別的WWN,有時沒有，這跟系統是否為叢集有關。如果是叢集，需要將所有叢集節點的HBA 卡及儲存前端埠

換分在以個Zone 裡面，所以上述命令大的輸出為Zone 裡面所有節點的WWN。Unknown Type 顯示的WWN 其實是叢集節點對端的HBA 卡WWN。

Disk device，表示儲存前端埠WWN。

root@prod # cfgadm -al

Ap_Id Type Receptacle Occupant Condition

c0 scsi-bus connected configured unknown

c0::dsk/c0t0d0 disk connected configured unknown

c1 fc-fabric connected configured unknown

c1::201300a0b8563d8e disk connected configured unknown

c1::201300a0b8564f0e disk connected configured unknown

c1::2100001b3282270a unknown connected unconfigured unknown

c1::2100001b32822d0a unknown connected unconfigured unknown

c1::2100001b32944fd3 disk connected configured unknown

c2 fc-fabric connected configured unknown

c2::201200a0b8563d8e disk connected configured unknown

c2::202200a0b8564f0e disk connected configured unknown

c2::2100001b32829b04 unknown connected unconfigured unknown

c2::2100001b3282ec05 unknown connected unconfigured unknown

c2::21000024ff062e5c disk connected configured unknown

usb0/1 unknown empty unconfigured ok

usb0/2 unknown empty unconfigured ok

usb0/3 unknown empty unconfigured ok

usb1/1 unknown empty unconfigured ok

usb1/2 unknown empty unconfigured ok

usb2/1 unknown empty unconfigured ok

usb2/2 usb-hub connected configured ok

usb2/2.1 unknown empty unconfigured ok

usb2/2.2 unknown empty unconfigured ok

usb2/2.3 usb-storage connected configured ok

usb2/2.4 unknown empty unconfigured ok

usb2/3 unknown empty unconfigured ok

usb2/4 unknown empty unconfigured ok

usb2/5 unknown empty unconfigured ok

root@prod # cfgadm -c configure c1::2100001b32944fd3

root@prod # cfgadm -c configure c3::21000024ff062e5c

# format

Searching for disks...done

c4t600144F0383E8200000050C00A8E0004d0: configured with capacity of 7168.00GB

AVAILABLE DISK SELECTIONS:

0. c0t0d0

/pci@400/pci@0/pci@1/scsi@0/sd@0,0

1. c0t1d0

/pci@400/pci@0/pci@1/scsi@0/sd@1,0

2. c0t2d0

/pci@400/pci@0/pci@1/scsi@0/sd@2,0

3. c1t201300A0B8563D8Ed31

/pci@400/pci@0/pci@8/pci@0/pci@9/SUNW,qlc@0/fp@0,0/ssd@w201300a0b8563d8e,1f

4. c1t201300A0B8564F0Ed31

/pci@400/pci@0/pci@8/pci@0/pci@9/SUNW,qlc@0/fp@0,0/ssd@w201300a0b8564f0e,1f

5. c3t201200A0B8563D8Ed31

/pci@500/pci@0/pci@d/SUNW,qlc@0/fp@0,0/ssd@w201200a0b8563d8e,1f

6. c3t202200A0B8564F0Ed31

/pci@500/pci@0/pci@d/SUNW,qlc@0/fp@0,0/ssd@w202200a0b8564f0e,1f

7. c4t600A0B8000563D8E000002F64AA02509d0

/scsi_vhci/ssd@g600a0b8000563d8e000002f64aa02509

8. c4t600A0B8000563D8E000002F94AA02524d0

/scsi_vhci/ssd@g600a0b8000563d8e000002f94aa02524

9. c4t600A0B8000563D8E000002FB4AA02699d0

/scsi_vhci/ssd@g600a0b8000563d8e000002fb4aa02699

10. c4t600A0B8000563DC6000005124BB1A243d0

/scsi_vhci/ssd@g600a0b8000563dc6000005124bb1a243

11. c4t600A0B8000564E18000006694BB30DB5d0

/scsi_vhci/ssd@g600a0b8000564e18000006694bb30db5

12. c4t600A0B8000564F0E000006EE4BB30D1Fd0

/scsi_vhci/ssd@g600a0b8000564f0e000006ee4bb30d1f

13. c4t600144F0383E8200000050C00A8E0004d0

/scsi_vhci/ssd@g600144f0383e8200000050c00a8e0004

14. c6t5d0

/iscsi/disk@0000iqn.2006-01.com.openfiler%3Atsn.3e8eb39d1ca50001,0

Specify disk (enter its number): ^C

# zpool list

NAME SIZE USED AVAIL CAP HEALTH ALTROOT

bi_bak 212G 159G 53.2G 74% ONLINE -

bipool 278G 81.1G 197G 29% ONLINE -

dbdata 3.25T 3.14T 114G 96% ONLINE -

iscsidata 1.09T 450G 662G 40% ONLINE -

u01 1.23T 782G 474G 62% ONLINE -

# zfs list

NAME USED AVAIL REFER MOUNTPOINT

bi_bak 159G 49.9G 159G /bi_bak

bipool 113G 161G 97K /bipool

bipool/ROOT 64.9G 161G 21K legacy

bipool/ROOT/s10s_u8wos_08a 64.9G 161G 64.9G /

bipool/dump 2.00G 161G 2.00G -

bipool/export 14.1G 161G 14.0G /export

bipool/export/home 65.8M 161G 65.8M /export/home

bipool/swap 32G 193G 98.5M -

dbdata 3.14T 56.7G 3.14T /dbdata

iscsidata 450G 645G 450G /iscsidata

u01 784G 452G 784G /u01

# zpool create dbdata_1 c4t600144F0383E8200000050C00A8E0004d0

# zpool list

NAME SIZE USED AVAIL CAP HEALTH ALTROOT

bi_bak 212G 159G 53.2G 74% ONLINE -

bipool 278G 81.1G 197G 29% ONLINE -

dbdata 3.25T 3.14T 109G 96% ONLINE -

dbdata_1 6.94T 76.5K 6.94T 0% ONLINE -

iscsidata 1.09T 450G 662G 40% ONLINE -

u01 1.23T 787G 469G 62% ONLINE -

# zfs list

NAME USED AVAIL REFER MOUNTPOINT

bi_bak 159G 49.9G 159G /bi_bak

bipool 113G 161G 97K /bipool

bipool/ROOT 64.9G 161G 21K legacy

bipool/ROOT/s10s_u8wos_08a 64.9G 161G 64.9G /

bipool/dump 2.00G 161G 2.00G -

bipool/export 14.1G 161G 14.0G /export

bipool/export/home 65.8M 161G 65.8M /export/home

bipool/swap 32G 193G 98.5M -

dbdata 3.14T 56.7G 3.14T /dbdata

dbdata_1 72K 6.83T 21K /dbdata_1

iscsidata 450G 645G 450G /iscsidata

u01 787G 449G 787G /u01

# luxadm probe

No Network Array enclosures found in /dev/es

Found Fibre Channel device(s):