大資料應用之HBase資料插入效能優化實測教程

 import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
 
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.util.Bytes;

/*
 * -------優化案例說明------------
 * 1.優化引數1：Autoflush                預設關閉，需要手動開啟
 * 2.優化引數2：put list size            支援單條與批量
 * 3.優化引數3：JVM heap size             預設值是平臺而不同，需要手動設定
 * 4.優化引數4：Write Buffer Size        預設值2M    
 * 5.優化引數5：Write Ahead Log             預設開啟，需要手動關閉
 * */

public class TestInsert {
    static Configuration hbaseConfig = null;

    public static void main(String[] args) throws Exception {
        Configuration HBASE_CONFIG = new Configuration();
        HBASE_CONFIG.set("hbase.master", "192.168.230.133:60000");
        HBASE_CONFIG.set("hbase.zookeeper.quorum", "192.168.230.133");
        HBASE_CONFIG.set("hbase.zookeeper.property.clientPort", "2181");
        hbaseConfig = HBaseConfiguration.create(HBASE_CONFIG);
        //關閉wal,autoflush,writebuffer = 24M
        insert(false,false,1024*1024*24);
        //開啟AutoFlush，writebuffer = 0
        insert(false,true,0);
        //預設值，全部開啟
        insert(true,true,0);
    }

    private static void insert(boolean wal,boolean autoFlush,long writeBuffer)
            throws IOException {
        String tableName="etltest";
        HBaseAdmin hAdmin = new HBaseAdmin(hbaseConfig);
        if (hAdmin.tableExists(tableName)) {
            hAdmin.disableTable(tableName);
            hAdmin.deleteTable(tableName);
        }

        HTableDescriptor t = new HTableDescriptor(tableName);
        t.addFamily(new HColumnDescriptor("f1"));
        t.addFamily(new HColumnDescriptor("f2"));
        t.addFamily(new HColumnDescriptor("f3"));
        t.addFamily(new HColumnDescriptor("f4"));
        hAdmin.createTable(t);
        System.out.println("table created");
        
        HTable table = new HTable(hbaseConfig, tableName);
        table.setAutoFlush(autoFlush);
        if(writeBuffer!=0){
            table.setWriteBufferSize(writeBuffer);
        }
        List<Put> lp = new ArrayList<Put>();
        long all = System.currentTimeMillis();
        
        System.out.println("start time = "+all);
        int count = 10000;
        byte[] buffer = new byte[128];
        Random r = new Random();
        for (int i = 1; i <= count; ++i) {
            Put p = new Put(String.format("row d",i).getBytes());
            r.nextBytes(buffer);
            p.add("f1".getBytes(), null, buffer);
            p.add("f2".getBytes(), null, buffer);
            p.add("f3".getBytes(), null, buffer);
            p.add("f4".getBytes(), null, buffer);
            p.setWriteToWAL(wal);
            lp.add(p);
            if(i%1000 == 0){
                table.put(lp);
                lp.clear();
            }
        }
        
        System.out.println("WAL="+wal+",autoFlush="+autoFlush+",buffer="+writeBuffer+",count="+count);
        long end = System.currentTimeMillis();
        System.out.println("total need time = "+ (end - all)*1.0/1000+"s");
        
        
        System.out.println("insert complete"+",costs:"+(System.currentTimeMillis()-all)*1.0/1000+"ms");
    }
}