客快物流大資料專案(四十二)：Java程式碼操作Kudu

十一vs十一發表於2024-03-29
大資料Java

Java程式碼操作Kudu

一、構建maven工程

二、匯入依賴

三、​​​​​​​建立包結構

四、​​​​​​​初始化方法

五、​​​​​​​建立表

六、​​​​​​​插入資料

七、​​​​​​​查詢資料

八、修改資料

九、​​​​​​​刪除資料

十、​​​​​​​修改表

十一、​​​​​​​刪除表

Java程式碼操作Kudu

一、​​​​​​​構建maven工程

客快物流大資料專案(四十二)：Java程式碼操作Kudu

二、匯入依賴

<repositories>
    <repository>
        <id>cloudera</id>
        <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
    </repository>
</repositories>

<dependencies>
    <dependency>
        <groupId>org.apache.kudu</groupId>
        <artifactId>kudu-client</artifactId>
        <version>1.9.0-cdh6.2.1</version>
    </dependency>

    <dependency>
        <groupId>junit</groupId>
        <artifactId>junit</artifactId>
        <version>4.12</version>
    </dependency>

    <dependency>
        <groupId>org.apache.kudu</groupId>
        <artifactId>kudu-client-tools</artifactId>
        <version>1.9.0-cdh6.2.1</version>
    </dependency>

    <!-- https://mvnrepository.com/artifact/org.apache.kudu/kudu-spark2 -->
    <dependency>
        <groupId>org.apache.kudu</groupId>
        <artifactId>kudu-spark2_2.11</artifactId>
        <version>1.9.0-cdh6.2.1</version>
    </dependency>

    <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-sql -->
    <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-sql_2.11</artifactId>
        <version>2.1.0</version>
    </dependency>
</dependencies>

三、​​​​​​​建立包結構

包名

說明

cn.it

程式碼所在的包目錄

四、​​​​​​​初始化方法

package cn.it;

import org.apache.kudu.ColumnSchema;
import org.apache.kudu.Type;
import org.apache.kudu.client.KuduClient;
import org.junit.Before;

public class TestKudu {
    //定義KuduClient客戶端物件
    private static KuduClient kuduClient;
    //定義表名
    private static String tableName = "person";

    /**
     * 初始化方法
     */
    @Before
    public void init() {
        //指定master地址
        String masterAddress = "node2.cn";
        //建立kudu的資料庫連線
        kuduClient = new KuduClient.KuduClientBuilder(masterAddress).defaultSocketReadTimeoutMs(6000).build();
    }

    //構建表schema的欄位資訊
    //欄位名稱   資料型別     是否為主鍵
    public ColumnSchema newColumn(String name, Type type, boolean isKey) {
        ColumnSchema.ColumnSchemaBuilder column = new ColumnSchema.ColumnSchemaBuilder(name, type);
        column.key(isKey);
        return column.build();
    }
}

五、​​​​​​​建立表

/**  使用junit進行測試
 *
 * 建立表
 * @throws KuduException
 */
@Test
public void createTable() throws KuduException {
    //設定表的schema
    List<ColumnSchema> columns = new LinkedList<ColumnSchema>();
    columns.add(newColumn("CompanyId", Type.INT32, true));
    columns.add(newColumn("WorkId", Type.INT32, false));
    columns.add(newColumn("Name", Type.STRING, false));
    columns.add(newColumn("Gender", Type.STRING, false));
    columns.add(newColumn("Photo", Type.STRING, false));
    Schema schema = new Schema(columns);
    //建立表時提供的所有選項
    CreateTableOptions tableOptions = new CreateTableOptions();
    //設定表的副本和分割槽規則
    LinkedList<String> list = new LinkedList<String>();
    list.add("CompanyId");
    //設定表副本數
    tableOptions.setNumReplicas(1);
    //設定range分割槽
    //tableOptions.setRangePartitionColumns(list);
    //設定hash分割槽和分割槽的數量
    tableOptions.addHashPartitions(list, 3);
    try {
        kuduClient.createTable("person", schema, tableOptions);
    } catch (Exception e) {
        e.printStackTrace();
    }
    kuduClient.close();
}

六、​​​​​​​插入資料

/**
 * 向表中載入資料
 * @throws KuduException
 */
@Test
public void loadData() throws KuduException {
    //開啟表
    KuduTable kuduTable = kuduClient.openTable(tableName);
    //建立KuduSession物件 kudu必須透過KuduSession寫入資料
    KuduSession kuduSession = kuduClient.newSession();
    //採用flush方式 手動重新整理
    kuduSession.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
    kuduSession.setMutationBufferSpace(3000);
    //準備資料
    for(int i=1; i<=10; i++){
        Insert insert = kuduTable.newInsert();
        //設定欄位的內容
        insert.getRow().addInt("CompanyId",i);
        insert.getRow().addInt("WorkId",i);
        insert.getRow().addString("Name","lisi"+i);
        insert.getRow().addString("Gender","male");
        insert.getRow().addString("Photo","person"+i);
        kuduSession.flush();
        kuduSession.apply(insert);
    }
    kuduSession.close();
    kuduClient.close();
}

七、​​​​​​​查詢資料

 /**
 * 查詢表資料
 * @throws KuduException
 */
@Test
public void queryData() throws KuduException {
    //開啟表
    KuduTable kuduTable = kuduClient.openTable(tableName);
    //獲取scanner掃描器
    KuduScanner.KuduScannerBuilder scannerBuilder = kuduClient.newScannerBuilder(kuduTable);
    KuduScanner scanner = scannerBuilder.build();
    //遍歷
    while(scanner.hasMoreRows()){
        RowResultIterator rowResults = scanner.nextRows();
        while (rowResults.hasNext()){
            RowResult result = rowResults.next();
            int companyId = result.getInt("CompanyId");
            int workId = result.getInt("WorkId");
            String name = result.getString("Name");
            String gender = result.getString("Gender");
            String photo = result.getString("Photo");
            System.out.print("companyId:"+companyId+" ");
            System.out.print("workId:"+workId+" ");
            System.out.print("name:"+name+" ");
            System.out.print("gender:"+gender+" ");
            System.out.println("photo:"+photo);
        }
    }
    //關閉
    scanner.close();
    kuduClient.close();
}

八、修改資料

/**
 * 修改資料
 * @throws KuduException
 */
@Test
public void upDATEData() throws KuduException {
    //開啟表
    KuduTable kuduTable = kuduClient.openTable(tableName);
    //構建kuduSession物件
    KuduSession kuduSession = kuduClient.newSession();
    //設定重新整理資料模式，自動提交
    kuduSession.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND);

    //更新資料需要獲取UpDATE物件
    UpDATE upDATE = kuduTable.newUpDATE();
    //獲取row物件
    PartialRow row = upDATE.getRow();
    //設定要更新的資料資訊
    row.addInt("CompanyId",1);
    row.addString("Name","kobe");
    //操作這個upDATE物件
    kuduSession.apply(upDATE);
    kuduSession.close();
}

九、​​​​​​​刪除資料

/**
 * 刪除表中的資料
 */
@Test
public void deleteData() throws KuduException {
    //開啟表
    KuduTable kuduTable = kuduClient.openTable(tableName);
    KuduSession kuduSession = kuduClient.newSession();
    //獲取Delete物件
    Delete delete = kuduTable.newDelete();
    //構建要刪除的行物件
    PartialRow row = delete.getRow();
    //設定刪除資料的條件
    row.addInt("CompanyId",2);
    kuduSession.flush();
    kuduSession.apply(delete);
    kuduSession.close();
    kuduClient.close();
}

十、​​​​​​​修改表

package cn.it.kudu;

import org.apache.kudu.ColumnSchema;
import org.apache.kudu.Type;
import org.apache.kudu.client.*;
import org.junit.Before;
import org.junit.Test;

import java.util.List;

/**
 * 修改表操作
 */
public class AlterTable {
    //定義kudu的客戶端物件
    private static KuduClient kuduClient;
    //定義一張表名稱
    private static String tableName = "person";

    /**
     * 初始化操作
     */
    @Before
    public void init() {
        //指定kudu的master地址
        String masterAddress = "node2.cn";
        //建立kudu的資料庫連線
        kuduClient = new KuduClient.KuduClientBuilder(masterAddress).defaultSocketReadTimeoutMs(6000).build();
    }

    /**
     * 新增列
     */
    @Test
    public void alterTableAddColumn() {
        AlterTableOptions alterTableOptions = new AlterTableOptions();
        alterTableOptions.addColumn(new ColumnSchema.ColumnSchemaBuilder("Address", Type.STRING).nullable(true).build());
        try {
            kuduClient.alterTable(tableName, alterTableOptions);
        } catch (KuduException e) {
            e.printStackTrace();
       }
    }

    /**
     * 刪除列
     */
    @Test
    public void alterTableDeleteColumn(){
        AlterTableOptions alterTableOptions = new AlterTableOptions().dropColumn("Address");
        try {
            kuduClient.alterTable(tableName, alterTableOptions);
        } catch (KuduException e) {
            e.printStackTrace();
       }
    }

    /**
     * 新增分割槽列
     */
    @Test
    public void alterTableAddRangePartition(){
        int lowerValue = 110;
        int upperValue = 120;
        try {
            KuduTable kuduTable = kuduClient.openTable(tableName);
            List<Partition> rangePartitions = kuduTable.getRangePartitions(6000);
            boolean flag = true;
            for (Partition rangePartition : rangePartitions) {
                int startKey = rangePartition.getDecodedRangeKeyStart(kuduTable).getInt("Id");
                if(startKey == lowerValue){
                    flag = false;
                }
            }
            if(flag) {
                PartialRow lower = kuduTable.getSchema().newPartialRow();
                lower.addInt("Id", lowerValue);
                PartialRow upper = kuduTable.getSchema().newPartialRow();
                upper.addInt("Id", upperValue);
                kuduClient.alterTable(tableName,new AlterTableOptions().addRangePartition(lower, upper));
            }else{
                System.out.println("分割槽已經存在，不能重複建立！");
            }
        } catch (KuduException e) {
            e.printStackTrace();
        } catch (Exception exception) {
            exception.printStackTrace();
        }
    }

    /**
     * 刪除表
     * @throws KuduException
     */
    @Test
    public void dropTable() throws KuduException {
        kuduClient.deleteTable(tableName);
    }
}

十一、​​​​​​​刪除表

/**
 * 刪除表
 */
@Test
public void dropTable() throws KuduException {
    //刪除表
    DeleteTableResponse response = kuduClient.deleteTable(tableName);
    //關閉客戶端連線
    kuduClient.close();
}

相關文章