記一次 Boomer 壓測 MQTT 過程

花菜發表於2020-08-04

OOMMQQT

壓測背景

接入第三方mqtt服務,目前公司裝置超過10w臺,併發預計4000tps

工具選擇

Jmeter

優點: 有現成Mqtt外掛,開箱即用,支援分散式
缺點: 施壓需要消耗很大效能,外掛不夠靈活(可能是我不熟悉)

Locust + Python

優點: 很靈活,有現成Web介面
缺點: 原生Locust不支援mqtt協議,需要重寫HTTPLocust這個類.Python受限於GLI,併發不給力.需要起多個slave

Locust + Boomer

優點: boomer是golang編寫的,效能強勁,可搭配locust實現Web介面
缺點: 缺少mqtt現成案例參考(我本身對於go也不算熟悉)

一開始測試選了Jmeter,因為簡單方便.但發現除錯不是很方便,還是上面的,可能不熟悉.另外,50個併發左右,我的MBP(19款16寸,6核),就開始咆哮了!時間關係,我沒深究原因.
後來選擇了Locust + Boomer.踩了不少坑,但最後總算完成了任務.

壓測分析

mqtt賬號和主題是一一繫結,因此需要批量生成大概20w個賬號

壓測場景

賬號建立連線

300rps,5分鐘
500rps,5分鐘
1000rps,5分鐘

傳送訊息

1000rps,5分鐘
2000rps,5分鐘
4000rps,5分鐘

指令碼設計

流程圖

實現程式碼

//  main.go
// 程式碼僅供參考,無法直接執行.
package main

import (
    "bytes"
    "encoding/csv"
    "fmt"
    MQTT "github.com/eclipse/paho.mqtt.golang"
    "github.com/myzhan/boomer"
    "io"
    "io/ioutil"
    "log"
    "os"
    "strconv"
    "strings"
    "sync"
    "time"
)

var rows [][]string // 讀取csv檔案儲存到這裡
var clientTopic []map[string]MQTT.Client
var conn = 0 // 除錯用
var failCount = 0 // 初始化失敗數量
var i = 0 // 控制併發
var j = 1 // 記錄訊息傳送成功
var f = 1 // 記錄訊息傳送失敗
var nowStr = strconv.Itoa(int(time.Now().Unix())) // 當前時間戳,用來做後續查詢的訊息的識別符號

func newConn(c MQTT.Client, clientId string, group *sync.WaitGroup) {
    defer func() {
        group.Add(-1)
        err := recover()
        if err != nil {
            failCount++
            fmt.Println("login fail clientId:  ", clientId)
        }
    }()
    token := c.Connect()
    if token.Wait() && token.Error() != nil {
        panic(token.Error())
    }
    // 組裝topic
    topic := fmt.Sprintf("msg/%s/supply", clientId)
    temp := make(map[string]MQTT.Client)
    temp[topic] = c
    clientTopic = append(clientTopic, temp)
    conn++ // 除錯用
}

func initClients() {
    var wg sync.WaitGroup 
    server := "server_ip:1883"
    for i := 0; i < len(rows); i++ {
        wg.Add(1)
        clientId, userName, passWord := rows[i][0], rows[i][1], rows[i][2]
        opts := MQTT.NewClientOptions().AddBroker(server)
        opts.SetUsername(userName)
        opts.SetPassword(passWord)
        opts.SetClientID(clientId)
        opts.SetKeepAlive(300 * time.Second)
        c := MQTT.NewClient(opts)
        go newConn(c, clientId, &wg)

    }
    wg.Wait() // 等到所有協程執行完成 
    fmt.Printf("init finish, clients len is %d \n", len(clientTopic))
    fmt.Printf("conn: %d \n", conn)
    fmt.Printf("failCount: %d \n", failCount)
}

func initCsvData() {
    pwd, _ := os.Getwd()
    b, err := ioutil.ReadFile(pwd + "/clients.csv")
    fs := bytes.NewBuffer(b)
    if err != nil {
        log.Fatalf("can not open the file, err is %+v", err)
    }

    r := csv.NewReader(fs)
    //針對大檔案，一行一行的讀取檔案
    for {
        row, err := r.Read()
        if err != nil && err != io.EOF {
            log.Fatalf("can not read, err is %+v", err)
        }
        if err == io.EOF {
            break
        }
        rows = append(rows, row)
    }
}



func login() {
    server := "server_ip:port"
    clientId, userName, passWord := rows[i][0], rows[i][1], rows[i][2]
    start := time.Now()
    opts := MQTT.NewClientOptions().AddBroker(server)
    opts.SetUsername(userName)
    opts.SetPassword(passWord)
    opts.SetClientID(clientId)
    c := MQTT.NewClient(opts)
    token := c.Connect()
    elapsed := time.Since(start)
    if token.Error() == nil {
        log.Println("success" + strconv.Itoa(j))
        boomer.RecordSuccess("tcp", "login", elapsed.Nanoseconds()/int64(time.Millisecond), int64(10))
    } else {
        log.Println(token.Error())
        boomer.RecordFailure("tcp", "login", elapsed.Nanoseconds()/int64(time.Millisecond), clientId)
    }
    c.Disconnect(5)
    // avoid out of array
    if i < len(clientTopic)-1 {
        i++
    } else {
        i = 0
    }
    j++
}

func sendMsg() {
    start := time.Now()
    msgId := "msg" + strconv.Itoa(i)
    var clientId string
    var topic string
    var c MQTT.Client
    for k, v := range clientTopic[i] {
        clientId = k[6:19]
        topic = k
        c = v // v就是一個connected的client
    }
    deviceTime := nowStr
    str := []string{msgId, clientId, deviceTime}
    msgPayload := strings.Join(str, "|")

    if c.IsConnected() == true {
        token := c.Publish(topic, 1, false, msgPayload)
        //token.Wait() 等待訊息傳送完成,這樣會極大拉低併發
        elapsed := time.Since(start)
        if token.Error() == nil {
            fmt.Printf("this topic name is: %s \n", topic)
            fmt.Printf("this topic payload is: %s \n", msgPayload)
            fmt.Printf("success msg index: %v elapsed: %v  \n", j, elapsed)
            j++ // 訊息傳送成功, 記錄一條,並且也給locust記錄一條,方便後續校對資料量
            boomer.RecordSuccess("tcp", "task", elapsed.Nanoseconds()/int64(time.Millisecond), int64(j))
            // 避免陣列越界
            if i < len(clientTopic)-1 {
                i++
            } else {
                i = 0
            }
        } else {
            boomer.RecordFailure("tcp", "task", elapsed.Nanoseconds()/int64(time.Millisecond), msgPayload)
            fmt.Printf("傳送失敗, fail msg index: %v \n", f)
        }

    } else {
        if token := c.Connect(); token.Wait() && token.Error() != nil {
            elapsed := time.Since(start)
            fmt.Printf("fail msg index: %v \n", f)
            f++
            boomer.RecordFailure("tcp", "task", elapsed.Nanoseconds()/int64(time.Millisecond), msgPayload)
        }
    }
}

func main() {
    initCsvData()
    initClients()

    task1 := &boomer.Task{
        Name:   "myTask",
        Weight: 1,
        Fn:     sendMsg,
    }

    //task2 := &boomer.Task{
    //  Name:   "login",
    //  Weight: 1,
    //  Fn:     login,
    //}
    boomer.Run(task1)
}