Es資料彙總不準確的問題

Es資料彙總不準確的問題

問題展示

資料超過1萬條的時候es查詢出的資料出現資料不準確的問題

參考方案

新版本中的hits.total匹配數說明

在7.0版釋出之前，hits.total始終用於表示符合查詢條件的文件的實際數量。在Elasticsearch 7.0版中，如果匹配數大於10,000，則不會計算hits.total。 這是為了避免為給定查詢計算精確匹配文件的不必要開銷。 我們可以通過將track_total_hits = true作為請求引數來強制進行精確匹配的計算。

它具有10,000和“ relation” =“ gte”的值。

程式碼層實現

本次以go程式碼為例

封裝的程式碼模組

package es

import (
    "context"
    "encoding/json"
    "fmt"
    "log"
    "os"
    "time"

    "collect/lib"

    "github.com/olivere/elastic/v7"
)

const (
    dateFormat = "2006-01-02"
)

type tracelog struct{}

func (tracelog) Printf(format string, v ...interface{}) {
    fmt.Printf(format, v...)
}

type Store struct {
    client       *elastic.Client
    indexPartter string
}

func NewStore(config lib.StoreConfig) *Store {
    client, err := elastic.NewClient(
        elastic.SetURL(config.Addrs...),
        elastic.SetBasicAuth(config.Username, config.Password),
        elastic.SetSniff(false),
        elastic.SetHealthcheckInterval(10*time.Second),
        elastic.SetErrorLog(log.New(os.Stderr, "ELASTIC ", log.LstdFlags)),
        elastic.SetTraceLog(log.New(os.Stdout, "", log.LstdFlags)))
    if err != nil {
        log.Fatal(err)
    }

    return &Store{
        client:       client,
        indexPartter: config.Index,
    }
}

type QueryReq struct {
    AppName   string
    BizName   string
    StartTime time.Time
    EndTime   time.Time
    DateTime  string
    Level     string
    Keyword   string
    Module    string
    Method    string
    TraceId   string
    SpanId    string
    Message   string
    Page      int
    PageSize  int
}

type QueryRes struct {
    Data      []*lib.Data
    TotalHits int64
}

func (s *Store) Search(q QueryReq) (*QueryRes, error) {
    var querys []elastic.Query
    if q.AppName != "" {
        querys = append(querys, elastic.NewTermQuery("AppName", q.AppName))
    }

    if q.BizName != "" {
        querys = append(querys, elastic.NewTermQuery("BizName", q.BizName))
    }

    if q.Level != "" {
        querys = append(querys, elastic.NewTermQuery("Level", q.Level))
    }

    if q.Keyword != "" {
        querys = append(querys, elastic.NewTermQuery("Keyword", q.Keyword))
    }

    if q.Module != "" {
        querys = append(querys, elastic.NewTermQuery("Module", q.Module))
    }

    if q.Method != "" {
        querys = append(querys, elastic.NewTermQuery("Method", q.Method))
    }

    if q.TraceId != "" {
        querys = append(querys, elastic.NewTermQuery("TraceId", q.TraceId))
    }

    if q.SpanId != "" {
        querys = append(querys, elastic.NewTermQuery("SpanId", q.SpanId))
    }

    querys = append(querys, elastic.NewRangeQuery("TimeStamp").Lt(q.EndTime).Gt(q.StartTime))

    if q.Message != "" {
        querys = append(querys, elastic.NewMatchQuery("Message", q.Message))
    }

    index := s.indexPartter + "-" + q.EndTime.Format(dateFormat)

    boolQuery := elastic.NewBoolQuery().Must(querys...)

    res, err := s.client.Search(index).Query(boolQuery).From(q.Page).Size(q.PageSize).TrackTotalHits(true).Sort("TimeStamp", false).Do(context.Background())
    if err != nil {
        // 索引不存在
        if elastic.IsNotFound(err) {
            return &QueryRes{}, nil
        }
        return nil, err
    }

    var data []*lib.Data
    for _, h := range res.Hits.Hits {
        b, err := h.Source.MarshalJSON()
        if err != nil {
            return nil, err
        }

        var d = new(lib.Data)
        if err := json.Unmarshal(b, d); err != nil {
            return nil, err
        }
        data = append(data, d)
    }
    return &QueryRes{Data: data, TotalHits: res.Hits.TotalHits.Value}, nil
}

type AggsReq struct {
    AppName  string
    BizName  string
    DateTime string
    Level    string
    Module   string
    Method   string
}

type Buctet struct {
    Key      string
    DocCount int64
}

type AggsRes struct {
    Buctets []*Buctet
}

func (s *Store) Aggs(q AggsReq) (*AggsRes, error) {
    var querys []elastic.Query
    if q.AppName != "" {
        querys = append(querys, elastic.NewTermQuery("AppName", q.AppName))
    }

    if q.BizName != "" {
        querys = append(querys, elastic.NewTermQuery("BizName", q.BizName))
    }

    if q.Level != "" {
        querys = append(querys, elastic.NewTermQuery("Level", q.Level))
    }

    if q.Module != "" {
        querys = append(querys, elastic.NewTermQuery("Module", q.Module))
    }

    index := s.indexPartter + "-" + q.DateTime

    boolQuery := elastic.NewBoolQuery().Must(querys...)

    // 只取出錯率最高的前20
    aggs := elastic.NewTermsAggregation().Field("Method").OrderByCountDesc().Size(20)

    // 以Method欄位分組聚合
    res, err := s.client.Search(index).Query(boolQuery).Aggregation("Method", aggs).Size(0).Do(context.Background())
    if err != nil {
        // 索引不存在
        if elastic.IsNotFound(err) {
            return &AggsRes{}, nil
        }
        return nil, err
    }

    var buckets []*Buctet
    methodItem, ok := res.Aggregations.Terms("Method")
    if ok {
        for _, data := range methodItem.Buckets {
            var d = new(Buctet)
            key, ok := data.Key.(string)
            if !ok {
                continue
            }

            d.Key = key

            d.DocCount = data.DocCount

            buckets = append(buckets, d)
        }
    }
    return &AggsRes{Buctets: buckets}, nil
}

修改後繼續嘗試

根據網上的說法仍然有問題

總結

有知道的歡迎評論

本作品採用《CC 協議》，轉載必須註明作者和本文連結