Go開源遊戲伺服器框架——Pitaya

歪歪梯發表於2020-09-23

Go開源遊戲伺服器框架——Pitaya

簡介

Pitaya是一款由國外遊戲公司topfreegames使用golang進行編寫,易於使用,快速且輕量級的開源分散式遊戲伺服器框架
Pitaya使用etcd作為預設的服務發現元件,提供使用nats和grpc進行遠端呼叫(server to server)的可選配置,並提供在docker中執行以上元件(etcd、nats)的docker-compose配置

抽象分析

  • PlayerConn
    PlayerConn是一個封裝的連線物件,繼承net.Conn,並提供一個獲取下一個資料包的方法
	type PlayerConn interface {
		GetNextMessage() (b []byte, err error)
		net.Conn
	}
  • Acceptor
    Acceptor代表一個服務端埠程式,接收客戶端連線,並用一個內部Chan來維護這些連線物件
	type Acceptor interface {
		ListenAndServe()
		Stop()
		GetAddr() string
		GetConnChan() chan PlayerConn
	}
  • Acceptorwrapper
    Acceptorwrapper義如其名就是Acceptor的包裝器,因為Acceptor的通過Chan來儲存連線
    所以wrapper可以通過遍歷這個Chan來實時包裝這些連線
	type Wrapper interface {
		Wrap(acceptor.Acceptor) acceptor.Acceptor
	}
  • Agent
    Agent是一個服務端的應用層連線物件,包含了:
    Session資訊
    伺服器預傳送訊息佇列
    拆解包物件
    最後心跳時間
    停止傳送心跳的chan
    關閉傳送資料的chan
    全域性的關閉訊號
    連線物件
    Agent當前狀態
    … …
	type (
		// Agent corresponds to a user and is used for storing raw Conn information
		Agent struct {
			Session            *session.Session  // session
			appDieChan         chan bool         // app die channel
			chDie              chan struct{}     // wait for close
			chSend             chan pendingWrite // push message queue
			chStopHeartbeat    chan struct{}     // stop heartbeats
			chStopWrite        chan struct{}     // stop writing messages
			closeMutex         sync.Mutex
			conn               net.Conn            // low-level conn fd
			decoder            codec.PacketDecoder // binary decoder
			encoder            codec.PacketEncoder // binary encoder
			heartbeatTimeout   time.Duration
			lastAt             int64 // last heartbeat unix time stamp
			messageEncoder     message.Encoder
			... ...
			state              int32                // current agent state
		}
	
		pendingWrite struct {
			ctx  context.Context
			data []byte
			err  error
		}
	)
  • Component
    Component代表業務元件,提供若干個介面
    通過Component生成處理請求的Service
	type Component interface {
		Init()
		AfterInit()
		BeforeShutdown()
		Shutdown()
	}
  • Handler、Remote、Service
    Handler和Remote分別代表本地邏輯執行器和遠端邏輯執行器
    Service是一組服務物件,包含若干Handler和Remote
    這裡有個溫柔的細節——Receiver reflect.Value
    pitaya的設計者為了降低引用,採取在邏輯執行器中保留方法的Receiver以達到在Handler和Remote物件中,只需要儲存型別的Method,而無需儲存帶物件引用的Value.Method
	type (
		//Handler represents a message.Message's handler's meta information.
		Handler struct {
			Receiver    reflect.Value  // receiver of method
			Method      reflect.Method // method stub
			Type        reflect.Type   // low-level type of method
			IsRawArg    bool           // whether the data need to serialize
			MessageType message.Type   // handler allowed message type (either request or notify)
		}
	
		//Remote represents remote's meta information.
		Remote struct {
			Receiver reflect.Value  // receiver of method
			Method   reflect.Method // method stub
			HasArgs  bool           // if remote has no args we won't try to serialize received data into arguments
			Type     reflect.Type   // low-level type of method
		}
	
		// Service implements a specific service, some of it's methods will be
		// called when the correspond events is occurred.
		Service struct {
			Name     string              // name of service
			Type     reflect.Type        // type of the receiver
			Receiver reflect.Value       // receiver of methods for the service
			Handlers map[string]*Handler // registered methods
			Remotes  map[string]*Remote  // registered remote methods
			Options  options             // options
		}
	)
  • Modules
    Modules模組和Component結構一致,唯一的區別在於使用上
    Modules主要是面向系統的一些全域性存活的物件
    方便在統一的時機,集中進行啟動和關閉
	type Base struct{}
	
	func (c *Base) Init() error {
		return nil
	}
	
	func (c *Base) AfterInit() {}
	
	func (c *Base) BeforeShutdown() {}
	
	func (c *Base) Shutdown() error {
		return nil
	}

集中管理的物件容器在外部module.go中定義

	var (
		modulesMap = make(map[string]interfaces.Module)
		modulesArr = []moduleWrapper{}
	)
	
	type moduleWrapper struct {
		module interfaces.Module
		name   string
	}
  • HandleService
    HandleService就是服務端的主邏輯物件,負責處理一切資料包
    chLocalProcess用於儲存待處理的客戶端資料包
    chRemoteProcess用於儲存待處理的來自其他伺服器的資料包
    services註冊了處理客戶端的服務
    內部聚合一個RemoteService物件,專門負責處理伺服器間的資料包
	type (
		HandlerService struct {
			appDieChan         chan bool             // die channel app
			chLocalProcess     chan unhandledMessage // channel of messages that will be processed locally
			chRemoteProcess    chan unhandledMessage // channel of messages that will be processed remotely
			decoder            codec.PacketDecoder   // binary decoder
			encoder            codec.PacketEncoder   // binary encoder
			heartbeatTimeout   time.Duration
			messagesBufferSize int
			remoteService      *RemoteService
			serializer         serialize.Serializer          // message serializer
			server             *cluster.Server               // server obj
			services           map[string]*component.Service // all registered service
			messageEncoder     message.Encoder
			metricsReporters   []metrics.Reporter
		}
	
		unhandledMessage struct {
			ctx   context.Context
			agent *agent.Agent
			route *route.Route
			msg   *message.Message
		}
	)
  • RemoteService
    RemoteService中維護服務發現和註冊提供的遠端服務
	type RemoteService struct {
		rpcServer              cluster.RPCServer
		serviceDiscovery       cluster.ServiceDiscovery
		serializer             serialize.Serializer
		encoder                codec.PacketEncoder
		rpcClient              cluster.RPCClient
		services               map[string]*component.Service // all registered service
		router                 *router.Router
		messageEncoder         message.Encoder
		server                 *cluster.Server // server obj
		remoteBindingListeners []cluster.RemoteBindingListener
	}
  • Timer
    Timer模組中維護一個全域性定時任務管理者,使用執行緒安全的map來儲存定時任務,通過time.Ticker的chan訊號來定期觸發
	var (
		// Manager manager for all Timers
		Manager = &struct {
			incrementID    int64      // auto increment id
			timers         sync.Map   // all Timers
			ChClosingTimer chan int64 // timer for closing
			ChCreatedTimer chan *Timer
		}{}
	
		// Precision indicates the precision of timer, default is time.Second
		Precision = time.Second
	
		// GlobalTicker represents global ticker that all cron job will be executed
		// in globalTicker.
		GlobalTicker *time.Ticker
	)
  • pipeline
    pipeline模組提供全域性鉤子函式的配置
    BeforeHandler 在業務邏輯之前執行
    AfterHandler 在業務邏輯之後執行
	var (
		BeforeHandler = &pipelineChannel{}
		AfterHandler = &pipelineAfterChannel{}
	)
	
	type (
		HandlerTempl func(ctx context.Context, in interface{}) (out interface{}, err error)
	
		AfterHandlerTempl func(ctx context.Context, out interface{}, err error) (interface{}, error)
	
		pipelineChannel struct {
			Handlers []HandlerTempl
		}
	
		pipelineAfterChannel struct {
			Handlers []AfterHandlerTempl
		}
	)

框架流程

app.go是系統啟動的入口
建立HandlerService
並根據啟動模式如果是叢集模式建立RemoteService
開啟服務端事件監聽
開啟監聽伺服器關閉訊號的Chan

	var (
		app = &App{
			... ..
		}
	
		remoteService  *service.RemoteService
		handlerService *service.HandlerService
	)
	func Start() {
		... ..
		if app.serverMode == Cluster {
			... ..
			app.router.SetServiceDiscovery(app.serviceDiscovery)
	
			remoteService = service.NewRemoteService(
				app.rpcClient,
				app.rpcServer,
				app.serviceDiscovery,
				app.router,
				... ..
			)
	
			app.rpcServer.SetPitayaServer(remoteService)
	
			initSysRemotes()
		}
	
		handlerService = service.NewHandlerService(
			app.dieChan,
			app.heartbeat,
			app.server,
			remoteService,
			... ..
		)
	
		... ..
	
		listen()
		... ..
		// stop server
		select {
		case <-app.dieChan:
			logger.Log.Warn("the app will shutdown in a few seconds")
		case s := <-sg:
			logger.Log.Warn("got signal: ", s, ", shutting down...")
			close(app.dieChan)
		}
		... ..
	}

listen方法也就是開啟服務,具體包括一下步驟:
1.註冊Component
2.註冊定時任務的GlobalTicker
3.開啟Dispatch處理業務和定時任務(ticket)的goroutine
4.開啟acceptor處理連線的goroutine
5.開啟主邏輯的goroutine
6.註冊Modules

	func listen() {
		startupComponents()
		// create global ticker instance, timer precision could be customized
		// by SetTimerPrecision
		timer.GlobalTicker = time.NewTicker(timer.Precision)
	
		logger.Log.Infof("starting server %s:%s", app.server.Type, app.server.ID)
		for i := 0; i < app.config.GetInt("pitaya.concurrency.handler.dispatch"); i++ {
			go handlerService.Dispatch(i)
		}
		for _, acc := range app.acceptors {
			a := acc
			go func() {
				for conn := range a.GetConnChan() {
					go handlerService.Handle(conn)
				}
			}()
	
			go func() {
				a.ListenAndServe()
			}()
	
			logger.Log.Infof("listening with acceptor %s on addr %s", reflect.TypeOf(a), a.GetAddr())
		}
		... ..
		startModules()
	
		logger.Log.Info("all modules started!")
	
		app.running = true
	}

startupComponents對Component進行初始化
然後把Component註冊到handlerService和remoteService上

	func startupComponents() {
		// component initialize hooks
		for _, c := range handlerComp {
			c.comp.Init()
		}
	
		// component after initialize hooks
		for _, c := range handlerComp {
			c.comp.AfterInit()
		}
	
		// register all components
		for _, c := range handlerComp {
			if err := handlerService.Register(c.comp, c.opts); err != nil {
				logger.Log.Errorf("Failed to register handler: %s", err.Error())
			}
		}
	
		// register all remote components
		for _, c := range remoteComp {
			if remoteService == nil {
				logger.Log.Warn("registered a remote component but remoteService is not running! skipping...")
			} else {
				if err := remoteService.Register(c.comp, c.opts); err != nil {
					logger.Log.Errorf("Failed to register remote: %s", err.Error())
				}
			}
		}
		... ..
	}

比如HandlerService的註冊,反射得到component型別的全部方法,判斷isHandlerMethod就加入services裡面
並聚合Component物件的反射Value物件為全部Handler的Method Receiver,減少了物件引用

	func NewService(comp Component, opts []Option) *Service {
		s := &Service{
			Type:     reflect.TypeOf(comp),
			Receiver: reflect.ValueOf(comp),
		}
		... ..
		return s
	}
	
	func (h *HandlerService) Register(comp component.Component, opts []component.Option) error {
		s := component.NewService(comp, opts)
		... ..
		if err := s.ExtractHandler(); err != nil {
			return err
		}
	
		h.services[s.Name] = s
		for name, handler := range s.Handlers {
			handlers[fmt.Sprintf("%s.%s", s.Name, name)] = handler
		}
		return nil
	}
	func (s *Service) ExtractHandler() error {
		typeName := reflect.Indirect(s.Receiver).Type().Name()
		... ..
		s.Handlers = suitableHandlerMethods(s.Type, s.Options.nameFunc)
		... ..
		for i := range s.Handlers {
			s.Handlers[i].Receiver = s.Receiver
		}
		return nil
	}
	func suitableHandlerMethods(typ reflect.Type, nameFunc func(string) string) map[string]*Handler {
		methods := make(map[string]*Handler)
		for m := 0; m < typ.NumMethod(); m++ {
			method := typ.Method(m)
			mt := method.Type
			mn := method.Name
			if isHandlerMethod(method) {
				... ..
				handler := &Handler{
					Method:      method,
					IsRawArg:    raw,
					MessageType: msgType,
				}
				... ..
				methods[mn] = handler
			}
		}
		return methods
	}

handlerService.Dispatch方法負責各種業務的處理,包括:
1.處理chLocalProcess中的本地Message
2.使用remoteService處理chRemoteProcess中的遠端Message
3.在定時ticket到達時呼叫timer.Cron執行定時任務
4.管理定時任務的建立
5.管理定時任務的刪除

	func (h *HandlerService) Dispatch(thread int) {
		defer timer.GlobalTicker.Stop()
	
		for {
			select {
			case lm := <-h.chLocalProcess:
				metrics.ReportMessageProcessDelayFromCtx(lm.ctx, h.metricsReporters, "local")
				h.localProcess(lm.ctx, lm.agent, lm.route, lm.msg)
	
			case rm := <-h.chRemoteProcess:
				metrics.ReportMessageProcessDelayFromCtx(rm.ctx, h.metricsReporters, "remote")
				h.remoteService.remoteProcess(rm.ctx, nil, rm.agent, rm.route, rm.msg)
	
			case <-timer.GlobalTicker.C: // execute cron task
				timer.Cron()
	
			case t := <-timer.Manager.ChCreatedTimer: // new Timers
				timer.AddTimer(t)
	
			case id := <-timer.Manager.ChClosingTimer: // closing Timers
				timer.RemoveTimer(id)
			}
		}
	}

接下來看看Acceptor的工作,以下為Tcp實現,就是負責接收連線,流入acceptor的Chan

	func (a *TCPAcceptor) ListenAndServe() {
		if a.hasTLSCertificates() {
			a.ListenAndServeTLS(a.certFile, a.keyFile)
			return
		}
	
		listener, err := net.Listen("tcp", a.addr)
		if err != nil {
			logger.Log.Fatalf("Failed to listen: %s", err.Error())
		}
		a.listener = listener
		a.running = true
		a.serve()
	}
	func (a *TCPAcceptor) serve() {
		defer a.Stop()
		for a.running {
			conn, err := a.listener.Accept()
			if err != nil {
				logger.Log.Errorf("Failed to accept TCP connection: %s", err.Error())
				continue
			}
	
			a.connChan <- &tcpPlayerConn{
				Conn: conn,
			}
		}
	}

前面講過對於每個Acceptor開啟了一個goroutine去處理連線,也就是下面程式碼

	for conn := range a.GetConnChan() {
		go handlerService.Handle(conn)
	}

所以流入Chan的連線就會被實時的開啟一個goroutine去處理,處理過程就是先建立一個Agent物件
並開啟一個goroutine給Agent負責維護連線的心跳
然後開啟死迴圈,讀取連線的資料processPacket

	func (h *HandlerService) Handle(conn acceptor.PlayerConn) {
		// create a client agent and startup write goroutine
		a := agent.NewAgent(conn, h.decoder, h.encoder, h.serializer, h.heartbeatTimeout, h.messagesBufferSize, h.appDieChan, h.messageEncoder, h.metricsReporters)
	
		// startup agent goroutine
		go a.Handle()
		... ..
		for {
			msg, err := conn.GetNextMessage()
	
			if err != nil {
				logger.Log.Errorf("Error reading next available message: %s", err.Error())
				return
			}
	
			packets, err := h.decoder.Decode(msg)
			if err != nil {
				logger.Log.Errorf("Failed to decode message: %s", err.Error())
				return
			}
	
			if len(packets) < 1 {
				logger.Log.Warnf("Read no packets, data: %v", msg)
				continue
			}
	
			// process all packet
			for i := range packets {
				if err := h.processPacket(a, packets[i]); err != nil {
					logger.Log.Errorf("Failed to process packet: %s", err.Error())
					return
				}
			}
		}
	}

這時如果使用了pitaya提供的漏桶演算法實現的限流wrap來包裝acceptor,則會對客戶端傳送的訊息進行限流限速
這裡也是靈活利用for迴圈遍歷chan的特性,所以也是實時地對連線進行包裝

	func (b *BaseWrapper) ListenAndServe() {
		go b.pipe()
		b.Acceptor.ListenAndServe()
	}
	
	// GetConnChan returns the wrapper conn chan
	func (b *BaseWrapper) GetConnChan() chan acceptor.PlayerConn {
		return b.connChan
	}
	
	func (b *BaseWrapper) pipe() {
		for conn := range b.Acceptor.GetConnChan() {
			b.connChan <- b.wrapConn(conn)
		}
	}
	type RateLimitingWrapper struct {
		BaseWrapper
	}
	
	func NewRateLimitingWrapper(c *config.Config) *RateLimitingWrapper {
		r := &RateLimitingWrapper{}
		r.BaseWrapper = NewBaseWrapper(func(conn acceptor.PlayerConn) acceptor.PlayerConn {
			... ..
			return NewRateLimiter(conn, limit, interval, forceDisable)
		})
		return r
	}
	
	func (r *RateLimitingWrapper) Wrap(a acceptor.Acceptor) acceptor.Acceptor {
		r.Acceptor = a
		return r
	}
	
	func (r *RateLimiter) GetNextMessage() (msg []byte, err error) {
		if r.forceDisable {
			return r.PlayerConn.GetNextMessage()
		}
	
		for {
			msg, err := r.PlayerConn.GetNextMessage()
			if err != nil {
				return nil, err
			}
	
			now := time.Now()
			if r.shouldRateLimit(now) {
				logger.Log.Errorf("Data=%s, Error=%s", msg, constants.ErrRateLimitExceeded)
				metrics.ReportExceededRateLimiting(pitaya.GetMetricsReporters())
				continue
			}
	
			return msg, err
		}
	}

processPacket對資料包解包後,執行processMessage

	func (h *HandlerService) processPacket(a *agent.Agent, p *packet.Packet) error {
		switch p.Type {
		case packet.Handshake:
			... ..
		case packet.HandshakeAck:
			... ..
		case packet.Data:
			if a.GetStatus() < constants.StatusWorking {
				return fmt.Errorf("receive data on socket which is not yet ACK, session will be closed immediately, remote=%s",
					a.RemoteAddr().String())
			}
			msg, err := message.Decode(p.Data)
			if err != nil {
				return err
			}
			h.processMessage(a, msg)
		case packet.Heartbeat:
			// expected
		}
		a.SetLastAt()
		return nil
	}

processMessage中包裝資料包為unHandledMessage
根據訊息型別,流入chLocalProcess 或者chRemoteProcess 也就轉交給上面提到的負責Dispatch的goroutine去處理了

	func (h *HandlerService) processMessage(a *agent.Agent, msg *message.Message) {
		requestID := uuid.New()
		ctx := pcontext.AddToPropagateCtx(context.Background(), constants.StartTimeKey, time.Now().UnixNano())
		ctx = pcontext.AddToPropagateCtx(ctx, constants.RouteKey, msg.Route)
		ctx = pcontext.AddToPropagateCtx(ctx, constants.RequestIDKey, requestID.String())
		tags := opentracing.Tags{
			"local.id":   h.server.ID,
			"span.kind":  "server",
			"msg.type":   strings.ToLower(msg.Type.String()),
			"user.id":    a.Session.UID(),
			"request.id": requestID.String(),
		}
		ctx = tracing.StartSpan(ctx, msg.Route, tags)
		ctx = context.WithValue(ctx, constants.SessionCtxKey, a.Session)
	
		r, err := route.Decode(msg.Route)
		... ..
		message := unhandledMessage{
			ctx:   ctx,
			agent: a,
			route: r,
			msg:   msg,
		}
		if r.SvType == h.server.Type {
			h.chLocalProcess <- message
		} else {
			if h.remoteService != nil {
				h.chRemoteProcess <- message
			} else {
				logger.Log.Warnf("request made to another server type but no remoteService running")
			}
		}
	}

伺服器程式啟動的最後一步是對全域性模組啟動

在外部的module.go檔案中,提供了對module的全域性註冊方法、全部順序啟動方法、全部順序關閉方法

	func RegisterModule(module interfaces.Module, name string) error {
		... ..
	}
	
	func startModules() {
		for _, modWrapper := range modulesArr {
			modWrapper.module.Init()
		}
		for _, modWrapper := range modulesArr {
			modWrapper.module.AfterInit()
		}
	}
	
	func shutdownModules() {
		for i := len(modulesArr) - 1; i >= 0; i-- {
			modulesArr[i].module.BeforeShutdown()
		}
	
		for i := len(modulesArr) - 1; i >= 0; i-- {
			mod := modulesArr[i].module
			mod.Shutdown()
		}
	}

處理細節

  • localProcess
    接下來看看localprocess對於訊息的處理細節(為了直觀省略部分異常處理程式碼)
    使用processHandlerMessagef方法對包裝出來的ctx物件進行業務操作
    最終根據訊息的型別 notify / Request 區分是否需要響應,執行不同處理
	func (h *HandlerService) localProcess(ctx context.Context, a *agent.Agent, route *route.Route, msg *message.Message) {
		var mid uint
		switch msg.Type {
		case message.Request:
			mid = msg.ID
		case message.Notify:
			mid = 0
		}
	
		ret, err := processHandlerMessage(ctx, route, h.serializer, a.Session, msg.Data, msg.Type, false)
		if msg.Type != message.Notify {
			... ..
			err := a.Session.ResponseMID(ctx, mid, ret)
			... ..
		} else {
			metrics.ReportTimingFromCtx(ctx, h.metricsReporters, handlerType, nil)
			tracing.FinishSpan(ctx, err)
		}
	}
  • processHandlerMessage
    這裡面負進行業務邏輯
    會先呼叫executeBeforePipeline(ctx, arg),執行前置的鉤子函式
    再通過util.Pcall(h.Method, args)反射呼叫handler方法
    再呼叫executeAfterPipeline(ctx, resp, err),執行後置的鉤子函式
    最後呼叫serializeReturn(serializer, resp),對請求結果進行序列化
	func processHandlerMessage(
		ctx context.Context,
		rt *route.Route,
		serializer serialize.Serializer,
		session *session.Session,
		data []byte,
		msgTypeIface interface{},
		remote bool,
	) ([]byte, error) {
		if ctx == nil {
			ctx = context.Background()
		}
		ctx = context.WithValue(ctx, constants.SessionCtxKey, session)
		ctx = util.CtxWithDefaultLogger(ctx, rt.String(), session.UID())
	
		h, err := getHandler(rt)
		... ..
		
		msgType, err := getMsgType(msgTypeIface)
		... ..
		
		logger := ctx.Value(constants.LoggerCtxKey).(logger.Logger)
		exit, err := h.ValidateMessageType(msgType)
		... ..
		
		arg, err := unmarshalHandlerArg(h, serializer, data)
		... ..
		
		if arg, err = executeBeforePipeline(ctx, arg); err != nil {
			return nil, err
		}
		... ..
		
		args := []reflect.Value{h.Receiver, reflect.ValueOf(ctx)}
		if arg != nil {
			args = append(args, reflect.ValueOf(arg))
		}
	
		resp, err := util.Pcall(h.Method, args)
		if remote && msgType == message.Notify {
			resp = []byte("ack")
		}
	
		resp, err = executeAfterPipeline(ctx, resp, err)
		... ..
	
		ret, err := serializeReturn(serializer, resp)
		... ..
		
		return ret, nil
	}
  • executeBeforePipeline
    實際就是執行pipeline的BeforeHandler
	func executeBeforePipeline(ctx context.Context, data interface{}) (interface{}, error) {
		var err error
		res := data
		if len(pipeline.BeforeHandler.Handlers) > 0 {
			for _, h := range pipeline.BeforeHandler.Handlers {
				res, err = h(ctx, res)
				if err != nil {
					logger.Log.Debugf("pitaya/handler: broken pipeline: %s", err.Error())
					return res, err
				}
			}
		}
		return res, nil
	}
  • executeAfterPipeline
    實際就是執行pipeline的AfterHandler
	func executeAfterPipeline(ctx context.Context, res interface{}, err error) (interface{}, error) {
		ret := res
		if len(pipeline.AfterHandler.Handlers) > 0 {
			for _, h := range pipeline.AfterHandler.Handlers {
				ret, err = h(ctx, ret, err)
			}
		}
		return ret, err
	}

util.pcall裡展示了golang反射的一種高階用法
method.Func.Call,第一個引數是Receiver,也就是呼叫物件方法的例項
這種設計對比直接儲存Value物件的method,反射時直接call,擁有的額外好處就是降低了物件引用,方法不和例項繫結

	func Pcall(method reflect.Method, args []reflect.Value) (rets interface{}, err error) {
		... ..
		r := method.Func.Call(args)
		if len(r) == 2 {
			if v := r[1].Interface(); v != nil {
				err = v.(error)
			} else if !r[0].IsNil() {
				rets = r[0].Interface()
			} else {
				err = constants.ErrReplyShouldBeNotNull
			}
		}
		return
	}

更多文章,請搜尋公眾號歪歪梯Club
更多資料,請搜尋公眾號歪歪梯Club

相關文章