Hi3559AV100 NNIE開發(6)RFCN中NNIE實現關鍵執行緒函式->SAMPLE_SVP_NNIE_Rfcn_ViToVo()進行資料流分析

流星斬月發表於2021-03-25

  前面隨筆給出了NNIE開發的基本知識,下面幾篇隨筆將著重於Mobilefacenet NNIE開發,實現mobilefacenet.wk的chip版本,並在Hi3559AV100上實現mobilefacenet網路功能,外接USB攝像頭通過MPP平臺輸出至VO HDMI顯示結果。下文是Hi3559AV100 NNIE開發(6)RFCN中實現關鍵執行緒函式->SAMPLE_SVP_NNIE_Rfcn_ViToVo()進行資料流分析,通過對執行緒函式分析,詳細瞭解如何對.wk模型資料進行處理並弄清楚檢測框繪製這些後處理的實現。

1、SAMPLE_SVP_NNIE_Rfcn_ViToVo()函式呼叫

  首先給出SAMPLE_SVP_NNIE_Rfcn_ViToVo()函式的呼叫,為後續分析提供參照:

 1 static pthread_t s_hNnieThread = 0; //全域性定義
 2 
 3 HI_CHAR acThreadName[16] = {0};//區域性定義
 4 
 5 
 6 /******************************************
 7         Create work thread
 8  ******************************************/
 9 snprintf(acThreadName, 16, "NNIE_ViToVo");
10 prctl(PR_SET_NAME, (unsigned long)acThreadName, 0,0,0);
11 pthread_create(&s_hNnieThread, 0, SAMPLE_SVP_NNIE_Rfcn_ViToVo, NULL);    

  其中prctl函式的定義如下,其中PR_SET_NAME表示使用(char *) arg2所指向的位置中的值設定呼叫執行緒的名稱。

1 int prctl(int option, unsigned long arg2, unsigned long arg3,
2                  unsigned long arg4, unsigned long arg5);

2、SAMPLE_SVP_NNIE_Rfcn_ViToVo()函式實現具體分析

  下面講分析SAMPLE_SVP_NNIE_Rfcn_ViToVo(HI_VOID* pArgs)函式的內部實現,具體設計到5個函式,實現的功能已經註釋出來,具體如下:

 1 static HI_VOID* SAMPLE_SVP_NNIE_Rfcn_ViToVo(HI_VOID* pArgs)
 2 {
 3 
 4     .... //引數定義
 5 
 6     while (HI_FALSE == s_bNnieStopSignal)
 7     {    
 8         //使用者從通道獲取一幀處理完成的影像 通道1-輸出stExtFrmInfo
 9         s32Ret = HI_MPI_VPSS_GetChnFrame(s32VpssGrp,
10                                          as32VpssChn[1],
11                                          &stExtFrmInfo,
12                                          s32MilliSec);
13         ......
14 
15         //使用者從通道獲取一幀處理完成的影像 通道0-輸出stBaseFrmInfo
16         s32Ret = HI_MPI_VPSS_GetChnFrame(s32VpssGrp, 
17                                          as32VpssChn[0], 
18                                          &stBaseFrmInfo, 
19                                          s32MilliSec);
20         ......
21 
22         //關鍵處理函式
23         s32Ret = SAMPLE_SVP_NNIE_Rfcn_Proc(
24                      pstParam,
25                      pstSwParam, 
26                      &stExtFrmInfo,
27                      stBaseFrmInfo.stVFrame.u32Width,
28                      stBaseFrmInfo.stVFrame.u32Height);
29         ......
30 
31         //Draw rect
32         s32Ret = SAMPLE_COMM_SVP_NNIE_FillRect(
33                      &stBaseFrmInfo, 
34                      &(pstSwParam->stRect), 
35                      0x0000FF00); //綠色
36     ......    
37 
38         //將視訊影像送入指定輸出通道顯示。
39         s32Ret = HI_MPI_VO_SendFrame(voLayer, 
40                                      voChn, 
41                                      &stBaseFrmInfo, 
42                                      s32MilliSec);
43     ......
44     
45 BASE_RELEASE:
46         s32Ret = HI_MPI_VPSS_ReleaseChnFrame(s32VpssGrp,as32VpssChn[0], &stBaseFrmInfo);
47     ......
48 
49 EXT_RELEASE:
50         s32Ret = HI_MPI_VPSS_ReleaseChnFrame(s32VpssGrp,as32VpssChn[1], &stExtFrmInfo);
51     .......
52 
53     }
54 
55     return HI_NULL;
56 }

2.1、HI_MPI_VPSS_GetChnFrame的實現

  下面給出HI_MPI_VPSS_GetChnFrame函式的引數呼叫與實現過程(其中VPSS雙通道輸出),具體功能為從通道獲取一幀處理完成的影像:

 1     HI_S32 s32VpssGrp = 0;
 2     HI_S32 as32VpssChn[] = {VPSS_CHN0, VPSS_CHN1};
 3     VIDEO_FRAME_INFO_S stBaseFrmInfo;
 4     VIDEO_FRAME_INFO_S stExtFrmInfo;
 5     HI_S32 s32MilliSec = 20000;
 6 
 7         //使用者從通道獲取一幀處理完成的影像 通道1-輸出stExtFrmInfo
 8      s32Ret = HI_MPI_VPSS_GetChnFrame(s32VpssGrp,
 9                                 as32VpssChn[1],
10                                 &stExtFrmInfo,
11                                 s32MilliSec);
12 
13         //使用者從通道獲取一幀處理完成的影像 通道0-輸出stBaseFrmInfo
14      s32Ret = HI_MPI_VPSS_GetChnFrame(s32VpssGrp, 
15                                 as32VpssChn[0], 
16                                 &stBaseFrmInfo, 
17                                 s32MilliSec);
18 
19     //函式定義
20     HI_S32 HI_MPI_VPSS_GetChnFrame(VPSS_GRP VpssGrp, VPSS_CHN VpssChn,
21 VIDEO_FRAME_INFO_S *pstVideoFrame, HI_S32 s32MilliSec);

2.2、SAMPLE_SVP_NNIE_Rfcn_Proc的實現-NNIE資料處理

  SAMPLE_SVP_NNIE_Rfcn_Proc函式實現是整個RFCN NNIE資料處理過程的Key Point,檢測加框等資訊來源處,現給出函式引數及實現分析:

 1      SAMPLE_SVP_NNIE_PARAM_S *pstParam;
 2     SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S *pstSwParam;
 3 
 4     pstParam = &s_stRfcnNnieParam;
 5     pstSwParam = &s_stRfcnSoftwareParam;
 6 /*  
 7     s_stRfcnNnieParam及s_stRfcnSoftwareParam引數涉及前述操作
 8 
 9     s_stRfcnNnieParam.pstModel = &s_stRfcnModel.stModel;
10     s_stRfcnSoftwareParam.apcRpnDataLayerName[0] = "rpn_cls_score";
11     s_stRfcnSoftwareParam.apcRpnDataLayerName[1] = "rpn_bbox_pred";
12     s32Ret = SAMPLE_SVP_NNIE_Rfcn_ParamInit(&stNnieCfg,
13                                     &s_stRfcnNnieParam,
14                                         &s_stRfcnSoftwareParam);
15 */
16 
17 
18     VIDEO_FRAME_INFO_S stBaseFrmInfo;
19     VIDEO_FRAME_INFO_S stExtFrmInfo;
20 
21 //其中stBaseFrmInfo引數與stExtFrmInfo引數是經過下述函式輸出得到
22 /*
23     s32Ret = HI_MPI_VPSS_GetChnFrame(s32VpssGrp,
24                              as32VpssChn[1],
25                             &stExtFrmInfo,
26                             s32MilliSec);
27     s32Ret = HI_MPI_VPSS_GetChnFrame(s32VpssGrp, 
28                              as32VpssChn[0], 
29                             &stBaseFrmInfo, 
30                                 s32MilliSec);
31 */
32 
33 
34     s32Ret = SAMPLE_SVP_NNIE_Rfcn_Proc(
35                      pstParam,
36                      pstSwParam, 
37                      &stExtFrmInfo,
38                      stBaseFrmInfo.stVFrame.u32Width,
39                      stBaseFrmInfo.stVFrame.u32Height);    

  隨之給出SAMPLE_SVP_NNIE_Rfcn_Proc函式的定義及內部實現過程:

 1 static HI_S32 SAMPLE_SVP_NNIE_Rfcn_Proc(
 2     SAMPLE_SVP_NNIE_PARAM_S *pstParam,
 3     SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S *pstSwParam, 
 4     VIDEO_FRAME_INFO_S* pstExtFrmInfo,
 5     HI_U32 u32BaseWidth,HI_U32 u32BaseHeight)
 6 {
 7 
 8     ......引數定義
 9 
10     s32Ret = SAMPLE_SVP_NNIE_Forward(pstParam,
11                                      &stInputDataIdx,
12                                      &stProcSegIdx,
13                                      HI_TRUE);
14     
15     ......
16 
17     /*RPN*/
18     s32Ret = SAMPLE_SVP_NNIE_Rfcn_Rpn(pstParam, 
19                                       pstSwParam);
20     
21 
22     if(0 != pstSwParam->stRpnBbox.unShape.stWhc.u32Height)
23     {
24 
25         s32Ret = SAMPLE_SVP_NNIE_ForwardWithBbox(
26                     pstParam,
27                     &stInputDataIdx,
28                     &pstSwParam->stRpnBbox,
29                     &stProcSegIdx,
30                     HI_TRUE);
31        ......
32         
33         s32Ret = SAMPLE_SVP_NNIE_ForwardWithBbox(
34                     pstParam,
35                     &stInputDataIdx,
36                     &pstSwParam->stRpnBbox,
37                     &stProcSegIdx,
38                     HI_TRUE);
39     ......
40 
41         s32Ret = SAMPLE_SVP_NNIE_Rfcn_GetResult(pstParam,
42                                     pstSwParam);
43         
44     }
45     else
46        { ...... }
47     s32Ret = SAMPLE_SVP_NNIE_RoiToRect(
48                 &(pstSwParam->stDstScore),
49                 &(pstSwParam->stDstRoi), 
50                 &(pstSwParam->stClassRoiNum), 
51                 pstSwParam->af32ScoreThr,
52                 HI_TRUE,
53                 &(pstSwParam->stRect),
54                 pstExtFrmInfo->stVFrame.u32Width, 
55                 pstExtFrmInfo->stVFrame.u32Height,
56                 u32BaseWidth,
57                 u32BaseHeight);
58     
59     ......
60 
61     return s32Ret;
62 
63 }

2.2.1、SAMPLE_SVP_NNIE_Forward子函式分析

   首先給出函式的呼叫及引數細節,便於分析函式功能:

 1     其中在SAMPLE_SVP_NNIE_Rfcn_ViToVo函式中:
 2     SAMPLE_SVP_NNIE_PARAM_S *pstParam;
 3     pstParam = &s_stRfcnNnieParam;  //此值傳給SAMPLE_SVP_NNIE_Rfcn_Proc
 4     
 5 
 6 
 7     static HI_S32 SAMPLE_SVP_NNIE_Rfcn_Proc(
 8         SAMPLE_SVP_NNIE_PARAM_S *pstParam,      /*這個引數傳進來之後經過了SP420賦值然後送入
 9                                                 SAMPLE_SVP_NNIE_Forward*/
10         SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S *pstSwParam, 
11         VIDEO_FRAME_INFO_S* pstExtFrmInfo,
12         HI_U32 u32BaseWidth,HI_U32 u32BaseHeight)
13 
14     ......
15 
16     SAMPLE_SVP_NNIE_INPUT_DATA_INDEX_S stInputDataIdx = {0};
17     SAMPLE_SVP_NNIE_PROCESS_SEG_INDEX_S stProcSegIdx = {0};
18 
19     stInputDataIdx.u32SegIdx = 0;
20     stInputDataIdx.u32NodeIdx = 0;
21 
22      /*SP420*/
23     pstParam->astSegData[stInputDataIdx.u32SegIdx].astSrc[stInputDataIdx.u32NodeIdx].u64VirAddr = pstExtFrmInfo->stVFrame.u64VirAddr[0];
24     pstParam->astSegData[stInputDataIdx.u32SegIdx].astSrc[stInputDataIdx.u32NodeIdx].u64PhyAddr = pstExtFrmInfo->stVFrame.u64PhyAddr[0];
25     pstParam->astSegData[stInputDataIdx.u32SegIdx].astSrc[stInputDataIdx.u32NodeIdx].u32Stride  = pstExtFrmInfo->stVFrame.u32Stride[0];
26 
27     /*NNIE process 0-th seg*/
28     stProcSegIdx.u32SegIdx = 0;
29 
30      /*NNIE process 0-th seg*/
31     stProcSegIdx.u32SegIdx = 0;
32     s32Ret = SAMPLE_SVP_NNIE_Forward(pstParam,
33                                      &stInputDataIdx,
34                                      &stProcSegIdx,
35                                      HI_TRUE);
36 
37     ......
38 
39     //函式定義
40     static HI_S32 SAMPLE_SVP_NNIE_Forward(
41                 SAMPLE_SVP_NNIE_PARAM_S *pstNnieParam,
42                 SAMPLE_SVP_NNIE_INPUT_DATA_INDEX_S* pstInputDataIdx,
43                 SAMPLE_SVP_NNIE_PROCESS_SEG_INDEX_S* pstProcSegIdx,
44                 HI_BOOL bInstant)

  此函式如其名,主要實現了NNIE forward功能,下面給出呼叫具體函式:

 1 /******************************************************************************
 2 * function : NNIE Forward
 3 ******************************************************************************/
 4 static HI_S32 SAMPLE_SVP_NNIE_Forward(SAMPLE_SVP_NNIE_PARAM_S *pstNnieParam,
 5     SAMPLE_SVP_NNIE_INPUT_DATA_INDEX_S* pstInputDataIdx,
 6     SAMPLE_SVP_NNIE_PROCESS_SEG_INDEX_S* pstProcSegIdx,HI_BOOL bInstant)
 7 {
 8     ......
 9 
10     SAMPLE_COMM_SVP_FlushCache(
11     pstNnieParam->astForwardCtrl[pstProcSegIdx->u32SegIdx].stTskBuf.u64PhyAddr,
12     (HI_VOID *) pstNnieParam->astForwardCtrl[pstProcSegIdx->u32SegIdx].stTskBuf.u64VirAddr,
13      pstNnieParam->astForwardCtrl[pstProcSegIdx->u32SegIdx].stTskBuf.u32Size);
14 
15     /*set input blob according to node name*/
16     if(pstInputDataIdx->u32SegIdx != pstProcSegIdx->u32SegIdx)
17     {
18         for(i = 0; i < pstNnieParam->pstModel->astSeg[pstProcSegIdx->u32SegIdx].u16SrcNum; i++)
19         {
20             ......
21         }
22     }
23 
24     /*NNIE_Forward  多節點輸入輸出的 CNN 型別網路預測。 
25     對輸入樣本(s)進行CNN預測,對對應樣本(s)進行輸出響應*/
26     s32Ret = HI_MPI_SVP_NNIE_Forward(
27         &hSvpNnieHandle,
28         pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astSrc,
29         pstNnieParam->pstModel, 
30         pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst,
31         &pstNnieParam->astForwardCtrl[pstProcSegIdx->u32SegIdx], 
32         bInstant);
33     
34 
35     if(bInstant)
36     {
37         /*Wait NNIE finish,,,,enNnieId 執行網路段的 NNIE 引擎 ID。 */
38         while(HI_ERR_SVP_NNIE_QUERY_TIMEOUT == (s32Ret = HI_MPI_SVP_NNIE_Query(pstNnieParam->astForwardCtrl[pstProcSegIdx->u32SegIdx].enNnieId,
39             hSvpNnieHandle, &bFinish, HI_TRUE)))
40         {
41             ......
42         }
43     }
44 
45     bFinish = HI_FALSE;
46     for(i = 0; i < pstNnieParam->astForwardCtrl[pstProcSegIdx->u32SegIdx].u32DstNum; i++)
47     {
48         if(SVP_BLOB_TYPE_SEQ_S32 == pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].enType)
49         {
50             ......
51             
52             SAMPLE_COMM_SVP_FlushCache(
53     pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u64PhyAddr,
54     (HI_VOID *) pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u64VirAddr,
55     u32TotalStepNum*pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u32Stride);
56 
57         }
58         else
59         {
60 
61             SAMPLE_COMM_SVP_FlushCache(pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u64PhyAddr,
62                 (HI_VOID *) pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u64VirAddr,
63                 pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u32Num*
64                 pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].unShape.stWhc.u32Chn*
65                 pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].unShape.stWhc.u32Height*
66                 pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u32Stride);
67         }
68     }
69 
70     return s32Ret;
71 }

2.2.2、SAMPLE_SVP_NNIE_Rfcn_Rpn子函式分析

   下面給出sample_svp_NNIE_Rfcn_Rpn子函式分析,此函式主要是用於rpn相關,下面給出引數呼叫及函式分析:

 1     其中在SAMPLE_SVP_NNIE_Rfcn_ViToVo函式中:
 2     SAMPLE_SVP_NNIE_PARAM_S *pstParam; //此值傳給SAMPLE_SVP_NNIE_Rfcn_Proc
 3     pstParam = &s_stRfcnNnieParam;  //此值傳給SAMPLE_SVP_NNIE_Rfcn_Proc
 4 
 5     SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S *pstSwParam;
 6     pstSwParam = &s_stRfcnSoftwareParam;
 7 
 8     static HI_S32 SAMPLE_SVP_NNIE_Rfcn_Proc(
 9         SAMPLE_SVP_NNIE_PARAM_S *pstParam,      /*這個引數傳進來之後經過了SP420賦值然後送入
10                                                 SAMPLE_SVP_NNIE_Forward*/
11         SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S *pstSwParam,    //直接傳給SAMPLE_SVP_NNIE_Rfcn_Rpn函式
12         VIDEO_FRAME_INFO_S* pstExtFrmInfo,
13         HI_U32 u32BaseWidth,HI_U32 u32BaseHeight)
14 
15      /*SP420*/
16     pstParam->astSegData[stInputDataIdx.u32SegIdx].astSrc[stInputDataIdx.u32NodeIdx].u64VirAddr = pstExtFrmInfo->stVFrame.u64VirAddr[0];
17     pstParam->astSegData[stInputDataIdx.u32SegIdx].astSrc[stInputDataIdx.u32NodeIdx].u64PhyAddr = pstExtFrmInfo->stVFrame.u64PhyAddr[0];
18     pstParam->astSegData[stInputDataIdx.u32SegIdx].astSrc[stInputDataIdx.u32NodeIdx].u32Stride  = pstExtFrmInfo->stVFrame.u32Stride[0];
19 
20     此後,pstParam引數先傳入SAMPLE_SVP_NNIE_Forward函式進行處理,隨後傳入SAMPLE_SVP_NNIE_Rfcn_Rpn函式
21 
22         /*RPN*/
23     s32Ret = SAMPLE_SVP_NNIE_Rfcn_Rpn(pstParam, 
24                                       pstSwParam);
25 
26     // 函式呼叫  ,用於used to do rpn
27     HI_S32 SAMPLE_SVP_NNIE_Rfcn_Rpn(
28         SAMPLE_SVP_NNIE_PARAM_S*pstNnieParam,
29         SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S* pstSoftwareParam)

  下面給出sample_svp_NNIE_Rfcn_Rpn子函式內部實現,此函式雖然傳入兩個引數,但是沒有對pstNnieParam引數進行任何處理,完成的pstSoftwareParam結構體內大量引數的的處理 ,主要是呼叫了SVP_NNIE_Rpn函式與SAMPLE_COMM_SVP_FlushCache函式:

 1 HI_S32 SAMPLE_SVP_NNIE_Rfcn_Rpn(SAMPLE_SVP_NNIE_PARAM_S*pstNnieParam,
 2     SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S* pstSoftwareParam)
 3 {
 4     HI_S32 s32Ret = HI_SUCCESS;
 5     s32Ret = SVP_NNIE_Rpn(pstSoftwareParam->aps32Conv,pstSoftwareParam->u32NumRatioAnchors,
 6         pstSoftwareParam->u32NumScaleAnchors,pstSoftwareParam->au32Scales,
 7         pstSoftwareParam->au32Ratios,pstSoftwareParam->u32OriImHeight,
 8         pstSoftwareParam->u32OriImWidth,pstSoftwareParam->au32ConvHeight,
 9         pstSoftwareParam->au32ConvWidth,pstSoftwareParam->au32ConvChannel,
10         pstSoftwareParam->u32ConvStride,pstSoftwareParam->u32MaxRoiNum,
11         pstSoftwareParam->u32MinSize,pstSoftwareParam->u32SpatialScale,
12         pstSoftwareParam->u32NmsThresh,pstSoftwareParam->u32FilterThresh,
13         pstSoftwareParam->u32NumBeforeNms,(HI_U32*)pstSoftwareParam->stRpnTmpBuf.u64VirAddr,
14         (HI_S32*)pstSoftwareParam->stRpnBbox.u64VirAddr,
15         &pstSoftwareParam->stRpnBbox.unShape.stWhc.u32Height);
16     SAMPLE_COMM_SVP_FlushCache(pstSoftwareParam->stRpnBbox.u64PhyAddr,
17         (HI_VOID *) pstSoftwareParam->stRpnBbox.u64VirAddr,
18         pstSoftwareParam->stRpnBbox.u32Num*
19         pstSoftwareParam->stRpnBbox.unShape.stWhc.u32Chn*
20         pstSoftwareParam->stRpnBbox.unShape.stWhc.u32Height*
21         pstSoftwareParam->stRpnBbox.u32Stride);
22     SAMPLE_SVP_CHECK_EXPR_RET(HI_SUCCESS != s32Ret,s32Ret,SAMPLE_SVP_ERR_LEVEL_ERROR,
23         "Error,SVP_NNIE_Rpn failed!\n");
24     return s32Ret;
25 }

2.2.3、SAMPLE_SVP_NNIE_ForwardWithBbox子函式分析

  完成前面幾個函式後,通過if判斷,當滿足條件後執行SAMPLE_SVP_NNIE_ForwardWithBbox函式:

1 if(0 != pstSwParam->stRpnBbox.unShape.stWhc.u32Height)

  否則不滿足if條件的時候,將執行下面賦值語句:

1         for (i = 0; i < pstSwParam->stClassRoiNum.unShape.stWhc.u32Width; i++)
2         {
3             *(((HI_U32*)(HI_UL)pstSwParam->stClassRoiNum.u64VirAddr)+i) = 0;
4         }

  而SAMPLE_SVP_NNIE_ForwardWithBbox執行了有兩次,分別是對不同NNIE process x-th seg進行處理,具體如下:

 1     其中在SAMPLE_SVP_NNIE_Rfcn_ViToVo函式中:
 2     SAMPLE_SVP_NNIE_PARAM_S *pstParam; //此值傳給SAMPLE_SVP_NNIE_Rfcn_Proc
 3     pstParam = &s_stRfcnNnieParam;  //此值傳給SAMPLE_SVP_NNIE_Rfcn_Proc
 4 
 5     SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S *pstSwParam;
 6     pstSwParam = &s_stRfcnSoftwareParam;
 7 
 8     static HI_S32 SAMPLE_SVP_NNIE_Rfcn_Proc(
 9         SAMPLE_SVP_NNIE_PARAM_S *pstParam,      /*這個引數傳進來之後經過了SP420賦值然後送入
10                                                 SAMPLE_SVP_NNIE_Forward*/
11         SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S *pstSwParam,    /*直接傳給SAMPLE_SVP_NNIE_Rfcn_Rpn函式
12                                                             隨後傳給SAMPLE_SVP_NNIE_ForwardWithBbox函式*/
13         VIDEO_FRAME_INFO_S* pstExtFrmInfo,
14         HI_U32 u32BaseWidth,HI_U32 u32BaseHeight)
15 
16     /*RPN*/
17     s32Ret = SAMPLE_SVP_NNIE_Rfcn_Rpn(pstParam, 
18                                       pstSwParam);  /*此函式完成pstSwParam的相關賦值,並
19                                                      傳給SAMPLE_SVP_NNIE_ForwardWithBbox函式*/
20 
21 
22     /*在SAMPLE_SVP_NNIE_ForwardWithBbox函式前面,已經有了下面引數的賦值,
23     並且傳入至SAMPLE_SVP_NNIE_Forward函式*/
24         stInputDataIdx.u32SegIdx = 0;
25         stInputDataIdx.u32NodeIdx = 0;
26 
27      s32Ret = SAMPLE_SVP_NNIE_Forward(pstParam,
28                                      &stInputDataIdx, //stInputDataIdx引數已經用於此函式中
29                                      &stProcSegIdx,
30                                      HI_TRUE);
31 
32 
33      //函式呼叫
34     /*NNIE process 1-th seg, the input data comes from 3-rd report node of 0-th seg,
35      the input roi comes from RPN results*/
36     stInputDataIdx.u32SegIdx = 0;
37     stInputDataIdx.u32NodeIdx = 3;
38 
39     stProcSegIdx.u32SegIdx = 1;
40 
41     s32Ret = SAMPLE_SVP_NNIE_ForwardWithBbox(
42                 pstParam,
43                 &stInputDataIdx,
44                 &pstSwParam->stRpnBbox,
45                 &stProcSegIdx,
46                 HI_TRUE);
47 
48 
49     /*NNIE process 2-nd seg, the input data comes from 4-th report node of 0-th seg
50       the input roi comes from RPN results*/
51     stInputDataIdx.u32SegIdx = 0;
52     stInputDataIdx.u32NodeIdx = 4;
53 
54     stProcSegIdx.u32SegIdx = 2;
55 
56     s32Ret = SAMPLE_SVP_NNIE_ForwardWithBbox(
57                 pstParam,
58                 &stInputDataIdx,
59                 &pstSwParam->stRpnBbox,
60                 &stProcSegIdx,
61                 HI_TRUE);

  下面給出函式具體實現:

 1 /******************************************************************************
 2 * function : NNIE ForwardWithBbox
 3 ******************************************************************************/
 4 static HI_S32 SAMPLE_SVP_NNIE_ForwardWithBbox(
 5             SAMPLE_SVP_NNIE_PARAM_S *pstNnieParam,
 6             SAMPLE_SVP_NNIE_INPUT_DATA_INDEX_S* pstInputDataIdx,
 7             SVP_SRC_BLOB_S astBbox[],
 8             SAMPLE_SVP_NNIE_PROCESS_SEG_INDEX_S* pstProcSegIdx,
 9             HI_BOOL bInstant)
10 {
11     ......
12 
13     SAMPLE_COMM_SVP_FlushCache(pstNnieParam->astForwardWithBboxCtrl[pstProcSegIdx->u32SegIdx].stTskBuf.u64PhyAddr,
14         (HI_VOID *) pstNnieParam->astForwardWithBboxCtrl[pstProcSegIdx->u32SegIdx].stTskBuf.u64VirAddr,
15         pstNnieParam->astForwardWithBboxCtrl[pstProcSegIdx->u32SegIdx].stTskBuf.u32Size);
16 
17     /*set input blob according to node name*/
18     if(pstInputDataIdx->u32SegIdx != pstProcSegIdx->u32SegIdx)
19     {
20         for(i = 0; i < pstNnieParam->pstModel->astSeg[pstProcSegIdx->u32SegIdx].u16SrcNum; i++)
21         {
22             for(j = 0; j < pstNnieParam->pstModel->astSeg[pstInputDataIdx->u32SegIdx].u16DstNum; j++)
23             {
24                 ......
25             }
26             ......
27         }
28     }
29     /*NNIE_ForwardWithBbox*/
30     s32Ret = HI_MPI_SVP_NNIE_ForwardWithBbox(
31         &hSvpNnieHandle,
32         pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astSrc,
33         astBbox,
34         pstNnieParam->pstModel,  //網路型別只支援ROI/PSROI
35         pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst,
36         &pstNnieParam->astForwardWithBboxCtrl[pstProcSegIdx->u32SegIdx], 
37         bInstant);
38 
39     ......
40 
41     if(bInstant)
42     {
43         /*Wait NNIE finish*/
44         while(HI_ERR_SVP_NNIE_QUERY_TIMEOUT == (s32Ret = HI_MPI_SVP_NNIE_Query(pstNnieParam->astForwardWithBboxCtrl[pstProcSegIdx->u32SegIdx].enNnieId,
45             hSvpNnieHandle, &bFinish, HI_TRUE)))
46         {
47             ......
48         }
49     }
50 
51     bFinish = HI_FALSE;
52 
53 
54     for(i = 0; i < pstNnieParam->astForwardWithBboxCtrl[pstProcSegIdx->u32SegIdx].u32DstNum; i++)
55     {
56         if(SVP_BLOB_TYPE_SEQ_S32 == pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].enType)
57         {
58             ......
59             
60             SAMPLE_COMM_SVP_FlushCache(pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u64PhyAddr,
61                 (HI_VOID *) pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u64VirAddr,
62                 u32TotalStepNum*pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u32Stride);
63         }
64         else
65         {
66             SAMPLE_COMM_SVP_FlushCache(pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u64PhyAddr,
67                 (HI_VOID *) pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u64VirAddr,
68                 pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u32Num*
69                 pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].unShape.stWhc.u32Chn*
70                 pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].unShape.stWhc.u32Height*
71                 pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u32Stride);
72         }
73     }
74 
75     return s32Ret;
76 }

2.2.4、SAMPLE_SVP_NNIE_Rfcn_GetResult子函式分析

  下一個子函式是獲得NNIE Rfcn的結果,前提需要保證網路結構和輸入資料保持一致,函式呼叫如下:

 

 1     其中在SAMPLE_SVP_NNIE_Rfcn_ViToVo函式中:
 2     SAMPLE_SVP_NNIE_PARAM_S *pstParam; //此值傳給SAMPLE_SVP_NNIE_Rfcn_Proc
 3     pstParam = &s_stRfcnNnieParam;  //此值傳給SAMPLE_SVP_NNIE_Rfcn_Proc
 4 
 5     SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S *pstSwParam;
 6     pstSwParam = &s_stRfcnSoftwareParam;
 7 
 8     static HI_S32 SAMPLE_SVP_NNIE_Rfcn_Proc(
 9         SAMPLE_SVP_NNIE_PARAM_S *pstParam,      /*這個引數傳進來之後經過了SP420賦值然後送入
10                                                 SAMPLE_SVP_NNIE_Forward*/
11         SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S *pstSwParam,    /*直接傳給SAMPLE_SVP_NNIE_Rfcn_Rpn函式
12                                                             隨後傳給SAMPLE_SVP_NNIE_ForwardWithBbox函式*/
13         VIDEO_FRAME_INFO_S* pstExtFrmInfo,
14         HI_U32 u32BaseWidth,HI_U32 u32BaseHeight)
15 
16     pstParam->astSegData[stInputDataIdx.u32SegIdx].astSrc[stInputDataIdx.u32NodeIdx].u64VirAddr = pstExtFrmInfo->stVFrame.u64VirAddr[0];
17     pstParam->astSegData[stInputDataIdx.u32SegIdx].astSrc[stInputDataIdx.u32NodeIdx].u64PhyAddr = pstExtFrmInfo->stVFrame.u64PhyAddr[0];
18     pstParam->astSegData[stInputDataIdx.u32SegIdx].astSrc[stInputDataIdx.u32NodeIdx].u32Stride  = pstExtFrmInfo->stVFrame.u32Stride[0];
19 
20 
21     s32Ret = SAMPLE_SVP_NNIE_Forward(pstParam,   //引數呼叫
22                                      &stInputDataIdx,
23                                      &stProcSegIdx,
24                                      HI_TRUE);
25     
26     /*RPN*/
27     s32Ret = SAMPLE_SVP_NNIE_Rfcn_Rpn(pstParam,   //引數呼叫
28                                       pstSwParam);  //引數呼叫
29 
30 
31     s32Ret = SAMPLE_SVP_NNIE_ForwardWithBbox(
32                 pstParam,    //引數呼叫
33                 &stInputDataIdx,
34                 &pstSwParam->stRpnBbox,
35                 &stProcSegIdx,
36                 HI_TRUE);
37 
38 
39 
40     //函式呼叫
41 
42     /*GetResult*/
43     /*if user has changed net struct, please make sure SAMPLE_SVP_NNIE_Rfcn_GetResult
44      function's input datas are correct*/
45     s32Ret = SAMPLE_SVP_NNIE_Rfcn_GetResult(pstParam,
46                                             pstSwParam);
47 
48     //函式呼叫
49     HI_S32 SAMPLE_SVP_NNIE_Rfcn_GetResult(SAMPLE_SVP_NNIE_PARAM_S*pstNnieParam,
50                                       SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S* pstSoftwareParam)

  此函式作用是為了獲取NNIE  RFCN的結果,其核心是實現了SVP_NNIE_Rfcn_GetResult函式,具體如下:

 1 HI_S32 SAMPLE_SVP_NNIE_Rfcn_GetResult(SAMPLE_SVP_NNIE_PARAM_S*pstNnieParam,
 2                                       SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S* pstSoftwareParam)
 3 {
 4     HI_S32 s32Ret = HI_SUCCESS;
 5     HI_U32 i = 0;
 6     HI_S32* ps32Proposal = (HI_S32*)pstSoftwareParam->stRpnBbox.u64VirAddr;
 7 
 8     ......
 9 
10     for(i = 0; i < pstSoftwareParam->stRpnBbox.unShape.stWhc.u32Height; i++)
11     {
12         *(ps32Proposal+SAMPLE_SVP_NNIE_COORDI_NUM*i) /= SAMPLE_SVP_NNIE_QUANT_BASE;
13         *(ps32Proposal+SAMPLE_SVP_NNIE_COORDI_NUM*i+1) /= SAMPLE_SVP_NNIE_QUANT_BASE;
14         *(ps32Proposal+SAMPLE_SVP_NNIE_COORDI_NUM*i+2) /= SAMPLE_SVP_NNIE_QUANT_BASE;
15         *(ps32Proposal+SAMPLE_SVP_NNIE_COORDI_NUM*i+3) /= SAMPLE_SVP_NNIE_QUANT_BASE;
16     }
17     //this function is used to get RFCN result
18     s32Ret = SVP_NNIE_Rfcn_GetResult(
19                  (HI_S32*)pstNnieParam->astSegData[1].astDst[0].u64VirAddr,
20                  pstNnieParam->astSegData[1].astDst[0].u32Stride,
21                  (HI_S32*)pstNnieParam->astSegData[2].astDst[0].u64VirAddr,
22                  pstNnieParam->astSegData[2].astDst[0].u32Stride,
23                  (HI_S32*)pstSoftwareParam->stRpnBbox.u64VirAddr,
24                  pstSoftwareParam->stRpnBbox.unShape.stWhc.u32Height,
25                  pstSoftwareParam->au32ConfThresh,pstSoftwareParam->u32MaxRoiNum,
26                  pstSoftwareParam->u32ClassNum,pstSoftwareParam->u32OriImWidth,
27                  pstSoftwareParam->u32OriImHeight,pstSoftwareParam->u32ValidNmsThresh,
28                  (HI_U32*)pstSoftwareParam->stGetResultTmpBuf.u64VirAddr,
29                  (HI_S32*)pstSoftwareParam->stDstScore.u64VirAddr,
30                  (HI_S32*)pstSoftwareParam->stDstRoi.u64VirAddr,
31                  (HI_S32*)pstSoftwareParam->stClassRoiNum.u64VirAddr);
32     ......
33     return s32Ret;
34 }

2.2.5、SAMPLE_SVP_NNIE_RoiToRect子函式分析

  SAMPLE_SVP_NNIE_Rfcn_Proc函式最後一個子函式為對ROI(感興趣區域畫框),RFNC對21類物體進行目標識別,具體呼叫及實現如下:

 1     /*draw result, this sample has 21 classes:
 2      class 0:background     class 1:plane           class 2:bicycle
 3      class 3:bird           class 4:boat            class 5:bottle
 4      class 6:bus            class 7:car             class 8:cat
 5      class 9:chair          class10:cow             class11:diningtable
 6      class 12:dog           class13:horse           class14:motorbike
 7      class 15:person        class16:pottedplant     class17:sheep
 8      class 18:sofa          class19:train           class20:tvmonitor*/
 9     s32Ret = SAMPLE_SVP_NNIE_RoiToRect(
10                  &(pstSwParam->stDstScore),
11                  &(pstSwParam->stDstRoi), 
12                  &(pstSwParam->stClassRoiNum), 
13                  pstSwParam->af32ScoreThr,
14                  HI_TRUE,
15                  &(pstSwParam->stRect),
16                  pstExtFrmInfo->stVFrame.u32Width, 
17                  pstExtFrmInfo->stVFrame.u32Height,
18                  u32BaseWidth,
19                  u32BaseHeight);
20 
21 /******************************************************************************
22 * function : roi to rect
23 ******************************************************************************/
24 HI_S32 SAMPLE_SVP_NNIE_RoiToRect(SVP_BLOB_S *pstDstScore,
25                                  SVP_BLOB_S *pstDstRoi, SVP_BLOB_S *pstClassRoiNum, HI_FLOAT *paf32ScoreThr,
26                                  HI_BOOL bRmBg,SAMPLE_SVP_NNIE_RECT_ARRAY_S *pstRect,
27                                  HI_U32 u32SrcWidth, HI_U32 u32SrcHeight,HI_U32 u32DstWidth,HI_U32 u32DstHeight)
28 {
29     HI_U32 i = 0, j = 0;
30     HI_U32 u32RoiNumBias = 0;
31     HI_U32 u32ScoreBias = 0;
32     HI_U32 u32BboxBias = 0;
33     HI_FLOAT f32Score = 0.0f;
34     HI_S32* ps32Score = (HI_S32*)pstDstScore->u64VirAddr;
35     HI_S32* ps32Roi = (HI_S32*)pstDstRoi->u64VirAddr;
36     HI_S32* ps32ClassRoiNum = (HI_S32*)pstClassRoiNum->u64VirAddr;
37     HI_U32 u32ClassNum = pstClassRoiNum->unShape.stWhc.u32Width;
38     HI_U32 u32RoiNumTmp = 0;
39 
40     .......
41     
42     pstRect->u32TotalNum = 0;
43     pstRect->u32ClsNum = u32ClassNum;
44     if (bRmBg)
45     {
46         pstRect->au32RoiNum[0] = 0;
47         u32RoiNumBias += ps32ClassRoiNum[0];
48         for (i = 1; i < u32ClassNum; i++)
49         {
50             u32ScoreBias = u32RoiNumBias;
51             u32BboxBias = u32RoiNumBias * SAMPLE_SVP_NNIE_COORDI_NUM;
52             u32RoiNumTmp = 0;
53             /*if the confidence score greater than result thresh, the result will be drawed*/
54             if(((HI_FLOAT)ps32Score[u32ScoreBias] / SAMPLE_SVP_NNIE_QUANT_BASE >=
55                     paf32ScoreThr[i])  &&  (ps32ClassRoiNum[i] != 0))
56             {
57                 for (j = 0; j < (HI_U32)ps32ClassRoiNum[i]; j++)
58                 {
59                     /*Score is descend order*/
60                     f32Score = (HI_FLOAT)ps32Score[u32ScoreBias + j] / SAMPLE_SVP_NNIE_QUANT_BASE;
61                     if ((f32Score < paf32ScoreThr[i]) || (u32RoiNumTmp >= SAMPLE_SVP_NNIE_MAX_ROI_NUM_OF_CLASS))
62                     {
63                         break;
64                     }
65 
66                     pstRect->astRect[i][u32RoiNumTmp].astPoint[0].s32X = (HI_U32)((HI_FLOAT)ps32Roi[u32BboxBias + j*SAMPLE_SVP_NNIE_COORDI_NUM] / (HI_FLOAT)u32SrcWidth * (HI_FLOAT)u32DstWidth) & (~1) ;
67                     pstRect->astRect[i][u32RoiNumTmp].astPoint[0].s32Y = (HI_U32)((HI_FLOAT)ps32Roi[u32BboxBias + j*SAMPLE_SVP_NNIE_COORDI_NUM + 1] / (HI_FLOAT)u32SrcHeight * (HI_FLOAT)u32DstHeight) & (~1);
68 
69                     pstRect->astRect[i][u32RoiNumTmp].astPoint[1].s32X = (HI_U32)((HI_FLOAT)ps32Roi[u32BboxBias + j*SAMPLE_SVP_NNIE_COORDI_NUM + 2]/ (HI_FLOAT)u32SrcWidth * (HI_FLOAT)u32DstWidth) & (~1);
70                     pstRect->astRect[i][u32RoiNumTmp].astPoint[1].s32Y = pstRect->astRect[i][u32RoiNumTmp].astPoint[0].s32Y;
71 
72                     pstRect->astRect[i][u32RoiNumTmp].astPoint[2].s32X = pstRect->astRect[i][u32RoiNumTmp].astPoint[1].s32X;
73                     pstRect->astRect[i][u32RoiNumTmp].astPoint[2].s32Y = (HI_U32)((HI_FLOAT)ps32Roi[u32BboxBias + j*SAMPLE_SVP_NNIE_COORDI_NUM + 3] / (HI_FLOAT)u32SrcHeight * (HI_FLOAT)u32DstHeight) & (~1);
74 
75                     pstRect->astRect[i][u32RoiNumTmp].astPoint[3].s32X =  pstRect->astRect[i][u32RoiNumTmp].astPoint[0].s32X;
76                     pstRect->astRect[i][u32RoiNumTmp].astPoint[3].s32Y =  pstRect->astRect[i][u32RoiNumTmp].astPoint[2].s32Y;
77 
78                     u32RoiNumTmp++;
79                 }
80 
81             }
82 
83             pstRect->au32RoiNum[i] = u32RoiNumTmp;
84             pstRect->u32TotalNum += u32RoiNumTmp;
85             u32RoiNumBias += ps32ClassRoiNum[i];
86         }
87 
88     }
89     return HI_SUCCESS;
90 }

3、SAMPLE_COMM_SVP_NNIE_FillRect函式分析

  SAMPLE_COMM_SVP_NNIE_FillRect主要是配合VGS實現畫框功能,具體呼叫和實現如下,函式功能已由註釋給出:

 1     //Draw rect
 2     s32Ret = SAMPLE_COMM_SVP_NNIE_FillRect(
 3                  &stBaseFrmInfo, 
 4                  &(pstSwParam->stRect), 
 5                  0x0000FF00); //綠色
 6 
 7 HI_S32 SAMPLE_COMM_SVP_NNIE_FillRect(
 8     VIDEO_FRAME_INFO_S *pstFrmInfo,
 9     SAMPLE_SVP_NNIE_RECT_ARRAY_S* pstRect, 
10     HI_U32 u32Color)
11 {
12     VGS_HANDLE VgsHandle = -1;
13     HI_S32 s32Ret = HI_SUCCESS;
14     HI_U32 i,j;
15     VGS_TASK_ATTR_S stVgsTask;//定義 VGS task 的屬性
16     VGS_ADD_COVER_S stVgsAddCover;//定義 VGS 上 COVER 的配置
17     static HI_U32 u32Frm = 0;
18     u32Frm++;
19     if (0 == pstRect->u32TotalNum)
20     {
21         return s32Ret;
22     }
23     s32Ret = HI_MPI_VGS_BeginJob(&VgsHandle); //啟動一個 job。
24     if (s32Ret != HI_SUCCESS)
25     {
26         ......
27         return s32Ret;
28     }
29 
30     memcpy(&stVgsTask.stImgIn, pstFrmInfo, sizeof(VIDEO_FRAME_INFO_S));
31     memcpy(&stVgsTask.stImgOut, pstFrmInfo, sizeof(VIDEO_FRAME_INFO_S));
32 
33     stVgsAddCover.enCoverType = COVER_QUAD_RANGLE;//任意四邊形COVER
34     stVgsAddCover.u32Color = u32Color;  //RGB888
35     stVgsAddCover.stQuadRangle.bSolid = HI_FALSE; //空心 COVER
36     stVgsAddCover.stQuadRangle.u32Thick = 2; //2 畫素對齊
37 
38     for (i = 0; i < pstRect->u32ClsNum; i++)
39     {
40         for (j = 0; j < pstRect->au32RoiNum[i]; j++)
41         {
42             memcpy(stVgsAddCover.stQuadRangle.stPoint, pstRect->astRect[i][j].astPoint, sizeof(pstRect->astRect[i][j].astPoint));
43             
44             //做 COVER 任務時,輸入輸出影像為同一塊 buffer
45             //往一個已經啟動的 job 裡新增打 COVER task。 task屬性必須滿足VGS的能力。
46             s32Ret = HI_MPI_VGS_AddCoverTask(VgsHandle, &stVgsTask, &stVgsAddCover);
47             if (s32Ret != HI_SUCCESS)
48             {
49                 SAMPLE_PRT("HI_MPI_VGS_AddCoverTask fail,Error(%#x)\n", s32Ret);
50                 HI_MPI_VGS_CancelJob(VgsHandle);
51                 return s32Ret;
52             }
53 
54         }
55 
56     }
57     //提交一個 job。
58     s32Ret = HI_MPI_VGS_EndJob(VgsHandle);
59     if (s32Ret != HI_SUCCESS)
60     {
61         SAMPLE_PRT("HI_MPI_VGS_EndJob fail,Error(%#x)\n", s32Ret);
62         HI_MPI_VGS_CancelJob(VgsHandle);
63         return s32Ret;
64     }
65 
66     return s32Ret;

4、HI_MPI_VO_SendFrame函式分析

  SAMPLE_SVP_NNIE_Rfcn_ViToVo執行緒函式執行的最後一個函式HI_MPI_VO_SendFrame,函式作用是將視訊影像送入指定輸出通道顯示,具體呼叫如下:

1         s32Ret = HI_MPI_VO_SendFrame(voLayer, 
2                                      voChn, 
3                                      &stBaseFrmInfo, 
4                                      s32MilliSec);

 

  

 

相關文章