前面隨筆給出了NNIE開發的基本知識,下面幾篇隨筆將著重於Mobilefacenet NNIE開發,實現mobilefacenet.wk的chip版本,並在Hi3559AV100上實現mobilefacenet網路功能,外接USB攝像頭通過MPP平臺輸出至VO HDMI顯示結果。下文是Hi3559AV100 NNIE開發(6)RFCN中實現關鍵執行緒函式->SAMPLE_SVP_NNIE_Rfcn_ViToVo()進行資料流分析,通過對執行緒函式分析,詳細瞭解如何對.wk模型資料進行處理並弄清楚檢測框繪製這些後處理的實現。
1、SAMPLE_SVP_NNIE_Rfcn_ViToVo()函式呼叫
首先給出SAMPLE_SVP_NNIE_Rfcn_ViToVo()函式的呼叫,為後續分析提供參照:
1 static pthread_t s_hNnieThread = 0; //全域性定義 2 3 HI_CHAR acThreadName[16] = {0};//區域性定義 4 5 6 /****************************************** 7 Create work thread 8 ******************************************/ 9 snprintf(acThreadName, 16, "NNIE_ViToVo"); 10 prctl(PR_SET_NAME, (unsigned long)acThreadName, 0,0,0); 11 pthread_create(&s_hNnieThread, 0, SAMPLE_SVP_NNIE_Rfcn_ViToVo, NULL);
其中prctl函式的定義如下,其中PR_SET_NAME表示使用(char *) arg2所指向的位置中的值設定呼叫執行緒的名稱。
1 int prctl(int option, unsigned long arg2, unsigned long arg3, 2 unsigned long arg4, unsigned long arg5);
2、SAMPLE_SVP_NNIE_Rfcn_ViToVo()函式實現具體分析
下面講分析SAMPLE_SVP_NNIE_Rfcn_ViToVo(HI_VOID* pArgs)函式的內部實現,具體設計到5個函式,實現的功能已經註釋出來,具體如下:
1 static HI_VOID* SAMPLE_SVP_NNIE_Rfcn_ViToVo(HI_VOID* pArgs) 2 { 3 4 .... //引數定義 5 6 while (HI_FALSE == s_bNnieStopSignal) 7 { 8 //使用者從通道獲取一幀處理完成的影像 通道1-輸出stExtFrmInfo 9 s32Ret = HI_MPI_VPSS_GetChnFrame(s32VpssGrp, 10 as32VpssChn[1], 11 &stExtFrmInfo, 12 s32MilliSec); 13 ...... 14 15 //使用者從通道獲取一幀處理完成的影像 通道0-輸出stBaseFrmInfo 16 s32Ret = HI_MPI_VPSS_GetChnFrame(s32VpssGrp, 17 as32VpssChn[0], 18 &stBaseFrmInfo, 19 s32MilliSec); 20 ...... 21 22 //關鍵處理函式 23 s32Ret = SAMPLE_SVP_NNIE_Rfcn_Proc( 24 pstParam, 25 pstSwParam, 26 &stExtFrmInfo, 27 stBaseFrmInfo.stVFrame.u32Width, 28 stBaseFrmInfo.stVFrame.u32Height); 29 ...... 30 31 //Draw rect 32 s32Ret = SAMPLE_COMM_SVP_NNIE_FillRect( 33 &stBaseFrmInfo, 34 &(pstSwParam->stRect), 35 0x0000FF00); //綠色 36 ...... 37 38 //將視訊影像送入指定輸出通道顯示。 39 s32Ret = HI_MPI_VO_SendFrame(voLayer, 40 voChn, 41 &stBaseFrmInfo, 42 s32MilliSec); 43 ...... 44 45 BASE_RELEASE: 46 s32Ret = HI_MPI_VPSS_ReleaseChnFrame(s32VpssGrp,as32VpssChn[0], &stBaseFrmInfo); 47 ...... 48 49 EXT_RELEASE: 50 s32Ret = HI_MPI_VPSS_ReleaseChnFrame(s32VpssGrp,as32VpssChn[1], &stExtFrmInfo); 51 ....... 52 53 } 54 55 return HI_NULL; 56 }
2.1、HI_MPI_VPSS_GetChnFrame的實現
下面給出HI_MPI_VPSS_GetChnFrame函式的引數呼叫與實現過程(其中VPSS雙通道輸出),具體功能為從通道獲取一幀處理完成的影像:
1 HI_S32 s32VpssGrp = 0; 2 HI_S32 as32VpssChn[] = {VPSS_CHN0, VPSS_CHN1}; 3 VIDEO_FRAME_INFO_S stBaseFrmInfo; 4 VIDEO_FRAME_INFO_S stExtFrmInfo; 5 HI_S32 s32MilliSec = 20000; 6 7 //使用者從通道獲取一幀處理完成的影像 通道1-輸出stExtFrmInfo 8 s32Ret = HI_MPI_VPSS_GetChnFrame(s32VpssGrp, 9 as32VpssChn[1], 10 &stExtFrmInfo, 11 s32MilliSec); 12 13 //使用者從通道獲取一幀處理完成的影像 通道0-輸出stBaseFrmInfo 14 s32Ret = HI_MPI_VPSS_GetChnFrame(s32VpssGrp, 15 as32VpssChn[0], 16 &stBaseFrmInfo, 17 s32MilliSec); 18 19 //函式定義 20 HI_S32 HI_MPI_VPSS_GetChnFrame(VPSS_GRP VpssGrp, VPSS_CHN VpssChn, 21 VIDEO_FRAME_INFO_S *pstVideoFrame, HI_S32 s32MilliSec);
2.2、SAMPLE_SVP_NNIE_Rfcn_Proc的實現-NNIE資料處理
SAMPLE_SVP_NNIE_Rfcn_Proc函式實現是整個RFCN NNIE資料處理過程的Key Point,檢測加框等資訊來源處,現給出函式引數及實現分析:
1 SAMPLE_SVP_NNIE_PARAM_S *pstParam; 2 SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S *pstSwParam; 3 4 pstParam = &s_stRfcnNnieParam; 5 pstSwParam = &s_stRfcnSoftwareParam; 6 /* 7 s_stRfcnNnieParam及s_stRfcnSoftwareParam引數涉及前述操作 8 9 s_stRfcnNnieParam.pstModel = &s_stRfcnModel.stModel; 10 s_stRfcnSoftwareParam.apcRpnDataLayerName[0] = "rpn_cls_score"; 11 s_stRfcnSoftwareParam.apcRpnDataLayerName[1] = "rpn_bbox_pred"; 12 s32Ret = SAMPLE_SVP_NNIE_Rfcn_ParamInit(&stNnieCfg, 13 &s_stRfcnNnieParam, 14 &s_stRfcnSoftwareParam); 15 */ 16 17 18 VIDEO_FRAME_INFO_S stBaseFrmInfo; 19 VIDEO_FRAME_INFO_S stExtFrmInfo; 20 21 //其中stBaseFrmInfo引數與stExtFrmInfo引數是經過下述函式輸出得到 22 /* 23 s32Ret = HI_MPI_VPSS_GetChnFrame(s32VpssGrp, 24 as32VpssChn[1], 25 &stExtFrmInfo, 26 s32MilliSec); 27 s32Ret = HI_MPI_VPSS_GetChnFrame(s32VpssGrp, 28 as32VpssChn[0], 29 &stBaseFrmInfo, 30 s32MilliSec); 31 */ 32 33 34 s32Ret = SAMPLE_SVP_NNIE_Rfcn_Proc( 35 pstParam, 36 pstSwParam, 37 &stExtFrmInfo, 38 stBaseFrmInfo.stVFrame.u32Width, 39 stBaseFrmInfo.stVFrame.u32Height);
隨之給出SAMPLE_SVP_NNIE_Rfcn_Proc函式的定義及內部實現過程:
1 static HI_S32 SAMPLE_SVP_NNIE_Rfcn_Proc( 2 SAMPLE_SVP_NNIE_PARAM_S *pstParam, 3 SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S *pstSwParam, 4 VIDEO_FRAME_INFO_S* pstExtFrmInfo, 5 HI_U32 u32BaseWidth,HI_U32 u32BaseHeight) 6 { 7 8 ......引數定義 9 10 s32Ret = SAMPLE_SVP_NNIE_Forward(pstParam, 11 &stInputDataIdx, 12 &stProcSegIdx, 13 HI_TRUE); 14 15 ...... 16 17 /*RPN*/ 18 s32Ret = SAMPLE_SVP_NNIE_Rfcn_Rpn(pstParam, 19 pstSwParam); 20 21 22 if(0 != pstSwParam->stRpnBbox.unShape.stWhc.u32Height) 23 { 24 25 s32Ret = SAMPLE_SVP_NNIE_ForwardWithBbox( 26 pstParam, 27 &stInputDataIdx, 28 &pstSwParam->stRpnBbox, 29 &stProcSegIdx, 30 HI_TRUE); 31 ...... 32 33 s32Ret = SAMPLE_SVP_NNIE_ForwardWithBbox( 34 pstParam, 35 &stInputDataIdx, 36 &pstSwParam->stRpnBbox, 37 &stProcSegIdx, 38 HI_TRUE); 39 ...... 40 41 s32Ret = SAMPLE_SVP_NNIE_Rfcn_GetResult(pstParam, 42 pstSwParam); 43 44 } 45 else 46 { ...... } 47 s32Ret = SAMPLE_SVP_NNIE_RoiToRect( 48 &(pstSwParam->stDstScore), 49 &(pstSwParam->stDstRoi), 50 &(pstSwParam->stClassRoiNum), 51 pstSwParam->af32ScoreThr, 52 HI_TRUE, 53 &(pstSwParam->stRect), 54 pstExtFrmInfo->stVFrame.u32Width, 55 pstExtFrmInfo->stVFrame.u32Height, 56 u32BaseWidth, 57 u32BaseHeight); 58 59 ...... 60 61 return s32Ret; 62 63 }
2.2.1、SAMPLE_SVP_NNIE_Forward子函式分析
首先給出函式的呼叫及引數細節,便於分析函式功能:
1 其中在SAMPLE_SVP_NNIE_Rfcn_ViToVo函式中: 2 SAMPLE_SVP_NNIE_PARAM_S *pstParam; 3 pstParam = &s_stRfcnNnieParam; //此值傳給SAMPLE_SVP_NNIE_Rfcn_Proc 4 5 6 7 static HI_S32 SAMPLE_SVP_NNIE_Rfcn_Proc( 8 SAMPLE_SVP_NNIE_PARAM_S *pstParam, /*這個引數傳進來之後經過了SP420賦值然後送入 9 SAMPLE_SVP_NNIE_Forward*/ 10 SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S *pstSwParam, 11 VIDEO_FRAME_INFO_S* pstExtFrmInfo, 12 HI_U32 u32BaseWidth,HI_U32 u32BaseHeight) 13 14 ...... 15 16 SAMPLE_SVP_NNIE_INPUT_DATA_INDEX_S stInputDataIdx = {0}; 17 SAMPLE_SVP_NNIE_PROCESS_SEG_INDEX_S stProcSegIdx = {0}; 18 19 stInputDataIdx.u32SegIdx = 0; 20 stInputDataIdx.u32NodeIdx = 0; 21 22 /*SP420*/ 23 pstParam->astSegData[stInputDataIdx.u32SegIdx].astSrc[stInputDataIdx.u32NodeIdx].u64VirAddr = pstExtFrmInfo->stVFrame.u64VirAddr[0]; 24 pstParam->astSegData[stInputDataIdx.u32SegIdx].astSrc[stInputDataIdx.u32NodeIdx].u64PhyAddr = pstExtFrmInfo->stVFrame.u64PhyAddr[0]; 25 pstParam->astSegData[stInputDataIdx.u32SegIdx].astSrc[stInputDataIdx.u32NodeIdx].u32Stride = pstExtFrmInfo->stVFrame.u32Stride[0]; 26 27 /*NNIE process 0-th seg*/ 28 stProcSegIdx.u32SegIdx = 0; 29 30 /*NNIE process 0-th seg*/ 31 stProcSegIdx.u32SegIdx = 0; 32 s32Ret = SAMPLE_SVP_NNIE_Forward(pstParam, 33 &stInputDataIdx, 34 &stProcSegIdx, 35 HI_TRUE); 36 37 ...... 38 39 //函式定義 40 static HI_S32 SAMPLE_SVP_NNIE_Forward( 41 SAMPLE_SVP_NNIE_PARAM_S *pstNnieParam, 42 SAMPLE_SVP_NNIE_INPUT_DATA_INDEX_S* pstInputDataIdx, 43 SAMPLE_SVP_NNIE_PROCESS_SEG_INDEX_S* pstProcSegIdx, 44 HI_BOOL bInstant)
此函式如其名,主要實現了NNIE forward功能,下面給出呼叫具體函式:
1 /****************************************************************************** 2 * function : NNIE Forward 3 ******************************************************************************/ 4 static HI_S32 SAMPLE_SVP_NNIE_Forward(SAMPLE_SVP_NNIE_PARAM_S *pstNnieParam, 5 SAMPLE_SVP_NNIE_INPUT_DATA_INDEX_S* pstInputDataIdx, 6 SAMPLE_SVP_NNIE_PROCESS_SEG_INDEX_S* pstProcSegIdx,HI_BOOL bInstant) 7 { 8 ...... 9 10 SAMPLE_COMM_SVP_FlushCache( 11 pstNnieParam->astForwardCtrl[pstProcSegIdx->u32SegIdx].stTskBuf.u64PhyAddr, 12 (HI_VOID *) pstNnieParam->astForwardCtrl[pstProcSegIdx->u32SegIdx].stTskBuf.u64VirAddr, 13 pstNnieParam->astForwardCtrl[pstProcSegIdx->u32SegIdx].stTskBuf.u32Size); 14 15 /*set input blob according to node name*/ 16 if(pstInputDataIdx->u32SegIdx != pstProcSegIdx->u32SegIdx) 17 { 18 for(i = 0; i < pstNnieParam->pstModel->astSeg[pstProcSegIdx->u32SegIdx].u16SrcNum; i++) 19 { 20 ...... 21 } 22 } 23 24 /*NNIE_Forward 多節點輸入輸出的 CNN 型別網路預測。 25 對輸入樣本(s)進行CNN預測,對對應樣本(s)進行輸出響應*/ 26 s32Ret = HI_MPI_SVP_NNIE_Forward( 27 &hSvpNnieHandle, 28 pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astSrc, 29 pstNnieParam->pstModel, 30 pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst, 31 &pstNnieParam->astForwardCtrl[pstProcSegIdx->u32SegIdx], 32 bInstant); 33 34 35 if(bInstant) 36 { 37 /*Wait NNIE finish,,,,enNnieId 執行網路段的 NNIE 引擎 ID。 */ 38 while(HI_ERR_SVP_NNIE_QUERY_TIMEOUT == (s32Ret = HI_MPI_SVP_NNIE_Query(pstNnieParam->astForwardCtrl[pstProcSegIdx->u32SegIdx].enNnieId, 39 hSvpNnieHandle, &bFinish, HI_TRUE))) 40 { 41 ...... 42 } 43 } 44 45 bFinish = HI_FALSE; 46 for(i = 0; i < pstNnieParam->astForwardCtrl[pstProcSegIdx->u32SegIdx].u32DstNum; i++) 47 { 48 if(SVP_BLOB_TYPE_SEQ_S32 == pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].enType) 49 { 50 ...... 51 52 SAMPLE_COMM_SVP_FlushCache( 53 pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u64PhyAddr, 54 (HI_VOID *) pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u64VirAddr, 55 u32TotalStepNum*pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u32Stride); 56 57 } 58 else 59 { 60 61 SAMPLE_COMM_SVP_FlushCache(pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u64PhyAddr, 62 (HI_VOID *) pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u64VirAddr, 63 pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u32Num* 64 pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].unShape.stWhc.u32Chn* 65 pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].unShape.stWhc.u32Height* 66 pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u32Stride); 67 } 68 } 69 70 return s32Ret; 71 }
2.2.2、SAMPLE_SVP_NNIE_Rfcn_Rpn子函式分析
下面給出sample_svp_NNIE_Rfcn_Rpn子函式分析,此函式主要是用於rpn相關,下面給出引數呼叫及函式分析:
1 其中在SAMPLE_SVP_NNIE_Rfcn_ViToVo函式中: 2 SAMPLE_SVP_NNIE_PARAM_S *pstParam; //此值傳給SAMPLE_SVP_NNIE_Rfcn_Proc 3 pstParam = &s_stRfcnNnieParam; //此值傳給SAMPLE_SVP_NNIE_Rfcn_Proc 4 5 SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S *pstSwParam; 6 pstSwParam = &s_stRfcnSoftwareParam; 7 8 static HI_S32 SAMPLE_SVP_NNIE_Rfcn_Proc( 9 SAMPLE_SVP_NNIE_PARAM_S *pstParam, /*這個引數傳進來之後經過了SP420賦值然後送入 10 SAMPLE_SVP_NNIE_Forward*/ 11 SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S *pstSwParam, //直接傳給SAMPLE_SVP_NNIE_Rfcn_Rpn函式 12 VIDEO_FRAME_INFO_S* pstExtFrmInfo, 13 HI_U32 u32BaseWidth,HI_U32 u32BaseHeight) 14 15 /*SP420*/ 16 pstParam->astSegData[stInputDataIdx.u32SegIdx].astSrc[stInputDataIdx.u32NodeIdx].u64VirAddr = pstExtFrmInfo->stVFrame.u64VirAddr[0]; 17 pstParam->astSegData[stInputDataIdx.u32SegIdx].astSrc[stInputDataIdx.u32NodeIdx].u64PhyAddr = pstExtFrmInfo->stVFrame.u64PhyAddr[0]; 18 pstParam->astSegData[stInputDataIdx.u32SegIdx].astSrc[stInputDataIdx.u32NodeIdx].u32Stride = pstExtFrmInfo->stVFrame.u32Stride[0]; 19 20 此後,pstParam引數先傳入SAMPLE_SVP_NNIE_Forward函式進行處理,隨後傳入SAMPLE_SVP_NNIE_Rfcn_Rpn函式 21 22 /*RPN*/ 23 s32Ret = SAMPLE_SVP_NNIE_Rfcn_Rpn(pstParam, 24 pstSwParam); 25 26 // 函式呼叫 ,用於used to do rpn 27 HI_S32 SAMPLE_SVP_NNIE_Rfcn_Rpn( 28 SAMPLE_SVP_NNIE_PARAM_S*pstNnieParam, 29 SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S* pstSoftwareParam)
下面給出sample_svp_NNIE_Rfcn_Rpn子函式內部實現,此函式雖然傳入兩個引數,但是沒有對pstNnieParam引數進行任何處理,完成的pstSoftwareParam結構體內大量引數的的處理 ,主要是呼叫了SVP_NNIE_Rpn函式與SAMPLE_COMM_SVP_FlushCache函式:
1 HI_S32 SAMPLE_SVP_NNIE_Rfcn_Rpn(SAMPLE_SVP_NNIE_PARAM_S*pstNnieParam, 2 SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S* pstSoftwareParam) 3 { 4 HI_S32 s32Ret = HI_SUCCESS; 5 s32Ret = SVP_NNIE_Rpn(pstSoftwareParam->aps32Conv,pstSoftwareParam->u32NumRatioAnchors, 6 pstSoftwareParam->u32NumScaleAnchors,pstSoftwareParam->au32Scales, 7 pstSoftwareParam->au32Ratios,pstSoftwareParam->u32OriImHeight, 8 pstSoftwareParam->u32OriImWidth,pstSoftwareParam->au32ConvHeight, 9 pstSoftwareParam->au32ConvWidth,pstSoftwareParam->au32ConvChannel, 10 pstSoftwareParam->u32ConvStride,pstSoftwareParam->u32MaxRoiNum, 11 pstSoftwareParam->u32MinSize,pstSoftwareParam->u32SpatialScale, 12 pstSoftwareParam->u32NmsThresh,pstSoftwareParam->u32FilterThresh, 13 pstSoftwareParam->u32NumBeforeNms,(HI_U32*)pstSoftwareParam->stRpnTmpBuf.u64VirAddr, 14 (HI_S32*)pstSoftwareParam->stRpnBbox.u64VirAddr, 15 &pstSoftwareParam->stRpnBbox.unShape.stWhc.u32Height); 16 SAMPLE_COMM_SVP_FlushCache(pstSoftwareParam->stRpnBbox.u64PhyAddr, 17 (HI_VOID *) pstSoftwareParam->stRpnBbox.u64VirAddr, 18 pstSoftwareParam->stRpnBbox.u32Num* 19 pstSoftwareParam->stRpnBbox.unShape.stWhc.u32Chn* 20 pstSoftwareParam->stRpnBbox.unShape.stWhc.u32Height* 21 pstSoftwareParam->stRpnBbox.u32Stride); 22 SAMPLE_SVP_CHECK_EXPR_RET(HI_SUCCESS != s32Ret,s32Ret,SAMPLE_SVP_ERR_LEVEL_ERROR, 23 "Error,SVP_NNIE_Rpn failed!\n"); 24 return s32Ret; 25 }
2.2.3、SAMPLE_SVP_NNIE_ForwardWithBbox子函式分析
完成前面幾個函式後,通過if判斷,當滿足條件後執行SAMPLE_SVP_NNIE_ForwardWithBbox函式:
1 if(0 != pstSwParam->stRpnBbox.unShape.stWhc.u32Height)
否則不滿足if條件的時候,將執行下面賦值語句:
1 for (i = 0; i < pstSwParam->stClassRoiNum.unShape.stWhc.u32Width; i++) 2 { 3 *(((HI_U32*)(HI_UL)pstSwParam->stClassRoiNum.u64VirAddr)+i) = 0; 4 }
而SAMPLE_SVP_NNIE_ForwardWithBbox執行了有兩次,分別是對不同NNIE process x-th seg進行處理,具體如下:
1 其中在SAMPLE_SVP_NNIE_Rfcn_ViToVo函式中: 2 SAMPLE_SVP_NNIE_PARAM_S *pstParam; //此值傳給SAMPLE_SVP_NNIE_Rfcn_Proc 3 pstParam = &s_stRfcnNnieParam; //此值傳給SAMPLE_SVP_NNIE_Rfcn_Proc 4 5 SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S *pstSwParam; 6 pstSwParam = &s_stRfcnSoftwareParam; 7 8 static HI_S32 SAMPLE_SVP_NNIE_Rfcn_Proc( 9 SAMPLE_SVP_NNIE_PARAM_S *pstParam, /*這個引數傳進來之後經過了SP420賦值然後送入 10 SAMPLE_SVP_NNIE_Forward*/ 11 SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S *pstSwParam, /*直接傳給SAMPLE_SVP_NNIE_Rfcn_Rpn函式 12 隨後傳給SAMPLE_SVP_NNIE_ForwardWithBbox函式*/ 13 VIDEO_FRAME_INFO_S* pstExtFrmInfo, 14 HI_U32 u32BaseWidth,HI_U32 u32BaseHeight) 15 16 /*RPN*/ 17 s32Ret = SAMPLE_SVP_NNIE_Rfcn_Rpn(pstParam, 18 pstSwParam); /*此函式完成pstSwParam的相關賦值,並 19 傳給SAMPLE_SVP_NNIE_ForwardWithBbox函式*/ 20 21 22 /*在SAMPLE_SVP_NNIE_ForwardWithBbox函式前面,已經有了下面引數的賦值, 23 並且傳入至SAMPLE_SVP_NNIE_Forward函式*/ 24 stInputDataIdx.u32SegIdx = 0; 25 stInputDataIdx.u32NodeIdx = 0; 26 27 s32Ret = SAMPLE_SVP_NNIE_Forward(pstParam, 28 &stInputDataIdx, //stInputDataIdx引數已經用於此函式中 29 &stProcSegIdx, 30 HI_TRUE); 31 32 33 //函式呼叫 34 /*NNIE process 1-th seg, the input data comes from 3-rd report node of 0-th seg, 35 the input roi comes from RPN results*/ 36 stInputDataIdx.u32SegIdx = 0; 37 stInputDataIdx.u32NodeIdx = 3; 38 39 stProcSegIdx.u32SegIdx = 1; 40 41 s32Ret = SAMPLE_SVP_NNIE_ForwardWithBbox( 42 pstParam, 43 &stInputDataIdx, 44 &pstSwParam->stRpnBbox, 45 &stProcSegIdx, 46 HI_TRUE); 47 48 49 /*NNIE process 2-nd seg, the input data comes from 4-th report node of 0-th seg 50 the input roi comes from RPN results*/ 51 stInputDataIdx.u32SegIdx = 0; 52 stInputDataIdx.u32NodeIdx = 4; 53 54 stProcSegIdx.u32SegIdx = 2; 55 56 s32Ret = SAMPLE_SVP_NNIE_ForwardWithBbox( 57 pstParam, 58 &stInputDataIdx, 59 &pstSwParam->stRpnBbox, 60 &stProcSegIdx, 61 HI_TRUE);
下面給出函式具體實現:
1 /****************************************************************************** 2 * function : NNIE ForwardWithBbox 3 ******************************************************************************/ 4 static HI_S32 SAMPLE_SVP_NNIE_ForwardWithBbox( 5 SAMPLE_SVP_NNIE_PARAM_S *pstNnieParam, 6 SAMPLE_SVP_NNIE_INPUT_DATA_INDEX_S* pstInputDataIdx, 7 SVP_SRC_BLOB_S astBbox[], 8 SAMPLE_SVP_NNIE_PROCESS_SEG_INDEX_S* pstProcSegIdx, 9 HI_BOOL bInstant) 10 { 11 ...... 12 13 SAMPLE_COMM_SVP_FlushCache(pstNnieParam->astForwardWithBboxCtrl[pstProcSegIdx->u32SegIdx].stTskBuf.u64PhyAddr, 14 (HI_VOID *) pstNnieParam->astForwardWithBboxCtrl[pstProcSegIdx->u32SegIdx].stTskBuf.u64VirAddr, 15 pstNnieParam->astForwardWithBboxCtrl[pstProcSegIdx->u32SegIdx].stTskBuf.u32Size); 16 17 /*set input blob according to node name*/ 18 if(pstInputDataIdx->u32SegIdx != pstProcSegIdx->u32SegIdx) 19 { 20 for(i = 0; i < pstNnieParam->pstModel->astSeg[pstProcSegIdx->u32SegIdx].u16SrcNum; i++) 21 { 22 for(j = 0; j < pstNnieParam->pstModel->astSeg[pstInputDataIdx->u32SegIdx].u16DstNum; j++) 23 { 24 ...... 25 } 26 ...... 27 } 28 } 29 /*NNIE_ForwardWithBbox*/ 30 s32Ret = HI_MPI_SVP_NNIE_ForwardWithBbox( 31 &hSvpNnieHandle, 32 pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astSrc, 33 astBbox, 34 pstNnieParam->pstModel, //網路型別只支援ROI/PSROI 35 pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst, 36 &pstNnieParam->astForwardWithBboxCtrl[pstProcSegIdx->u32SegIdx], 37 bInstant); 38 39 ...... 40 41 if(bInstant) 42 { 43 /*Wait NNIE finish*/ 44 while(HI_ERR_SVP_NNIE_QUERY_TIMEOUT == (s32Ret = HI_MPI_SVP_NNIE_Query(pstNnieParam->astForwardWithBboxCtrl[pstProcSegIdx->u32SegIdx].enNnieId, 45 hSvpNnieHandle, &bFinish, HI_TRUE))) 46 { 47 ...... 48 } 49 } 50 51 bFinish = HI_FALSE; 52 53 54 for(i = 0; i < pstNnieParam->astForwardWithBboxCtrl[pstProcSegIdx->u32SegIdx].u32DstNum; i++) 55 { 56 if(SVP_BLOB_TYPE_SEQ_S32 == pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].enType) 57 { 58 ...... 59 60 SAMPLE_COMM_SVP_FlushCache(pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u64PhyAddr, 61 (HI_VOID *) pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u64VirAddr, 62 u32TotalStepNum*pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u32Stride); 63 } 64 else 65 { 66 SAMPLE_COMM_SVP_FlushCache(pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u64PhyAddr, 67 (HI_VOID *) pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u64VirAddr, 68 pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u32Num* 69 pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].unShape.stWhc.u32Chn* 70 pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].unShape.stWhc.u32Height* 71 pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u32Stride); 72 } 73 } 74 75 return s32Ret; 76 }
2.2.4、SAMPLE_SVP_NNIE_Rfcn_GetResult子函式分析
下一個子函式是獲得NNIE Rfcn的結果,前提需要保證網路結構和輸入資料保持一致,函式呼叫如下:
1 其中在SAMPLE_SVP_NNIE_Rfcn_ViToVo函式中: 2 SAMPLE_SVP_NNIE_PARAM_S *pstParam; //此值傳給SAMPLE_SVP_NNIE_Rfcn_Proc 3 pstParam = &s_stRfcnNnieParam; //此值傳給SAMPLE_SVP_NNIE_Rfcn_Proc 4 5 SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S *pstSwParam; 6 pstSwParam = &s_stRfcnSoftwareParam; 7 8 static HI_S32 SAMPLE_SVP_NNIE_Rfcn_Proc( 9 SAMPLE_SVP_NNIE_PARAM_S *pstParam, /*這個引數傳進來之後經過了SP420賦值然後送入 10 SAMPLE_SVP_NNIE_Forward*/ 11 SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S *pstSwParam, /*直接傳給SAMPLE_SVP_NNIE_Rfcn_Rpn函式 12 隨後傳給SAMPLE_SVP_NNIE_ForwardWithBbox函式*/ 13 VIDEO_FRAME_INFO_S* pstExtFrmInfo, 14 HI_U32 u32BaseWidth,HI_U32 u32BaseHeight) 15 16 pstParam->astSegData[stInputDataIdx.u32SegIdx].astSrc[stInputDataIdx.u32NodeIdx].u64VirAddr = pstExtFrmInfo->stVFrame.u64VirAddr[0]; 17 pstParam->astSegData[stInputDataIdx.u32SegIdx].astSrc[stInputDataIdx.u32NodeIdx].u64PhyAddr = pstExtFrmInfo->stVFrame.u64PhyAddr[0]; 18 pstParam->astSegData[stInputDataIdx.u32SegIdx].astSrc[stInputDataIdx.u32NodeIdx].u32Stride = pstExtFrmInfo->stVFrame.u32Stride[0]; 19 20 21 s32Ret = SAMPLE_SVP_NNIE_Forward(pstParam, //引數呼叫 22 &stInputDataIdx, 23 &stProcSegIdx, 24 HI_TRUE); 25 26 /*RPN*/ 27 s32Ret = SAMPLE_SVP_NNIE_Rfcn_Rpn(pstParam, //引數呼叫 28 pstSwParam); //引數呼叫 29 30 31 s32Ret = SAMPLE_SVP_NNIE_ForwardWithBbox( 32 pstParam, //引數呼叫 33 &stInputDataIdx, 34 &pstSwParam->stRpnBbox, 35 &stProcSegIdx, 36 HI_TRUE); 37 38 39 40 //函式呼叫 41 42 /*GetResult*/ 43 /*if user has changed net struct, please make sure SAMPLE_SVP_NNIE_Rfcn_GetResult 44 function's input datas are correct*/ 45 s32Ret = SAMPLE_SVP_NNIE_Rfcn_GetResult(pstParam, 46 pstSwParam); 47 48 //函式呼叫 49 HI_S32 SAMPLE_SVP_NNIE_Rfcn_GetResult(SAMPLE_SVP_NNIE_PARAM_S*pstNnieParam, 50 SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S* pstSoftwareParam)
此函式作用是為了獲取NNIE RFCN的結果,其核心是實現了SVP_NNIE_Rfcn_GetResult函式,具體如下:
1 HI_S32 SAMPLE_SVP_NNIE_Rfcn_GetResult(SAMPLE_SVP_NNIE_PARAM_S*pstNnieParam, 2 SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S* pstSoftwareParam) 3 { 4 HI_S32 s32Ret = HI_SUCCESS; 5 HI_U32 i = 0; 6 HI_S32* ps32Proposal = (HI_S32*)pstSoftwareParam->stRpnBbox.u64VirAddr; 7 8 ...... 9 10 for(i = 0; i < pstSoftwareParam->stRpnBbox.unShape.stWhc.u32Height; i++) 11 { 12 *(ps32Proposal+SAMPLE_SVP_NNIE_COORDI_NUM*i) /= SAMPLE_SVP_NNIE_QUANT_BASE; 13 *(ps32Proposal+SAMPLE_SVP_NNIE_COORDI_NUM*i+1) /= SAMPLE_SVP_NNIE_QUANT_BASE; 14 *(ps32Proposal+SAMPLE_SVP_NNIE_COORDI_NUM*i+2) /= SAMPLE_SVP_NNIE_QUANT_BASE; 15 *(ps32Proposal+SAMPLE_SVP_NNIE_COORDI_NUM*i+3) /= SAMPLE_SVP_NNIE_QUANT_BASE; 16 } 17 //this function is used to get RFCN result 18 s32Ret = SVP_NNIE_Rfcn_GetResult( 19 (HI_S32*)pstNnieParam->astSegData[1].astDst[0].u64VirAddr, 20 pstNnieParam->astSegData[1].astDst[0].u32Stride, 21 (HI_S32*)pstNnieParam->astSegData[2].astDst[0].u64VirAddr, 22 pstNnieParam->astSegData[2].astDst[0].u32Stride, 23 (HI_S32*)pstSoftwareParam->stRpnBbox.u64VirAddr, 24 pstSoftwareParam->stRpnBbox.unShape.stWhc.u32Height, 25 pstSoftwareParam->au32ConfThresh,pstSoftwareParam->u32MaxRoiNum, 26 pstSoftwareParam->u32ClassNum,pstSoftwareParam->u32OriImWidth, 27 pstSoftwareParam->u32OriImHeight,pstSoftwareParam->u32ValidNmsThresh, 28 (HI_U32*)pstSoftwareParam->stGetResultTmpBuf.u64VirAddr, 29 (HI_S32*)pstSoftwareParam->stDstScore.u64VirAddr, 30 (HI_S32*)pstSoftwareParam->stDstRoi.u64VirAddr, 31 (HI_S32*)pstSoftwareParam->stClassRoiNum.u64VirAddr); 32 ...... 33 return s32Ret; 34 }
2.2.5、SAMPLE_SVP_NNIE_RoiToRect子函式分析
SAMPLE_SVP_NNIE_Rfcn_Proc函式最後一個子函式為對ROI(感興趣區域畫框),RFNC對21類物體進行目標識別,具體呼叫及實現如下:
1 /*draw result, this sample has 21 classes: 2 class 0:background class 1:plane class 2:bicycle 3 class 3:bird class 4:boat class 5:bottle 4 class 6:bus class 7:car class 8:cat 5 class 9:chair class10:cow class11:diningtable 6 class 12:dog class13:horse class14:motorbike 7 class 15:person class16:pottedplant class17:sheep 8 class 18:sofa class19:train class20:tvmonitor*/ 9 s32Ret = SAMPLE_SVP_NNIE_RoiToRect( 10 &(pstSwParam->stDstScore), 11 &(pstSwParam->stDstRoi), 12 &(pstSwParam->stClassRoiNum), 13 pstSwParam->af32ScoreThr, 14 HI_TRUE, 15 &(pstSwParam->stRect), 16 pstExtFrmInfo->stVFrame.u32Width, 17 pstExtFrmInfo->stVFrame.u32Height, 18 u32BaseWidth, 19 u32BaseHeight); 20 21 /****************************************************************************** 22 * function : roi to rect 23 ******************************************************************************/ 24 HI_S32 SAMPLE_SVP_NNIE_RoiToRect(SVP_BLOB_S *pstDstScore, 25 SVP_BLOB_S *pstDstRoi, SVP_BLOB_S *pstClassRoiNum, HI_FLOAT *paf32ScoreThr, 26 HI_BOOL bRmBg,SAMPLE_SVP_NNIE_RECT_ARRAY_S *pstRect, 27 HI_U32 u32SrcWidth, HI_U32 u32SrcHeight,HI_U32 u32DstWidth,HI_U32 u32DstHeight) 28 { 29 HI_U32 i = 0, j = 0; 30 HI_U32 u32RoiNumBias = 0; 31 HI_U32 u32ScoreBias = 0; 32 HI_U32 u32BboxBias = 0; 33 HI_FLOAT f32Score = 0.0f; 34 HI_S32* ps32Score = (HI_S32*)pstDstScore->u64VirAddr; 35 HI_S32* ps32Roi = (HI_S32*)pstDstRoi->u64VirAddr; 36 HI_S32* ps32ClassRoiNum = (HI_S32*)pstClassRoiNum->u64VirAddr; 37 HI_U32 u32ClassNum = pstClassRoiNum->unShape.stWhc.u32Width; 38 HI_U32 u32RoiNumTmp = 0; 39 40 ....... 41 42 pstRect->u32TotalNum = 0; 43 pstRect->u32ClsNum = u32ClassNum; 44 if (bRmBg) 45 { 46 pstRect->au32RoiNum[0] = 0; 47 u32RoiNumBias += ps32ClassRoiNum[0]; 48 for (i = 1; i < u32ClassNum; i++) 49 { 50 u32ScoreBias = u32RoiNumBias; 51 u32BboxBias = u32RoiNumBias * SAMPLE_SVP_NNIE_COORDI_NUM; 52 u32RoiNumTmp = 0; 53 /*if the confidence score greater than result thresh, the result will be drawed*/ 54 if(((HI_FLOAT)ps32Score[u32ScoreBias] / SAMPLE_SVP_NNIE_QUANT_BASE >= 55 paf32ScoreThr[i]) && (ps32ClassRoiNum[i] != 0)) 56 { 57 for (j = 0; j < (HI_U32)ps32ClassRoiNum[i]; j++) 58 { 59 /*Score is descend order*/ 60 f32Score = (HI_FLOAT)ps32Score[u32ScoreBias + j] / SAMPLE_SVP_NNIE_QUANT_BASE; 61 if ((f32Score < paf32ScoreThr[i]) || (u32RoiNumTmp >= SAMPLE_SVP_NNIE_MAX_ROI_NUM_OF_CLASS)) 62 { 63 break; 64 } 65 66 pstRect->astRect[i][u32RoiNumTmp].astPoint[0].s32X = (HI_U32)((HI_FLOAT)ps32Roi[u32BboxBias + j*SAMPLE_SVP_NNIE_COORDI_NUM] / (HI_FLOAT)u32SrcWidth * (HI_FLOAT)u32DstWidth) & (~1) ; 67 pstRect->astRect[i][u32RoiNumTmp].astPoint[0].s32Y = (HI_U32)((HI_FLOAT)ps32Roi[u32BboxBias + j*SAMPLE_SVP_NNIE_COORDI_NUM + 1] / (HI_FLOAT)u32SrcHeight * (HI_FLOAT)u32DstHeight) & (~1); 68 69 pstRect->astRect[i][u32RoiNumTmp].astPoint[1].s32X = (HI_U32)((HI_FLOAT)ps32Roi[u32BboxBias + j*SAMPLE_SVP_NNIE_COORDI_NUM + 2]/ (HI_FLOAT)u32SrcWidth * (HI_FLOAT)u32DstWidth) & (~1); 70 pstRect->astRect[i][u32RoiNumTmp].astPoint[1].s32Y = pstRect->astRect[i][u32RoiNumTmp].astPoint[0].s32Y; 71 72 pstRect->astRect[i][u32RoiNumTmp].astPoint[2].s32X = pstRect->astRect[i][u32RoiNumTmp].astPoint[1].s32X; 73 pstRect->astRect[i][u32RoiNumTmp].astPoint[2].s32Y = (HI_U32)((HI_FLOAT)ps32Roi[u32BboxBias + j*SAMPLE_SVP_NNIE_COORDI_NUM + 3] / (HI_FLOAT)u32SrcHeight * (HI_FLOAT)u32DstHeight) & (~1); 74 75 pstRect->astRect[i][u32RoiNumTmp].astPoint[3].s32X = pstRect->astRect[i][u32RoiNumTmp].astPoint[0].s32X; 76 pstRect->astRect[i][u32RoiNumTmp].astPoint[3].s32Y = pstRect->astRect[i][u32RoiNumTmp].astPoint[2].s32Y; 77 78 u32RoiNumTmp++; 79 } 80 81 } 82 83 pstRect->au32RoiNum[i] = u32RoiNumTmp; 84 pstRect->u32TotalNum += u32RoiNumTmp; 85 u32RoiNumBias += ps32ClassRoiNum[i]; 86 } 87 88 } 89 return HI_SUCCESS; 90 }
3、SAMPLE_COMM_SVP_NNIE_FillRect函式分析
SAMPLE_COMM_SVP_NNIE_FillRect主要是配合VGS實現畫框功能,具體呼叫和實現如下,函式功能已由註釋給出:
1 //Draw rect 2 s32Ret = SAMPLE_COMM_SVP_NNIE_FillRect( 3 &stBaseFrmInfo, 4 &(pstSwParam->stRect), 5 0x0000FF00); //綠色 6 7 HI_S32 SAMPLE_COMM_SVP_NNIE_FillRect( 8 VIDEO_FRAME_INFO_S *pstFrmInfo, 9 SAMPLE_SVP_NNIE_RECT_ARRAY_S* pstRect, 10 HI_U32 u32Color) 11 { 12 VGS_HANDLE VgsHandle = -1; 13 HI_S32 s32Ret = HI_SUCCESS; 14 HI_U32 i,j; 15 VGS_TASK_ATTR_S stVgsTask;//定義 VGS task 的屬性 16 VGS_ADD_COVER_S stVgsAddCover;//定義 VGS 上 COVER 的配置 17 static HI_U32 u32Frm = 0; 18 u32Frm++; 19 if (0 == pstRect->u32TotalNum) 20 { 21 return s32Ret; 22 } 23 s32Ret = HI_MPI_VGS_BeginJob(&VgsHandle); //啟動一個 job。 24 if (s32Ret != HI_SUCCESS) 25 { 26 ...... 27 return s32Ret; 28 } 29 30 memcpy(&stVgsTask.stImgIn, pstFrmInfo, sizeof(VIDEO_FRAME_INFO_S)); 31 memcpy(&stVgsTask.stImgOut, pstFrmInfo, sizeof(VIDEO_FRAME_INFO_S)); 32 33 stVgsAddCover.enCoverType = COVER_QUAD_RANGLE;//任意四邊形COVER 34 stVgsAddCover.u32Color = u32Color; //RGB888 35 stVgsAddCover.stQuadRangle.bSolid = HI_FALSE; //空心 COVER 36 stVgsAddCover.stQuadRangle.u32Thick = 2; //2 畫素對齊 37 38 for (i = 0; i < pstRect->u32ClsNum; i++) 39 { 40 for (j = 0; j < pstRect->au32RoiNum[i]; j++) 41 { 42 memcpy(stVgsAddCover.stQuadRangle.stPoint, pstRect->astRect[i][j].astPoint, sizeof(pstRect->astRect[i][j].astPoint)); 43 44 //做 COVER 任務時,輸入輸出影像為同一塊 buffer 45 //往一個已經啟動的 job 裡新增打 COVER task。 task屬性必須滿足VGS的能力。 46 s32Ret = HI_MPI_VGS_AddCoverTask(VgsHandle, &stVgsTask, &stVgsAddCover); 47 if (s32Ret != HI_SUCCESS) 48 { 49 SAMPLE_PRT("HI_MPI_VGS_AddCoverTask fail,Error(%#x)\n", s32Ret); 50 HI_MPI_VGS_CancelJob(VgsHandle); 51 return s32Ret; 52 } 53 54 } 55 56 } 57 //提交一個 job。 58 s32Ret = HI_MPI_VGS_EndJob(VgsHandle); 59 if (s32Ret != HI_SUCCESS) 60 { 61 SAMPLE_PRT("HI_MPI_VGS_EndJob fail,Error(%#x)\n", s32Ret); 62 HI_MPI_VGS_CancelJob(VgsHandle); 63 return s32Ret; 64 } 65 66 return s32Ret;
4、HI_MPI_VO_SendFrame函式分析
SAMPLE_SVP_NNIE_Rfcn_ViToVo執行緒函式執行的最後一個函式HI_MPI_VO_SendFrame,函式作用是將視訊影像送入指定輸出通道顯示,具體呼叫如下:
1 s32Ret = HI_MPI_VO_SendFrame(voLayer, 2 voChn, 3 &stBaseFrmInfo, 4 s32MilliSec);