VideoPipe視覺化影片結構化框架更新總結（2023-3-30）

專案地址：https://github.com/sherlockchou86/video_pipe_c

往期文章：https://www.cnblogs.com/xiaozhi_5638/p/16969546.html

最近有多個更新，有興趣的掃碼加群交流。

新增例項分割相關支援

增加了基於mask-rcnn的例項分割外掛和相關sample。

#include "VP.h"

#include "../nodes/vp_file_src_node.h"
#include "../nodes/infers/vp_mask_rcnn_detector_node.h"
#include "../nodes/track/vp_sort_track_node.h"
#include "../nodes/osd/vp_osd_node_v3.h"
#include "../nodes/vp_screen_des_node.h"

#include "../utils/analysis_board/vp_analysis_board.h"

/*
* ## mask rcnn sample ##
* image segmentation using mask rcnn.
*/

#if mask_rcnn_sample

int main() {
    VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO);
    VP_LOGGER_INIT();

    // create nodes
    auto file_src_0 = std::make_shared<vp_nodes::vp_file_src_node>("file_src_0", 0, "./test_video/19.mp4", 0.6);
    auto mask_rcnn_detector = std::make_shared<vp_nodes::vp_mask_rcnn_detector_node>("mask_rcnn_detector", "./models/mask_rcnn/frozen_inference_graph.pb", "./models/mask_rcnn/mask_rcnn_inception_v2_coco_2018_01_28.pbtxt", "./models/coco_80classes.txt");
    auto track_0 = std::make_shared<vp_nodes::vp_sort_track_node>("sort_track_0");
    auto osd_v3_0 = std::make_shared<vp_nodes::vp_osd_node_v3>("osd_v3_0", "../third_party/paddle_ocr/font/NotoSansCJKsc-Medium.otf");
    auto screen_des_0 = std::make_shared<vp_nodes::vp_screen_des_node>("screen_des_0", 0);

    // construct pipeline
    mask_rcnn_detector->attach_to({file_src_0});
    track_0->attach_to({mask_rcnn_detector});
    osd_v3_0->attach_to({track_0});
    screen_des_0->attach_to({osd_v3_0});

    file_src_0->start();

    // for debug purpose
    vp_utils::vp_analysis_board board({file_src_0});
    board.display();
}


#endif

上面程式碼效果圖如下：

 

 

新增語義分割相關支援

新增了基於ENet網路的語義分割外掛和sample。

#include "VP.h"

#include "../nodes/vp_file_src_node.h"
#include "../nodes/infers/vp_enet_seg_node.h"
#include "../nodes/osd/vp_seg_osd_node.h"
#include "../nodes/vp_screen_des_node.h"

#include "../utils/analysis_board/vp_analysis_board.h"

/*
* ## enet seg sample ##
* semantic segmentation based on ENet.
* 1 input, 2 outputs including orignal frame and mask frame.
*/

#if enet_seg_sample

int main() {
    VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO);
    VP_LOGGER_INIT();

    // create nodes
    auto file_src_0 = std::make_shared<vp_nodes::vp_file_src_node>("file_src_0", 0, "./test_video/21.mp4");
    auto enet_seg = std::make_shared<vp_nodes::vp_enet_seg_node>("enet_seg", "models/enet-cityscapes/enet-model.net");
    auto seg_osd_0 = std::make_shared<vp_nodes::vp_seg_osd_node>("seg_osd_0", "models/enet-cityscapes/enet-classes.txt", "models/enet-cityscapes/enet-colors.txt");
    auto screen_des_mask = std::make_shared<vp_nodes::vp_screen_des_node>("screen_des_mask", 0, true, vp_objects::vp_size(400, 225));
    auto screen_des_original = std::make_shared<vp_nodes::vp_screen_des_node>("screen_des_original", 0, false, vp_objects::vp_size(400, 225));

    // construct pipeline
    enet_seg->attach_to({file_src_0});
    seg_osd_0->attach_to({enet_seg});
    screen_des_mask->attach_to({seg_osd_0});
    screen_des_original->attach_to({seg_osd_0});

    file_src_0->start();

    // for debug purpose
    vp_utils::vp_analysis_board board({file_src_0});
    board.display();
}

#endif

上面程式碼效果圖如下：

 

新增多級推理外掛sample

多個檢測、分類外掛串聯，不同分類器作用於不同的主目標：

#include "VP.h"

#include "../nodes/vp_file_src_node.h"
#include "../nodes/infers/vp_yolo_detector_node.h"
#include "../nodes/infers/vp_classifier_node.h"
#include "../nodes/osd/vp_osd_node.h"
#include "../nodes/vp_screen_des_node.h"
#include "../utils/analysis_board/vp_analysis_board.h"

/*
* ## multi detectors and classifiers sample ##
* show multi infer nodes work together.
* 1 detector and 2 classifiers applied on primary class ids(1/2/3).
*/

#if multi_detectors_and_classifiers_sample

int main() {
    VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO);
    VP_LOGGER_INIT();

    // create nodes
    auto file_src_0 = std::make_shared<vp_nodes::vp_file_src_node>("file_src_0", 0, "test_video/20.mp4", 0.6);
    /* primary detector */
    // labels for detector model
    // 0 - person
    // 1 - car
    // 2 - bus
    // 3 - truck
    // 4 - 2wheel
    auto primary_detector = std::make_shared<vp_nodes::vp_yolo_detector_node>("primary_detector", "models/det_cls/yolov3-tiny-2022-0721_best.weights", "models/det_cls/yolov3-tiny-2022-0721.cfg", "models/det_cls/yolov3_tiny_5classes.txt", 416, 416, 1);
    /* secondary classifier 1, applied to car(1)/bus(2)/truck(3) only */
    auto _1st_classifier = std::make_shared<vp_nodes::vp_classifier_node>("1st_classifier", "models/det_cls/vehicle/resnet18-batch=N-type_view_0322_nhwc.onnx", "", "models/det_cls/vehicle/vehicle_types.txt", 224, 224, 1, std::vector<int>{1, 2, 3}, 10, false, 1, cv::Scalar(), cv::Scalar(), true, true);
    /* secondary classifier 2, applied to car(1)/bus(2)/truck(3) only */
    auto _2nd_classifier = std::make_shared<vp_nodes::vp_classifier_node>("2nd_classifier", "models/det_cls/vehicle/resnet18-batch=N-color_view_0322_nhwc.onnx", "", "models/det_cls/vehicle/vehicle_colors.txt", 224, 224, 1, std::vector<int>{1, 2, 3}, 10, false, 1, cv::Scalar(), cv::Scalar(), true, true);
    auto osd_0 = std::make_shared<vp_nodes::vp_osd_node>("osd_0");
    auto screen_des_0 = std::make_shared<vp_nodes::vp_screen_des_node>("screen_des_o", 0);

    // construct pipeline
    primary_detector->attach_to({file_src_0});
    _1st_classifier->attach_to({primary_detector});
    _2nd_classifier->attach_to({_1st_classifier});
    osd_0->attach_to({_2nd_classifier});
    screen_des_0->attach_to({osd_0});

    // start
    file_src_0->start();

    // for debug purpose
    vp_utils::vp_analysis_board board({file_src_0});
    board.display();
}

#endif

上面程式碼執行效果如下：

 

新增圖片源輸入外掛

支援以圖片方式輸入（檔案或UDP），頻率可調、各個通道互相獨立。

#include "VP.h"

#include "../nodes/vp_image_src_node.h"
#include "../nodes/infers/vp_yolo_detector_node.h"
#include "../nodes/osd/vp_osd_node.h"
#include "../nodes/vp_split_node.h"
#include "../nodes/vp_screen_des_node.h"

#include "../utils/analysis_board/vp_analysis_board.h"

/*
* ## image_src_sample ##
* show how vp_image_src_node works, read image from local file or receive image from remote via udp.
*/

#if image_src_sample

int main() {
    VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO);
    VP_LOGGER_INIT();

    // create nodes
    auto image_src_0 = std::make_shared<vp_nodes::vp_image_src_node>("image_file_src_0", 0, "./images/test_%d.jpg", 1, 0.4); // read 1 image EVERY 1 second from local files, such as test_0.jpg,test_1.jpg
    /* sending command for test: `gst-launch-1.0 filesrc location=16.mp4 ! qtdemux ! avdec_h264 ! videoconvert ! videoscale ! video/x-raw,width=416,height=416 ! videorate ! video/x-raw,framerate=1/1 ! jpegenc ! rtpjpegpay ! udpsink host=ip port=6000` */
    auto image_src_1 = std::make_shared<vp_nodes::vp_image_src_node>("image_udp_src_1", 1, "6000", 3);                       // receive 1 image EVERY 3 seconds from remote via udp , such as 127.0.0.1:6000
    auto yolo_detector = std::make_shared<vp_nodes::vp_yolo_detector_node>("yolo_detector", "models/det_cls/yolov3-tiny-2022-0721_best.weights", "models/det_cls/yolov3-tiny-2022-0721.cfg", "models/det_cls/yolov3_tiny_5classes.txt");
    auto osd = std::make_shared<vp_nodes::vp_osd_node>("osd");
    auto split = std::make_shared<vp_nodes::vp_split_node>("split_by_channel", true);    // split by channel index (important!)
    auto screen_des_0 = std::make_shared<vp_nodes::vp_screen_des_node>("screen_des_0", 0);
    auto screen_des_1 = std::make_shared<vp_nodes::vp_screen_des_node>("screen_des_1", 1);
    
    // construct pipeline
    yolo_detector->attach_to({image_src_0, image_src_1});
    osd->attach_to({yolo_detector});
    split->attach_to({osd});
    screen_des_0->attach_to({split});
    screen_des_1->attach_to({split});

    image_src_0->start();  // start read from local file
    image_src_1->start();  // start receive from remote via udp

    // for debug purpose
    vp_utils::vp_analysis_board board({image_src_0, image_src_1});
    board.display();
}

#endif

上面程式碼執行效果如下：

 

新增圖片結果輸出外掛

支援以圖片格式輸出結果（檔案或UDP），頻率可調、各通道互相獨立。

#include "VP.h"

#include "../nodes/vp_file_src_node.h"
#include "../nodes/infers/vp_yunet_face_detector_node.h"
#include "../nodes/infers/vp_sface_feature_encoder_node.h"
#include "../nodes/osd/vp_face_osd_node_v2.h"
#include "../nodes/vp_screen_des_node.h"
#include "../nodes/vp_image_des_node.h"

#include "../utils/analysis_board/vp_analysis_board.h"

/*
* ## image_des_sample ##
* show how vp_image_des_node works, save image to local file or push image to remote via udp.
*/

#if image_des_sample

int main() {
    VP_SET_LOG_LEVEL(vp_utils::vp_log_level::INFO);
    VP_LOGGER_INIT();

    // create nodes
    auto file_src_0 = std::make_shared<vp_nodes::vp_file_src_node>("file_src_0", 0, "./test_video/10.mp4", 0.6);
    auto yunet_face_detector_0 = std::make_shared<vp_nodes::vp_yunet_face_detector_node>("yunet_face_detector_0", "./models/face/face_detection_yunet_2022mar.onnx");
    auto sface_face_encoder_0 = std::make_shared<vp_nodes::vp_sface_feature_encoder_node>("sface_face_encoder_0", "./models/face/face_recognition_sface_2021dec.onnx");
    auto osd_0 = std::make_shared<vp_nodes::vp_face_osd_node_v2>("osd_0");
    auto screen_des_0 = std::make_shared<vp_nodes::vp_screen_des_node>("screen_des_0", 0);
    
    /* save to file, `%d` is placeholder for filename index */
    //auto image_des_0 = std::make_shared<vp_nodes::vp_image_des_node>("image_file_des_0", 0, "./images/%d.jpg", 3);
    
    /* push via udp,  receiving command for test: `gst-launch-1.0 udpsrc port=6000 ! application/x-rtp,encoding-name=jpeg ! rtpjpegdepay ! jpegparse ! jpegdec ! queue ! videoconvert ! autovideosink` */
    auto image_des_0 = std::make_shared<vp_nodes::vp_image_des_node>("image_udp_des_0", 0, "192.168.77.248:6000", 3, vp_objects::vp_size(400, 200));

    // construct pipeline
    yunet_face_detector_0->attach_to({file_src_0});
    sface_face_encoder_0->attach_to({yunet_face_detector_0});
    osd_0->attach_to({sface_face_encoder_0});
    screen_des_0->attach_to({osd_0});
    image_des_0->attach_to({osd_0});

    file_src_0->start();

    // for debug purpose
    vp_utils::vp_analysis_board board({file_src_0});
    board.display();
}

#endif

上面程式碼執行效果如下：

 

更多更新掃碼加入微信群，及時獲取通知。