Metal實現YUV轉RGB渲染視訊

OrangesChen發表於2017-12-27

效果圖.PNG
本次例子使用的是AVFoundation提供的AVCaptureVideoDataOutput獲取每一幀的CVPixelBufferRef，詳細步驟就不說了，網上有很多例子，這篇文章主要是介紹Metal中實現YUV轉RGB格式的一些主要步驟，和OpenGL中的步驟差不多，主要是API和著色器不同，思路是一樣的，這篇文章適合熟悉OpenGL視訊渲染和有Metal基礎的人觀看，程式碼就不一一註釋了，主要是本人理解的也不是很深，怕誤人子弟。 原始碼下載地址 ######1 首先是shader上的片元著色器轉換YUV到RGB

#include <metal_stdlib>
using namespace metal;
#define YUV_SHADER_ARGS  VertexOut      inFrag    [[ stage_in ]],\
texture2d<float>  lumaTex     [[ texture(0) ]],\
texture2d<float>  chromaTex     [[ texture(1) ]],\
sampler bilinear [[ sampler(0) ]], \
constant ColorParameters *colorParameters [[ buffer(0) ]]// RGB到YUV的轉換矩陣
struct VertexIn{
    packed_float3 position;
    packed_float4 color;
    packed_float2 st;
};

struct VertexOut{
    float4 position [[position]];  //1
    float4 color;
    float2 st;
};
struct ColorParameters
{
    float3x3 yuvToRGB;
};
vertex VertexOut texture_vertex(
                              const device VertexIn* vertex_array [[ buffer(0) ]],           //1
                              unsigned int vid [[ vertex_id ]]) {
    
    
    VertexIn VertexIn = vertex_array[vid];
    
    VertexOut VertexOut;
    VertexOut.position = float4(VertexIn.position,1);  //3
    VertexOut.color = VertexIn.color;
    VertexOut.st = VertexIn.st;
    return VertexOut;
}
fragment half4 yuv_rgb(YUV_SHADER_ARGS)
{
    float3 yuv;
    yuv.x = lumaTex.sample(bilinear, inFrag.st).r;
    yuv.yz = chromaTex.sample(bilinear,inFrag.st).rg - float2(0.5);
    return half4(half3(colorParameters->yuvToRGB * yuv),yuv.x);
}
複製程式碼

######2 新增紋理快取CVMetalTextureCacheRef和紋理MTLTexture變數

  CVMetalTextureCacheRef _videoTextureCache;
   id<MTLTexture> _videoTexture[2];
   CVPixelBufferRef _pixelBuffer;
複製程式碼

新增轉換矩陣的接收變數

@property (nonatomic, strong)  id<MTLBuffer> parametersBuffer;
複製程式碼

以下幾個都是YUV轉RGB的矩陣演算法，給parametersBuffer賦值，拷貝到GPU中計算

 _parametersBuffer = [_device newBufferWithLength:sizeof(ColorParameters) * 2 options:MTLResourceOptionCPUCacheModeDefault];
     ColorParameters matrix;
     simd::float3 A;
     simd::float3 B;
     simd::float3 C;
     
     // 1
     //    A.x = 1;
     //    A.y = 1;
     //    A.z = 1;
     //
     //    B.x = 0;
     //    B.y = -0.343;
     //    B.z = 1.765;
     //
     //    C.x = 1.4;
     //    C.y = -0.765;
     //    C.z = 0;
     
     // 2
     //    A.x = 1.164;
     //    A.y = 1.164;
     //    A.z = 1.164;
     //
     //    B.x = 0;
     //    B.y = -0.392;
     //    B.z = 2.017;
     //
     //    C.x = 1.596;
     //    C.y = -0.813;
     //    C.z = 0;

     // 3
     A.x = 1.164;
     A.y = 1.164;
     A.z = 1.164;

     B.x = 0;
     B.y = -0.231;
     B.z = 2.112;

     C.x = 1.793;
     C.y = -0.533;
     C.z = 0;
     matrix.yuvToRGB = simd::float3x3{A, B, C}; 
     memcpy(self.parametersBuffer.contents, &matrix, sizeof(ColorParameters));
複製程式碼

獲取每一幀視訊資訊生成紋理的程式碼

- (void)makeYUVTexture:(CVPixelBufferRef)pixelBuffer {
    CVMetalTextureRef y_texture ;
    float y_width = CVPixelBufferGetWidthOfPlane(pixelBuffer, 0);
    float y_height = CVPixelBufferGetHeightOfPlane(pixelBuffer, 0);
 CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, _videoTextureCache, pixelBuffer, nil, MTLPixelFormatR8Unorm, y_width, y_height, 0, &y_texture);
 
    CVMetalTextureRef uv_texture;
    float uv_width = CVPixelBufferGetWidthOfPlane(pixelBuffer, 1);
    float uv_height = CVPixelBufferGetHeightOfPlane(pixelBuffer, 1);
    CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, _videoTextureCache, pixelBuffer, nil, MTLPixelFormatRG8Unorm, uv_width, uv_height, 1, &uv_texture);
    
    id<MTLTexture> luma = CVMetalTextureGetTexture(y_texture);
    id<MTLTexture> chroma = CVMetalTextureGetTexture(uv_texture);
    
    _videoTexture[0] = luma;
    _videoTexture[1] = chroma;
    
    CVBufferRelease(y_texture);
    CVBufferRelease(uv_texture);
}

- (void)display:(CVPixelBufferRef)overlay {
    if (!overlay) {
        return;
    }
    if (!_videoTextureCache) {
        NSLog(@"No video texture cache");
        return;
    }
    [self makeYUVTexture:overlay];
}
- (void)setVideoTexture {
 CVMetalTextureCacheFlush(_videoTextureCache, 0);
    CVReturn err = CVMetalTextureCacheCreate(kCFAllocatorDefault, NULL, _device, NULL, &_videoTextureCache);
    if (err) {
        NSLog(@">> ERROR: Could not create a texture cache");
        assert(0);
    }
}
複製程式碼

#（最後PS： 有大神知道怎麼使用Metal實現渲染到紋理的麼，求指導） 使用

相關文章