using System; using System.Runtime.InteropServices; using Sdcb.FFmpeg.Raw; using Sdcb.FFmpeg.Utils; using SharpDX.Direct3D; using SharpDX.Direct3D11; using SharpDX.DXGI; using T3.Core.Resource; using ComputeShader = T3.Core.DataTypes.ComputeShader; using CoreTexture2D = T3.Core.DataTypes.Texture2D; using SharpDxTexture2D = SharpDX.Direct3D11.Texture2D; namespace T3.VideoServices; /// /// GPU YUV→RGBA converter for the D3D11VA zero-copy path. A hardware carries the decoder /// surface in data[0] (an ID3D11Texture2D array) and the array slice index in data[1]. This wraps /// that surface, builds luma + chroma plane SRVs for the slice — R8/R8G8 for NV12 (8-bit), R16/R16G16 for /// P010/P016 (10/12-bit) — and dispatches a compute shader into an RGBA output texture's UAV (RGBA8 for 8-bit, /// RGBA16 for deeper) — no CPU round-trip. /// /// Must run on the render thread (immediate context). Owns the output texture + UAV; the caller reads /// 's result and must not dispose it. /// internal sealed class HardwareFrameConverter : IDisposable { public unsafe CoreTexture2D? Convert(Frame gpuFrame, int width, int height) { var shader = ShaderResource?.Value; if (shader == null || width <= 0 || height <= 0) return _output; // Only a real D3D11VA surface carries an ID3D11Texture2D in data[0]. A hardware decoder can still emit a // software-format frame — e.g. the first frame right after opening, while the hwaccel initialises — even // though the session already reports zero-copy. Treating that frame's data[0] (a CPU pixel buffer) as a COM // object would AddRef a non-pointer and crash hard (AccessViolationException, uncatchable). Skip it; the // next real surface converts normally and the last-valid texture stays on screen meanwhile. if ((AVPixelFormat)gpuFrame.Format != AVPixelFormat.D3d11) return _output; var texturePtr = (IntPtr)gpuFrame.Data[0]; var sliceIndex = (int)gpuFrame.Data[1]; if (texturePtr == IntPtr.Zero) return _output; var device = ResourceManager.Device; var csStage = device.ImmediateContext.ComputeShader; // Decode (worker) and convert (here) share one D3D11 device. FFmpeg guards its decode with DeviceLock // (its lock/unlock callbacks take it — see VideoDecoderSession), so take the same lock around the // convert: the two never touch the GPU at once. lock (DeviceLock) { // Borrow the FFmpeg-owned decoder texture: AddRef so the SharpDX wrapper's Dispose-time Release is // balanced and FFmpeg's own reference is left intact. (Risk spot: if the pool exhausts after ~20 // frames and decode stalls, this AddRef is doubling a ref the ctor already takes — drop it then.) Marshal.AddRef(texturePtr); using var decoderTexture = new SharpDxTexture2D(texturePtr); // NV12 (8-bit) reads R8/R8G8 plane SRVs into an RGBA8 output; P010/P016 (10/12-bit) read R16/R16G16 // into RGBA16 so the extra bits survive. The plane is selected by the SRV format on the decoder // texture-array slice (D3D11 maps the format to the plane; PlaneSlice is a D3D12 concept). var tenBit = decoderTexture.Description.Format != Format.NV12; var lumaFormat = tenBit ? Format.R16_UNorm : Format.R8_UNorm; var chromaFormat = tenBit ? Format.R16G16_UNorm : Format.R8G8_UNorm; var outputFormat = tenBit ? Format.R16G16B16A16_UNorm : Format.R8G8B8A8_UNorm; EnsureOutput(width, height, outputFormat); using var lumaSrv = new ShaderResourceView(device, decoderTexture, PlaneSrvDesc(lumaFormat, sliceIndex)); using var chromaSrv = new ShaderResourceView(device, decoderTexture, PlaneSrvDesc(chromaFormat, sliceIndex)); var prevShader = csStage.Get(); var prevUavs = csStage.GetUnorderedAccessViews(0, 1); var prevSrvs = csStage.GetShaderResources(0, 2); csStage.Set(shader); csStage.SetShaderResource(0, lumaSrv); csStage.SetShaderResource(1, chromaSrv); csStage.SetUnorderedAccessView(0, _outputUav); device.ImmediateContext.Dispatch((width + 15) / 16, (height + 15) / 16, 1); // Restore previous compute-stage bindings, then release the refs Get* handed us (else they leak per frame). csStage.Set(prevShader); csStage.SetUnorderedAccessView(0, prevUavs[0]); csStage.SetShaderResource(0, prevSrvs[0]); csStage.SetShaderResource(1, prevSrvs[1]); prevShader?.Dispose(); prevUavs[0]?.Dispose(); prevSrvs[0]?.Dispose(); prevSrvs[1]?.Dispose(); } return _output; } private static ShaderResourceViewDescription PlaneSrvDesc(Format format, int slice) => new() { Format = format, Dimension = ShaderResourceViewDimension.Texture2DArray, Texture2DArray = new ShaderResourceViewDescription.Texture2DArrayResource { FirstArraySlice = slice, ArraySize = 1, MipLevels = 1, MostDetailedMip = 0, }, }; private void EnsureOutput(int width, int height, Format format) { if (_output != null && _output.Description.Width == width && _output.Description.Height == height && _output.Description.Format == format) return; DisposeOutput(); _output = CoreTexture2D.CreateTexture2D(new Texture2DDescription { Width = width, Height = height, ArraySize = 1, MipLevels = 1, Format = format, BindFlags = BindFlags.ShaderResource | BindFlags.RenderTarget | BindFlags.UnorderedAccess, CpuAccessFlags = CpuAccessFlags.None, OptionFlags = ResourceOptionFlags.None, Usage = ResourceUsage.Default, SampleDescription = new SampleDescription(1, 0), }); _outputUav = new UnorderedAccessView(ResourceManager.Device, _output); } private Resource? ShaderResource => _shaderResource ??= ResourceManager.CreateShaderResource(ShaderPath, null, () => "main"); // Held by FFmpeg's decode (its D3D11VA lock/unlock callbacks) and by this converter's dispatch, so decode // and convert never run on the shared device simultaneously. internal static readonly object DeviceLock = new(); public void Dispose() => DisposeOutput(); private void DisposeOutput() { _outputUav?.Dispose(); _outputUav = null; _output?.Dispose(); _output = null; } private const string ShaderPath = "Lib:shaders/img/Nv12ToRgba-cs.hlsl"; private Resource? _shaderResource; private CoreTexture2D? _output; private UnorderedAccessView? _outputUav; }