using System.Runtime.InteropServices;
using Sdcb.FFmpeg.Codecs;
using Sdcb.FFmpeg.Formats;
using Sdcb.FFmpeg.Raw;
using Sdcb.FFmpeg.Swscales;
using Sdcb.FFmpeg.Utils;
using T3.Core.Logging;
namespace T3.VideoServices;
///
/// Settings for one . Pixel/codec types are FFmpeg-level on purpose — the
/// editor-facing Core facade maps its own FFmpeg-free settings onto these.
///
public readonly record struct VideoEncoderSettings
{
public required string FilePath { get; init; }
/// Muxer name (e.g. mp4); null infers it from 's extension.
public string? FormatName { get; init; }
public required int Width { get; init; }
public required int Height { get; init; }
/// Frames per second as a rational (e.g. 60/1, 60000/1001).
public required AVRational FrameRate { get; init; }
public long BitRate { get; init; }
///
/// Explicit encoder name (e.g. h264_nvenc) — wins over when set. Lets the
/// caller target a specific hardware encoder; null falls back to the codec id's default encoder.
///
public string? VideoEncoderName { get; init; }
/// Used when is null (e.g. Mpeg4 for the LGPL fallback).
public AVCodecID VideoCodecId { get; init; }
/// Codec-private AVOptions applied when the encoder is opened (e.g. HAP's format =
/// hap_q). Optional; null applies none.
public IReadOnlyList>? VideoCodecOptions { get; init; }
///
/// Pixel format fed to the encoder; the swscale step converts into it, so it must be
/// one the encoder accepts (e.g. Yuv420p for H.264/MPEG-4, Yuv422p10le for ProRes). Defaults to
/// Yuv420p (the enum's zero value); is treated the same.
///
public AVPixelFormat EncoderPixelFormat { get; init; }
/// Pixel format of the bytes passed to (e.g. Rgba).
public AVPixelFormat SourceFormat { get; init; }
/// Bytes per pixel of (e.g. 4 for RGBA8).
public int SourceBytesPerPixel { get; init; }
/// When true, an AAC audio stream is muxed in; feed it interleaved-float PCM via
/// .
public bool EncodeAudio { get; init; }
public int AudioSampleRate { get; init; }
public int AudioChannels { get; init; }
/// AAC target bit rate; ≤ 0 uses a 192 kbit default.
public long AudioBitRate { get; init; }
}
///
/// Encodes a sequence of CPU frames (plus optional audio) into a muxed video file via FFmpeg (libav*), for
/// render-export. CPU-byte level on purpose: the GPU texture read-back stays in the caller, so this is
/// unit-testable without a D3D device. The video codec is caller-chosen — a hardware H.264 encoder
/// (h264_nvenc/_qsv/_amf) on capable machines, or a software LGPL codec otherwise
/// (software H.264 is libx264 = GPL and absent from the bundled LGPL build). Audio is native AAC (LGPL).
///
/// Not thread-safe: one encoder is driven by one caller. Per frame, call and (if
/// audio) , then (or ) exactly once.
///
public sealed class VideoFileEncoder : IDisposable
{
// FFmpeg's native AAC encoder consumes planar float.
private const AVSampleFormat AudioSampleFormat = AVSampleFormat.Fltp;
public unsafe VideoFileEncoder(in VideoEncoderSettings settings)
{
if (!FfmpegLibrary.EnsureInitialized())
throw new InvalidOperationException(FfmpegLibrary.StatusError ?? "FFmpeg is not available");
_settings = settings;
// None (and the struct's zero default) means "let the encoder use planar 4:2:0".
var encoderPixelFormat = settings.EncoderPixelFormat == AVPixelFormat.None
? AVPixelFormat.Yuv420p
: settings.EncoderPixelFormat;
var videoCodec = !string.IsNullOrEmpty(settings.VideoEncoderName)
? Codec.FindEncoderByName(settings.VideoEncoderName)
: Codec.FindEncoderById(settings.VideoCodecId);
_formatContext = FormatContext.AllocOutput(formatName: settings.FormatName, fileName: settings.FilePath);
// Some muxers (mp4/mov) keep codec headers in the container, not the bitstream — required for a playable file.
var outputFormat = _formatContext.OutputFormat;
var needsGlobalHeader = outputFormat != null && outputFormat.Value.Flags.HasFlag(AVFMT.Globalheader);
_videoStream = _formatContext.NewStream(videoCodec);
_videoCodecContext = new CodecContext(videoCodec)
{
Width = settings.Width,
Height = settings.Height,
TimeBase = new AVRational(settings.FrameRate.Den, settings.FrameRate.Num),
Framerate = settings.FrameRate,
PixelFormat = encoderPixelFormat,
BitRate = settings.BitRate,
// Tag BT.709 / limited range, matching the previous Media Foundation output, so the
// file isn't untagged (players then guess BT.601 vs 709 and the colour shifts).
ColorRange = AVColorRange.Mpeg,
ColorPrimaries = AVColorPrimaries.Bt709,
ColorTrc = AVColorTransferCharacteristic.Bt709,
Colorspace = AVColorSpace.Bt709,
// 0 = auto-select thread count; lets the software encoders (VP9/AV1/kvazaar/FFV1) use
// all cores instead of one. Hardware encoders ignore it.
ThreadCount = 0,
};
if (needsGlobalHeader)
_videoCodecContext.Flags |= AV_CODEC_FLAG.GlobalHeader;
OpenVideoCodec(settings.VideoCodecOptions);
_videoStream.Codecpar!.CopyFrom(_videoCodecContext);
_videoStream.TimeBase = _videoCodecContext.TimeBase;
if (settings.EncodeAudio)
SetupAudioStream(settings, needsGlobalHeader);
// All streams added — now open the file and write the header (must precede any packet).
_ioContext = IOContext.OpenWrite(settings.FilePath, null);
_formatContext.Pb = _ioContext;
_formatContext.WriteHeader(null);
_sourceFrame = Frame.CreateVideo(settings.Width, settings.Height, settings.SourceFormat);
_sourceFrame.EnsureBuffer(1);
_sourceFrame.ColorRange = AVColorRange.Jpeg; // source RGB is full-range
_encoderFrame = Frame.CreateVideo(settings.Width, settings.Height, encoderPixelFormat);
_encoderFrame.EnsureBuffer(1);
_encoderFrame.ColorRange = AVColorRange.Mpeg;
_encoderFrame.ColorPrimaries = AVColorPrimaries.Bt709;
_encoderFrame.ColorTrc = AVColorTransferCharacteristic.Bt709;
_encoderFrame.Colorspace = AVColorSpace.Bt709;
// Drive swscale directly so we can force BT.709: its RGB→YUV default is BT.601, and Sdcb's
// VideoFrameConverter doesn't expose the colorspace. (No-op for HAP's RGBA→RGBA, where there's no matrix.)
_swsContext = ffmpeg.sws_getContext(settings.Width, settings.Height, settings.SourceFormat,
settings.Width, settings.Height, encoderPixelFormat,
(int)SWS.Bilinear, null, null, null);
if (_swsContext == null)
throw new InvalidOperationException("sws_getContext failed");
if (encoderPixelFormat != settings.SourceFormat)
{
var coeff709 = ffmpeg.sws_getCoefficients(1); // SWS_CS_ITU709
// full-range RGB in (1) → limited-range YUV out (0), BT.709 coefficients.
ffmpeg.sws_setColorspaceDetails(_swsContext, coeff709, 1, coeff709, 0, 0, 1 << 16, 1 << 16);
}
}
// Opens the video encoder, applying any codec-private AVOptions (e.g. HAP's "format") first. Children search
// descends into the codec's private data, where these options live (mirrors the CLI's `-format hap_q`).
private void OpenVideoCodec(IReadOnlyList>? options)
{
if (options != null)
{
foreach (var (key, value) in options)
_videoCodecContext.Options.Set(key, value, AV_OPT_SEARCH.Children);
}
_videoCodecContext.Open();
}
private unsafe void SetupAudioStream(in VideoEncoderSettings settings, bool needsGlobalHeader)
{
var audioCodec = Codec.FindEncoderById(AVCodecID.Aac);
_audioStream = _formatContext.NewStream(audioCodec);
AVChannelLayout layout = default;
ffmpeg.av_channel_layout_default(&layout, settings.AudioChannels);
_audioLayout = layout;
_audioCodecContext = new CodecContext(audioCodec)
{
SampleRate = settings.AudioSampleRate,
SampleFormat = AudioSampleFormat,
ChLayout = layout,
BitRate = settings.AudioBitRate > 0 ? settings.AudioBitRate : 192_000,
TimeBase = new AVRational(1, settings.AudioSampleRate),
};
if (needsGlobalHeader)
_audioCodecContext.Flags |= AV_CODEC_FLAG.GlobalHeader;
_audioCodecContext.Open();
_audioStream.Codecpar!.CopyFrom(_audioCodecContext);
_audioStream.TimeBase = _audioCodecContext.TimeBase;
// AAC fixes its frame size (1024 samples); buffer incoming PCM and emit exactly that many per frame.
_audioFrameSize = _audioCodecContext.FrameSize > 0 ? _audioCodecContext.FrameSize : 1024;
_audioFrame = Frame.CreateAudio(AudioSampleFormat, layout, settings.AudioSampleRate, _audioFrameSize);
_audioFrame.EnsureBuffer(0);
}
///
/// Encodes one video frame. holds
/// rows of Width * SourceBytesPerPixel bytes in , with
/// bytes per row (≥ the row's pixel bytes; padding ignored).
///
public unsafe void WriteVideoFrame(ReadOnlySpan sourcePixels, int sourceStride)
{
if (_finished)
throw new InvalidOperationException("Encoder already finished");
var rowBytes = _settings.Width * _settings.SourceBytesPerPixel;
var destStride = _sourceFrame.Linesize[0];
var dest = (byte*)_sourceFrame.Data[0];
fixed (byte* src = sourcePixels)
{
for (var y = 0; y < _settings.Height; y++)
Buffer.MemoryCopy(src + (long)y * sourceStride, dest + (long)y * destStride, destStride, rowBytes);
}
ScaleSourceFrame();
_encoderFrame.Pts = _nextVideoPts++;
_videoCodecContext.SendFrame(_encoderFrame);
DrainEncoderToMuxer(_videoCodecContext, _videoStream);
}
// RGBA → encoder pixel format via our BT.709-configured swscale context.
private unsafe void ScaleSourceFrame()
{
var srcData = new byte*[4];
var srcStride = new int[4];
var dstData = new byte*[4];
var dstStride = new int[4];
for (var i = 0; i < 4; i++)
{
srcData[i] = (byte*)_sourceFrame.Data[i];
srcStride[i] = _sourceFrame.Linesize[i];
dstData[i] = (byte*)_encoderFrame.Data[i];
dstStride[i] = _encoderFrame.Linesize[i];
}
ffmpeg.sws_scale(_swsContext, srcData, srcStride, 0, _settings.Height, dstData, dstStride);
}
///
/// Buffers and encodes audio. is interleaved 32-bit-float samples
/// (channel-interleaved) matching /
/// . No-op when the encoder has no audio stream.
///
public void WriteAudioSamples(ReadOnlySpan interleavedFloatPcm)
{
if (_audioCodecContext == null)
return;
if (_finished)
throw new InvalidOperationException("Encoder already finished");
_audioBuffer.AddRange(MemoryMarshal.Cast(interleavedFloatPcm));
var channels = _settings.AudioChannels;
var floatsPerFrame = _audioFrameSize * channels;
var consumedFloats = 0;
while (_audioBuffer.Count - consumedFloats >= floatsPerFrame)
{
EncodeAudioFrame(consumedFloats, _audioFrameSize);
consumedFloats += floatsPerFrame;
}
if (consumedFloats > 0)
_audioBuffer.RemoveRange(0, consumedFloats);
}
/// Flushes both encoders and writes the container trailer. Idempotent; also called by .
public unsafe void Finish()
{
if (_finished)
return;
_finished = true;
if (_audioCodecContext != null)
{
var remainingSamples = _audioBuffer.Count / _settings.AudioChannels;
if (remainingSamples > 0)
EncodeAudioFrame(0, remainingSamples);
_audioBuffer.Clear();
ffmpeg.avcodec_send_frame(_audioCodecContext, null); // drain the audio encoder
DrainEncoderToMuxer(_audioCodecContext, _audioStream);
}
ffmpeg.avcodec_send_frame(_videoCodecContext, null); // drain the video encoder
DrainEncoderToMuxer(_videoCodecContext, _videoStream);
_formatContext.WriteTrailer();
}
// Deinterleaves nbSamples from _audioBuffer (starting at floatOffset) into the planar frame, then encodes it.
private unsafe void EncodeAudioFrame(int floatOffset, int nbSamples)
{
ffmpeg.av_frame_make_writable(_audioFrame!);
_audioFrame!.NbSamples = nbSamples;
var channels = _settings.AudioChannels;
for (var ch = 0; ch < channels; ch++)
{
var plane = (float*)_audioFrame.Data[ch];
for (var i = 0; i < nbSamples; i++)
plane[i] = _audioBuffer[floatOffset + i * channels + ch];
}
_audioFrame.Pts = _audioSamplesEncoded;
_audioSamplesEncoded += nbSamples;
_audioCodecContext!.SendFrame(_audioFrame);
DrainEncoderToMuxer(_audioCodecContext, _audioStream);
}
private unsafe void DrainEncoderToMuxer(CodecContext codecContext, MediaStream stream)
{
while (codecContext.ReceivePacket(_packet) == CodecResult.Success)
{
_packet.StreamIndex = stream.Index;
ffmpeg.av_packet_rescale_ts(_packet, codecContext.TimeBase, stream.TimeBase);
_formatContext.InterleavedWritePacket(_packet);
_packet.Unref();
}
}
public unsafe void Dispose()
{
try
{
Finish();
}
catch (Exception e)
{
Log.Warning($"VideoFileEncoder: finishing the output failed - {e.Message}");
}
if (_swsContext != null)
ffmpeg.sws_freeContext(_swsContext);
_sourceFrame.Dispose();
_encoderFrame.Dispose();
_audioFrame?.Dispose();
_packet.Dispose();
_videoCodecContext.Dispose();
_audioCodecContext?.Dispose();
_formatContext.Dispose();
_ioContext.Dispose();
}
private readonly VideoEncoderSettings _settings;
private readonly FormatContext _formatContext;
private readonly IOContext _ioContext;
private readonly MediaStream _videoStream;
private readonly CodecContext _videoCodecContext;
private readonly unsafe SwsContext* _swsContext;
private readonly Frame _sourceFrame;
private readonly Frame _encoderFrame;
private long _nextVideoPts;
private MediaStream _audioStream;
private CodecContext? _audioCodecContext;
private Frame? _audioFrame;
private AVChannelLayout _audioLayout;
private int _audioFrameSize;
private long _audioSamplesEncoded;
private readonly List _audioBuffer = new();
private readonly Packet _packet = new();
private bool _finished;
}