From 496eb670ab0d765f3031fe9d97f158efcdb4f1b5 Mon Sep 17 00:00:00 2001 From: cosmonaut Date: Wed, 7 Jun 2023 21:18:44 +0000 Subject: [PATCH] AV1 Video instead of Theora (#49) VideoPlayer now takes AV1 video instead of Ogg Theora. This brings a significant decode speed improvement. The decoder now also operates in a threaded manner, which should prevent runtime stalls when fetching video frames. Reviewed-on: https://gitea.moonside.games/MoonsideGames/MoonWorks/pulls/49 --- .gitmodules | 6 +- MoonWorks.csproj | 2 +- MoonWorks.dll.config | 4 + lib/RefreshCS | 2 +- lib/Theorafile | 1 - lib/dav1dfile | 1 + src/Graphics/CommandBuffer.cs | 21 +++- src/Video/StreamingSoundTheora.cs | 67 ------------ src/Video/Video.cs | 127 ----------------------- src/Video/VideoAV1.cs | 71 +++++++++++++ src/Video/VideoAV1Stream.cs | 91 +++++++++++++++++ src/Video/VideoPlayer.cs | 163 +++++++++--------------------- src/Video/VideoState.cs | 9 ++ 13 files changed, 245 insertions(+), 320 deletions(-) delete mode 160000 lib/Theorafile create mode 160000 lib/dav1dfile delete mode 100644 src/Video/StreamingSoundTheora.cs delete mode 100644 src/Video/Video.cs create mode 100644 src/Video/VideoAV1.cs create mode 100644 src/Video/VideoAV1Stream.cs create mode 100644 src/Video/VideoState.cs diff --git a/.gitmodules b/.gitmodules index e323a49..4f87810 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,6 +10,6 @@ [submodule "lib/WellspringCS"] path = lib/WellspringCS url = https://gitea.moonside.games/MoonsideGames/WellspringCS.git -[submodule "lib/Theorafile"] - path = lib/Theorafile - url = https://github.com/FNA-XNA/Theorafile.git +[submodule "lib/dav1dfile"] + path = lib/dav1dfile + url = git@github.com:MoonsideGames/dav1dfile.git diff --git a/MoonWorks.csproj b/MoonWorks.csproj index 9346bc8..a7d3ecd 100644 --- a/MoonWorks.csproj +++ b/MoonWorks.csproj @@ -15,8 +15,8 @@ - + diff --git a/MoonWorks.dll.config b/MoonWorks.dll.config index c076485..cc9a116 100644 --- a/MoonWorks.dll.config +++ b/MoonWorks.dll.config @@ -19,4 +19,8 @@ + + + + diff --git a/lib/RefreshCS b/lib/RefreshCS index ebf5111..60a7523 160000 --- a/lib/RefreshCS +++ b/lib/RefreshCS @@ -1 +1 @@ -Subproject commit ebf511133aa6f567c004d687acac474e1649bbde +Subproject commit 60a7523fac254d5e2d89185392e8c1afd8581aa9 diff --git a/lib/Theorafile b/lib/Theorafile deleted file mode 160000 index 8f9419e..0000000 --- a/lib/Theorafile +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8f9419ea856480e08294698e1d6be8752df3710b diff --git a/lib/dav1dfile b/lib/dav1dfile new file mode 160000 index 0000000..859f47f --- /dev/null +++ b/lib/dav1dfile @@ -0,0 +1 @@ +Subproject commit 859f47f6fa0dfa0f7f941dcced6664fa83736202 diff --git a/src/Graphics/CommandBuffer.cs b/src/Graphics/CommandBuffer.cs index 8992819..a664791 100644 --- a/src/Graphics/CommandBuffer.cs +++ b/src/Graphics/CommandBuffer.cs @@ -1946,7 +1946,17 @@ namespace MoonWorks.Graphics /// /// Asynchronously copies YUV data into three textures. Use with compressed video. /// - public void SetTextureDataYUV(Texture yTexture, Texture uTexture, Texture vTexture, IntPtr dataPtr, uint dataLengthInBytes) + public void SetTextureDataYUV( + Texture yTexture, + Texture uTexture, + Texture vTexture, + IntPtr yDataPtr, + IntPtr uDataPtr, + IntPtr vDataPtr, + uint yDataLengthInBytes, + uint uvDataLengthInBytes, + uint yStride, + uint uvStride) { #if DEBUG AssertRenderPassInactive("Cannot copy during render pass!"); @@ -1962,8 +1972,13 @@ namespace MoonWorks.Graphics yTexture.Height, uTexture.Width, uTexture.Height, - dataPtr, - dataLengthInBytes + yDataPtr, + uDataPtr, + vDataPtr, + yDataLengthInBytes, + uvDataLengthInBytes, + yStride, + uvStride ); } diff --git a/src/Video/StreamingSoundTheora.cs b/src/Video/StreamingSoundTheora.cs deleted file mode 100644 index 95f8e57..0000000 --- a/src/Video/StreamingSoundTheora.cs +++ /dev/null @@ -1,67 +0,0 @@ -using System; -using MoonWorks.Audio; - -namespace MoonWorks.Video -{ - // TODO: should we just not handle theora sound? it sucks! - internal unsafe class StreamingSoundTheora : StreamingSound - { - private IntPtr VideoHandle; - public override bool Loaded => true; - - internal StreamingSoundTheora( - AudioDevice device, - IntPtr videoHandle, - int channels, - uint sampleRate, - uint bufferSize = 8192 - ) : base( - device, - 3, /* float type */ - 32, /* size of float */ - (ushort) (4 * channels), - (ushort) channels, - sampleRate, - bufferSize, - false // Theorafile is not thread safe, so let's update on the main thread - ) { - VideoHandle = videoHandle; - } - - public override unsafe void Load() - { - // no-op - } - - public override unsafe void Unload() - { - // no-op - } - - protected override unsafe void FillBuffer( - void* buffer, - int bufferLengthInBytes, - out int filledLengthInBytes, - out bool reachedEnd - ) { - var lengthInFloats = bufferLengthInBytes / sizeof(float); - - // FIXME: this gets gnarly with theorafile being not thread safe - // is there some way we could just manually update in VideoPlayer - // instead of going through AudioDevice? - lock (Device.StateLock) - { - int samples = Theorafile.tf_readaudio( - VideoHandle, - (IntPtr) buffer, - lengthInFloats - ); - - filledLengthInBytes = samples * sizeof(float); - reachedEnd = Theorafile.tf_eos(VideoHandle) == 1; - } - } - - protected override void OnReachedEnd() { } - } -} diff --git a/src/Video/Video.cs b/src/Video/Video.cs deleted file mode 100644 index 38feed4..0000000 --- a/src/Video/Video.cs +++ /dev/null @@ -1,127 +0,0 @@ -/* Heavily based on https://github.com/FNA-XNA/FNA/blob/master/src/Media/Xiph/VideoPlayer.cs */ -using System; -using System.IO; -using System.Runtime.InteropServices; -using SDL2; - -namespace MoonWorks.Video -{ - public enum VideoState - { - Playing, - Paused, - Stopped - } - - public unsafe class Video : IDisposable - { - internal IntPtr Handle; - private IntPtr rwData; - private void* videoData; - private int videoDataLength; - - public double FramesPerSecond => fps; - public int Width => yWidth; - public int Height => yHeight; - public int UVWidth { get; } - public int UVHeight { get; } - - private double fps; - private int yWidth; - private int yHeight; - - private bool IsDisposed; - - public Video(string filename) - { - if (!File.Exists(filename)) - { - throw new ArgumentException("Video file not found!"); - } - - var fileStream = new FileStream(filename, FileMode.Open, FileAccess.Read); - videoDataLength = (int) fileStream.Length; - videoData = NativeMemory.Alloc((nuint) videoDataLength); - var fileBufferSpan = new Span(videoData, videoDataLength); - fileStream.ReadExactly(fileBufferSpan); - fileStream.Close(); - - rwData = SDL.SDL_RWFromMem((IntPtr) videoData, videoDataLength); - if (Theorafile.tf_open_callbacks(rwData, out Handle, callbacks) < 0) - { - throw new ArgumentException("Invalid video file!"); - } - - Theorafile.th_pixel_fmt format; - Theorafile.tf_videoinfo( - Handle, - out yWidth, - out yHeight, - out fps, - out format - ); - - if (format == Theorafile.th_pixel_fmt.TH_PF_420) - { - UVWidth = Width / 2; - UVHeight = Height / 2; - } - else if (format == Theorafile.th_pixel_fmt.TH_PF_422) - { - UVWidth = Width / 2; - UVHeight = Height; - } - else if (format == Theorafile.th_pixel_fmt.TH_PF_444) - { - UVWidth = Width; - UVHeight = Height; - } - else - { - throw new NotSupportedException("Unrecognized YUV format!"); - } - } - - private static IntPtr Read(IntPtr ptr, IntPtr size, IntPtr nmemb, IntPtr datasource) => (IntPtr) SDL2.SDL.SDL_RWread(datasource, ptr, size, nmemb); - private static int Seek(IntPtr datasource, long offset, Theorafile.SeekWhence whence) => (int) SDL2.SDL.SDL_RWseek(datasource, offset, (int) whence); - private static int Close(IntPtr datasource) => (int) SDL2.SDL.SDL_RWclose(datasource); - - private static Theorafile.tf_callbacks callbacks = new Theorafile.tf_callbacks - { - read_func = Read, - seek_func = Seek, - close_func = Close - }; - - protected virtual void Dispose(bool disposing) - { - if (!IsDisposed) - { - if (disposing) - { - // dispose managed state (managed objects) - } - - // free unmanaged resources (unmanaged objects) - Theorafile.tf_close(ref Handle); - SDL.SDL_RWclose(rwData); - NativeMemory.Free(videoData); - - IsDisposed = true; - } - } - - ~Video() - { - // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method - Dispose(disposing: false); - } - - public void Dispose() - { - // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method - Dispose(disposing: true); - GC.SuppressFinalize(this); - } - } -} diff --git a/src/Video/VideoAV1.cs b/src/Video/VideoAV1.cs new file mode 100644 index 0000000..38aebee --- /dev/null +++ b/src/Video/VideoAV1.cs @@ -0,0 +1,71 @@ +using System; +using System.IO; + +namespace MoonWorks.Video +{ + /// + /// This class takes in a filename for AV1 data in .obu (open bitstream unit) format + /// + public unsafe class VideoAV1 + { + public string Filename { get; } + + // "double buffering" so we can loop without a stutter + internal VideoAV1Stream StreamA { get; } + internal VideoAV1Stream StreamB { get; } + + public int Width => width; + public int Height => height; + public double FramesPerSecond { get; set; } + public Dav1dfile.PixelLayout PixelLayout => pixelLayout; + public int UVWidth { get; } + public int UVHeight { get; } + + private int width; + private int height; + private Dav1dfile.PixelLayout pixelLayout; + + public VideoAV1(string filename, double framesPerSecond) + { + if (!File.Exists(filename)) + { + throw new ArgumentException("Video file not found!"); + } + + if (Dav1dfile.df_fopen(filename, out var handle) == 0) + { + throw new Exception("Failed to open video file!"); + } + + Dav1dfile.df_videoinfo(handle, out width, out height, out pixelLayout); + Dav1dfile.df_close(handle); + + if (pixelLayout == Dav1dfile.PixelLayout.I420) + { + UVWidth = Width / 2; + UVHeight = Height / 2; + } + else if (pixelLayout == Dav1dfile.PixelLayout.I422) + { + UVWidth = Width / 2; + UVHeight = Height; + } + else if (pixelLayout == Dav1dfile.PixelLayout.I444) + { + UVWidth = width; + UVHeight = height; + } + else + { + throw new NotSupportedException("Unrecognized YUV format!"); + } + + FramesPerSecond = framesPerSecond; + + Filename = filename; + + StreamA = new VideoAV1Stream(this); + StreamB = new VideoAV1Stream(this); + } + } +} diff --git a/src/Video/VideoAV1Stream.cs b/src/Video/VideoAV1Stream.cs new file mode 100644 index 0000000..5ac443c --- /dev/null +++ b/src/Video/VideoAV1Stream.cs @@ -0,0 +1,91 @@ +using System; + +namespace MoonWorks.Video +{ + internal class VideoAV1Stream + { + public IntPtr Handle => handle; + IntPtr handle; + + public bool Ended => Dav1dfile.df_eos(Handle) == 1; + + public IntPtr yDataHandle; + public IntPtr uDataHandle; + public IntPtr vDataHandle; + public uint yDataLength; + public uint uvDataLength; + public uint yStride; + public uint uvStride; + + public bool FrameDataUpdated { get; private set; } + + bool IsDisposed; + + public VideoAV1Stream(VideoAV1 video) + { + if (Dav1dfile.df_fopen(video.Filename, out handle) == 0) + { + throw new Exception("Failed to open video file!"); + } + + Reset(); + } + + public void Reset() + { + lock (this) + { + Dav1dfile.df_reset(Handle); + ReadNextFrame(); + } + } + + public void ReadNextFrame() + { + lock (this) + { + if (Dav1dfile.df_readvideo( + Handle, + 1, + out yDataHandle, + out uDataHandle, + out vDataHandle, + out yDataLength, + out uvDataLength, + out yStride, + out uvStride) == 1 + ) { + FrameDataUpdated = true; + } + } + } + + protected virtual void Dispose(bool disposing) + { + if (!IsDisposed) + { + if (disposing) + { + // dispose managed state (managed objects) + } + + // free unmanaged resources (unmanaged objects) + Dav1dfile.df_close(Handle); + + IsDisposed = true; + } + } + + ~VideoAV1Stream() + { + Dispose(disposing: false); + } + + public void Dispose() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + Dispose(disposing: true); + GC.SuppressFinalize(this); + } + } +} diff --git a/src/Video/VideoPlayer.cs b/src/Video/VideoPlayer.cs index d4c52ad..7c33532 100644 --- a/src/Video/VideoPlayer.cs +++ b/src/Video/VideoPlayer.cs @@ -1,5 +1,6 @@ using System; using System.Diagnostics; +using System.Threading.Tasks; using System.Runtime.InteropServices; using MoonWorks.Audio; using MoonWorks.Graphics; @@ -11,20 +12,10 @@ namespace MoonWorks.Video public Texture RenderTexture { get; private set; } = null; public VideoState State { get; private set; } = VideoState.Stopped; public bool Loop { get; set; } - public float Volume { - get => volume; - set - { - volume = value; - if (audioStream != null) - { - audioStream.Volume = value; - } - } - } public float PlaybackSpeed { get; set; } = 1; - private Video Video = null; + private VideoAV1 Video = null; + private VideoAV1Stream CurrentStream = null; private GraphicsDevice GraphicsDevice; private Texture yTexture = null; @@ -32,22 +23,15 @@ namespace MoonWorks.Video private Texture vTexture = null; private Sampler LinearSampler; - private void* yuvData = null; - private int yuvDataLength = 0; - private int currentFrame; - private AudioDevice AudioDevice; - private StreamingSoundTheora audioStream = null; - private float volume = 1.0f; - private Stopwatch timer; private double lastTimestamp; private double timeElapsed; private bool disposed; - public VideoPlayer(GraphicsDevice graphicsDevice, AudioDevice audioDevice) + public VideoPlayer(GraphicsDevice graphicsDevice) { GraphicsDevice = graphicsDevice; if (GraphicsDevice.VideoPipeline == null) @@ -55,13 +39,12 @@ namespace MoonWorks.Video throw new InvalidOperationException("Missing video shaders!"); } - AudioDevice = audioDevice; LinearSampler = new Sampler(graphicsDevice, SamplerCreateInfo.LinearClamp); timer = new Stopwatch(); } - public void Load(Video video) + public void Load(VideoAV1 video) { if (Video != video) { @@ -111,20 +94,9 @@ namespace MoonWorks.Video vTexture = CreateSubTexture(GraphicsDevice, video.UVWidth, video.UVHeight); } - var newDataLength = ( - (video.Width * video.Height) + - (video.UVWidth * video.UVHeight * 2) - ); - - if (newDataLength != yuvDataLength) - { - yuvData = NativeMemory.Realloc(yuvData, (nuint) newDataLength); - yuvDataLength = newDataLength; - } - Video = video; - InitializeTheoraStream(); + InitializeDav1dStream(); } } @@ -139,11 +111,6 @@ namespace MoonWorks.Video timer.Start(); - if (audioStream != null) - { - audioStream.Play(); - } - State = VideoState.Playing; } @@ -158,11 +125,6 @@ namespace MoonWorks.Video timer.Stop(); - if (audioStream != null) - { - audioStream.Pause(); - } - State = VideoState.Paused; } @@ -181,9 +143,7 @@ namespace MoonWorks.Video lastTimestamp = 0; timeElapsed = 0; - DestroyAudioStream(); - - Theorafile.tf_reset(Video.Handle); + InitializeDav1dStream(); State = VideoState.Stopped; } @@ -194,16 +154,6 @@ namespace MoonWorks.Video Video = null; } - public void Update() - { - if (Video == null) { return; } - - if (audioStream != null) - { - audioStream.Update(); - } - } - public void Render() { if (Video == null || State == VideoState.Stopped) @@ -217,33 +167,27 @@ namespace MoonWorks.Video int thisFrame = ((int) (timeElapsed / (1000.0 / Video.FramesPerSecond))); if (thisFrame > currentFrame) { - if (Theorafile.tf_readvideo( - Video.Handle, - (IntPtr) yuvData, - thisFrame - currentFrame - ) == 1 || currentFrame == -1) + if (CurrentStream.FrameDataUpdated) { UpdateRenderTexture(); } currentFrame = thisFrame; + Task.Run(CurrentStream.ReadNextFrame); } - bool ended = Theorafile.tf_eos(Video.Handle) == 1; - if (ended) + if (CurrentStream.Ended) { timer.Stop(); timer.Reset(); - DestroyAudioStream(); - - Theorafile.tf_reset(Video.Handle); + Task.Run(CurrentStream.Reset); if (Loop) { - // Start over! - InitializeTheoraStream(); - + // Start over on the next stream! + CurrentStream = (CurrentStream == Video.StreamA) ? Video.StreamB : Video.StreamA; + currentFrame = -1; timer.Start(); } else @@ -255,32 +199,40 @@ namespace MoonWorks.Video private void UpdateRenderTexture() { - var commandBuffer = GraphicsDevice.AcquireCommandBuffer(); + lock (CurrentStream) + { + var commandBuffer = GraphicsDevice.AcquireCommandBuffer(); - commandBuffer.SetTextureDataYUV( - yTexture, - uTexture, - vTexture, - (IntPtr) yuvData, - (uint) yuvDataLength - ); + commandBuffer.SetTextureDataYUV( + yTexture, + uTexture, + vTexture, + CurrentStream.yDataHandle, + CurrentStream.uDataHandle, + CurrentStream.vDataHandle, + CurrentStream.yDataLength, + CurrentStream.uvDataLength, + CurrentStream.yStride, + CurrentStream.uvStride + ); - commandBuffer.BeginRenderPass( - new ColorAttachmentInfo(RenderTexture, Color.Black) - ); + commandBuffer.BeginRenderPass( + new ColorAttachmentInfo(RenderTexture, Color.Black) + ); - commandBuffer.BindGraphicsPipeline(GraphicsDevice.VideoPipeline); - commandBuffer.BindFragmentSamplers( - new TextureSamplerBinding(yTexture, LinearSampler), - new TextureSamplerBinding(uTexture, LinearSampler), - new TextureSamplerBinding(vTexture, LinearSampler) - ); + commandBuffer.BindGraphicsPipeline(GraphicsDevice.VideoPipeline); + commandBuffer.BindFragmentSamplers( + new TextureSamplerBinding(yTexture, LinearSampler), + new TextureSamplerBinding(uTexture, LinearSampler), + new TextureSamplerBinding(vTexture, LinearSampler) + ); - commandBuffer.DrawPrimitives(0, 1, 0, 0); + commandBuffer.DrawPrimitives(0, 1, 0, 0); - commandBuffer.EndRenderPass(); + commandBuffer.EndRenderPass(); - GraphicsDevice.Submit(commandBuffer); + GraphicsDevice.Submit(commandBuffer); + } } private static Texture CreateRenderTexture(GraphicsDevice graphicsDevice, int width, int height) @@ -305,35 +257,15 @@ namespace MoonWorks.Video ); } - private void InitializeTheoraStream() + private void InitializeDav1dStream() { - // Grab the first video frame ASAP. - while (Theorafile.tf_readvideo(Video.Handle, (IntPtr) yuvData, 1) == 0); - - // Grab the first bit of audio. We're trying to start the decoding ASAP. - if (AudioDevice != null && Theorafile.tf_hasaudio(Video.Handle) == 1) - { - DestroyAudioStream(); - - int channels, sampleRate; - Theorafile.tf_audioinfo(Video.Handle, out channels, out sampleRate); - - audioStream = new StreamingSoundTheora(AudioDevice, Video.Handle, channels, (uint) sampleRate); - } + Task.Run(Video.StreamA.Reset); + Task.Run(Video.StreamB.Reset); + CurrentStream = Video.StreamA; currentFrame = -1; } - private void DestroyAudioStream() - { - if (audioStream != null) - { - audioStream.StopImmediate(); - audioStream.Dispose(); - audioStream = null; - } - } - protected virtual void Dispose(bool disposing) { if (!disposed) @@ -347,9 +279,6 @@ namespace MoonWorks.Video vTexture.Dispose(); } - // free unmanaged resources (unmanaged objects) and override finalizer - NativeMemory.Free(yuvData); - disposed = true; } } diff --git a/src/Video/VideoState.cs b/src/Video/VideoState.cs new file mode 100644 index 0000000..dd51fb1 --- /dev/null +++ b/src/Video/VideoState.cs @@ -0,0 +1,9 @@ +namespace MoonWorks.Video +{ + public enum VideoState + { + Playing, + Paused, + Stopped + } +}