From 0f9511f0f6712a30e49f299362f2b7c239c3ed23 Mon Sep 17 00:00:00 2001 From: cosmonaut Date: Thu, 1 Jun 2023 16:31:32 -0700 Subject: [PATCH 1/6] AV1 decoder implementation --- .gitmodules | 3 + MoonWorks.csproj | 1 + MoonWorks.dll.config | 4 + lib/RefreshCS | 2 +- lib/dav1dfile | 1 + src/Graphics/CommandBuffer.cs | 21 +++++- src/Video/VideoAV1.cs | 88 ++++++++++++++++++++++ src/Video/VideoPlayer.cs | 135 +++++++++++----------------------- 8 files changed, 160 insertions(+), 95 deletions(-) create mode 160000 lib/dav1dfile create mode 100644 src/Video/VideoAV1.cs diff --git a/.gitmodules b/.gitmodules index e323a49..932cd36 100644 --- a/.gitmodules +++ b/.gitmodules @@ -13,3 +13,6 @@ [submodule "lib/Theorafile"] path = lib/Theorafile url = https://github.com/FNA-XNA/Theorafile.git +[submodule "lib/dav1dfile"] + path = lib/dav1dfile + url = git@github.com:MoonsideGames/dav1dfile.git diff --git a/MoonWorks.csproj b/MoonWorks.csproj index 9346bc8..52b7977 100644 --- a/MoonWorks.csproj +++ b/MoonWorks.csproj @@ -17,6 +17,7 @@ + diff --git a/MoonWorks.dll.config b/MoonWorks.dll.config index c076485..cc9a116 100644 --- a/MoonWorks.dll.config +++ b/MoonWorks.dll.config @@ -19,4 +19,8 @@ + + + + diff --git a/lib/RefreshCS b/lib/RefreshCS index ebf5111..9edb2d1 160000 --- a/lib/RefreshCS +++ b/lib/RefreshCS @@ -1 +1 @@ -Subproject commit ebf511133aa6f567c004d687acac474e1649bbde +Subproject commit 9edb2d1c97f4a38dd3d6d87298d6c604defa4fc8 diff --git a/lib/dav1dfile b/lib/dav1dfile new file mode 160000 index 0000000..8ba817f --- /dev/null +++ b/lib/dav1dfile @@ -0,0 +1 @@ +Subproject commit 8ba817ff9bf65ed2fa7b9c12d70112b29c450cf4 diff --git a/src/Graphics/CommandBuffer.cs b/src/Graphics/CommandBuffer.cs index 8992819..a664791 100644 --- a/src/Graphics/CommandBuffer.cs +++ b/src/Graphics/CommandBuffer.cs @@ -1946,7 +1946,17 @@ namespace MoonWorks.Graphics /// /// Asynchronously copies YUV data into three textures. Use with compressed video. /// - public void SetTextureDataYUV(Texture yTexture, Texture uTexture, Texture vTexture, IntPtr dataPtr, uint dataLengthInBytes) + public void SetTextureDataYUV( + Texture yTexture, + Texture uTexture, + Texture vTexture, + IntPtr yDataPtr, + IntPtr uDataPtr, + IntPtr vDataPtr, + uint yDataLengthInBytes, + uint uvDataLengthInBytes, + uint yStride, + uint uvStride) { #if DEBUG AssertRenderPassInactive("Cannot copy during render pass!"); @@ -1962,8 +1972,13 @@ namespace MoonWorks.Graphics yTexture.Height, uTexture.Width, uTexture.Height, - dataPtr, - dataLengthInBytes + yDataPtr, + uDataPtr, + vDataPtr, + yDataLengthInBytes, + uvDataLengthInBytes, + yStride, + uvStride ); } diff --git a/src/Video/VideoAV1.cs b/src/Video/VideoAV1.cs new file mode 100644 index 0000000..e423284 --- /dev/null +++ b/src/Video/VideoAV1.cs @@ -0,0 +1,88 @@ +using System; +using System.IO; + +namespace MoonWorks.Video +{ + public unsafe class VideoAV1 : IDisposable + { + public IntPtr Handle => handle; + IntPtr handle; + + public int Width => width; + public int Height => height; + public double FramesPerSecond => 28.97; + public Dav1dfile.PixelLayout PixelLayout => pixelLayout; + public int UVWidth { get; } + public int UVHeight { get; } + + private int width; + private int height; + private double fps; + private Dav1dfile.PixelLayout pixelLayout; + + bool IsDisposed; + + public VideoAV1(string filename) + { + if (!File.Exists(filename)) + { + throw new ArgumentException("Video file not found!"); + } + + if (Dav1dfile.df_fopen(filename, out handle) == 0) + { + throw new Exception("Failed to open video file!"); + } + + Dav1dfile.df_videoinfo(Handle, out width, out height, out pixelLayout); + + if (pixelLayout == Dav1dfile.PixelLayout.I420) + { + UVWidth = Width / 2; + UVHeight = Height / 2; + } + else if (pixelLayout == Dav1dfile.PixelLayout.I422) + { + UVWidth = Width / 2; + UVHeight = Height; + } + else if (pixelLayout == Dav1dfile.PixelLayout.I444) + { + UVWidth = width; + UVHeight = height; + } + else + { + throw new NotSupportedException("Unrecognized YUV format!"); + } + } + + protected virtual void Dispose(bool disposing) + { + if (!IsDisposed) + { + if (disposing) + { + // dispose managed state (managed objects) + } + + // free unmanaged resources (unmanaged objects) + Dav1dfile.df_close(Handle); + + IsDisposed = true; + } + } + + ~VideoAV1() + { + Dispose(disposing: false); + } + + public void Dispose() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + Dispose(disposing: true); + GC.SuppressFinalize(this); + } + } +} diff --git a/src/Video/VideoPlayer.cs b/src/Video/VideoPlayer.cs index d4c52ad..5372768 100644 --- a/src/Video/VideoPlayer.cs +++ b/src/Video/VideoPlayer.cs @@ -11,20 +11,9 @@ namespace MoonWorks.Video public Texture RenderTexture { get; private set; } = null; public VideoState State { get; private set; } = VideoState.Stopped; public bool Loop { get; set; } - public float Volume { - get => volume; - set - { - volume = value; - if (audioStream != null) - { - audioStream.Volume = value; - } - } - } public float PlaybackSpeed { get; set; } = 1; - private Video Video = null; + private VideoAV1 Video = null; private GraphicsDevice GraphicsDevice; private Texture yTexture = null; @@ -32,19 +21,14 @@ namespace MoonWorks.Video private Texture vTexture = null; private Sampler LinearSampler; - private void* yuvData = null; - private int yuvDataLength = 0; - private int currentFrame; - private AudioDevice AudioDevice; - private StreamingSoundTheora audioStream = null; - private float volume = 1.0f; - private Stopwatch timer; private double lastTimestamp; private double timeElapsed; + private Stopwatch profilingTimer = new Stopwatch(); + private bool disposed; public VideoPlayer(GraphicsDevice graphicsDevice, AudioDevice audioDevice) @@ -55,13 +39,12 @@ namespace MoonWorks.Video throw new InvalidOperationException("Missing video shaders!"); } - AudioDevice = audioDevice; LinearSampler = new Sampler(graphicsDevice, SamplerCreateInfo.LinearClamp); timer = new Stopwatch(); } - public void Load(Video video) + public void Load(VideoAV1 video) { if (Video != video) { @@ -111,20 +94,9 @@ namespace MoonWorks.Video vTexture = CreateSubTexture(GraphicsDevice, video.UVWidth, video.UVHeight); } - var newDataLength = ( - (video.Width * video.Height) + - (video.UVWidth * video.UVHeight * 2) - ); - - if (newDataLength != yuvDataLength) - { - yuvData = NativeMemory.Realloc(yuvData, (nuint) newDataLength); - yuvDataLength = newDataLength; - } - Video = video; - InitializeTheoraStream(); + InitializeDav1dStream(); } } @@ -139,11 +111,6 @@ namespace MoonWorks.Video timer.Start(); - if (audioStream != null) - { - audioStream.Play(); - } - State = VideoState.Playing; } @@ -158,11 +125,6 @@ namespace MoonWorks.Video timer.Stop(); - if (audioStream != null) - { - audioStream.Pause(); - } - State = VideoState.Paused; } @@ -181,9 +143,7 @@ namespace MoonWorks.Video lastTimestamp = 0; timeElapsed = 0; - DestroyAudioStream(); - - Theorafile.tf_reset(Video.Handle); + Dav1dfile.df_reset(Video.Handle); State = VideoState.Stopped; } @@ -197,11 +157,6 @@ namespace MoonWorks.Video public void Update() { if (Video == null) { return; } - - if (audioStream != null) - { - audioStream.Update(); - } } public void Render() @@ -217,32 +172,41 @@ namespace MoonWorks.Video int thisFrame = ((int) (timeElapsed / (1000.0 / Video.FramesPerSecond))); if (thisFrame > currentFrame) { - if (Theorafile.tf_readvideo( + profilingTimer.Restart(); + int readResult = Dav1dfile.df_readvideo( Video.Handle, - (IntPtr) yuvData, - thisFrame - currentFrame - ) == 1 || currentFrame == -1) + thisFrame - currentFrame, + out var yData, + out var uData, + out var vData, + out var yDataLength, + out var uvDataLength, + out var yStride, + out var uvStride + ); + profilingTimer.Stop(); + System.Console.WriteLine($"Frame decoded in: {profilingTimer.Elapsed.TotalMilliseconds}"); + + if (readResult == 1 || currentFrame == -1) { - UpdateRenderTexture(); + UpdateRenderTexture(yData, uData, vData, yDataLength, uvDataLength, yStride, uvStride); } currentFrame = thisFrame; } - bool ended = Theorafile.tf_eos(Video.Handle) == 1; + bool ended = Dav1dfile.df_eos(Video.Handle) == 1; if (ended) { timer.Stop(); timer.Reset(); - DestroyAudioStream(); - - Theorafile.tf_reset(Video.Handle); + Dav1dfile.df_reset(Video.Handle); if (Loop) { // Start over! - InitializeTheoraStream(); + InitializeDav1dStream(); timer.Start(); } @@ -253,7 +217,7 @@ namespace MoonWorks.Video } } - private void UpdateRenderTexture() + private void UpdateRenderTexture(IntPtr yData, IntPtr uData, IntPtr vData, uint yDataLength, uint uvDataLength, uint yStride, uint uvStride) { var commandBuffer = GraphicsDevice.AcquireCommandBuffer(); @@ -261,8 +225,13 @@ namespace MoonWorks.Video yTexture, uTexture, vTexture, - (IntPtr) yuvData, - (uint) yuvDataLength + yData, + uData, + vData, + yDataLength, + uvDataLength, + yStride, + uvStride ); commandBuffer.BeginRenderPass( @@ -305,35 +274,22 @@ namespace MoonWorks.Video ); } - private void InitializeTheoraStream() + private void InitializeDav1dStream() { // Grab the first video frame ASAP. - while (Theorafile.tf_readvideo(Video.Handle, (IntPtr) yuvData, 1) == 0); - - // Grab the first bit of audio. We're trying to start the decoding ASAP. - if (AudioDevice != null && Theorafile.tf_hasaudio(Video.Handle) == 1) - { - DestroyAudioStream(); - - int channels, sampleRate; - Theorafile.tf_audioinfo(Video.Handle, out channels, out sampleRate); - - audioStream = new StreamingSoundTheora(AudioDevice, Video.Handle, channels, (uint) sampleRate); - } - + Dav1dfile.df_readvideo( + Video.Handle, + 1, + out var _, + out var _, + out var _, + out var _, + out var _, + out var _, + out var _); currentFrame = -1; } - private void DestroyAudioStream() - { - if (audioStream != null) - { - audioStream.StopImmediate(); - audioStream.Dispose(); - audioStream = null; - } - } - protected virtual void Dispose(bool disposing) { if (!disposed) @@ -347,9 +303,6 @@ namespace MoonWorks.Video vTexture.Dispose(); } - // free unmanaged resources (unmanaged objects) and override finalizer - NativeMemory.Free(yuvData); - disposed = true; } } -- 2.25.1 From 1954665c938cc70c90c6398010f29cfb8e1da708 Mon Sep 17 00:00:00 2001 From: cosmonaut Date: Fri, 2 Jun 2023 14:43:00 -0700 Subject: [PATCH 2/6] threaded video decoding --- src/Video/VideoAV1.cs | 7 ++- src/Video/VideoPlayer.cs | 127 ++++++++++++++++++++------------------- 2 files changed, 69 insertions(+), 65 deletions(-) diff --git a/src/Video/VideoAV1.cs b/src/Video/VideoAV1.cs index e423284..6cb238b 100644 --- a/src/Video/VideoAV1.cs +++ b/src/Video/VideoAV1.cs @@ -10,19 +10,18 @@ namespace MoonWorks.Video public int Width => width; public int Height => height; - public double FramesPerSecond => 28.97; + public double FramesPerSecond { get; set; } public Dav1dfile.PixelLayout PixelLayout => pixelLayout; public int UVWidth { get; } public int UVHeight { get; } private int width; private int height; - private double fps; private Dav1dfile.PixelLayout pixelLayout; bool IsDisposed; - public VideoAV1(string filename) + public VideoAV1(string filename, double framesPerSecond) { if (!File.Exists(filename)) { @@ -55,6 +54,8 @@ namespace MoonWorks.Video { throw new NotSupportedException("Unrecognized YUV format!"); } + + FramesPerSecond = framesPerSecond; } protected virtual void Dispose(bool disposing) diff --git a/src/Video/VideoPlayer.cs b/src/Video/VideoPlayer.cs index 5372768..ce95001 100644 --- a/src/Video/VideoPlayer.cs +++ b/src/Video/VideoPlayer.cs @@ -21,17 +21,26 @@ namespace MoonWorks.Video private Texture vTexture = null; private Sampler LinearSampler; + private object readLock = new object(); + private bool frameDataUpdated; + + private IntPtr yDataHandle; + private IntPtr uDataHandle; + private IntPtr vDataHandle; + private uint yDataLength; + private uint uvDataLength; + private uint yStride; + private uint uvStride; + private int currentFrame; private Stopwatch timer; private double lastTimestamp; private double timeElapsed; - private Stopwatch profilingTimer = new Stopwatch(); - private bool disposed; - public VideoPlayer(GraphicsDevice graphicsDevice, AudioDevice audioDevice) + public VideoPlayer(GraphicsDevice graphicsDevice) { GraphicsDevice = graphicsDevice; if (GraphicsDevice.VideoPipeline == null) @@ -154,11 +163,6 @@ namespace MoonWorks.Video Video = null; } - public void Update() - { - if (Video == null) { return; } - } - public void Render() { if (Video == null || State == VideoState.Stopped) @@ -172,27 +176,13 @@ namespace MoonWorks.Video int thisFrame = ((int) (timeElapsed / (1000.0 / Video.FramesPerSecond))); if (thisFrame > currentFrame) { - profilingTimer.Restart(); - int readResult = Dav1dfile.df_readvideo( - Video.Handle, - thisFrame - currentFrame, - out var yData, - out var uData, - out var vData, - out var yDataLength, - out var uvDataLength, - out var yStride, - out var uvStride - ); - profilingTimer.Stop(); - System.Console.WriteLine($"Frame decoded in: {profilingTimer.Elapsed.TotalMilliseconds}"); - - if (readResult == 1 || currentFrame == -1) + if (frameDataUpdated) { - UpdateRenderTexture(yData, uData, vData, yDataLength, uvDataLength, yStride, uvStride); + UpdateRenderTexture(); } currentFrame = thisFrame; + System.Threading.Tasks.Task.Run(ReadNextFrame); } bool ended = Dav1dfile.df_eos(Video.Handle) == 1; @@ -217,39 +207,42 @@ namespace MoonWorks.Video } } - private void UpdateRenderTexture(IntPtr yData, IntPtr uData, IntPtr vData, uint yDataLength, uint uvDataLength, uint yStride, uint uvStride) + private void UpdateRenderTexture() { - var commandBuffer = GraphicsDevice.AcquireCommandBuffer(); + lock (readLock) + { + var commandBuffer = GraphicsDevice.AcquireCommandBuffer(); - commandBuffer.SetTextureDataYUV( - yTexture, - uTexture, - vTexture, - yData, - uData, - vData, - yDataLength, - uvDataLength, - yStride, - uvStride - ); + commandBuffer.SetTextureDataYUV( + yTexture, + uTexture, + vTexture, + yDataHandle, + uDataHandle, + vDataHandle, + yDataLength, + uvDataLength, + yStride, + uvStride + ); - commandBuffer.BeginRenderPass( - new ColorAttachmentInfo(RenderTexture, Color.Black) - ); + commandBuffer.BeginRenderPass( + new ColorAttachmentInfo(RenderTexture, Color.Black) + ); - commandBuffer.BindGraphicsPipeline(GraphicsDevice.VideoPipeline); - commandBuffer.BindFragmentSamplers( - new TextureSamplerBinding(yTexture, LinearSampler), - new TextureSamplerBinding(uTexture, LinearSampler), - new TextureSamplerBinding(vTexture, LinearSampler) - ); + commandBuffer.BindGraphicsPipeline(GraphicsDevice.VideoPipeline); + commandBuffer.BindFragmentSamplers( + new TextureSamplerBinding(yTexture, LinearSampler), + new TextureSamplerBinding(uTexture, LinearSampler), + new TextureSamplerBinding(vTexture, LinearSampler) + ); - commandBuffer.DrawPrimitives(0, 1, 0, 0); + commandBuffer.DrawPrimitives(0, 1, 0, 0); - commandBuffer.EndRenderPass(); + commandBuffer.EndRenderPass(); - GraphicsDevice.Submit(commandBuffer); + GraphicsDevice.Submit(commandBuffer); + } } private static Texture CreateRenderTexture(GraphicsDevice graphicsDevice, int width, int height) @@ -276,20 +269,30 @@ namespace MoonWorks.Video private void InitializeDav1dStream() { - // Grab the first video frame ASAP. - Dav1dfile.df_readvideo( - Video.Handle, - 1, - out var _, - out var _, - out var _, - out var _, - out var _, - out var _, - out var _); + System.Threading.Tasks.Task.Run(ReadNextFrame); currentFrame = -1; } + private void ReadNextFrame() + { + lock (readLock) + { + if (Dav1dfile.df_readvideo( + Video.Handle, + 1, + out yDataHandle, + out uDataHandle, + out vDataHandle, + out yDataLength, + out uvDataLength, + out yStride, + out uvStride) == 1 + ) { + frameDataUpdated = true; + } + } + } + protected virtual void Dispose(bool disposing) { if (!disposed) -- 2.25.1 From c70560025d006a666e7975dc1b1e6fa887c0544b Mon Sep 17 00:00:00 2001 From: cosmonaut Date: Wed, 7 Jun 2023 12:32:27 -0700 Subject: [PATCH 3/6] double buffered video for fast loops --- src/Video/VideoAV1.cs | 45 +++++------------- src/Video/VideoAV1Stream.cs | 91 +++++++++++++++++++++++++++++++++++++ src/Video/VideoPlayer.cs | 71 +++++++++-------------------- 3 files changed, 125 insertions(+), 82 deletions(-) create mode 100644 src/Video/VideoAV1Stream.cs diff --git a/src/Video/VideoAV1.cs b/src/Video/VideoAV1.cs index 6cb238b..2f485a5 100644 --- a/src/Video/VideoAV1.cs +++ b/src/Video/VideoAV1.cs @@ -3,10 +3,13 @@ using System.IO; namespace MoonWorks.Video { - public unsafe class VideoAV1 : IDisposable + public unsafe class VideoAV1 { - public IntPtr Handle => handle; - IntPtr handle; + public string Filename { get; } + + // "double buffering" so we can loop without a stutter + internal VideoAV1Stream StreamA { get; } + internal VideoAV1Stream StreamB { get; } public int Width => width; public int Height => height; @@ -19,8 +22,6 @@ namespace MoonWorks.Video private int height; private Dav1dfile.PixelLayout pixelLayout; - bool IsDisposed; - public VideoAV1(string filename, double framesPerSecond) { if (!File.Exists(filename)) @@ -28,12 +29,13 @@ namespace MoonWorks.Video throw new ArgumentException("Video file not found!"); } - if (Dav1dfile.df_fopen(filename, out handle) == 0) + if (Dav1dfile.df_fopen(filename, out var handle) == 0) { throw new Exception("Failed to open video file!"); } - Dav1dfile.df_videoinfo(Handle, out width, out height, out pixelLayout); + Dav1dfile.df_videoinfo(handle, out width, out height, out pixelLayout); + Dav1dfile.df_close(handle); if (pixelLayout == Dav1dfile.PixelLayout.I420) { @@ -56,34 +58,11 @@ namespace MoonWorks.Video } FramesPerSecond = framesPerSecond; - } - protected virtual void Dispose(bool disposing) - { - if (!IsDisposed) - { - if (disposing) - { - // dispose managed state (managed objects) - } + Filename = filename; - // free unmanaged resources (unmanaged objects) - Dav1dfile.df_close(Handle); - - IsDisposed = true; - } - } - - ~VideoAV1() - { - Dispose(disposing: false); - } - - public void Dispose() - { - // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method - Dispose(disposing: true); - GC.SuppressFinalize(this); + StreamA = new VideoAV1Stream(this); + StreamB = new VideoAV1Stream(this); } } } diff --git a/src/Video/VideoAV1Stream.cs b/src/Video/VideoAV1Stream.cs new file mode 100644 index 0000000..5ac443c --- /dev/null +++ b/src/Video/VideoAV1Stream.cs @@ -0,0 +1,91 @@ +using System; + +namespace MoonWorks.Video +{ + internal class VideoAV1Stream + { + public IntPtr Handle => handle; + IntPtr handle; + + public bool Ended => Dav1dfile.df_eos(Handle) == 1; + + public IntPtr yDataHandle; + public IntPtr uDataHandle; + public IntPtr vDataHandle; + public uint yDataLength; + public uint uvDataLength; + public uint yStride; + public uint uvStride; + + public bool FrameDataUpdated { get; private set; } + + bool IsDisposed; + + public VideoAV1Stream(VideoAV1 video) + { + if (Dav1dfile.df_fopen(video.Filename, out handle) == 0) + { + throw new Exception("Failed to open video file!"); + } + + Reset(); + } + + public void Reset() + { + lock (this) + { + Dav1dfile.df_reset(Handle); + ReadNextFrame(); + } + } + + public void ReadNextFrame() + { + lock (this) + { + if (Dav1dfile.df_readvideo( + Handle, + 1, + out yDataHandle, + out uDataHandle, + out vDataHandle, + out yDataLength, + out uvDataLength, + out yStride, + out uvStride) == 1 + ) { + FrameDataUpdated = true; + } + } + } + + protected virtual void Dispose(bool disposing) + { + if (!IsDisposed) + { + if (disposing) + { + // dispose managed state (managed objects) + } + + // free unmanaged resources (unmanaged objects) + Dav1dfile.df_close(Handle); + + IsDisposed = true; + } + } + + ~VideoAV1Stream() + { + Dispose(disposing: false); + } + + public void Dispose() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + Dispose(disposing: true); + GC.SuppressFinalize(this); + } + } +} diff --git a/src/Video/VideoPlayer.cs b/src/Video/VideoPlayer.cs index ce95001..7c33532 100644 --- a/src/Video/VideoPlayer.cs +++ b/src/Video/VideoPlayer.cs @@ -1,5 +1,6 @@ using System; using System.Diagnostics; +using System.Threading.Tasks; using System.Runtime.InteropServices; using MoonWorks.Audio; using MoonWorks.Graphics; @@ -14,6 +15,7 @@ namespace MoonWorks.Video public float PlaybackSpeed { get; set; } = 1; private VideoAV1 Video = null; + private VideoAV1Stream CurrentStream = null; private GraphicsDevice GraphicsDevice; private Texture yTexture = null; @@ -21,17 +23,6 @@ namespace MoonWorks.Video private Texture vTexture = null; private Sampler LinearSampler; - private object readLock = new object(); - private bool frameDataUpdated; - - private IntPtr yDataHandle; - private IntPtr uDataHandle; - private IntPtr vDataHandle; - private uint yDataLength; - private uint uvDataLength; - private uint yStride; - private uint uvStride; - private int currentFrame; private Stopwatch timer; @@ -152,7 +143,7 @@ namespace MoonWorks.Video lastTimestamp = 0; timeElapsed = 0; - Dav1dfile.df_reset(Video.Handle); + InitializeDav1dStream(); State = VideoState.Stopped; } @@ -176,28 +167,27 @@ namespace MoonWorks.Video int thisFrame = ((int) (timeElapsed / (1000.0 / Video.FramesPerSecond))); if (thisFrame > currentFrame) { - if (frameDataUpdated) + if (CurrentStream.FrameDataUpdated) { UpdateRenderTexture(); } currentFrame = thisFrame; - System.Threading.Tasks.Task.Run(ReadNextFrame); + Task.Run(CurrentStream.ReadNextFrame); } - bool ended = Dav1dfile.df_eos(Video.Handle) == 1; - if (ended) + if (CurrentStream.Ended) { timer.Stop(); timer.Reset(); - Dav1dfile.df_reset(Video.Handle); + Task.Run(CurrentStream.Reset); if (Loop) { - // Start over! - InitializeDav1dStream(); - + // Start over on the next stream! + CurrentStream = (CurrentStream == Video.StreamA) ? Video.StreamB : Video.StreamA; + currentFrame = -1; timer.Start(); } else @@ -209,7 +199,7 @@ namespace MoonWorks.Video private void UpdateRenderTexture() { - lock (readLock) + lock (CurrentStream) { var commandBuffer = GraphicsDevice.AcquireCommandBuffer(); @@ -217,13 +207,13 @@ namespace MoonWorks.Video yTexture, uTexture, vTexture, - yDataHandle, - uDataHandle, - vDataHandle, - yDataLength, - uvDataLength, - yStride, - uvStride + CurrentStream.yDataHandle, + CurrentStream.uDataHandle, + CurrentStream.vDataHandle, + CurrentStream.yDataLength, + CurrentStream.uvDataLength, + CurrentStream.yStride, + CurrentStream.uvStride ); commandBuffer.BeginRenderPass( @@ -269,28 +259,11 @@ namespace MoonWorks.Video private void InitializeDav1dStream() { - System.Threading.Tasks.Task.Run(ReadNextFrame); - currentFrame = -1; - } + Task.Run(Video.StreamA.Reset); + Task.Run(Video.StreamB.Reset); - private void ReadNextFrame() - { - lock (readLock) - { - if (Dav1dfile.df_readvideo( - Video.Handle, - 1, - out yDataHandle, - out uDataHandle, - out vDataHandle, - out yDataLength, - out uvDataLength, - out yStride, - out uvStride) == 1 - ) { - frameDataUpdated = true; - } - } + CurrentStream = Video.StreamA; + currentFrame = -1; } protected virtual void Dispose(bool disposing) -- 2.25.1 From e1f934d445f13f2c10bf9c76b90da272da88a263 Mon Sep 17 00:00:00 2001 From: cosmonaut Date: Wed, 7 Jun 2023 14:10:48 -0700 Subject: [PATCH 4/6] drop Theorafile --- .gitmodules | 3 - MoonWorks.csproj | 1 - lib/Theorafile | 1 - lib/dav1dfile | 2 +- src/Video/StreamingSoundTheora.cs | 67 ---------------- src/Video/Video.cs | 127 ------------------------------ src/Video/VideoState.cs | 9 +++ 7 files changed, 10 insertions(+), 200 deletions(-) delete mode 160000 lib/Theorafile delete mode 100644 src/Video/StreamingSoundTheora.cs delete mode 100644 src/Video/Video.cs create mode 100644 src/Video/VideoState.cs diff --git a/.gitmodules b/.gitmodules index 932cd36..4f87810 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,9 +10,6 @@ [submodule "lib/WellspringCS"] path = lib/WellspringCS url = https://gitea.moonside.games/MoonsideGames/WellspringCS.git -[submodule "lib/Theorafile"] - path = lib/Theorafile - url = https://github.com/FNA-XNA/Theorafile.git [submodule "lib/dav1dfile"] path = lib/dav1dfile url = git@github.com:MoonsideGames/dav1dfile.git diff --git a/MoonWorks.csproj b/MoonWorks.csproj index 52b7977..a7d3ecd 100644 --- a/MoonWorks.csproj +++ b/MoonWorks.csproj @@ -15,7 +15,6 @@ - diff --git a/lib/Theorafile b/lib/Theorafile deleted file mode 160000 index 8f9419e..0000000 --- a/lib/Theorafile +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8f9419ea856480e08294698e1d6be8752df3710b diff --git a/lib/dav1dfile b/lib/dav1dfile index 8ba817f..859f47f 160000 --- a/lib/dav1dfile +++ b/lib/dav1dfile @@ -1 +1 @@ -Subproject commit 8ba817ff9bf65ed2fa7b9c12d70112b29c450cf4 +Subproject commit 859f47f6fa0dfa0f7f941dcced6664fa83736202 diff --git a/src/Video/StreamingSoundTheora.cs b/src/Video/StreamingSoundTheora.cs deleted file mode 100644 index 95f8e57..0000000 --- a/src/Video/StreamingSoundTheora.cs +++ /dev/null @@ -1,67 +0,0 @@ -using System; -using MoonWorks.Audio; - -namespace MoonWorks.Video -{ - // TODO: should we just not handle theora sound? it sucks! - internal unsafe class StreamingSoundTheora : StreamingSound - { - private IntPtr VideoHandle; - public override bool Loaded => true; - - internal StreamingSoundTheora( - AudioDevice device, - IntPtr videoHandle, - int channels, - uint sampleRate, - uint bufferSize = 8192 - ) : base( - device, - 3, /* float type */ - 32, /* size of float */ - (ushort) (4 * channels), - (ushort) channels, - sampleRate, - bufferSize, - false // Theorafile is not thread safe, so let's update on the main thread - ) { - VideoHandle = videoHandle; - } - - public override unsafe void Load() - { - // no-op - } - - public override unsafe void Unload() - { - // no-op - } - - protected override unsafe void FillBuffer( - void* buffer, - int bufferLengthInBytes, - out int filledLengthInBytes, - out bool reachedEnd - ) { - var lengthInFloats = bufferLengthInBytes / sizeof(float); - - // FIXME: this gets gnarly with theorafile being not thread safe - // is there some way we could just manually update in VideoPlayer - // instead of going through AudioDevice? - lock (Device.StateLock) - { - int samples = Theorafile.tf_readaudio( - VideoHandle, - (IntPtr) buffer, - lengthInFloats - ); - - filledLengthInBytes = samples * sizeof(float); - reachedEnd = Theorafile.tf_eos(VideoHandle) == 1; - } - } - - protected override void OnReachedEnd() { } - } -} diff --git a/src/Video/Video.cs b/src/Video/Video.cs deleted file mode 100644 index 38feed4..0000000 --- a/src/Video/Video.cs +++ /dev/null @@ -1,127 +0,0 @@ -/* Heavily based on https://github.com/FNA-XNA/FNA/blob/master/src/Media/Xiph/VideoPlayer.cs */ -using System; -using System.IO; -using System.Runtime.InteropServices; -using SDL2; - -namespace MoonWorks.Video -{ - public enum VideoState - { - Playing, - Paused, - Stopped - } - - public unsafe class Video : IDisposable - { - internal IntPtr Handle; - private IntPtr rwData; - private void* videoData; - private int videoDataLength; - - public double FramesPerSecond => fps; - public int Width => yWidth; - public int Height => yHeight; - public int UVWidth { get; } - public int UVHeight { get; } - - private double fps; - private int yWidth; - private int yHeight; - - private bool IsDisposed; - - public Video(string filename) - { - if (!File.Exists(filename)) - { - throw new ArgumentException("Video file not found!"); - } - - var fileStream = new FileStream(filename, FileMode.Open, FileAccess.Read); - videoDataLength = (int) fileStream.Length; - videoData = NativeMemory.Alloc((nuint) videoDataLength); - var fileBufferSpan = new Span(videoData, videoDataLength); - fileStream.ReadExactly(fileBufferSpan); - fileStream.Close(); - - rwData = SDL.SDL_RWFromMem((IntPtr) videoData, videoDataLength); - if (Theorafile.tf_open_callbacks(rwData, out Handle, callbacks) < 0) - { - throw new ArgumentException("Invalid video file!"); - } - - Theorafile.th_pixel_fmt format; - Theorafile.tf_videoinfo( - Handle, - out yWidth, - out yHeight, - out fps, - out format - ); - - if (format == Theorafile.th_pixel_fmt.TH_PF_420) - { - UVWidth = Width / 2; - UVHeight = Height / 2; - } - else if (format == Theorafile.th_pixel_fmt.TH_PF_422) - { - UVWidth = Width / 2; - UVHeight = Height; - } - else if (format == Theorafile.th_pixel_fmt.TH_PF_444) - { - UVWidth = Width; - UVHeight = Height; - } - else - { - throw new NotSupportedException("Unrecognized YUV format!"); - } - } - - private static IntPtr Read(IntPtr ptr, IntPtr size, IntPtr nmemb, IntPtr datasource) => (IntPtr) SDL2.SDL.SDL_RWread(datasource, ptr, size, nmemb); - private static int Seek(IntPtr datasource, long offset, Theorafile.SeekWhence whence) => (int) SDL2.SDL.SDL_RWseek(datasource, offset, (int) whence); - private static int Close(IntPtr datasource) => (int) SDL2.SDL.SDL_RWclose(datasource); - - private static Theorafile.tf_callbacks callbacks = new Theorafile.tf_callbacks - { - read_func = Read, - seek_func = Seek, - close_func = Close - }; - - protected virtual void Dispose(bool disposing) - { - if (!IsDisposed) - { - if (disposing) - { - // dispose managed state (managed objects) - } - - // free unmanaged resources (unmanaged objects) - Theorafile.tf_close(ref Handle); - SDL.SDL_RWclose(rwData); - NativeMemory.Free(videoData); - - IsDisposed = true; - } - } - - ~Video() - { - // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method - Dispose(disposing: false); - } - - public void Dispose() - { - // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method - Dispose(disposing: true); - GC.SuppressFinalize(this); - } - } -} diff --git a/src/Video/VideoState.cs b/src/Video/VideoState.cs new file mode 100644 index 0000000..dd51fb1 --- /dev/null +++ b/src/Video/VideoState.cs @@ -0,0 +1,9 @@ +namespace MoonWorks.Video +{ + public enum VideoState + { + Playing, + Paused, + Stopped + } +} -- 2.25.1 From c1fae8b4333d92726a8cd43975d380a95b9d4776 Mon Sep 17 00:00:00 2001 From: cosmonaut Date: Wed, 7 Jun 2023 14:12:13 -0700 Subject: [PATCH 5/6] Note about VideoAV1 format --- src/Video/VideoAV1.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Video/VideoAV1.cs b/src/Video/VideoAV1.cs index 2f485a5..38aebee 100644 --- a/src/Video/VideoAV1.cs +++ b/src/Video/VideoAV1.cs @@ -3,6 +3,9 @@ using System.IO; namespace MoonWorks.Video { + /// + /// This class takes in a filename for AV1 data in .obu (open bitstream unit) format + /// public unsafe class VideoAV1 { public string Filename { get; } -- 2.25.1 From 71b09ec577724a6d5095a680c8cad8905ae4315b Mon Sep 17 00:00:00 2001 From: cosmonaut Date: Wed, 7 Jun 2023 14:16:49 -0700 Subject: [PATCH 6/6] update RefreshCS --- lib/RefreshCS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/RefreshCS b/lib/RefreshCS index 9edb2d1..60a7523 160000 --- a/lib/RefreshCS +++ b/lib/RefreshCS @@ -1 +1 @@ -Subproject commit 9edb2d1c97f4a38dd3d6d87298d6c604defa4fc8 +Subproject commit 60a7523fac254d5e2d89185392e8c1afd8581aa9 -- 2.25.1