From 845881533bf0b8d218fe931b0ee128b28c906cc2 Mon Sep 17 00:00:00 2001 From: cosmonaut Date: Fri, 29 Jul 2022 18:24:05 -0700 Subject: [PATCH] initial video support --- .gitmodules | 3 + MoonWorks.csproj | 10 + MoonWorks.dll.config | 4 + lib/Theorafile | 1 + src/Graphics/CommandBuffer.cs | 22 +- src/Graphics/GraphicsDevice.cs | 31 ++ src/MoonWorksDllMap.cs | 1 + src/Video/Shaders/Compiled/FullscreenVert.spv | Bin 0 -> 1164 bytes src/Video/Shaders/Compiled/YUV2RGBAFrag.spv | Bin 0 -> 1644 bytes src/Video/Shaders/Source/Fullscreen.vert | 9 + src/Video/Shaders/Source/YUV2RGBA.frag | 38 +++ src/Video/Video.cs | 94 ++++++ src/Video/VideoPlayer.cs | 285 ++++++++++++++++++ 13 files changed, 496 insertions(+), 2 deletions(-) create mode 160000 lib/Theorafile create mode 100644 src/Video/Shaders/Compiled/FullscreenVert.spv create mode 100644 src/Video/Shaders/Compiled/YUV2RGBAFrag.spv create mode 100644 src/Video/Shaders/Source/Fullscreen.vert create mode 100644 src/Video/Shaders/Source/YUV2RGBA.frag create mode 100644 src/Video/Video.cs create mode 100644 src/Video/VideoPlayer.cs diff --git a/.gitmodules b/.gitmodules index 481755f..e323a49 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,3 +10,6 @@ [submodule "lib/WellspringCS"] path = lib/WellspringCS url = https://gitea.moonside.games/MoonsideGames/WellspringCS.git +[submodule "lib/Theorafile"] + path = lib/Theorafile + url = https://github.com/FNA-XNA/Theorafile.git diff --git a/MoonWorks.csproj b/MoonWorks.csproj index 88105d4..31c1fce 100644 --- a/MoonWorks.csproj +++ b/MoonWorks.csproj @@ -15,6 +15,7 @@ + @@ -22,4 +23,13 @@ PreserveNewest + + + + MoonWorks.Shaders.FullscreenVert.spv + + + MoonWorks.Shaders.YUV2RGBAFrag.spv + + diff --git a/MoonWorks.dll.config b/MoonWorks.dll.config index d384162..c076485 100644 --- a/MoonWorks.dll.config +++ b/MoonWorks.dll.config @@ -15,4 +15,8 @@ + + + + diff --git a/lib/Theorafile b/lib/Theorafile new file mode 160000 index 0000000..3ed1726 --- /dev/null +++ b/lib/Theorafile @@ -0,0 +1 @@ +Subproject commit 3ed1726b1e294799e85c3b597b114fb3b21cba72 diff --git a/src/Graphics/CommandBuffer.cs b/src/Graphics/CommandBuffer.cs index bcd698a..598d50b 100644 --- a/src/Graphics/CommandBuffer.cs +++ b/src/Graphics/CommandBuffer.cs @@ -1,6 +1,4 @@ using System; -using System.Runtime.InteropServices; -using MoonWorks.Math; using RefreshCS; namespace MoonWorks.Graphics @@ -835,6 +833,26 @@ namespace MoonWorks.Graphics SetTextureData(new TextureSlice(texture), dataPtr, dataLengthInBytes); } + /// + /// Asynchronously copies YUV data into three textures. Use with compressed video. + /// + public void SetTextureDataYUV(Texture yTexture, Texture uTexture, Texture vTexture, IntPtr dataPtr, uint dataLengthInBytes) + { + Refresh.Refresh_SetTextureDataYUV( + Device.Handle, + Handle, + yTexture.Handle, + uTexture.Handle, + vTexture.Handle, + yTexture.Width, + yTexture.Height, + uTexture.Width, + uTexture.Height, + dataPtr, + dataLengthInBytes + ); + } + /// /// Performs an asynchronous texture-to-texture copy on the GPU. /// diff --git a/src/Graphics/GraphicsDevice.cs b/src/Graphics/GraphicsDevice.cs index 99915b4..c111c9e 100644 --- a/src/Graphics/GraphicsDevice.cs +++ b/src/Graphics/GraphicsDevice.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.IO; using RefreshCS; namespace MoonWorks.Graphics @@ -8,6 +9,11 @@ namespace MoonWorks.Graphics { public IntPtr Handle { get; } + // Built-in video pipeline + private ShaderModule VideoVertexShader { get; } + private ShaderModule VideoFragmentShader { get; } + internal GraphicsPipeline VideoPipeline { get; } + public bool IsDisposed { get; private set; } private readonly List> resources = new List>(); @@ -28,6 +34,26 @@ namespace MoonWorks.Graphics presentationParameters, Conversions.BoolToByte(debugMode) ); + + VideoVertexShader = new ShaderModule(this, GetEmbeddedResource("MoonWorks.Shaders.FullscreenVert.spv")); + VideoFragmentShader = new ShaderModule(this, GetEmbeddedResource("MoonWorks.Shaders.YUV2RGBAFrag.spv")); + + VideoPipeline = new GraphicsPipeline( + this, + new GraphicsPipelineCreateInfo + { + AttachmentInfo = new GraphicsPipelineAttachmentInfo( + new ColorAttachmentDescription(TextureFormat.R8G8B8A8, ColorAttachmentBlendState.None) + ), + DepthStencilState = DepthStencilState.Disable, + VertexShaderInfo = GraphicsShaderInfo.Create(VideoVertexShader, "main", 0), + FragmentShaderInfo = GraphicsShaderInfo.Create(VideoFragmentShader, "main", 3), + VertexInputState = VertexInputState.Empty, + RasterizerState = RasterizerState.CCW_CullNone, + PrimitiveType = PrimitiveType.TriangleList, + MultisampleState = MultisampleState.None + } + ); } public CommandBuffer AcquireCommandBuffer() @@ -77,6 +103,11 @@ namespace MoonWorks.Graphics } } + private static Stream GetEmbeddedResource(string name) + { + return typeof(GraphicsDevice).Assembly.GetManifestResourceStream(name); + } + protected virtual void Dispose(bool disposing) { if (!IsDisposed) diff --git a/src/MoonWorksDllMap.cs b/src/MoonWorksDllMap.cs index 5164838..fd9b0f3 100644 --- a/src/MoonWorksDllMap.cs +++ b/src/MoonWorksDllMap.cs @@ -196,6 +196,7 @@ namespace MoonWorks NativeLibrary.SetDllImportResolver(typeof(RefreshCS.Refresh).Assembly, MapAndLoad); NativeLibrary.SetDllImportResolver(typeof(FAudio).Assembly, MapAndLoad); NativeLibrary.SetDllImportResolver(typeof(WellspringCS.Wellspring).Assembly, MapAndLoad); + NativeLibrary.SetDllImportResolver(typeof(Theorafile).Assembly, MapAndLoad); } #endregion diff --git a/src/Video/Shaders/Compiled/FullscreenVert.spv b/src/Video/Shaders/Compiled/FullscreenVert.spv new file mode 100644 index 0000000000000000000000000000000000000000..ffc57de4aad5b75ae64168b06875645c4b611764 GIT binary patch literal 1164 zcmZ9KTWb_S5QW=accY0W#+bxQjFZG`0>K9rLA()R74onl;;V%0AOq`+>`q|uN$}kt zC4zs+7s2nFnNEVe)K;BSr>m>0r`_6^3gJwc4(GyFsMmaG!Gtgqx|Q$k?(c4o%fa@e z#}8G^hIS=XGsiyHS?u#cmK&ePE?_sYHoq4BC*lsO)4C3f+Ty5upH6m)Vl;s1R(n4U zKYmC@WjcA657LQyyWs{}_c!qqZmZGH{)aDtJwESS-q8Da2bcUGm=69qth2d@IhS9s z;<<)%HoqHgzTuru(e!kgSU%QSAhu52bA?#$^qxhqe4o0$;}T{a`4@EOd&*hU{4GrG zN26bBxC9?-Y!EvS?R%rUO{~`39-K40i^;npcaK=kH|%qwGrNzeHRoKd`{K2tN%-dr`SI%Z%D2H literal 0 HcmV?d00001 diff --git a/src/Video/Shaders/Compiled/YUV2RGBAFrag.spv b/src/Video/Shaders/Compiled/YUV2RGBAFrag.spv new file mode 100644 index 0000000000000000000000000000000000000000..c9fbf324eb9aadc03e7947d6af078c753e429c93 GIT binary patch literal 1644 zcmZ9L$xc*J5QY!kHngA&A|Qg=fO8{GfG9Gy%7%@>A-W@h7!$BfJ4AQ#4m^enUqDTK z7vI2z8xp_oc9*%~B-N+>zy4EoPTlHaVWK0Yw$z!*>0xTFzEprosVlW-``Y@(`qaDk zTT|0BH&k?|VkT76lRDB7c5Qd_^&10y*c8@_PM!jIMY5*!i$f*YyuL*2D6!8UJ`yXX z##V5rah@L1jln%T#CexTjm=lJ`cB=1YgD(!IOZDy zd)6hk-q{Q_v9sK7XfviKh_e-&9|^Ua9OX1-F)6leJ{JYGCc6;jmtt0l^$VDE9{LjJuQsA3lSKwRt?R&qMqRwrwXVe+9&J4Rby=R=8!~Z(^ z@<-si;BSrc|NGtpn@63o&@JK@svXNwZwahU-kPCX#it+Hz5O0F%sb5T53+B5E0^3~ zm=e1^aJKya*|P<(Gt!MYA7W?V1m^dD&MA9$5>q4p7|wSNVe+mOIKTNYW^QL;k;-Cq zQD+3-Ify!=`1v|#z-r{9j{hThSJbKCTgMr)j#ypP8OL{?qRs?;dtS)lg z!O!QI2g^B6k>ftTIif#eb&+EMKc8bci>2hOMUEAGbHv-l%UHx8;2W=DzE!-69q_N7 G!~OxAcvYnU literal 0 HcmV?d00001 diff --git a/src/Video/Shaders/Source/Fullscreen.vert b/src/Video/Shaders/Source/Fullscreen.vert new file mode 100644 index 0000000..2f3c315 --- /dev/null +++ b/src/Video/Shaders/Source/Fullscreen.vert @@ -0,0 +1,9 @@ +#version 450 + +layout(location = 0) out vec2 outTexCoord; + +void main() +{ + outTexCoord = vec2((gl_VertexIndex << 1) & 2, gl_VertexIndex & 2); + gl_Position = vec4(outTexCoord * 2.0 - 1.0, 0.0, 1.0); +} diff --git a/src/Video/Shaders/Source/YUV2RGBA.frag b/src/Video/Shaders/Source/YUV2RGBA.frag new file mode 100644 index 0000000..fe2b5a1 --- /dev/null +++ b/src/Video/Shaders/Source/YUV2RGBA.frag @@ -0,0 +1,38 @@ +/* + * This effect is based on the YUV-to-RGBA GLSL shader found in SDL. + * Thus, it also released under the zlib license: + * http://libsdl.org/license.php + */ +#version 450 + +layout(location = 0) in vec2 TexCoord; + +layout(location = 0) out vec4 FragColor; + +layout(binding = 0, set = 1) uniform sampler2D YSampler; +layout(binding = 1, set = 1) uniform sampler2D USampler; +layout(binding = 2, set = 1) uniform sampler2D VSampler; + +/* More info about colorspace conversion: + * http://www.equasys.de/colorconversion.html + * http://www.equasys.de/colorformat.html + */ + +const vec3 offset = vec3(-0.0625, -0.5, -0.5); +const vec3 Rcoeff = vec3(1.164, 0.000, 1.793); +const vec3 Gcoeff = vec3(1.164, -0.213, -0.533); +const vec3 Bcoeff = vec3(1.164, 2.112, 0.000); + +void main() +{ + vec3 yuv; + yuv.x = texture(YSampler, TexCoord).r; + yuv.y = texture(USampler, TexCoord).r; + yuv.z = texture(VSampler, TexCoord).r; + yuv += offset; + + FragColor.r = dot(yuv, Rcoeff); + FragColor.g = dot(yuv, Gcoeff); + FragColor.b = dot(yuv, Bcoeff); + FragColor.a = 1.0; +} diff --git a/src/Video/Video.cs b/src/Video/Video.cs new file mode 100644 index 0000000..5dc4e55 --- /dev/null +++ b/src/Video/Video.cs @@ -0,0 +1,94 @@ +/* Heavily based on https://github.com/FNA-XNA/FNA/blob/master/src/Media/Xiph/Video.cs */ +using System; + +namespace MoonWorks.Video +{ + public class Video : IDisposable + { + public IntPtr Handle => handle; + public int YWidth => yWidth; + public int YHeight => yHeight; + public int UVWidth => uvWidth; + public int UVHeight => uvHeight; + public double FramesPerSecond => fps; + + private IntPtr handle; + private int yWidth; + private int yHeight; + private int uvWidth; + private int uvHeight; + private double fps; + + private bool disposed; + + public Video(string filename) + { + Theorafile.th_pixel_fmt format; + + if (!System.IO.File.Exists(filename)) + { + throw new ArgumentException("Video file not found!"); + } + + if (Theorafile.tf_fopen(filename, out handle) < 0) + { + throw new ArgumentException("Invalid video file!"); + } + + Theorafile.tf_videoinfo( + handle, + out yWidth, + out yHeight, + out fps, + out format + ); + + if (format == Theorafile.th_pixel_fmt.TH_PF_420) + { + uvWidth = yWidth / 2; + uvHeight = yHeight / 2; + } + else if (format == Theorafile.th_pixel_fmt.TH_PF_422) + { + uvWidth = yWidth / 2; + uvHeight = yHeight; + } + else if (format == Theorafile.th_pixel_fmt.TH_PF_444) + { + uvWidth = yWidth; + uvHeight = yHeight; + } + else + { + throw new NotSupportedException("Unrecognized YUV format!"); + } + } + + protected virtual void Dispose(bool disposing) + { + if (!disposed) + { + if (disposing) + { + // TODO: dispose managed state (managed objects) + } + + Theorafile.tf_close(ref handle); + disposed = true; + } + } + + ~Video() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + Dispose(disposing: false); + } + + public void Dispose() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + Dispose(disposing: true); + GC.SuppressFinalize(this); + } + } +} diff --git a/src/Video/VideoPlayer.cs b/src/Video/VideoPlayer.cs new file mode 100644 index 0000000..c8d275a --- /dev/null +++ b/src/Video/VideoPlayer.cs @@ -0,0 +1,285 @@ +/* Heavily based on https://github.com/FNA-XNA/FNA/blob/master/src/Media/Xiph/VideoPlayer.cs */ +using System; +using System.Diagnostics; +using System.Runtime.InteropServices; +using MoonWorks.Graphics; + +namespace MoonWorks.Video +{ + public enum VideoState + { + Playing, + Paused, + Stopped + } + + public unsafe class VideoPlayer : IDisposable + { + public bool Loop { get; set; } + public bool Mute { get; set; } + public float Volume { get; set; } + + private Video Video = null; + private VideoState State = VideoState.Stopped; + + private void* yuvData = null; + private int yuvDataLength; + private int currentFrame; + + private GraphicsDevice GraphicsDevice; + private Texture RenderTexture = null; + private Texture[] YUVTextures = new Texture[3]; + private Sampler LinearSampler; + + private Stopwatch timer; + + private bool disposed; + + public VideoPlayer(GraphicsDevice graphicsDevice) + { + GraphicsDevice = graphicsDevice; + timer = new Stopwatch(); + + LinearSampler = new Sampler(GraphicsDevice, SamplerCreateInfo.LinearClamp); + } + + public void Load(Video video) + { + Video = video; + State = VideoState.Stopped; + + if (yuvData != null) + { + NativeMemory.Free(yuvData); + } + + yuvDataLength = ( + (Video.YWidth * Video.YHeight) + + (Video.UVWidth * video.UVHeight * 2) + ); + + yuvData = NativeMemory.Alloc((nuint) yuvDataLength); + + InitializeTheoraStream(); + + if (Theorafile.tf_hasvideo(Video.Handle) == 1) + { + if (RenderTexture != null) + { + RenderTexture.Dispose(); + } + + RenderTexture = Texture.CreateTexture2D( + GraphicsDevice, + (uint) Video.YWidth, + (uint) Video.YHeight, + TextureFormat.R8G8B8A8, + TextureUsageFlags.ColorTarget | TextureUsageFlags.Sampler + ); + + for (int i = 0; i < 3; i += 1) + { + if (YUVTextures[i] != null) + { + YUVTextures[i].Dispose(); + } + } + + YUVTextures[0] = Texture.CreateTexture2D( + GraphicsDevice, + (uint) Video.YWidth, + (uint) Video.YHeight, + TextureFormat.R8, + TextureUsageFlags.Sampler + ); + + YUVTextures[1] = Texture.CreateTexture2D( + GraphicsDevice, + (uint) Video.UVWidth, + (uint) Video.UVHeight, + TextureFormat.R8, + TextureUsageFlags.Sampler + ); + + YUVTextures[2] = Texture.CreateTexture2D( + GraphicsDevice, + (uint) Video.UVWidth, + (uint) Video.UVHeight, + TextureFormat.R8, + TextureUsageFlags.Sampler + ); + } + } + + public void Play(bool loop = false) + { + if (State == VideoState.Playing) + { + return; + } + + Loop = loop; + timer.Start(); + + State = VideoState.Playing; + } + + public void Pause() + { + if (State == VideoState.Paused) + { + return; + } + + timer.Stop(); + + State = VideoState.Paused; + } + + public void Stop() + { + if (State == VideoState.Stopped) + { + return; + } + + timer.Stop(); + timer.Reset(); + + Theorafile.tf_reset(Video.Handle); + + State = VideoState.Stopped; + } + + public Texture GetTexture() + { + if (Video == null) + { + throw new InvalidOperationException(); + } + + if (State == VideoState.Stopped || Video.Handle == IntPtr.Zero || Theorafile.tf_hasvideo(Video.Handle) == 0) + { + return RenderTexture; + } + + int thisFrame = (int) (timer.Elapsed.TotalMilliseconds / (1000.0 / Video.FramesPerSecond)); + if (thisFrame > currentFrame) + { + if (Theorafile.tf_readvideo( + Video.Handle, + (IntPtr) yuvData, + thisFrame - currentFrame + ) == 1 || currentFrame == -1) { + UpdateTexture(); + } + + currentFrame = thisFrame; + } + + bool ended = Theorafile.tf_eos(Video.Handle) == 1; + if (ended) + { + timer.Stop(); + timer.Reset(); + + Theorafile.tf_reset(Video.Handle); + + if (Loop) + { + // Start over! + InitializeTheoraStream(); + + timer.Start(); + + } + else + { + State = VideoState.Stopped; + } + } + + return RenderTexture; + } + + private void InitializeTheoraStream() + { + // Grab the first video frame ASAP. + while (Theorafile.tf_readvideo(Video.Handle, (IntPtr) yuvData, 1) == 0); + + // Grab the first bit of audio. We're trying to start the decoding ASAP. + if (Theorafile.tf_hasaudio(Video.Handle) == 1) + { + int channels, samplerate; + Theorafile.tf_audioinfo(Video.Handle, out channels, out samplerate); + + // TODO: audio stream + } + + currentFrame = -1; + } + + private void UpdateTexture() + { + var commandBuffer = GraphicsDevice.AcquireCommandBuffer(); + + commandBuffer.SetTextureDataYUV( + YUVTextures[0], + YUVTextures[1], + YUVTextures[2], + (IntPtr) yuvData, + (uint) yuvDataLength + ); + + commandBuffer.BeginRenderPass( + new ColorAttachmentInfo(RenderTexture, Color.Black) + ); + + commandBuffer.BindGraphicsPipeline(GraphicsDevice.VideoPipeline); + commandBuffer.BindFragmentSamplers( + new TextureSamplerBinding(YUVTextures[0], LinearSampler), + new TextureSamplerBinding(YUVTextures[1], LinearSampler), + new TextureSamplerBinding(YUVTextures[2], LinearSampler) + ); + + commandBuffer.DrawPrimitives(0, 1, 0, 0); + + commandBuffer.EndRenderPass(); + + GraphicsDevice.Submit(commandBuffer); + } + + protected virtual void Dispose(bool disposing) + { + if (!disposed) + { + if (disposing) + { + // dispose managed state (managed objects) + RenderTexture.Dispose(); + YUVTextures[0].Dispose(); + YUVTextures[1].Dispose(); + YUVTextures[2].Dispose(); + } + + // free unmanaged resources (unmanaged objects) and override finalizer + + NativeMemory.Free(yuvData); + disposed = true; + } + } + + ~VideoPlayer() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + Dispose(disposing: false); + } + + public void Dispose() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + Dispose(disposing: true); + GC.SuppressFinalize(this); + } + } +}