diff --git a/.gitmodules b/.gitmodules index 481755f..e323a49 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,3 +10,6 @@ [submodule "lib/WellspringCS"] path = lib/WellspringCS url = https://gitea.moonside.games/MoonsideGames/WellspringCS.git +[submodule "lib/Theorafile"] + path = lib/Theorafile + url = https://github.com/FNA-XNA/Theorafile.git diff --git a/MoonWorks.csproj b/MoonWorks.csproj index 88105d4..31c1fce 100644 --- a/MoonWorks.csproj +++ b/MoonWorks.csproj @@ -15,6 +15,7 @@ + @@ -22,4 +23,13 @@ PreserveNewest + + + + MoonWorks.Shaders.FullscreenVert.spv + + + MoonWorks.Shaders.YUV2RGBAFrag.spv + + diff --git a/MoonWorks.dll.config b/MoonWorks.dll.config index d384162..c076485 100644 --- a/MoonWorks.dll.config +++ b/MoonWorks.dll.config @@ -15,4 +15,8 @@ + + + + diff --git a/lib/Theorafile b/lib/Theorafile new file mode 160000 index 0000000..3ed1726 --- /dev/null +++ b/lib/Theorafile @@ -0,0 +1 @@ +Subproject commit 3ed1726b1e294799e85c3b597b114fb3b21cba72 diff --git a/src/Graphics/CommandBuffer.cs b/src/Graphics/CommandBuffer.cs index bcd698a..598d50b 100644 --- a/src/Graphics/CommandBuffer.cs +++ b/src/Graphics/CommandBuffer.cs @@ -1,6 +1,4 @@ using System; -using System.Runtime.InteropServices; -using MoonWorks.Math; using RefreshCS; namespace MoonWorks.Graphics @@ -835,6 +833,26 @@ namespace MoonWorks.Graphics SetTextureData(new TextureSlice(texture), dataPtr, dataLengthInBytes); } + /// + /// Asynchronously copies YUV data into three textures. Use with compressed video. + /// + public void SetTextureDataYUV(Texture yTexture, Texture uTexture, Texture vTexture, IntPtr dataPtr, uint dataLengthInBytes) + { + Refresh.Refresh_SetTextureDataYUV( + Device.Handle, + Handle, + yTexture.Handle, + uTexture.Handle, + vTexture.Handle, + yTexture.Width, + yTexture.Height, + uTexture.Width, + uTexture.Height, + dataPtr, + dataLengthInBytes + ); + } + /// /// Performs an asynchronous texture-to-texture copy on the GPU. /// diff --git a/src/Graphics/GraphicsDevice.cs b/src/Graphics/GraphicsDevice.cs index 99915b4..c111c9e 100644 --- a/src/Graphics/GraphicsDevice.cs +++ b/src/Graphics/GraphicsDevice.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.IO; using RefreshCS; namespace MoonWorks.Graphics @@ -8,6 +9,11 @@ namespace MoonWorks.Graphics { public IntPtr Handle { get; } + // Built-in video pipeline + private ShaderModule VideoVertexShader { get; } + private ShaderModule VideoFragmentShader { get; } + internal GraphicsPipeline VideoPipeline { get; } + public bool IsDisposed { get; private set; } private readonly List> resources = new List>(); @@ -28,6 +34,26 @@ namespace MoonWorks.Graphics presentationParameters, Conversions.BoolToByte(debugMode) ); + + VideoVertexShader = new ShaderModule(this, GetEmbeddedResource("MoonWorks.Shaders.FullscreenVert.spv")); + VideoFragmentShader = new ShaderModule(this, GetEmbeddedResource("MoonWorks.Shaders.YUV2RGBAFrag.spv")); + + VideoPipeline = new GraphicsPipeline( + this, + new GraphicsPipelineCreateInfo + { + AttachmentInfo = new GraphicsPipelineAttachmentInfo( + new ColorAttachmentDescription(TextureFormat.R8G8B8A8, ColorAttachmentBlendState.None) + ), + DepthStencilState = DepthStencilState.Disable, + VertexShaderInfo = GraphicsShaderInfo.Create(VideoVertexShader, "main", 0), + FragmentShaderInfo = GraphicsShaderInfo.Create(VideoFragmentShader, "main", 3), + VertexInputState = VertexInputState.Empty, + RasterizerState = RasterizerState.CCW_CullNone, + PrimitiveType = PrimitiveType.TriangleList, + MultisampleState = MultisampleState.None + } + ); } public CommandBuffer AcquireCommandBuffer() @@ -77,6 +103,11 @@ namespace MoonWorks.Graphics } } + private static Stream GetEmbeddedResource(string name) + { + return typeof(GraphicsDevice).Assembly.GetManifestResourceStream(name); + } + protected virtual void Dispose(bool disposing) { if (!IsDisposed) diff --git a/src/MoonWorksDllMap.cs b/src/MoonWorksDllMap.cs index 5164838..fd9b0f3 100644 --- a/src/MoonWorksDllMap.cs +++ b/src/MoonWorksDllMap.cs @@ -196,6 +196,7 @@ namespace MoonWorks NativeLibrary.SetDllImportResolver(typeof(RefreshCS.Refresh).Assembly, MapAndLoad); NativeLibrary.SetDllImportResolver(typeof(FAudio).Assembly, MapAndLoad); NativeLibrary.SetDllImportResolver(typeof(WellspringCS.Wellspring).Assembly, MapAndLoad); + NativeLibrary.SetDllImportResolver(typeof(Theorafile).Assembly, MapAndLoad); } #endregion diff --git a/src/Video/Shaders/Compiled/FullscreenVert.spv b/src/Video/Shaders/Compiled/FullscreenVert.spv new file mode 100644 index 0000000..ffc57de Binary files /dev/null and b/src/Video/Shaders/Compiled/FullscreenVert.spv differ diff --git a/src/Video/Shaders/Compiled/YUV2RGBAFrag.spv b/src/Video/Shaders/Compiled/YUV2RGBAFrag.spv new file mode 100644 index 0000000..c9fbf32 Binary files /dev/null and b/src/Video/Shaders/Compiled/YUV2RGBAFrag.spv differ diff --git a/src/Video/Shaders/Source/Fullscreen.vert b/src/Video/Shaders/Source/Fullscreen.vert new file mode 100644 index 0000000..2f3c315 --- /dev/null +++ b/src/Video/Shaders/Source/Fullscreen.vert @@ -0,0 +1,9 @@ +#version 450 + +layout(location = 0) out vec2 outTexCoord; + +void main() +{ + outTexCoord = vec2((gl_VertexIndex << 1) & 2, gl_VertexIndex & 2); + gl_Position = vec4(outTexCoord * 2.0 - 1.0, 0.0, 1.0); +} diff --git a/src/Video/Shaders/Source/YUV2RGBA.frag b/src/Video/Shaders/Source/YUV2RGBA.frag new file mode 100644 index 0000000..fe2b5a1 --- /dev/null +++ b/src/Video/Shaders/Source/YUV2RGBA.frag @@ -0,0 +1,38 @@ +/* + * This effect is based on the YUV-to-RGBA GLSL shader found in SDL. + * Thus, it also released under the zlib license: + * http://libsdl.org/license.php + */ +#version 450 + +layout(location = 0) in vec2 TexCoord; + +layout(location = 0) out vec4 FragColor; + +layout(binding = 0, set = 1) uniform sampler2D YSampler; +layout(binding = 1, set = 1) uniform sampler2D USampler; +layout(binding = 2, set = 1) uniform sampler2D VSampler; + +/* More info about colorspace conversion: + * http://www.equasys.de/colorconversion.html + * http://www.equasys.de/colorformat.html + */ + +const vec3 offset = vec3(-0.0625, -0.5, -0.5); +const vec3 Rcoeff = vec3(1.164, 0.000, 1.793); +const vec3 Gcoeff = vec3(1.164, -0.213, -0.533); +const vec3 Bcoeff = vec3(1.164, 2.112, 0.000); + +void main() +{ + vec3 yuv; + yuv.x = texture(YSampler, TexCoord).r; + yuv.y = texture(USampler, TexCoord).r; + yuv.z = texture(VSampler, TexCoord).r; + yuv += offset; + + FragColor.r = dot(yuv, Rcoeff); + FragColor.g = dot(yuv, Gcoeff); + FragColor.b = dot(yuv, Bcoeff); + FragColor.a = 1.0; +} diff --git a/src/Video/Video.cs b/src/Video/Video.cs new file mode 100644 index 0000000..5dc4e55 --- /dev/null +++ b/src/Video/Video.cs @@ -0,0 +1,94 @@ +/* Heavily based on https://github.com/FNA-XNA/FNA/blob/master/src/Media/Xiph/Video.cs */ +using System; + +namespace MoonWorks.Video +{ + public class Video : IDisposable + { + public IntPtr Handle => handle; + public int YWidth => yWidth; + public int YHeight => yHeight; + public int UVWidth => uvWidth; + public int UVHeight => uvHeight; + public double FramesPerSecond => fps; + + private IntPtr handle; + private int yWidth; + private int yHeight; + private int uvWidth; + private int uvHeight; + private double fps; + + private bool disposed; + + public Video(string filename) + { + Theorafile.th_pixel_fmt format; + + if (!System.IO.File.Exists(filename)) + { + throw new ArgumentException("Video file not found!"); + } + + if (Theorafile.tf_fopen(filename, out handle) < 0) + { + throw new ArgumentException("Invalid video file!"); + } + + Theorafile.tf_videoinfo( + handle, + out yWidth, + out yHeight, + out fps, + out format + ); + + if (format == Theorafile.th_pixel_fmt.TH_PF_420) + { + uvWidth = yWidth / 2; + uvHeight = yHeight / 2; + } + else if (format == Theorafile.th_pixel_fmt.TH_PF_422) + { + uvWidth = yWidth / 2; + uvHeight = yHeight; + } + else if (format == Theorafile.th_pixel_fmt.TH_PF_444) + { + uvWidth = yWidth; + uvHeight = yHeight; + } + else + { + throw new NotSupportedException("Unrecognized YUV format!"); + } + } + + protected virtual void Dispose(bool disposing) + { + if (!disposed) + { + if (disposing) + { + // TODO: dispose managed state (managed objects) + } + + Theorafile.tf_close(ref handle); + disposed = true; + } + } + + ~Video() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + Dispose(disposing: false); + } + + public void Dispose() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + Dispose(disposing: true); + GC.SuppressFinalize(this); + } + } +} diff --git a/src/Video/VideoPlayer.cs b/src/Video/VideoPlayer.cs new file mode 100644 index 0000000..c8d275a --- /dev/null +++ b/src/Video/VideoPlayer.cs @@ -0,0 +1,285 @@ +/* Heavily based on https://github.com/FNA-XNA/FNA/blob/master/src/Media/Xiph/VideoPlayer.cs */ +using System; +using System.Diagnostics; +using System.Runtime.InteropServices; +using MoonWorks.Graphics; + +namespace MoonWorks.Video +{ + public enum VideoState + { + Playing, + Paused, + Stopped + } + + public unsafe class VideoPlayer : IDisposable + { + public bool Loop { get; set; } + public bool Mute { get; set; } + public float Volume { get; set; } + + private Video Video = null; + private VideoState State = VideoState.Stopped; + + private void* yuvData = null; + private int yuvDataLength; + private int currentFrame; + + private GraphicsDevice GraphicsDevice; + private Texture RenderTexture = null; + private Texture[] YUVTextures = new Texture[3]; + private Sampler LinearSampler; + + private Stopwatch timer; + + private bool disposed; + + public VideoPlayer(GraphicsDevice graphicsDevice) + { + GraphicsDevice = graphicsDevice; + timer = new Stopwatch(); + + LinearSampler = new Sampler(GraphicsDevice, SamplerCreateInfo.LinearClamp); + } + + public void Load(Video video) + { + Video = video; + State = VideoState.Stopped; + + if (yuvData != null) + { + NativeMemory.Free(yuvData); + } + + yuvDataLength = ( + (Video.YWidth * Video.YHeight) + + (Video.UVWidth * video.UVHeight * 2) + ); + + yuvData = NativeMemory.Alloc((nuint) yuvDataLength); + + InitializeTheoraStream(); + + if (Theorafile.tf_hasvideo(Video.Handle) == 1) + { + if (RenderTexture != null) + { + RenderTexture.Dispose(); + } + + RenderTexture = Texture.CreateTexture2D( + GraphicsDevice, + (uint) Video.YWidth, + (uint) Video.YHeight, + TextureFormat.R8G8B8A8, + TextureUsageFlags.ColorTarget | TextureUsageFlags.Sampler + ); + + for (int i = 0; i < 3; i += 1) + { + if (YUVTextures[i] != null) + { + YUVTextures[i].Dispose(); + } + } + + YUVTextures[0] = Texture.CreateTexture2D( + GraphicsDevice, + (uint) Video.YWidth, + (uint) Video.YHeight, + TextureFormat.R8, + TextureUsageFlags.Sampler + ); + + YUVTextures[1] = Texture.CreateTexture2D( + GraphicsDevice, + (uint) Video.UVWidth, + (uint) Video.UVHeight, + TextureFormat.R8, + TextureUsageFlags.Sampler + ); + + YUVTextures[2] = Texture.CreateTexture2D( + GraphicsDevice, + (uint) Video.UVWidth, + (uint) Video.UVHeight, + TextureFormat.R8, + TextureUsageFlags.Sampler + ); + } + } + + public void Play(bool loop = false) + { + if (State == VideoState.Playing) + { + return; + } + + Loop = loop; + timer.Start(); + + State = VideoState.Playing; + } + + public void Pause() + { + if (State == VideoState.Paused) + { + return; + } + + timer.Stop(); + + State = VideoState.Paused; + } + + public void Stop() + { + if (State == VideoState.Stopped) + { + return; + } + + timer.Stop(); + timer.Reset(); + + Theorafile.tf_reset(Video.Handle); + + State = VideoState.Stopped; + } + + public Texture GetTexture() + { + if (Video == null) + { + throw new InvalidOperationException(); + } + + if (State == VideoState.Stopped || Video.Handle == IntPtr.Zero || Theorafile.tf_hasvideo(Video.Handle) == 0) + { + return RenderTexture; + } + + int thisFrame = (int) (timer.Elapsed.TotalMilliseconds / (1000.0 / Video.FramesPerSecond)); + if (thisFrame > currentFrame) + { + if (Theorafile.tf_readvideo( + Video.Handle, + (IntPtr) yuvData, + thisFrame - currentFrame + ) == 1 || currentFrame == -1) { + UpdateTexture(); + } + + currentFrame = thisFrame; + } + + bool ended = Theorafile.tf_eos(Video.Handle) == 1; + if (ended) + { + timer.Stop(); + timer.Reset(); + + Theorafile.tf_reset(Video.Handle); + + if (Loop) + { + // Start over! + InitializeTheoraStream(); + + timer.Start(); + + } + else + { + State = VideoState.Stopped; + } + } + + return RenderTexture; + } + + private void InitializeTheoraStream() + { + // Grab the first video frame ASAP. + while (Theorafile.tf_readvideo(Video.Handle, (IntPtr) yuvData, 1) == 0); + + // Grab the first bit of audio. We're trying to start the decoding ASAP. + if (Theorafile.tf_hasaudio(Video.Handle) == 1) + { + int channels, samplerate; + Theorafile.tf_audioinfo(Video.Handle, out channels, out samplerate); + + // TODO: audio stream + } + + currentFrame = -1; + } + + private void UpdateTexture() + { + var commandBuffer = GraphicsDevice.AcquireCommandBuffer(); + + commandBuffer.SetTextureDataYUV( + YUVTextures[0], + YUVTextures[1], + YUVTextures[2], + (IntPtr) yuvData, + (uint) yuvDataLength + ); + + commandBuffer.BeginRenderPass( + new ColorAttachmentInfo(RenderTexture, Color.Black) + ); + + commandBuffer.BindGraphicsPipeline(GraphicsDevice.VideoPipeline); + commandBuffer.BindFragmentSamplers( + new TextureSamplerBinding(YUVTextures[0], LinearSampler), + new TextureSamplerBinding(YUVTextures[1], LinearSampler), + new TextureSamplerBinding(YUVTextures[2], LinearSampler) + ); + + commandBuffer.DrawPrimitives(0, 1, 0, 0); + + commandBuffer.EndRenderPass(); + + GraphicsDevice.Submit(commandBuffer); + } + + protected virtual void Dispose(bool disposing) + { + if (!disposed) + { + if (disposing) + { + // dispose managed state (managed objects) + RenderTexture.Dispose(); + YUVTextures[0].Dispose(); + YUVTextures[1].Dispose(); + YUVTextures[2].Dispose(); + } + + // free unmanaged resources (unmanaged objects) and override finalizer + + NativeMemory.Free(yuvData); + disposed = true; + } + } + + ~VideoPlayer() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + Dispose(disposing: false); + } + + public void Dispose() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + Dispose(disposing: true); + GC.SuppressFinalize(this); + } + } +}