initial video support

.gitmodules

@@ -10,3 +10,6 @@
 [submodule "lib/WellspringCS"]
 	path = lib/WellspringCS
 	url = https://gitea.moonside.games/MoonsideGames/WellspringCS.git
+[submodule "lib/Theorafile"]
+	path = lib/Theorafile
+	url = https://github.com/FNA-XNA/Theorafile.git

MoonWorks.csproj

@@ -15,6 +15,7 @@
 		<ProjectReference Include=".\lib\RefreshCS\RefreshCS.csproj" />
 		<ProjectReference Include=".\lib\FAudio\csharp\FAudio-CS.Core.csproj" />
 		<ProjectReference Include=".\lib\WellspringCS\WellspringCS.csproj" />
+		<ProjectReference Include=".\lib\Theorafile\csharp\Theorafile-CS.Core.csproj" />
 	</ItemGroup>
 
 	<ItemGroup>
@@ -22,4 +23,13 @@
 			<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
 		</None>
 	</ItemGroup>
+
+	<ItemGroup>
+		<EmbeddedResource Include="src\Video\Shaders\Compiled\FullscreenVert.spv">
+			<LogicalName>MoonWorks.Shaders.FullscreenVert.spv</LogicalName>
+		</EmbeddedResource>
+		<EmbeddedResource Include="src\Video\Shaders\Compiled\YUV2RGBAFrag.spv">
+			<LogicalName>MoonWorks.Shaders.YUV2RGBAFrag.spv</LogicalName>
+		</EmbeddedResource>
+	</ItemGroup>
 </Project>

MoonWorks.dll.config

@@ -15,4 +15,8 @@
 	<dllmap dll="Wellspring" os="windows" target="Wellspring.dll"/>
 	<dllmap dll="Wellspring" os="osx" target="libWellspring.0.dylib"/>
 	<dllmap dll="Wellspring" os="linux,freebsd,netbsd" target="libWellspring.so.0"/>
+
+	<dllmap dll="Theorafile" os="windows" target="libtheorafile.dll"/>
+	<dllmap dll="Theorafile" os="osx" target="libtheorafile.dylib"/>
+	<dllmap dll="Theorafile" os="linux,freebsd,netbsd" target="libtheorafile.so"/>
 </configuration>

lib/Theorafile

@@ -0,0 +1 @@
+Subproject commit 3ed1726b1e294799e85c3b597b114fb3b21cba72

src/Graphics/CommandBuffer.cs

@@ -1,6 +1,4 @@
 using System;
-using System.Runtime.InteropServices;
-using MoonWorks.Math;
 using RefreshCS;
 
 namespace MoonWorks.Graphics
@@ -835,6 +833,26 @@ namespace MoonWorks.Graphics
 			SetTextureData(new TextureSlice(texture), dataPtr, dataLengthInBytes);
 		}
 
+		/// <summary>
+		/// Asynchronously copies YUV data into three textures. Use with compressed video.
+		/// </summary>
+		public void SetTextureDataYUV(Texture yTexture, Texture uTexture, Texture vTexture, IntPtr dataPtr, uint dataLengthInBytes)
+		{
+			Refresh.Refresh_SetTextureDataYUV(
+				Device.Handle,
+				Handle,
+				yTexture.Handle,
+				uTexture.Handle,
+				vTexture.Handle,
+				yTexture.Width,
+				yTexture.Height,
+				uTexture.Width,
+				uTexture.Height,
+				dataPtr,
+				dataLengthInBytes
+			);
+		}
+
 		/// <summary>
 		/// Performs an asynchronous texture-to-texture copy on the GPU.
 		/// </summary>

src/Graphics/GraphicsDevice.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.IO;
 using RefreshCS;
 
 namespace MoonWorks.Graphics
@@ -8,6 +9,11 @@ namespace MoonWorks.Graphics
 	{
 		public IntPtr Handle { get; }
 
+		// Built-in video pipeline
+		private ShaderModule VideoVertexShader { get; }
+		private ShaderModule VideoFragmentShader { get; }
+		internal GraphicsPipeline VideoPipeline { get; }
+
 		public bool IsDisposed { get; private set; }
 
 		private readonly List<WeakReference<GraphicsResource>> resources = new List<WeakReference<GraphicsResource>>();
@@ -28,6 +34,26 @@ namespace MoonWorks.Graphics
 				presentationParameters,
 				Conversions.BoolToByte(debugMode)
 			);
+
+			VideoVertexShader = new ShaderModule(this, GetEmbeddedResource("MoonWorks.Shaders.FullscreenVert.spv"));
+			VideoFragmentShader = new ShaderModule(this, GetEmbeddedResource("MoonWorks.Shaders.YUV2RGBAFrag.spv"));
+
+			VideoPipeline = new GraphicsPipeline(
+				this,
+				new GraphicsPipelineCreateInfo
+				{
+					AttachmentInfo = new GraphicsPipelineAttachmentInfo(
+						new ColorAttachmentDescription(TextureFormat.R8G8B8A8, ColorAttachmentBlendState.None)
+					),
+					DepthStencilState = DepthStencilState.Disable,
+					VertexShaderInfo = GraphicsShaderInfo.Create(VideoVertexShader, "main", 0),
+					FragmentShaderInfo = GraphicsShaderInfo.Create(VideoFragmentShader, "main", 3),
+					VertexInputState = VertexInputState.Empty,
+					RasterizerState = RasterizerState.CCW_CullNone,
+					PrimitiveType = PrimitiveType.TriangleList,
+					MultisampleState = MultisampleState.None
+				}
+			);
 		}
 
 		public CommandBuffer AcquireCommandBuffer()
@@ -77,6 +103,11 @@ namespace MoonWorks.Graphics
 			}
 		}
 
+		private static Stream GetEmbeddedResource(string name)
+		{
+			return typeof(GraphicsDevice).Assembly.GetManifestResourceStream(name);
+		}
+
 		protected virtual void Dispose(bool disposing)
 		{
 			if (!IsDisposed)

src/MoonWorksDllMap.cs

@@ -196,6 +196,7 @@ namespace MoonWorks
 			NativeLibrary.SetDllImportResolver(typeof(RefreshCS.Refresh).Assembly, MapAndLoad);
 			NativeLibrary.SetDllImportResolver(typeof(FAudio).Assembly, MapAndLoad);
 			NativeLibrary.SetDllImportResolver(typeof(WellspringCS.Wellspring).Assembly, MapAndLoad);
+			NativeLibrary.SetDllImportResolver(typeof(Theorafile).Assembly, MapAndLoad);
 		}
 
 		#endregion

src/Video/Shaders/Source/Fullscreen.vert

@@ -0,0 +1,9 @@
+#version 450
+
+layout(location = 0) out vec2 outTexCoord;
+
+void main()
+{
+	outTexCoord = vec2((gl_VertexIndex << 1) & 2, gl_VertexIndex & 2);
+	gl_Position = vec4(outTexCoord * 2.0 - 1.0, 0.0, 1.0);
+}

src/Video/Shaders/Source/YUV2RGBA.frag

@@ -0,0 +1,38 @@
+/*
+ * This effect is based on the YUV-to-RGBA GLSL shader found in SDL.
+ * Thus, it also released under the zlib license:
+ * http://libsdl.org/license.php
+ */
+#version 450
+
+layout(location = 0) in vec2 TexCoord;
+
+layout(location = 0) out vec4 FragColor;
+
+layout(binding = 0, set = 1) uniform sampler2D YSampler;
+layout(binding = 1, set = 1) uniform sampler2D USampler;
+layout(binding = 2, set = 1) uniform sampler2D VSampler;
+
+/* More info about colorspace conversion:
+ * http://www.equasys.de/colorconversion.html
+ * http://www.equasys.de/colorformat.html
+ */
+
+const vec3 offset = vec3(-0.0625, -0.5, -0.5);
+const vec3 Rcoeff = vec3(1.164,  0.000,  1.793);
+const vec3 Gcoeff = vec3(1.164, -0.213, -0.533);
+const vec3 Bcoeff = vec3(1.164,  2.112,  0.000);
+
+void main()
+{
+	vec3 yuv;
+	yuv.x = texture(YSampler, TexCoord).r;
+	yuv.y = texture(USampler, TexCoord).r;
+	yuv.z = texture(VSampler, TexCoord).r;
+	yuv += offset;
+
+	FragColor.r = dot(yuv, Rcoeff);
+	FragColor.g = dot(yuv, Gcoeff);
+	FragColor.b = dot(yuv, Bcoeff);
+	FragColor.a = 1.0;
+}

src/Video/Video.cs

@@ -0,0 +1,94 @@
+/* Heavily based on https://github.com/FNA-XNA/FNA/blob/master/src/Media/Xiph/Video.cs */
+using System;
+
+namespace MoonWorks.Video
+{
+	public class Video : IDisposable
+	{
+		public IntPtr Handle => handle;
+		public int YWidth => yWidth;
+		public int YHeight => yHeight;
+		public int UVWidth => uvWidth;
+		public int UVHeight => uvHeight;
+		public double FramesPerSecond => fps;
+
+		private IntPtr handle;
+		private int yWidth;
+		private int yHeight;
+		private int uvWidth;
+		private int uvHeight;
+		private double fps;
+
+		private bool disposed;
+
+		public Video(string filename)
+		{
+			Theorafile.th_pixel_fmt format;
+
+			if (!System.IO.File.Exists(filename))
+			{
+				throw new ArgumentException("Video file not found!");
+			}
+
+			if (Theorafile.tf_fopen(filename, out handle) < 0)
+			{
+				throw new ArgumentException("Invalid video file!");
+			}
+
+			Theorafile.tf_videoinfo(
+				handle,
+				out yWidth,
+				out yHeight,
+				out fps,
+				out format
+			);
+
+			if (format == Theorafile.th_pixel_fmt.TH_PF_420)
+			{
+				uvWidth = yWidth / 2;
+				uvHeight = yHeight / 2;
+			}
+			else if (format == Theorafile.th_pixel_fmt.TH_PF_422)
+			{
+				uvWidth = yWidth / 2;
+				uvHeight = yHeight;
+			}
+			else if (format == Theorafile.th_pixel_fmt.TH_PF_444)
+			{
+				uvWidth = yWidth;
+				uvHeight = yHeight;
+			}
+			else
+			{
+				throw new NotSupportedException("Unrecognized YUV format!");
+			}
+		}
+
+		protected virtual void Dispose(bool disposing)
+		{
+			if (!disposed)
+			{
+				if (disposing)
+				{
+					// TODO: dispose managed state (managed objects)
+				}
+
+				Theorafile.tf_close(ref handle);
+				disposed = true;
+			}
+		}
+
+		~Video()
+		{
+		    // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method
+		    Dispose(disposing: false);
+		}
+
+		public void Dispose()
+		{
+			// Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method
+			Dispose(disposing: true);
+			GC.SuppressFinalize(this);
+		}
+	}
+}

src/Video/VideoPlayer.cs

@@ -0,0 +1,285 @@
+/* Heavily based on https://github.com/FNA-XNA/FNA/blob/master/src/Media/Xiph/VideoPlayer.cs */
+using System;
+using System.Diagnostics;
+using System.Runtime.InteropServices;
+using MoonWorks.Graphics;
+
+namespace MoonWorks.Video
+{
+	public enum VideoState
+	{
+		Playing,
+		Paused,
+		Stopped
+	}
+
+	public unsafe class VideoPlayer : IDisposable
+	{
+		public bool Loop { get; set; }
+		public bool Mute { get; set; }
+		public float Volume { get; set; }
+
+		private Video Video = null;
+		private VideoState State = VideoState.Stopped;
+
+		private void* yuvData = null;
+		private int yuvDataLength;
+		private int currentFrame;
+
+		private GraphicsDevice GraphicsDevice;
+		private Texture RenderTexture = null;
+		private Texture[] YUVTextures = new Texture[3];
+		private Sampler LinearSampler;
+
+		private Stopwatch timer;
+
+		private bool disposed;
+
+		public VideoPlayer(GraphicsDevice graphicsDevice)
+		{
+			GraphicsDevice = graphicsDevice;
+			timer = new Stopwatch();
+
+			LinearSampler = new Sampler(GraphicsDevice, SamplerCreateInfo.LinearClamp);
+		}
+
+		public void Load(Video video)
+		{
+			Video = video;
+			State = VideoState.Stopped;
+
+			if (yuvData != null)
+			{
+				NativeMemory.Free(yuvData);
+			}
+
+			yuvDataLength = (
+				(Video.YWidth * Video.YHeight) +
+				(Video.UVWidth * video.UVHeight * 2)
+			);
+
+			yuvData = NativeMemory.Alloc((nuint) yuvDataLength);
+
+			InitializeTheoraStream();
+
+			if (Theorafile.tf_hasvideo(Video.Handle) == 1)
+			{
+				if (RenderTexture != null)
+				{
+					RenderTexture.Dispose();
+				}
+
+				RenderTexture = Texture.CreateTexture2D(
+					GraphicsDevice,
+					(uint) Video.YWidth,
+					(uint) Video.YHeight,
+					TextureFormat.R8G8B8A8,
+					TextureUsageFlags.ColorTarget | TextureUsageFlags.Sampler
+				);
+
+				for (int i = 0; i < 3; i += 1)
+				{
+					if (YUVTextures[i] != null)
+					{
+						YUVTextures[i].Dispose();
+					}
+				}
+
+				YUVTextures[0] = Texture.CreateTexture2D(
+					GraphicsDevice,
+					(uint) Video.YWidth,
+					(uint) Video.YHeight,
+					TextureFormat.R8,
+					TextureUsageFlags.Sampler
+				);
+
+				YUVTextures[1] = Texture.CreateTexture2D(
+					GraphicsDevice,
+					(uint) Video.UVWidth,
+					(uint) Video.UVHeight,
+					TextureFormat.R8,
+					TextureUsageFlags.Sampler
+				);
+
+				YUVTextures[2] = Texture.CreateTexture2D(
+					GraphicsDevice,
+					(uint) Video.UVWidth,
+					(uint) Video.UVHeight,
+					TextureFormat.R8,
+					TextureUsageFlags.Sampler
+				);
+			}
+		}
+
+		public void Play(bool loop = false)
+		{
+			if (State == VideoState.Playing)
+			{
+				return;
+			}
+
+			Loop = loop;
+			timer.Start();
+
+			State = VideoState.Playing;
+		}
+
+		public void Pause()
+		{
+			if (State == VideoState.Paused)
+			{
+				return;
+			}
+
+			timer.Stop();
+
+			State = VideoState.Paused;
+		}
+
+		public void Stop()
+		{
+			if (State == VideoState.Stopped)
+			{
+				return;
+			}
+
+			timer.Stop();
+			timer.Reset();
+
+			Theorafile.tf_reset(Video.Handle);
+
+			State = VideoState.Stopped;
+		}
+
+		public Texture GetTexture()
+		{
+			if (Video == null)
+			{
+				throw new InvalidOperationException();
+			}
+
+			if (State == VideoState.Stopped || Video.Handle == IntPtr.Zero || Theorafile.tf_hasvideo(Video.Handle) == 0)
+			{
+				return RenderTexture;
+			}
+
+			int thisFrame = (int) (timer.Elapsed.TotalMilliseconds / (1000.0 / Video.FramesPerSecond));
+			if (thisFrame > currentFrame)
+			{
+				if (Theorafile.tf_readvideo(
+					Video.Handle,
+					(IntPtr) yuvData,
+					thisFrame - currentFrame
+				) == 1 || currentFrame == -1) {
+					UpdateTexture();
+				}
+
+				currentFrame = thisFrame;
+			}
+
+			bool ended = Theorafile.tf_eos(Video.Handle) == 1;
+			if (ended)
+			{
+				timer.Stop();
+				timer.Reset();
+
+				Theorafile.tf_reset(Video.Handle);
+
+				if (Loop)
+				{
+					// Start over!
+					InitializeTheoraStream();
+
+					timer.Start();
+
+				}
+				else
+				{
+					State = VideoState.Stopped;
+				}
+			}
+
+			return RenderTexture;
+		}
+
+		private void InitializeTheoraStream()
+		{
+			// Grab the first video frame ASAP.
+			while (Theorafile.tf_readvideo(Video.Handle, (IntPtr) yuvData, 1) == 0);
+
+			// Grab the first bit of audio. We're trying to start the decoding ASAP.
+			if (Theorafile.tf_hasaudio(Video.Handle) == 1)
+			{
+				int channels, samplerate;
+				Theorafile.tf_audioinfo(Video.Handle, out channels, out samplerate);
+
+				// TODO: audio stream
+			}
+
+			currentFrame = -1;
+		}
+
+		private void UpdateTexture()
+		{
+			var commandBuffer = GraphicsDevice.AcquireCommandBuffer();
+
+			commandBuffer.SetTextureDataYUV(
+				YUVTextures[0],
+				YUVTextures[1],
+				YUVTextures[2],
+				(IntPtr) yuvData,
+				(uint) yuvDataLength
+			);
+
+			commandBuffer.BeginRenderPass(
+				new ColorAttachmentInfo(RenderTexture, Color.Black)
+			);
+
+			commandBuffer.BindGraphicsPipeline(GraphicsDevice.VideoPipeline);
+			commandBuffer.BindFragmentSamplers(
+				new TextureSamplerBinding(YUVTextures[0], LinearSampler),
+				new TextureSamplerBinding(YUVTextures[1], LinearSampler),
+				new TextureSamplerBinding(YUVTextures[2], LinearSampler)
+			);
+
+			commandBuffer.DrawPrimitives(0, 1, 0, 0);
+
+			commandBuffer.EndRenderPass();
+
+			GraphicsDevice.Submit(commandBuffer);
+		}
+
+		protected virtual void Dispose(bool disposing)
+		{
+			if (!disposed)
+			{
+				if (disposing)
+				{
+					// dispose managed state (managed objects)
+					RenderTexture.Dispose();
+					YUVTextures[0].Dispose();
+					YUVTextures[1].Dispose();
+					YUVTextures[2].Dispose();
+				}
+
+				// free unmanaged resources (unmanaged objects) and override finalizer
+
+				NativeMemory.Free(yuvData);
+				disposed = true;
+			}
+		}
+
+		~VideoPlayer()
+		{
+		    // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method
+		    Dispose(disposing: false);
+		}
+
+		public void Dispose()
+		{
+			// Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method
+			Dispose(disposing: true);
+			GC.SuppressFinalize(this);
+		}
+	}
+}