AV1 decoder implementation

.gitmodules

@@ -13,3 +13,6 @@
 [submodule "lib/Theorafile"]
 	path = lib/Theorafile
 	url = https://github.com/FNA-XNA/Theorafile.git
+[submodule "lib/dav1dfile"]
+	path = lib/dav1dfile
+	url = git@github.com:MoonsideGames/dav1dfile.git

MoonWorks.csproj

@@ -17,6 +17,7 @@
 		<Compile Include="lib\SDL2-CS\src\SDL2.cs" />
 		<Compile Include="lib\Theorafile\csharp\Theorafile.cs" />
 		<Compile Include="lib\WellspringCS\WellspringCS.cs" />
+		<Compile Include="lib\dav1dfile\csharp\dav1dfile.cs" />
 	</ItemGroup>
 
 	<ItemGroup>

MoonWorks.dll.config

@@ -19,4 +19,8 @@
 	<dllmap dll="Theorafile" os="windows" target="libtheorafile.dll"/>
 	<dllmap dll="Theorafile" os="osx" target="libtheorafile.dylib"/>
 	<dllmap dll="Theorafile" os="linux,freebsd,netbsd" target="libtheorafile.so"/>
+
+	<dllmap dll="dav1dfile" os="windows" target="dav1dfile.dll"/>
+	<dllmap dll="dav1dfile" os="osx" target="libdav1dfile.0.dylib"/>
+	<dllmap dll="dav1dfile" os="linux,freebsd,netbsd,openbsd" target="libdav1dfile.so.0"/>
 </configuration>

lib/RefreshCS

@@ -1 +1 @@
-Subproject commit ebf511133aa6f567c004d687acac474e1649bbde
+Subproject commit 9edb2d1c97f4a38dd3d6d87298d6c604defa4fc8

lib/dav1dfile

@@ -0,0 +1 @@
+Subproject commit 8ba817ff9bf65ed2fa7b9c12d70112b29c450cf4

src/Graphics/CommandBuffer.cs

@@ -1946,7 +1946,17 @@ namespace MoonWorks.Graphics
 		/// <summary>
 		/// Asynchronously copies YUV data into three textures. Use with compressed video.
 		/// </summary>
-		public void SetTextureDataYUV(Texture yTexture, Texture uTexture, Texture vTexture, IntPtr dataPtr, uint dataLengthInBytes)
+		public void SetTextureDataYUV(
+			Texture yTexture,
+			Texture uTexture,
+			Texture vTexture,
+			IntPtr yDataPtr,
+			IntPtr uDataPtr,
+			IntPtr vDataPtr,
+			uint yDataLengthInBytes,
+			uint uvDataLengthInBytes,
+			uint yStride,
+			uint uvStride)
 		{
 #if DEBUG
 			AssertRenderPassInactive("Cannot copy during render pass!");
@@ -1962,8 +1972,13 @@ namespace MoonWorks.Graphics
 				yTexture.Height,
 				uTexture.Width,
 				uTexture.Height,
-				dataPtr,
+				yDataPtr,
-				dataLengthInBytes
+				uDataPtr,
+				vDataPtr,
+				yDataLengthInBytes,
+				uvDataLengthInBytes,
+				yStride,
+				uvStride
 			);
 		}
 

src/Video/VideoAV1.cs

@@ -0,0 +1,88 @@
+using System;
+using System.IO;
+
+namespace MoonWorks.Video
+{
+	public unsafe class VideoAV1 : IDisposable
+	{
+		public IntPtr Handle => handle;
+		IntPtr handle;
+
+		public int Width => width;
+		public int Height => height;
+		public double FramesPerSecond => 28.97;
+		public Dav1dfile.PixelLayout PixelLayout => pixelLayout;
+		public int UVWidth { get; }
+		public int UVHeight { get; }
+
+		private int width;
+		private int height;
+		private double fps;
+		private Dav1dfile.PixelLayout pixelLayout;
+
+		bool IsDisposed;
+
+		public VideoAV1(string filename)
+		{
+			if (!File.Exists(filename))
+			{
+				throw new ArgumentException("Video file not found!");
+			}
+
+			if (Dav1dfile.df_fopen(filename, out handle) == 0)
+			{
+				throw new Exception("Failed to open video file!");
+			}
+
+			Dav1dfile.df_videoinfo(Handle, out width, out height, out pixelLayout);
+
+			if (pixelLayout == Dav1dfile.PixelLayout.I420)
+			{
+				UVWidth = Width / 2;
+				UVHeight = Height / 2;
+			}
+			else if (pixelLayout == Dav1dfile.PixelLayout.I422)
+			{
+				UVWidth = Width / 2;
+				UVHeight = Height;
+			}
+			else if (pixelLayout == Dav1dfile.PixelLayout.I444)
+			{
+				UVWidth = width;
+				UVHeight = height;
+			}
+			else
+			{
+				throw new NotSupportedException("Unrecognized YUV format!");
+			}
+		}
+
+		protected virtual void Dispose(bool disposing)
+		{
+			if (!IsDisposed)
+			{
+				if (disposing)
+				{
+					// dispose managed state (managed objects)
+				}
+
+				// free unmanaged resources (unmanaged objects)
+				Dav1dfile.df_close(Handle);
+
+				IsDisposed = true;
+			}
+		}
+
+		~VideoAV1()
+		{
+			Dispose(disposing: false);
+		}
+
+		public void Dispose()
+		{
+			// Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method
+			Dispose(disposing: true);
+			GC.SuppressFinalize(this);
+		}
+	}
+}

src/Video/VideoPlayer.cs

@@ -11,20 +11,9 @@ namespace MoonWorks.Video
 		public Texture RenderTexture { get; private set; } = null;
 		public VideoState State { get; private set; } = VideoState.Stopped;
 		public bool Loop { get; set; }
-		public float Volume {
-			get => volume;
-			set
-			{
-				volume = value;
-				if (audioStream != null)
-				{
-					audioStream.Volume = value;
-				}
-			}
-		}
 		public float PlaybackSpeed { get; set; } = 1;
 
-		private Video Video = null;
+		private VideoAV1 Video = null;
 
 		private GraphicsDevice GraphicsDevice;
 		private Texture yTexture = null;
@@ -32,19 +21,14 @@ namespace MoonWorks.Video
 		private Texture vTexture = null;
 		private Sampler LinearSampler;
 
-		private void* yuvData = null;
-		private int yuvDataLength = 0;
-
 		private int currentFrame;
 
-		private AudioDevice AudioDevice;
-		private StreamingSoundTheora audioStream = null;
-		private float volume = 1.0f;
-
 		private Stopwatch timer;
 		private double lastTimestamp;
 		private double timeElapsed;
 
+		private Stopwatch profilingTimer = new Stopwatch();
+
 		private bool disposed;
 
 		public VideoPlayer(GraphicsDevice graphicsDevice, AudioDevice audioDevice)
@@ -55,13 +39,12 @@ namespace MoonWorks.Video
 				throw new InvalidOperationException("Missing video shaders!");
 			}
 
-			AudioDevice = audioDevice;
 			LinearSampler = new Sampler(graphicsDevice, SamplerCreateInfo.LinearClamp);
 
 			timer = new Stopwatch();
 		}
 
-		public void Load(Video video)
+		public void Load(VideoAV1 video)
 		{
 			if (Video != video)
 			{
@@ -111,20 +94,9 @@ namespace MoonWorks.Video
 					vTexture = CreateSubTexture(GraphicsDevice, video.UVWidth, video.UVHeight);
 				}
 
-				var newDataLength = (
-					(video.Width * video.Height) +
-					(video.UVWidth * video.UVHeight * 2)
-				);
-
-				if (newDataLength != yuvDataLength)
-				{
-					yuvData = NativeMemory.Realloc(yuvData, (nuint) newDataLength);
-					yuvDataLength = newDataLength;
-				}
-
 				Video = video;
 
-				InitializeTheoraStream();
+				InitializeDav1dStream();
 			}
 		}
 
@@ -139,11 +111,6 @@ namespace MoonWorks.Video
 
 			timer.Start();
 
-			if (audioStream != null)
-			{
-				audioStream.Play();
-			}
-
 			State = VideoState.Playing;
 		}
 
@@ -158,11 +125,6 @@ namespace MoonWorks.Video
 
 			timer.Stop();
 
-			if (audioStream != null)
-			{
-				audioStream.Pause();
-			}
-
 			State = VideoState.Paused;
 		}
 
@@ -181,9 +143,7 @@ namespace MoonWorks.Video
 			lastTimestamp = 0;
 			timeElapsed = 0;
 
-			DestroyAudioStream();
+			Dav1dfile.df_reset(Video.Handle);
-
-			Theorafile.tf_reset(Video.Handle);
 
 			State = VideoState.Stopped;
 		}
@@ -197,11 +157,6 @@ namespace MoonWorks.Video
 		public void Update()
 		{
 			if (Video == null) { return; }
-
-			if (audioStream != null)
-			{
-				audioStream.Update();
-			}
 		}
 
 		public void Render()
@@ -217,32 +172,41 @@ namespace MoonWorks.Video
 			int thisFrame = ((int) (timeElapsed / (1000.0 / Video.FramesPerSecond)));
 			if (thisFrame > currentFrame)
 			{
-				if (Theorafile.tf_readvideo(
+				profilingTimer.Restart();
+				int readResult = Dav1dfile.df_readvideo(
 					Video.Handle,
-					(IntPtr) yuvData,
+					thisFrame - currentFrame,
-					thisFrame - currentFrame
+					out var yData,
-				) == 1 || currentFrame == -1)
+					out var uData,
+					out var vData,
+					out var yDataLength,
+					out var uvDataLength,
+					out var yStride,
+					out var uvStride
+				);
+				profilingTimer.Stop();
+				System.Console.WriteLine($"Frame decoded in: {profilingTimer.Elapsed.TotalMilliseconds}");
+
+				if (readResult == 1 || currentFrame == -1)
 				{
-					UpdateRenderTexture();
+					UpdateRenderTexture(yData, uData, vData, yDataLength, uvDataLength, yStride, uvStride);
 				}
 
 				currentFrame = thisFrame;
 			}
 
-			bool ended = Theorafile.tf_eos(Video.Handle) == 1;
+			bool ended = Dav1dfile.df_eos(Video.Handle) == 1;
 			if (ended)
 			{
 				timer.Stop();
 				timer.Reset();
 
-				DestroyAudioStream();
+				Dav1dfile.df_reset(Video.Handle);
-
-				Theorafile.tf_reset(Video.Handle);
 
 				if (Loop)
 				{
 					// Start over!
-					InitializeTheoraStream();
+					InitializeDav1dStream();
 
 					timer.Start();
 				}
@@ -253,7 +217,7 @@ namespace MoonWorks.Video
 			}
 		}
 
-		private void UpdateRenderTexture()
+		private void UpdateRenderTexture(IntPtr yData, IntPtr uData, IntPtr vData, uint yDataLength, uint uvDataLength, uint yStride, uint uvStride)
 		{
 			var commandBuffer = GraphicsDevice.AcquireCommandBuffer();
 
@@ -261,8 +225,13 @@ namespace MoonWorks.Video
 				yTexture,
 				uTexture,
 				vTexture,
-				(IntPtr) yuvData,
+				yData,
-				(uint) yuvDataLength
+				uData,
+				vData,
+				yDataLength,
+				uvDataLength,
+				yStride,
+				uvStride
 			);
 
 			commandBuffer.BeginRenderPass(
@@ -305,35 +274,22 @@ namespace MoonWorks.Video
 			);
 		}
 
-		private void InitializeTheoraStream()
+		private void InitializeDav1dStream()
 		{
 			// Grab the first video frame ASAP.
-			while (Theorafile.tf_readvideo(Video.Handle, (IntPtr) yuvData, 1) == 0);
+			Dav1dfile.df_readvideo(
-
+				Video.Handle,
-			// Grab the first bit of audio. We're trying to start the decoding ASAP.
+				1,
-			if (AudioDevice != null && Theorafile.tf_hasaudio(Video.Handle) == 1)
+				out var _,
-			{
+				out var _,
-				DestroyAudioStream();
+				out var _,
-
+				out var _,
-				int channels, sampleRate;
+				out var _,
-				Theorafile.tf_audioinfo(Video.Handle, out channels, out sampleRate);
+				out var _,
-
+				out var _);
-				audioStream = new StreamingSoundTheora(AudioDevice, Video.Handle, channels, (uint) sampleRate);
-			}
-
 			currentFrame = -1;
 		}
 
-		private void DestroyAudioStream()
-		{
-			if (audioStream != null)
-			{
-				audioStream.StopImmediate();
-				audioStream.Dispose();
-				audioStream = null;
-			}
-		}
-
 		protected virtual void Dispose(bool disposing)
 		{
 			if (!disposed)
@@ -347,9 +303,6 @@ namespace MoonWorks.Video
 					vTexture.Dispose();
 				}
 
-				// free unmanaged resources (unmanaged objects) and override finalizer
-				NativeMemory.Free(yuvData);
-
 				disposed = true;
 			}
 		}