restructure audio API to support theora audio stream

lib/Theorafile

@@ -1 +1 @@
-Subproject commit 3ed1726b1e294799e85c3b597b114fb3b21cba72
+Subproject commit dd8c7fa69e678b6182cdaa71458ad08dd31c65da

src/Audio/SoundInstance.cs

@@ -1,6 +1,5 @@
 using System;
 using System.Runtime.InteropServices;
-using MoonWorks.Math.Float;
 
 namespace MoonWorks.Audio
 {
@@ -8,7 +7,6 @@ namespace MoonWorks.Audio
 	{
 		internal IntPtr Handle;
 		internal FAudio.FAudioWaveFormatEx Format;
-		public bool Loop { get; protected set; } = false;
 
 		protected FAudio.F3DAUDIO_DSP_SETTINGS dspSettings;
 
@@ -238,11 +236,9 @@ namespace MoonWorks.Audio
 			);
 		}
 
-		public abstract void Play(bool loop);
+		public abstract void Play();
 		public abstract void Pause();
 		public abstract void Stop(bool immediate);
-		public abstract void Seek(float seconds);
-		public abstract void Seek(uint sampleFrame);
 
 		private void InitDSPSettings(uint srcChannels)
 		{

src/Audio/StaticSoundInstance.cs

@@ -6,6 +6,8 @@ namespace MoonWorks.Audio
 	{
 		public StaticSound Parent { get; }
 
+		public bool Loop { get; set; }
+
 		private SoundState _state = SoundState.Stopped;
 		public override SoundState State
 		{
@@ -38,15 +40,13 @@ namespace MoonWorks.Audio
 			Parent = parent;
 		}
 
-		public override void Play(bool loop = false)
+		public override void Play()
 		{
 			if (State == SoundState.Playing)
 			{
 				return;
 			}
 
-			Loop = loop;
-
 			if (Loop)
 			{
 				Parent.Handle.LoopCount = 255;
@@ -93,7 +93,7 @@ namespace MoonWorks.Audio
 			}
 		}
 
-		private void PerformSeek(uint sampleFrame)
+		public void Seek(uint sampleFrame)
 		{
 			if (State == SoundState.Playing)
 			{
@@ -102,20 +102,6 @@ namespace MoonWorks.Audio
 			}
 
 			Parent.Handle.PlayBegin = sampleFrame;
-			Play();
-		}
-
-		public override void Seek(float seconds)
-		{
-			uint sampleFrame =
-				(uint) (Parent.SamplesPerSecond * seconds);
-
-			PerformSeek(sampleFrame);
-		}
-
-		public override void Seek(uint sampleFrame)
-		{
-			PerformSeek(sampleFrame);
 		}
 
 		public void Free()

src/Audio/StreamingSound.cs

@@ -11,10 +11,11 @@ namespace MoonWorks.Audio
 	public abstract class StreamingSound : SoundInstance
 	{
 		private readonly List<IntPtr> queuedBuffers = new List<IntPtr>();
-		private readonly List<uint> queuedSizes = new List<uint>();
 		private const int MINIMUM_BUFFER_CHECK = 3;
 
-		public int PendingBufferCount => queuedBuffers.Count;
+		private int PendingBufferCount => queuedBuffers.Count;
+
+		public abstract int BUFFER_SIZE { get; }
 
 		public StreamingSound(
 			AudioDevice device,
@@ -23,16 +24,18 @@ namespace MoonWorks.Audio
 			ushort blockAlign,
 			ushort channels,
 			uint samplesPerSecond
-		) : base(device, formatTag, bitsPerSample, blockAlign, channels, samplesPerSecond) { }
+		) : base(device, formatTag, bitsPerSample, blockAlign, channels, samplesPerSecond)
+		{
+			device.AddDynamicSoundInstance(this);
+		}
 
-		public override void Play(bool loop = false)
+		public override void Play()
 		{
 			if (State == SoundState.Playing)
 			{
 				return;
 			}
 
-			Loop = loop;
 			State = SoundState.Playing;
 
 			Update();
@@ -60,7 +63,7 @@ namespace MoonWorks.Audio
 			State = SoundState.Stopped;
 		}
 
-		internal void Update()
+		internal unsafe void Update()
 		{
 			if (State != SoundState.Playing)
 			{
@@ -74,11 +77,13 @@ namespace MoonWorks.Audio
 			);
 
 			while (PendingBufferCount > state.BuffersQueued)
+			{
 				lock (queuedBuffers)
 				{
-					Marshal.FreeHGlobal(queuedBuffers[0]);
+					NativeMemory.Free((void*) queuedBuffers[0]);
 					queuedBuffers.RemoveAt(0);
 				}
+			}
 
 			QueueBuffers();
 		}
@@ -95,46 +100,42 @@ namespace MoonWorks.Audio
 			}
 		}
 
-		protected void ClearBuffers()
+		protected unsafe void ClearBuffers()
 		{
 			lock (queuedBuffers)
 			{
 				foreach (IntPtr buf in queuedBuffers)
 				{
-					Marshal.FreeHGlobal(buf);
+					NativeMemory.Free((void*) buf);
 				}
 				queuedBuffers.Clear();
-				queuedSizes.Clear();
 			}
 		}
 
-		protected void AddBuffer()
+		protected unsafe void AddBuffer()
 		{
+			void* buffer = NativeMemory.Alloc((nuint) BUFFER_SIZE);
+
 			AddBuffer(
-				out var buffer,
+				buffer,
-				out var bufferOffset,
+				BUFFER_SIZE,
-				out var bufferLength,
+				out int filledLengthInBytes,
-				out var reachedEnd
+				out bool reachedEnd
 			);
 
-			var lengthInBytes = bufferLength * sizeof(float);
-
-			IntPtr next = Marshal.AllocHGlobal((int) lengthInBytes);
-			Marshal.Copy(buffer, (int) bufferOffset, next, (int) bufferLength);
-
 			lock (queuedBuffers)
 			{
-				queuedBuffers.Add(next);
+				queuedBuffers.Add((IntPtr) buffer);
 				if (State != SoundState.Stopped)
 				{
 					FAudio.FAudioBuffer buf = new FAudio.FAudioBuffer
 					{
-						AudioBytes = lengthInBytes,
+						AudioBytes = (uint) filledLengthInBytes,
-						pAudioData = next,
+						pAudioData = (IntPtr) buffer,
 						PlayLength = (
-							lengthInBytes /
+							(uint) (filledLengthInBytes /
 							Format.nChannels /
-							(uint) (Format.wBitsPerSample / 8)
+							(uint) (Format.wBitsPerSample / 8))
 						)
 					};
 
@@ -144,35 +145,28 @@ namespace MoonWorks.Audio
 						IntPtr.Zero
 					);
 				}
-				else
-				{
-					queuedSizes.Add(lengthInBytes);
-				}
 			}
 
+
 			/* We have reached the end of the file, what do we do? */
 			if (reachedEnd)
 			{
-				if (Loop)
+				OnReachedEnd();
-				{
-					SeekStart();
-				}
-				else
-				{
-					Stop(false);
-				}
 			}
 		}
 
-		protected abstract void AddBuffer(
+		protected virtual void OnReachedEnd()
-			out float[] buffer,
+		{
-			out uint bufferOffset, /* in floats */
+			Stop(false);
-			out uint bufferLength, /* in floats */
+		}
+
+		protected unsafe abstract void AddBuffer(
+			void* buffer,
+			int bufferLength, /* in bytes */
+			out int filledLength, /* in bytes */
 			out bool reachedEnd
 		);
 
-		protected abstract void SeekStart();
-
 		protected override void Destroy()
 		{
 			Stop(true);

src/Audio/StreamingSoundOgg.cs

@@ -1,13 +1,14 @@
 using System;
 using System.IO;
+using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
 
 namespace MoonWorks.Audio
 {
-	public class StreamingSoundOgg : StreamingSound
+	public class StreamingSoundOgg : StreamingSoundSeekable
 	{
 		// FIXME: what should this value be?
-		public const int BUFFER_SIZE = 1024 * 128;
+		public override int BUFFER_SIZE => 1024 * 128;
 
 		private IntPtr VorbisHandle;
 		private IntPtr FileDataPtr;
@@ -17,15 +18,15 @@ namespace MoonWorks.Audio
 
 		public override SoundState State { get; protected set; }
 
-		public static StreamingSoundOgg Load(AudioDevice device, string filePath)
+		public unsafe static StreamingSoundOgg Load(AudioDevice device, string filePath)
 		{
 			var fileData = File.ReadAllBytes(filePath);
-			var fileDataPtr = Marshal.AllocHGlobal(fileData.Length);
+			var fileDataPtr = NativeMemory.Alloc((nuint) fileData.Length);
-			Marshal.Copy(fileData, 0, fileDataPtr, fileData.Length);
+			Marshal.Copy(fileData, 0, (IntPtr) fileDataPtr, fileData.Length);
-			var vorbisHandle = FAudio.stb_vorbis_open_memory(fileDataPtr, fileData.Length, out int error, IntPtr.Zero);
+			var vorbisHandle = FAudio.stb_vorbis_open_memory((IntPtr) fileDataPtr, fileData.Length, out int error, IntPtr.Zero);
 			if (error != 0)
 			{
-				((GCHandle) fileDataPtr).Free();
+				NativeMemory.Free(fileDataPtr);
 				Logger.LogError("Error opening OGG file!");
 				Logger.LogError("Error: " + error);
 				throw new AudioLoadException("Error opening OGG file!");
@@ -34,7 +35,7 @@ namespace MoonWorks.Audio
 
 			return new StreamingSoundOgg(
 				device,
-				fileDataPtr,
+				(IntPtr) fileDataPtr,
 				vorbisHandle,
 				info
 			);
@@ -42,7 +43,7 @@ namespace MoonWorks.Audio
 
 		internal StreamingSoundOgg(
 			AudioDevice device,
-			IntPtr fileDataPtr, // MUST BE AN ALLOCHGLOBAL HANDLE!!
+			IntPtr fileDataPtr, // MUST BE A NATIVE MEMORY HANDLE!!
 			IntPtr vorbisHandle,
 			FAudio.stb_vorbis_info info
 		) : base(
@@ -58,11 +59,9 @@ namespace MoonWorks.Audio
 			VorbisHandle = vorbisHandle;
 			Info = info;
 			buffer = new float[BUFFER_SIZE];
-
-			device.AddDynamicSoundInstance(this);
 		}
 
-		private void PerformSeek(uint sampleFrame)
+		public override void Seek(uint sampleFrame)
 		{
 			if (State == SoundState.Playing)
 			{
@@ -80,49 +79,32 @@ namespace MoonWorks.Audio
 			}
 		}
 
-		public override void Seek(float seconds)
+		protected unsafe override void AddBuffer(
-		{
+			void* buffer,
-			uint sampleFrame = (uint) (Info.sample_rate * seconds);
+			int bufferLengthInBytes,
-			PerformSeek(sampleFrame);
+			out int filledLengthInBytes,
-		}
-
-		public override void Seek(uint sampleFrame)
-		{
-			PerformSeek(sampleFrame);
-		}
-
-		protected override void AddBuffer(
-			out float[] buffer,
-			out uint bufferOffset,
-			out uint bufferLength,
 			out bool reachedEnd
 		)
 		{
-			buffer = this.buffer;
+			var lengthInFloats = bufferLengthInBytes / sizeof(float);
 
 			/* NOTE: this function returns samples per channel, not total samples */
 			var samples = FAudio.stb_vorbis_get_samples_float_interleaved(
 				VorbisHandle,
 				Info.channels,
-				buffer,
+				(IntPtr) buffer,
-				buffer.Length
+				lengthInFloats
 			);
 
 			var sampleCount = samples * Info.channels;
-			bufferOffset = 0;
+			reachedEnd = sampleCount < lengthInFloats;
-			bufferLength = (uint) sampleCount;
+			filledLengthInBytes = sampleCount * sizeof(float);
-			reachedEnd = sampleCount < buffer.Length;
 		}
 
-		protected override void SeekStart()
+		protected unsafe override void Destroy()
-		{
-			FAudio.stb_vorbis_seek_start(VorbisHandle);
-		}
-
-		protected override void Destroy()
 		{
 			FAudio.stb_vorbis_close(VorbisHandle);
-			Marshal.FreeHGlobal(FileDataPtr);
+			NativeMemory.Free((void*) FileDataPtr);
 		}
 	}
 }

src/Audio/StreamingSoundSeekable.cs

@@ -0,0 +1,25 @@
+namespace MoonWorks.Audio
+{
+	public abstract class StreamingSoundSeekable : StreamingSound
+	{
+		public bool Loop { get; set; }
+
+		protected StreamingSoundSeekable(AudioDevice device, ushort formatTag, ushort bitsPerSample, ushort blockAlign, ushort channels, uint samplesPerSecond) : base(device, formatTag, bitsPerSample, blockAlign, channels, samplesPerSecond)
+		{
+		}
+
+		public abstract void Seek(uint sampleFrame);
+
+		protected override void OnReachedEnd()
+		{
+			if (Loop)
+			{
+				Seek(0);
+			}
+			else
+			{
+				Stop(false);
+			}
+		}
+	}
+}

src/Video/StreamingSoundTheora.cs

@@ -0,0 +1,46 @@
+using System;
+using MoonWorks.Audio;
+
+namespace MoonWorks.Video
+{
+	public unsafe class StreamingSoundTheora : StreamingSound
+	{
+		public IntPtr VideoHandle;
+
+		public override SoundState State { get => throw new System.NotImplementedException(); protected set => throw new System.NotImplementedException(); }
+
+		public override int BUFFER_SIZE => 4096 * 2;
+
+		internal StreamingSoundTheora(
+			AudioDevice device,
+			IntPtr videoHandle,
+			int channels,
+			uint sampleRate
+		) : base(
+			device,
+			3, /* float type */
+			32, /* size of float */
+			(ushort) (4 * channels),
+			(ushort) channels,
+			sampleRate
+		) {
+			VideoHandle = videoHandle;
+		}
+
+		protected override unsafe void AddBuffer(
+			void* buffer,
+			int bufferLength,
+			out int filledLength,
+			out bool reachedEnd
+		) {
+			int samples = Theorafile.tf_readaudio(
+				VideoHandle,
+				(IntPtr) buffer,
+				bufferLength
+			);
+
+			filledLength = samples * sizeof(float);
+			reachedEnd = Theorafile.tf_eos(VideoHandle) == 1;
+		}
+	}
+}

src/Video/Video.cs

@@ -2,6 +2,7 @@
 using System;
 using System.Diagnostics;
 using System.Runtime.InteropServices;
+using MoonWorks.Audio;
 using MoonWorks.Graphics;
 
 namespace MoonWorks.Video
@@ -15,7 +16,7 @@ namespace MoonWorks.Video
 
 	public unsafe class Video : IDisposable
 	{
-		private IntPtr Handle;
+		internal IntPtr Handle;
 
 		public bool Loop { get; private set; }
 		public float Volume { get; private set; }
@@ -40,15 +41,20 @@ namespace MoonWorks.Video
 		private Texture vTexture = null;
 		private Sampler LinearSampler;
 
+		private AudioDevice AudioDevice = null;
+		private StreamingSoundTheora audioStream = null;
+
 		private Stopwatch timer;
 		private double lastTimestamp;
 		private double timeElapsed;
 
 		private bool disposed;
 
-		public Video(GraphicsDevice graphicsDevice, string filename)
+		/* TODO: is there some way for us to load the data into memory? */
+		public Video(GraphicsDevice graphicsDevice, AudioDevice audioDevice, string filename)
 		{
 			GraphicsDevice = graphicsDevice;
+			AudioDevice = audioDevice;
 
 			if (!System.IO.File.Exists(filename))
 			{
@@ -148,10 +154,14 @@ namespace MoonWorks.Video
 			Loop = loop;
 			timer.Start();
 
+			if (audioStream != null)
+			{
+				audioStream.Play();
+			}
+
 			State = VideoState.Playing;
 		}
 
-
 		public void Pause()
 		{
 			if (State != VideoState.Playing)
@@ -161,6 +171,11 @@ namespace MoonWorks.Video
 
 			timer.Stop();
 
+			if (audioStream != null)
+			{
+				audioStream.Pause();
+			}
+
 			State = VideoState.Paused;
 		}
 
@@ -178,6 +193,13 @@ namespace MoonWorks.Video
 			lastTimestamp = 0;
 			timeElapsed = 0;
 
+			if (audioStream != null)
+			{
+				audioStream.Stop(true);
+				audioStream.Dispose();
+				audioStream = null;
+			}
+
 			State = VideoState.Stopped;
 		}
 
@@ -216,6 +238,13 @@ namespace MoonWorks.Video
 				timer.Stop();
 				timer.Reset();
 
+				if (audioStream != null)
+				{
+					audioStream.Stop();
+					audioStream.Dispose();
+					audioStream = null;
+				}
+
 				Theorafile.tf_reset(Handle);
 
 				if (Loop)
@@ -224,7 +253,6 @@ namespace MoonWorks.Video
 					InitializeTheoraStream();
 
 					timer.Start();
-
 				}
 				else
 				{
@@ -271,12 +299,11 @@ namespace MoonWorks.Video
 			while (Theorafile.tf_readvideo(Handle, (IntPtr) yuvData, 1) == 0);
 
 			// Grab the first bit of audio. We're trying to start the decoding ASAP.
-			if (Theorafile.tf_hasaudio(Handle) == 1)
+			if (AudioDevice != null && Theorafile.tf_hasaudio(Handle) == 1)
 			{
-				int channels, samplerate;
+				int channels, sampleRate;
-				Theorafile.tf_audioinfo(Handle, out channels, out samplerate);
+				Theorafile.tf_audioinfo(Handle, out channels, out sampleRate);
-
+				audioStream = new StreamingSoundTheora(AudioDevice, Handle, channels, (uint) sampleRate);
-				// TODO: audio stream
 			}
 
 			currentFrame = -1;