From 4d9d3e6422f43b05362d152dd237f94aa5d5bfb0 Mon Sep 17 00:00:00 2001 From: cosmonaut Date: Mon, 1 Aug 2022 23:02:13 -0700 Subject: [PATCH] restructure audio API to support theora audio stream --- lib/Theorafile | 2 +- src/Audio/SoundInstance.cs | 6 +-- src/Audio/StaticSoundInstance.cs | 22 ++------ src/Audio/StreamingSound.cs | 80 +++++++++++++---------------- src/Audio/StreamingSoundOgg.cs | 62 ++++++++-------------- src/Audio/StreamingSoundSeekable.cs | 25 +++++++++ src/Video/StreamingSoundTheora.cs | 46 +++++++++++++++++ src/Video/Video.cs | 45 ++++++++++++---- 8 files changed, 172 insertions(+), 116 deletions(-) create mode 100644 src/Audio/StreamingSoundSeekable.cs create mode 100644 src/Video/StreamingSoundTheora.cs diff --git a/lib/Theorafile b/lib/Theorafile index 3ed1726..dd8c7fa 160000 --- a/lib/Theorafile +++ b/lib/Theorafile @@ -1 +1 @@ -Subproject commit 3ed1726b1e294799e85c3b597b114fb3b21cba72 +Subproject commit dd8c7fa69e678b6182cdaa71458ad08dd31c65da diff --git a/src/Audio/SoundInstance.cs b/src/Audio/SoundInstance.cs index 03a09ec..8210d2a 100644 --- a/src/Audio/SoundInstance.cs +++ b/src/Audio/SoundInstance.cs @@ -1,6 +1,5 @@ using System; using System.Runtime.InteropServices; -using MoonWorks.Math.Float; namespace MoonWorks.Audio { @@ -8,7 +7,6 @@ namespace MoonWorks.Audio { internal IntPtr Handle; internal FAudio.FAudioWaveFormatEx Format; - public bool Loop { get; protected set; } = false; protected FAudio.F3DAUDIO_DSP_SETTINGS dspSettings; @@ -238,11 +236,9 @@ namespace MoonWorks.Audio ); } - public abstract void Play(bool loop); + public abstract void Play(); public abstract void Pause(); public abstract void Stop(bool immediate); - public abstract void Seek(float seconds); - public abstract void Seek(uint sampleFrame); private void InitDSPSettings(uint srcChannels) { diff --git a/src/Audio/StaticSoundInstance.cs b/src/Audio/StaticSoundInstance.cs index 1afbb0d..685e350 100644 --- a/src/Audio/StaticSoundInstance.cs +++ b/src/Audio/StaticSoundInstance.cs @@ -6,6 +6,8 @@ namespace MoonWorks.Audio { public StaticSound Parent { get; } + public bool Loop { get; set; } + private SoundState _state = SoundState.Stopped; public override SoundState State { @@ -38,15 +40,13 @@ namespace MoonWorks.Audio Parent = parent; } - public override void Play(bool loop = false) + public override void Play() { if (State == SoundState.Playing) { return; } - Loop = loop; - if (Loop) { Parent.Handle.LoopCount = 255; @@ -93,7 +93,7 @@ namespace MoonWorks.Audio } } - private void PerformSeek(uint sampleFrame) + public void Seek(uint sampleFrame) { if (State == SoundState.Playing) { @@ -102,20 +102,6 @@ namespace MoonWorks.Audio } Parent.Handle.PlayBegin = sampleFrame; - Play(); - } - - public override void Seek(float seconds) - { - uint sampleFrame = - (uint) (Parent.SamplesPerSecond * seconds); - - PerformSeek(sampleFrame); - } - - public override void Seek(uint sampleFrame) - { - PerformSeek(sampleFrame); } public void Free() diff --git a/src/Audio/StreamingSound.cs b/src/Audio/StreamingSound.cs index 18a5a33..9dd954d 100644 --- a/src/Audio/StreamingSound.cs +++ b/src/Audio/StreamingSound.cs @@ -11,10 +11,11 @@ namespace MoonWorks.Audio public abstract class StreamingSound : SoundInstance { private readonly List queuedBuffers = new List(); - private readonly List queuedSizes = new List(); private const int MINIMUM_BUFFER_CHECK = 3; - public int PendingBufferCount => queuedBuffers.Count; + private int PendingBufferCount => queuedBuffers.Count; + + public abstract int BUFFER_SIZE { get; } public StreamingSound( AudioDevice device, @@ -23,16 +24,18 @@ namespace MoonWorks.Audio ushort blockAlign, ushort channels, uint samplesPerSecond - ) : base(device, formatTag, bitsPerSample, blockAlign, channels, samplesPerSecond) { } + ) : base(device, formatTag, bitsPerSample, blockAlign, channels, samplesPerSecond) + { + device.AddDynamicSoundInstance(this); + } - public override void Play(bool loop = false) + public override void Play() { if (State == SoundState.Playing) { return; } - Loop = loop; State = SoundState.Playing; Update(); @@ -60,7 +63,7 @@ namespace MoonWorks.Audio State = SoundState.Stopped; } - internal void Update() + internal unsafe void Update() { if (State != SoundState.Playing) { @@ -74,11 +77,13 @@ namespace MoonWorks.Audio ); while (PendingBufferCount > state.BuffersQueued) + { lock (queuedBuffers) { - Marshal.FreeHGlobal(queuedBuffers[0]); + NativeMemory.Free((void*) queuedBuffers[0]); queuedBuffers.RemoveAt(0); } + } QueueBuffers(); } @@ -95,46 +100,42 @@ namespace MoonWorks.Audio } } - protected void ClearBuffers() + protected unsafe void ClearBuffers() { lock (queuedBuffers) { foreach (IntPtr buf in queuedBuffers) { - Marshal.FreeHGlobal(buf); + NativeMemory.Free((void*) buf); } queuedBuffers.Clear(); - queuedSizes.Clear(); } } - protected void AddBuffer() + protected unsafe void AddBuffer() { + void* buffer = NativeMemory.Alloc((nuint) BUFFER_SIZE); + AddBuffer( - out var buffer, - out var bufferOffset, - out var bufferLength, - out var reachedEnd + buffer, + BUFFER_SIZE, + out int filledLengthInBytes, + out bool reachedEnd ); - var lengthInBytes = bufferLength * sizeof(float); - - IntPtr next = Marshal.AllocHGlobal((int) lengthInBytes); - Marshal.Copy(buffer, (int) bufferOffset, next, (int) bufferLength); - lock (queuedBuffers) { - queuedBuffers.Add(next); + queuedBuffers.Add((IntPtr) buffer); if (State != SoundState.Stopped) { FAudio.FAudioBuffer buf = new FAudio.FAudioBuffer { - AudioBytes = lengthInBytes, - pAudioData = next, + AudioBytes = (uint) filledLengthInBytes, + pAudioData = (IntPtr) buffer, PlayLength = ( - lengthInBytes / + (uint) (filledLengthInBytes / Format.nChannels / - (uint) (Format.wBitsPerSample / 8) + (uint) (Format.wBitsPerSample / 8)) ) }; @@ -144,35 +145,28 @@ namespace MoonWorks.Audio IntPtr.Zero ); } - else - { - queuedSizes.Add(lengthInBytes); - } } + /* We have reached the end of the file, what do we do? */ if (reachedEnd) { - if (Loop) - { - SeekStart(); - } - else - { - Stop(false); - } + OnReachedEnd(); } } - protected abstract void AddBuffer( - out float[] buffer, - out uint bufferOffset, /* in floats */ - out uint bufferLength, /* in floats */ + protected virtual void OnReachedEnd() + { + Stop(false); + } + + protected unsafe abstract void AddBuffer( + void* buffer, + int bufferLength, /* in bytes */ + out int filledLength, /* in bytes */ out bool reachedEnd ); - protected abstract void SeekStart(); - protected override void Destroy() { Stop(true); diff --git a/src/Audio/StreamingSoundOgg.cs b/src/Audio/StreamingSoundOgg.cs index 8aa4721..0c3bb7f 100644 --- a/src/Audio/StreamingSoundOgg.cs +++ b/src/Audio/StreamingSoundOgg.cs @@ -1,13 +1,14 @@ using System; using System.IO; +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; namespace MoonWorks.Audio { - public class StreamingSoundOgg : StreamingSound + public class StreamingSoundOgg : StreamingSoundSeekable { // FIXME: what should this value be? - public const int BUFFER_SIZE = 1024 * 128; + public override int BUFFER_SIZE => 1024 * 128; private IntPtr VorbisHandle; private IntPtr FileDataPtr; @@ -17,15 +18,15 @@ namespace MoonWorks.Audio public override SoundState State { get; protected set; } - public static StreamingSoundOgg Load(AudioDevice device, string filePath) + public unsafe static StreamingSoundOgg Load(AudioDevice device, string filePath) { var fileData = File.ReadAllBytes(filePath); - var fileDataPtr = Marshal.AllocHGlobal(fileData.Length); - Marshal.Copy(fileData, 0, fileDataPtr, fileData.Length); - var vorbisHandle = FAudio.stb_vorbis_open_memory(fileDataPtr, fileData.Length, out int error, IntPtr.Zero); + var fileDataPtr = NativeMemory.Alloc((nuint) fileData.Length); + Marshal.Copy(fileData, 0, (IntPtr) fileDataPtr, fileData.Length); + var vorbisHandle = FAudio.stb_vorbis_open_memory((IntPtr) fileDataPtr, fileData.Length, out int error, IntPtr.Zero); if (error != 0) { - ((GCHandle) fileDataPtr).Free(); + NativeMemory.Free(fileDataPtr); Logger.LogError("Error opening OGG file!"); Logger.LogError("Error: " + error); throw new AudioLoadException("Error opening OGG file!"); @@ -34,7 +35,7 @@ namespace MoonWorks.Audio return new StreamingSoundOgg( device, - fileDataPtr, + (IntPtr) fileDataPtr, vorbisHandle, info ); @@ -42,7 +43,7 @@ namespace MoonWorks.Audio internal StreamingSoundOgg( AudioDevice device, - IntPtr fileDataPtr, // MUST BE AN ALLOCHGLOBAL HANDLE!! + IntPtr fileDataPtr, // MUST BE A NATIVE MEMORY HANDLE!! IntPtr vorbisHandle, FAudio.stb_vorbis_info info ) : base( @@ -58,11 +59,9 @@ namespace MoonWorks.Audio VorbisHandle = vorbisHandle; Info = info; buffer = new float[BUFFER_SIZE]; - - device.AddDynamicSoundInstance(this); } - private void PerformSeek(uint sampleFrame) + public override void Seek(uint sampleFrame) { if (State == SoundState.Playing) { @@ -80,49 +79,32 @@ namespace MoonWorks.Audio } } - public override void Seek(float seconds) - { - uint sampleFrame = (uint) (Info.sample_rate * seconds); - PerformSeek(sampleFrame); - } - - public override void Seek(uint sampleFrame) - { - PerformSeek(sampleFrame); - } - - protected override void AddBuffer( - out float[] buffer, - out uint bufferOffset, - out uint bufferLength, + protected unsafe override void AddBuffer( + void* buffer, + int bufferLengthInBytes, + out int filledLengthInBytes, out bool reachedEnd ) { - buffer = this.buffer; + var lengthInFloats = bufferLengthInBytes / sizeof(float); /* NOTE: this function returns samples per channel, not total samples */ var samples = FAudio.stb_vorbis_get_samples_float_interleaved( VorbisHandle, Info.channels, - buffer, - buffer.Length + (IntPtr) buffer, + lengthInFloats ); var sampleCount = samples * Info.channels; - bufferOffset = 0; - bufferLength = (uint) sampleCount; - reachedEnd = sampleCount < buffer.Length; + reachedEnd = sampleCount < lengthInFloats; + filledLengthInBytes = sampleCount * sizeof(float); } - protected override void SeekStart() - { - FAudio.stb_vorbis_seek_start(VorbisHandle); - } - - protected override void Destroy() + protected unsafe override void Destroy() { FAudio.stb_vorbis_close(VorbisHandle); - Marshal.FreeHGlobal(FileDataPtr); + NativeMemory.Free((void*) FileDataPtr); } } } diff --git a/src/Audio/StreamingSoundSeekable.cs b/src/Audio/StreamingSoundSeekable.cs new file mode 100644 index 0000000..ad159c1 --- /dev/null +++ b/src/Audio/StreamingSoundSeekable.cs @@ -0,0 +1,25 @@ +namespace MoonWorks.Audio +{ + public abstract class StreamingSoundSeekable : StreamingSound + { + public bool Loop { get; set; } + + protected StreamingSoundSeekable(AudioDevice device, ushort formatTag, ushort bitsPerSample, ushort blockAlign, ushort channels, uint samplesPerSecond) : base(device, formatTag, bitsPerSample, blockAlign, channels, samplesPerSecond) + { + } + + public abstract void Seek(uint sampleFrame); + + protected override void OnReachedEnd() + { + if (Loop) + { + Seek(0); + } + else + { + Stop(false); + } + } + } +} diff --git a/src/Video/StreamingSoundTheora.cs b/src/Video/StreamingSoundTheora.cs new file mode 100644 index 0000000..aeab3e1 --- /dev/null +++ b/src/Video/StreamingSoundTheora.cs @@ -0,0 +1,46 @@ +using System; +using MoonWorks.Audio; + +namespace MoonWorks.Video +{ + public unsafe class StreamingSoundTheora : StreamingSound + { + public IntPtr VideoHandle; + + public override SoundState State { get => throw new System.NotImplementedException(); protected set => throw new System.NotImplementedException(); } + + public override int BUFFER_SIZE => 4096 * 2; + + internal StreamingSoundTheora( + AudioDevice device, + IntPtr videoHandle, + int channels, + uint sampleRate + ) : base( + device, + 3, /* float type */ + 32, /* size of float */ + (ushort) (4 * channels), + (ushort) channels, + sampleRate + ) { + VideoHandle = videoHandle; + } + + protected override unsafe void AddBuffer( + void* buffer, + int bufferLength, + out int filledLength, + out bool reachedEnd + ) { + int samples = Theorafile.tf_readaudio( + VideoHandle, + (IntPtr) buffer, + bufferLength + ); + + filledLength = samples * sizeof(float); + reachedEnd = Theorafile.tf_eos(VideoHandle) == 1; + } + } +} diff --git a/src/Video/Video.cs b/src/Video/Video.cs index f2df5ad..679c603 100644 --- a/src/Video/Video.cs +++ b/src/Video/Video.cs @@ -2,6 +2,7 @@ using System; using System.Diagnostics; using System.Runtime.InteropServices; +using MoonWorks.Audio; using MoonWorks.Graphics; namespace MoonWorks.Video @@ -15,7 +16,7 @@ namespace MoonWorks.Video public unsafe class Video : IDisposable { - private IntPtr Handle; + internal IntPtr Handle; public bool Loop { get; private set; } public float Volume { get; private set; } @@ -40,15 +41,20 @@ namespace MoonWorks.Video private Texture vTexture = null; private Sampler LinearSampler; + private AudioDevice AudioDevice = null; + private StreamingSoundTheora audioStream = null; + private Stopwatch timer; private double lastTimestamp; private double timeElapsed; private bool disposed; - public Video(GraphicsDevice graphicsDevice, string filename) + /* TODO: is there some way for us to load the data into memory? */ + public Video(GraphicsDevice graphicsDevice, AudioDevice audioDevice, string filename) { GraphicsDevice = graphicsDevice; + AudioDevice = audioDevice; if (!System.IO.File.Exists(filename)) { @@ -148,10 +154,14 @@ namespace MoonWorks.Video Loop = loop; timer.Start(); + if (audioStream != null) + { + audioStream.Play(); + } + State = VideoState.Playing; } - public void Pause() { if (State != VideoState.Playing) @@ -161,6 +171,11 @@ namespace MoonWorks.Video timer.Stop(); + if (audioStream != null) + { + audioStream.Pause(); + } + State = VideoState.Paused; } @@ -178,6 +193,13 @@ namespace MoonWorks.Video lastTimestamp = 0; timeElapsed = 0; + if (audioStream != null) + { + audioStream.Stop(true); + audioStream.Dispose(); + audioStream = null; + } + State = VideoState.Stopped; } @@ -216,6 +238,13 @@ namespace MoonWorks.Video timer.Stop(); timer.Reset(); + if (audioStream != null) + { + audioStream.Stop(); + audioStream.Dispose(); + audioStream = null; + } + Theorafile.tf_reset(Handle); if (Loop) @@ -224,7 +253,6 @@ namespace MoonWorks.Video InitializeTheoraStream(); timer.Start(); - } else { @@ -271,12 +299,11 @@ namespace MoonWorks.Video while (Theorafile.tf_readvideo(Handle, (IntPtr) yuvData, 1) == 0); // Grab the first bit of audio. We're trying to start the decoding ASAP. - if (Theorafile.tf_hasaudio(Handle) == 1) + if (AudioDevice != null && Theorafile.tf_hasaudio(Handle) == 1) { - int channels, samplerate; - Theorafile.tf_audioinfo(Handle, out channels, out samplerate); - - // TODO: audio stream + int channels, sampleRate; + Theorafile.tf_audioinfo(Handle, out channels, out sampleRate); + audioStream = new StreamingSoundTheora(AudioDevice, Handle, channels, (uint) sampleRate); } currentFrame = -1;