AV1 decoder implementation

pull/49/head
cosmonaut 2023-06-01 16:31:32 -07:00
parent 00f4bfdeae
commit 0f9511f0f6
8 changed files with 160 additions and 95 deletions

3
.gitmodules vendored
View File

@ -13,3 +13,6 @@
[submodule "lib/Theorafile"] [submodule "lib/Theorafile"]
path = lib/Theorafile path = lib/Theorafile
url = https://github.com/FNA-XNA/Theorafile.git url = https://github.com/FNA-XNA/Theorafile.git
[submodule "lib/dav1dfile"]
path = lib/dav1dfile
url = git@github.com:MoonsideGames/dav1dfile.git

View File

@ -17,6 +17,7 @@
<Compile Include="lib\SDL2-CS\src\SDL2.cs" /> <Compile Include="lib\SDL2-CS\src\SDL2.cs" />
<Compile Include="lib\Theorafile\csharp\Theorafile.cs" /> <Compile Include="lib\Theorafile\csharp\Theorafile.cs" />
<Compile Include="lib\WellspringCS\WellspringCS.cs" /> <Compile Include="lib\WellspringCS\WellspringCS.cs" />
<Compile Include="lib\dav1dfile\csharp\dav1dfile.cs" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>

View File

@ -19,4 +19,8 @@
<dllmap dll="Theorafile" os="windows" target="libtheorafile.dll"/> <dllmap dll="Theorafile" os="windows" target="libtheorafile.dll"/>
<dllmap dll="Theorafile" os="osx" target="libtheorafile.dylib"/> <dllmap dll="Theorafile" os="osx" target="libtheorafile.dylib"/>
<dllmap dll="Theorafile" os="linux,freebsd,netbsd" target="libtheorafile.so"/> <dllmap dll="Theorafile" os="linux,freebsd,netbsd" target="libtheorafile.so"/>
<dllmap dll="dav1dfile" os="windows" target="dav1dfile.dll"/>
<dllmap dll="dav1dfile" os="osx" target="libdav1dfile.0.dylib"/>
<dllmap dll="dav1dfile" os="linux,freebsd,netbsd,openbsd" target="libdav1dfile.so.0"/>
</configuration> </configuration>

@ -1 +1 @@
Subproject commit ebf511133aa6f567c004d687acac474e1649bbde Subproject commit 9edb2d1c97f4a38dd3d6d87298d6c604defa4fc8

1
lib/dav1dfile Submodule

@ -0,0 +1 @@
Subproject commit 8ba817ff9bf65ed2fa7b9c12d70112b29c450cf4

View File

@ -1946,7 +1946,17 @@ namespace MoonWorks.Graphics
/// <summary> /// <summary>
/// Asynchronously copies YUV data into three textures. Use with compressed video. /// Asynchronously copies YUV data into three textures. Use with compressed video.
/// </summary> /// </summary>
public void SetTextureDataYUV(Texture yTexture, Texture uTexture, Texture vTexture, IntPtr dataPtr, uint dataLengthInBytes) public void SetTextureDataYUV(
Texture yTexture,
Texture uTexture,
Texture vTexture,
IntPtr yDataPtr,
IntPtr uDataPtr,
IntPtr vDataPtr,
uint yDataLengthInBytes,
uint uvDataLengthInBytes,
uint yStride,
uint uvStride)
{ {
#if DEBUG #if DEBUG
AssertRenderPassInactive("Cannot copy during render pass!"); AssertRenderPassInactive("Cannot copy during render pass!");
@ -1962,8 +1972,13 @@ namespace MoonWorks.Graphics
yTexture.Height, yTexture.Height,
uTexture.Width, uTexture.Width,
uTexture.Height, uTexture.Height,
dataPtr, yDataPtr,
dataLengthInBytes uDataPtr,
vDataPtr,
yDataLengthInBytes,
uvDataLengthInBytes,
yStride,
uvStride
); );
} }

88
src/Video/VideoAV1.cs Normal file
View File

@ -0,0 +1,88 @@
using System;
using System.IO;
namespace MoonWorks.Video
{
public unsafe class VideoAV1 : IDisposable
{
public IntPtr Handle => handle;
IntPtr handle;
public int Width => width;
public int Height => height;
public double FramesPerSecond => 28.97;
public Dav1dfile.PixelLayout PixelLayout => pixelLayout;
public int UVWidth { get; }
public int UVHeight { get; }
private int width;
private int height;
private double fps;
private Dav1dfile.PixelLayout pixelLayout;
bool IsDisposed;
public VideoAV1(string filename)
{
if (!File.Exists(filename))
{
throw new ArgumentException("Video file not found!");
}
if (Dav1dfile.df_fopen(filename, out handle) == 0)
{
throw new Exception("Failed to open video file!");
}
Dav1dfile.df_videoinfo(Handle, out width, out height, out pixelLayout);
if (pixelLayout == Dav1dfile.PixelLayout.I420)
{
UVWidth = Width / 2;
UVHeight = Height / 2;
}
else if (pixelLayout == Dav1dfile.PixelLayout.I422)
{
UVWidth = Width / 2;
UVHeight = Height;
}
else if (pixelLayout == Dav1dfile.PixelLayout.I444)
{
UVWidth = width;
UVHeight = height;
}
else
{
throw new NotSupportedException("Unrecognized YUV format!");
}
}
protected virtual void Dispose(bool disposing)
{
if (!IsDisposed)
{
if (disposing)
{
// dispose managed state (managed objects)
}
// free unmanaged resources (unmanaged objects)
Dav1dfile.df_close(Handle);
IsDisposed = true;
}
}
~VideoAV1()
{
Dispose(disposing: false);
}
public void Dispose()
{
// Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method
Dispose(disposing: true);
GC.SuppressFinalize(this);
}
}
}

View File

@ -11,20 +11,9 @@ namespace MoonWorks.Video
public Texture RenderTexture { get; private set; } = null; public Texture RenderTexture { get; private set; } = null;
public VideoState State { get; private set; } = VideoState.Stopped; public VideoState State { get; private set; } = VideoState.Stopped;
public bool Loop { get; set; } public bool Loop { get; set; }
public float Volume {
get => volume;
set
{
volume = value;
if (audioStream != null)
{
audioStream.Volume = value;
}
}
}
public float PlaybackSpeed { get; set; } = 1; public float PlaybackSpeed { get; set; } = 1;
private Video Video = null; private VideoAV1 Video = null;
private GraphicsDevice GraphicsDevice; private GraphicsDevice GraphicsDevice;
private Texture yTexture = null; private Texture yTexture = null;
@ -32,19 +21,14 @@ namespace MoonWorks.Video
private Texture vTexture = null; private Texture vTexture = null;
private Sampler LinearSampler; private Sampler LinearSampler;
private void* yuvData = null;
private int yuvDataLength = 0;
private int currentFrame; private int currentFrame;
private AudioDevice AudioDevice;
private StreamingSoundTheora audioStream = null;
private float volume = 1.0f;
private Stopwatch timer; private Stopwatch timer;
private double lastTimestamp; private double lastTimestamp;
private double timeElapsed; private double timeElapsed;
private Stopwatch profilingTimer = new Stopwatch();
private bool disposed; private bool disposed;
public VideoPlayer(GraphicsDevice graphicsDevice, AudioDevice audioDevice) public VideoPlayer(GraphicsDevice graphicsDevice, AudioDevice audioDevice)
@ -55,13 +39,12 @@ namespace MoonWorks.Video
throw new InvalidOperationException("Missing video shaders!"); throw new InvalidOperationException("Missing video shaders!");
} }
AudioDevice = audioDevice;
LinearSampler = new Sampler(graphicsDevice, SamplerCreateInfo.LinearClamp); LinearSampler = new Sampler(graphicsDevice, SamplerCreateInfo.LinearClamp);
timer = new Stopwatch(); timer = new Stopwatch();
} }
public void Load(Video video) public void Load(VideoAV1 video)
{ {
if (Video != video) if (Video != video)
{ {
@ -111,20 +94,9 @@ namespace MoonWorks.Video
vTexture = CreateSubTexture(GraphicsDevice, video.UVWidth, video.UVHeight); vTexture = CreateSubTexture(GraphicsDevice, video.UVWidth, video.UVHeight);
} }
var newDataLength = (
(video.Width * video.Height) +
(video.UVWidth * video.UVHeight * 2)
);
if (newDataLength != yuvDataLength)
{
yuvData = NativeMemory.Realloc(yuvData, (nuint) newDataLength);
yuvDataLength = newDataLength;
}
Video = video; Video = video;
InitializeTheoraStream(); InitializeDav1dStream();
} }
} }
@ -139,11 +111,6 @@ namespace MoonWorks.Video
timer.Start(); timer.Start();
if (audioStream != null)
{
audioStream.Play();
}
State = VideoState.Playing; State = VideoState.Playing;
} }
@ -158,11 +125,6 @@ namespace MoonWorks.Video
timer.Stop(); timer.Stop();
if (audioStream != null)
{
audioStream.Pause();
}
State = VideoState.Paused; State = VideoState.Paused;
} }
@ -181,9 +143,7 @@ namespace MoonWorks.Video
lastTimestamp = 0; lastTimestamp = 0;
timeElapsed = 0; timeElapsed = 0;
DestroyAudioStream(); Dav1dfile.df_reset(Video.Handle);
Theorafile.tf_reset(Video.Handle);
State = VideoState.Stopped; State = VideoState.Stopped;
} }
@ -197,11 +157,6 @@ namespace MoonWorks.Video
public void Update() public void Update()
{ {
if (Video == null) { return; } if (Video == null) { return; }
if (audioStream != null)
{
audioStream.Update();
}
} }
public void Render() public void Render()
@ -217,32 +172,41 @@ namespace MoonWorks.Video
int thisFrame = ((int) (timeElapsed / (1000.0 / Video.FramesPerSecond))); int thisFrame = ((int) (timeElapsed / (1000.0 / Video.FramesPerSecond)));
if (thisFrame > currentFrame) if (thisFrame > currentFrame)
{ {
if (Theorafile.tf_readvideo( profilingTimer.Restart();
int readResult = Dav1dfile.df_readvideo(
Video.Handle, Video.Handle,
(IntPtr) yuvData, thisFrame - currentFrame,
thisFrame - currentFrame out var yData,
) == 1 || currentFrame == -1) out var uData,
out var vData,
out var yDataLength,
out var uvDataLength,
out var yStride,
out var uvStride
);
profilingTimer.Stop();
System.Console.WriteLine($"Frame decoded in: {profilingTimer.Elapsed.TotalMilliseconds}");
if (readResult == 1 || currentFrame == -1)
{ {
UpdateRenderTexture(); UpdateRenderTexture(yData, uData, vData, yDataLength, uvDataLength, yStride, uvStride);
} }
currentFrame = thisFrame; currentFrame = thisFrame;
} }
bool ended = Theorafile.tf_eos(Video.Handle) == 1; bool ended = Dav1dfile.df_eos(Video.Handle) == 1;
if (ended) if (ended)
{ {
timer.Stop(); timer.Stop();
timer.Reset(); timer.Reset();
DestroyAudioStream(); Dav1dfile.df_reset(Video.Handle);
Theorafile.tf_reset(Video.Handle);
if (Loop) if (Loop)
{ {
// Start over! // Start over!
InitializeTheoraStream(); InitializeDav1dStream();
timer.Start(); timer.Start();
} }
@ -253,7 +217,7 @@ namespace MoonWorks.Video
} }
} }
private void UpdateRenderTexture() private void UpdateRenderTexture(IntPtr yData, IntPtr uData, IntPtr vData, uint yDataLength, uint uvDataLength, uint yStride, uint uvStride)
{ {
var commandBuffer = GraphicsDevice.AcquireCommandBuffer(); var commandBuffer = GraphicsDevice.AcquireCommandBuffer();
@ -261,8 +225,13 @@ namespace MoonWorks.Video
yTexture, yTexture,
uTexture, uTexture,
vTexture, vTexture,
(IntPtr) yuvData, yData,
(uint) yuvDataLength uData,
vData,
yDataLength,
uvDataLength,
yStride,
uvStride
); );
commandBuffer.BeginRenderPass( commandBuffer.BeginRenderPass(
@ -305,35 +274,22 @@ namespace MoonWorks.Video
); );
} }
private void InitializeTheoraStream() private void InitializeDav1dStream()
{ {
// Grab the first video frame ASAP. // Grab the first video frame ASAP.
while (Theorafile.tf_readvideo(Video.Handle, (IntPtr) yuvData, 1) == 0); Dav1dfile.df_readvideo(
Video.Handle,
// Grab the first bit of audio. We're trying to start the decoding ASAP. 1,
if (AudioDevice != null && Theorafile.tf_hasaudio(Video.Handle) == 1) out var _,
{ out var _,
DestroyAudioStream(); out var _,
out var _,
int channels, sampleRate; out var _,
Theorafile.tf_audioinfo(Video.Handle, out channels, out sampleRate); out var _,
out var _);
audioStream = new StreamingSoundTheora(AudioDevice, Video.Handle, channels, (uint) sampleRate);
}
currentFrame = -1; currentFrame = -1;
} }
private void DestroyAudioStream()
{
if (audioStream != null)
{
audioStream.StopImmediate();
audioStream.Dispose();
audioStream = null;
}
}
protected virtual void Dispose(bool disposing) protected virtual void Dispose(bool disposing)
{ {
if (!disposed) if (!disposed)
@ -347,9 +303,6 @@ namespace MoonWorks.Video
vTexture.Dispose(); vTexture.Dispose();
} }
// free unmanaged resources (unmanaged objects) and override finalizer
NativeMemory.Free(yuvData);
disposed = true; disposed = true;
} }
} }