initial video support

pull/20/head
cosmonaut 2022-07-29 18:24:05 -07:00
parent 5a5fbc0c77
commit 845881533b
13 changed files with 496 additions and 2 deletions

3
.gitmodules vendored
View File

@ -10,3 +10,6 @@
[submodule "lib/WellspringCS"] [submodule "lib/WellspringCS"]
path = lib/WellspringCS path = lib/WellspringCS
url = https://gitea.moonside.games/MoonsideGames/WellspringCS.git url = https://gitea.moonside.games/MoonsideGames/WellspringCS.git
[submodule "lib/Theorafile"]
path = lib/Theorafile
url = https://github.com/FNA-XNA/Theorafile.git

View File

@ -15,6 +15,7 @@
<ProjectReference Include=".\lib\RefreshCS\RefreshCS.csproj" /> <ProjectReference Include=".\lib\RefreshCS\RefreshCS.csproj" />
<ProjectReference Include=".\lib\FAudio\csharp\FAudio-CS.Core.csproj" /> <ProjectReference Include=".\lib\FAudio\csharp\FAudio-CS.Core.csproj" />
<ProjectReference Include=".\lib\WellspringCS\WellspringCS.csproj" /> <ProjectReference Include=".\lib\WellspringCS\WellspringCS.csproj" />
<ProjectReference Include=".\lib\Theorafile\csharp\Theorafile-CS.Core.csproj" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
@ -22,4 +23,13 @@
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None> </None>
</ItemGroup> </ItemGroup>
<ItemGroup>
<EmbeddedResource Include="src\Video\Shaders\Compiled\FullscreenVert.spv">
<LogicalName>MoonWorks.Shaders.FullscreenVert.spv</LogicalName>
</EmbeddedResource>
<EmbeddedResource Include="src\Video\Shaders\Compiled\YUV2RGBAFrag.spv">
<LogicalName>MoonWorks.Shaders.YUV2RGBAFrag.spv</LogicalName>
</EmbeddedResource>
</ItemGroup>
</Project> </Project>

View File

@ -15,4 +15,8 @@
<dllmap dll="Wellspring" os="windows" target="Wellspring.dll"/> <dllmap dll="Wellspring" os="windows" target="Wellspring.dll"/>
<dllmap dll="Wellspring" os="osx" target="libWellspring.0.dylib"/> <dllmap dll="Wellspring" os="osx" target="libWellspring.0.dylib"/>
<dllmap dll="Wellspring" os="linux,freebsd,netbsd" target="libWellspring.so.0"/> <dllmap dll="Wellspring" os="linux,freebsd,netbsd" target="libWellspring.so.0"/>
<dllmap dll="Theorafile" os="windows" target="libtheorafile.dll"/>
<dllmap dll="Theorafile" os="osx" target="libtheorafile.dylib"/>
<dllmap dll="Theorafile" os="linux,freebsd,netbsd" target="libtheorafile.so"/>
</configuration> </configuration>

1
lib/Theorafile Submodule

@ -0,0 +1 @@
Subproject commit 3ed1726b1e294799e85c3b597b114fb3b21cba72

View File

@ -1,6 +1,4 @@
using System; using System;
using System.Runtime.InteropServices;
using MoonWorks.Math;
using RefreshCS; using RefreshCS;
namespace MoonWorks.Graphics namespace MoonWorks.Graphics
@ -835,6 +833,26 @@ namespace MoonWorks.Graphics
SetTextureData(new TextureSlice(texture), dataPtr, dataLengthInBytes); SetTextureData(new TextureSlice(texture), dataPtr, dataLengthInBytes);
} }
/// <summary>
/// Asynchronously copies YUV data into three textures. Use with compressed video.
/// </summary>
public void SetTextureDataYUV(Texture yTexture, Texture uTexture, Texture vTexture, IntPtr dataPtr, uint dataLengthInBytes)
{
Refresh.Refresh_SetTextureDataYUV(
Device.Handle,
Handle,
yTexture.Handle,
uTexture.Handle,
vTexture.Handle,
yTexture.Width,
yTexture.Height,
uTexture.Width,
uTexture.Height,
dataPtr,
dataLengthInBytes
);
}
/// <summary> /// <summary>
/// Performs an asynchronous texture-to-texture copy on the GPU. /// Performs an asynchronous texture-to-texture copy on the GPU.
/// </summary> /// </summary>

View File

@ -1,5 +1,6 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.IO;
using RefreshCS; using RefreshCS;
namespace MoonWorks.Graphics namespace MoonWorks.Graphics
@ -8,6 +9,11 @@ namespace MoonWorks.Graphics
{ {
public IntPtr Handle { get; } public IntPtr Handle { get; }
// Built-in video pipeline
private ShaderModule VideoVertexShader { get; }
private ShaderModule VideoFragmentShader { get; }
internal GraphicsPipeline VideoPipeline { get; }
public bool IsDisposed { get; private set; } public bool IsDisposed { get; private set; }
private readonly List<WeakReference<GraphicsResource>> resources = new List<WeakReference<GraphicsResource>>(); private readonly List<WeakReference<GraphicsResource>> resources = new List<WeakReference<GraphicsResource>>();
@ -28,6 +34,26 @@ namespace MoonWorks.Graphics
presentationParameters, presentationParameters,
Conversions.BoolToByte(debugMode) Conversions.BoolToByte(debugMode)
); );
VideoVertexShader = new ShaderModule(this, GetEmbeddedResource("MoonWorks.Shaders.FullscreenVert.spv"));
VideoFragmentShader = new ShaderModule(this, GetEmbeddedResource("MoonWorks.Shaders.YUV2RGBAFrag.spv"));
VideoPipeline = new GraphicsPipeline(
this,
new GraphicsPipelineCreateInfo
{
AttachmentInfo = new GraphicsPipelineAttachmentInfo(
new ColorAttachmentDescription(TextureFormat.R8G8B8A8, ColorAttachmentBlendState.None)
),
DepthStencilState = DepthStencilState.Disable,
VertexShaderInfo = GraphicsShaderInfo.Create(VideoVertexShader, "main", 0),
FragmentShaderInfo = GraphicsShaderInfo.Create(VideoFragmentShader, "main", 3),
VertexInputState = VertexInputState.Empty,
RasterizerState = RasterizerState.CCW_CullNone,
PrimitiveType = PrimitiveType.TriangleList,
MultisampleState = MultisampleState.None
}
);
} }
public CommandBuffer AcquireCommandBuffer() public CommandBuffer AcquireCommandBuffer()
@ -77,6 +103,11 @@ namespace MoonWorks.Graphics
} }
} }
private static Stream GetEmbeddedResource(string name)
{
return typeof(GraphicsDevice).Assembly.GetManifestResourceStream(name);
}
protected virtual void Dispose(bool disposing) protected virtual void Dispose(bool disposing)
{ {
if (!IsDisposed) if (!IsDisposed)

View File

@ -196,6 +196,7 @@ namespace MoonWorks
NativeLibrary.SetDllImportResolver(typeof(RefreshCS.Refresh).Assembly, MapAndLoad); NativeLibrary.SetDllImportResolver(typeof(RefreshCS.Refresh).Assembly, MapAndLoad);
NativeLibrary.SetDllImportResolver(typeof(FAudio).Assembly, MapAndLoad); NativeLibrary.SetDllImportResolver(typeof(FAudio).Assembly, MapAndLoad);
NativeLibrary.SetDllImportResolver(typeof(WellspringCS.Wellspring).Assembly, MapAndLoad); NativeLibrary.SetDllImportResolver(typeof(WellspringCS.Wellspring).Assembly, MapAndLoad);
NativeLibrary.SetDllImportResolver(typeof(Theorafile).Assembly, MapAndLoad);
} }
#endregion #endregion

Binary file not shown.

Binary file not shown.

View File

@ -0,0 +1,9 @@
#version 450
layout(location = 0) out vec2 outTexCoord;
void main()
{
outTexCoord = vec2((gl_VertexIndex << 1) & 2, gl_VertexIndex & 2);
gl_Position = vec4(outTexCoord * 2.0 - 1.0, 0.0, 1.0);
}

View File

@ -0,0 +1,38 @@
/*
* This effect is based on the YUV-to-RGBA GLSL shader found in SDL.
* Thus, it also released under the zlib license:
* http://libsdl.org/license.php
*/
#version 450
layout(location = 0) in vec2 TexCoord;
layout(location = 0) out vec4 FragColor;
layout(binding = 0, set = 1) uniform sampler2D YSampler;
layout(binding = 1, set = 1) uniform sampler2D USampler;
layout(binding = 2, set = 1) uniform sampler2D VSampler;
/* More info about colorspace conversion:
* http://www.equasys.de/colorconversion.html
* http://www.equasys.de/colorformat.html
*/
const vec3 offset = vec3(-0.0625, -0.5, -0.5);
const vec3 Rcoeff = vec3(1.164, 0.000, 1.793);
const vec3 Gcoeff = vec3(1.164, -0.213, -0.533);
const vec3 Bcoeff = vec3(1.164, 2.112, 0.000);
void main()
{
vec3 yuv;
yuv.x = texture(YSampler, TexCoord).r;
yuv.y = texture(USampler, TexCoord).r;
yuv.z = texture(VSampler, TexCoord).r;
yuv += offset;
FragColor.r = dot(yuv, Rcoeff);
FragColor.g = dot(yuv, Gcoeff);
FragColor.b = dot(yuv, Bcoeff);
FragColor.a = 1.0;
}

94
src/Video/Video.cs Normal file
View File

@ -0,0 +1,94 @@
/* Heavily based on https://github.com/FNA-XNA/FNA/blob/master/src/Media/Xiph/Video.cs */
using System;
namespace MoonWorks.Video
{
public class Video : IDisposable
{
public IntPtr Handle => handle;
public int YWidth => yWidth;
public int YHeight => yHeight;
public int UVWidth => uvWidth;
public int UVHeight => uvHeight;
public double FramesPerSecond => fps;
private IntPtr handle;
private int yWidth;
private int yHeight;
private int uvWidth;
private int uvHeight;
private double fps;
private bool disposed;
public Video(string filename)
{
Theorafile.th_pixel_fmt format;
if (!System.IO.File.Exists(filename))
{
throw new ArgumentException("Video file not found!");
}
if (Theorafile.tf_fopen(filename, out handle) < 0)
{
throw new ArgumentException("Invalid video file!");
}
Theorafile.tf_videoinfo(
handle,
out yWidth,
out yHeight,
out fps,
out format
);
if (format == Theorafile.th_pixel_fmt.TH_PF_420)
{
uvWidth = yWidth / 2;
uvHeight = yHeight / 2;
}
else if (format == Theorafile.th_pixel_fmt.TH_PF_422)
{
uvWidth = yWidth / 2;
uvHeight = yHeight;
}
else if (format == Theorafile.th_pixel_fmt.TH_PF_444)
{
uvWidth = yWidth;
uvHeight = yHeight;
}
else
{
throw new NotSupportedException("Unrecognized YUV format!");
}
}
protected virtual void Dispose(bool disposing)
{
if (!disposed)
{
if (disposing)
{
// TODO: dispose managed state (managed objects)
}
Theorafile.tf_close(ref handle);
disposed = true;
}
}
~Video()
{
// Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method
Dispose(disposing: false);
}
public void Dispose()
{
// Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method
Dispose(disposing: true);
GC.SuppressFinalize(this);
}
}
}

285
src/Video/VideoPlayer.cs Normal file
View File

@ -0,0 +1,285 @@
/* Heavily based on https://github.com/FNA-XNA/FNA/blob/master/src/Media/Xiph/VideoPlayer.cs */
using System;
using System.Diagnostics;
using System.Runtime.InteropServices;
using MoonWorks.Graphics;
namespace MoonWorks.Video
{
public enum VideoState
{
Playing,
Paused,
Stopped
}
public unsafe class VideoPlayer : IDisposable
{
public bool Loop { get; set; }
public bool Mute { get; set; }
public float Volume { get; set; }
private Video Video = null;
private VideoState State = VideoState.Stopped;
private void* yuvData = null;
private int yuvDataLength;
private int currentFrame;
private GraphicsDevice GraphicsDevice;
private Texture RenderTexture = null;
private Texture[] YUVTextures = new Texture[3];
private Sampler LinearSampler;
private Stopwatch timer;
private bool disposed;
public VideoPlayer(GraphicsDevice graphicsDevice)
{
GraphicsDevice = graphicsDevice;
timer = new Stopwatch();
LinearSampler = new Sampler(GraphicsDevice, SamplerCreateInfo.LinearClamp);
}
public void Load(Video video)
{
Video = video;
State = VideoState.Stopped;
if (yuvData != null)
{
NativeMemory.Free(yuvData);
}
yuvDataLength = (
(Video.YWidth * Video.YHeight) +
(Video.UVWidth * video.UVHeight * 2)
);
yuvData = NativeMemory.Alloc((nuint) yuvDataLength);
InitializeTheoraStream();
if (Theorafile.tf_hasvideo(Video.Handle) == 1)
{
if (RenderTexture != null)
{
RenderTexture.Dispose();
}
RenderTexture = Texture.CreateTexture2D(
GraphicsDevice,
(uint) Video.YWidth,
(uint) Video.YHeight,
TextureFormat.R8G8B8A8,
TextureUsageFlags.ColorTarget | TextureUsageFlags.Sampler
);
for (int i = 0; i < 3; i += 1)
{
if (YUVTextures[i] != null)
{
YUVTextures[i].Dispose();
}
}
YUVTextures[0] = Texture.CreateTexture2D(
GraphicsDevice,
(uint) Video.YWidth,
(uint) Video.YHeight,
TextureFormat.R8,
TextureUsageFlags.Sampler
);
YUVTextures[1] = Texture.CreateTexture2D(
GraphicsDevice,
(uint) Video.UVWidth,
(uint) Video.UVHeight,
TextureFormat.R8,
TextureUsageFlags.Sampler
);
YUVTextures[2] = Texture.CreateTexture2D(
GraphicsDevice,
(uint) Video.UVWidth,
(uint) Video.UVHeight,
TextureFormat.R8,
TextureUsageFlags.Sampler
);
}
}
public void Play(bool loop = false)
{
if (State == VideoState.Playing)
{
return;
}
Loop = loop;
timer.Start();
State = VideoState.Playing;
}
public void Pause()
{
if (State == VideoState.Paused)
{
return;
}
timer.Stop();
State = VideoState.Paused;
}
public void Stop()
{
if (State == VideoState.Stopped)
{
return;
}
timer.Stop();
timer.Reset();
Theorafile.tf_reset(Video.Handle);
State = VideoState.Stopped;
}
public Texture GetTexture()
{
if (Video == null)
{
throw new InvalidOperationException();
}
if (State == VideoState.Stopped || Video.Handle == IntPtr.Zero || Theorafile.tf_hasvideo(Video.Handle) == 0)
{
return RenderTexture;
}
int thisFrame = (int) (timer.Elapsed.TotalMilliseconds / (1000.0 / Video.FramesPerSecond));
if (thisFrame > currentFrame)
{
if (Theorafile.tf_readvideo(
Video.Handle,
(IntPtr) yuvData,
thisFrame - currentFrame
) == 1 || currentFrame == -1) {
UpdateTexture();
}
currentFrame = thisFrame;
}
bool ended = Theorafile.tf_eos(Video.Handle) == 1;
if (ended)
{
timer.Stop();
timer.Reset();
Theorafile.tf_reset(Video.Handle);
if (Loop)
{
// Start over!
InitializeTheoraStream();
timer.Start();
}
else
{
State = VideoState.Stopped;
}
}
return RenderTexture;
}
private void InitializeTheoraStream()
{
// Grab the first video frame ASAP.
while (Theorafile.tf_readvideo(Video.Handle, (IntPtr) yuvData, 1) == 0);
// Grab the first bit of audio. We're trying to start the decoding ASAP.
if (Theorafile.tf_hasaudio(Video.Handle) == 1)
{
int channels, samplerate;
Theorafile.tf_audioinfo(Video.Handle, out channels, out samplerate);
// TODO: audio stream
}
currentFrame = -1;
}
private void UpdateTexture()
{
var commandBuffer = GraphicsDevice.AcquireCommandBuffer();
commandBuffer.SetTextureDataYUV(
YUVTextures[0],
YUVTextures[1],
YUVTextures[2],
(IntPtr) yuvData,
(uint) yuvDataLength
);
commandBuffer.BeginRenderPass(
new ColorAttachmentInfo(RenderTexture, Color.Black)
);
commandBuffer.BindGraphicsPipeline(GraphicsDevice.VideoPipeline);
commandBuffer.BindFragmentSamplers(
new TextureSamplerBinding(YUVTextures[0], LinearSampler),
new TextureSamplerBinding(YUVTextures[1], LinearSampler),
new TextureSamplerBinding(YUVTextures[2], LinearSampler)
);
commandBuffer.DrawPrimitives(0, 1, 0, 0);
commandBuffer.EndRenderPass();
GraphicsDevice.Submit(commandBuffer);
}
protected virtual void Dispose(bool disposing)
{
if (!disposed)
{
if (disposing)
{
// dispose managed state (managed objects)
RenderTexture.Dispose();
YUVTextures[0].Dispose();
YUVTextures[1].Dispose();
YUVTextures[2].Dispose();
}
// free unmanaged resources (unmanaged objects) and override finalizer
NativeMemory.Free(yuvData);
disposed = true;
}
}
~VideoPlayer()
{
// Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method
Dispose(disposing: false);
}
public void Dispose()
{
// Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method
Dispose(disposing: true);
GC.SuppressFinalize(this);
}
}
}