using System; using System.Diagnostics; using System.Threading.Tasks; using MoonWorks.Graphics; namespace MoonWorks.Video { /// /// A structure for continuous decoding of AV1 videos and rendering them into a texture. /// public unsafe class VideoPlayer : GraphicsResource { public Texture RenderTexture { get; private set; } = null; public VideoState State { get; private set; } = VideoState.Stopped; public bool Loop { get; set; } public float PlaybackSpeed { get; set; } = 1; private VideoAV1 Video = null; private VideoAV1Stream CurrentStream = null; private Task ReadNextFrameTask; private Task ResetTask; private Task ResetSecondaryStreamTask; private Texture yTexture = null; private Texture uTexture = null; private Texture vTexture = null; private Sampler LinearSampler; private TransferBuffer TransferBuffer; private int currentFrame; private Stopwatch timer; private double lastTimestamp; private double timeElapsed; public VideoPlayer(GraphicsDevice device) : base(device) { LinearSampler = new Sampler(device, SamplerCreateInfo.LinearClamp); timer = new Stopwatch(); } /// /// Prepares a VideoAV1 for decoding and rendering. /// /// public void Load(VideoAV1 video) { if (Video != video) { Stop(); if (RenderTexture == null) { RenderTexture = CreateRenderTexture(Device, video.Width, video.Height); } if (yTexture == null) { yTexture = CreateSubTexture(Device, video.Width, video.Height); } if (uTexture == null) { uTexture = CreateSubTexture(Device, video.UVWidth, video.UVHeight); } if (vTexture == null) { vTexture = CreateSubTexture(Device, video.UVWidth, video.UVHeight); } if (video.Width != RenderTexture.Width || video.Height != RenderTexture.Height) { RenderTexture.Dispose(); RenderTexture = CreateRenderTexture(Device, video.Width, video.Height); } if (video.Width != yTexture.Width || video.Height != yTexture.Height) { yTexture.Dispose(); yTexture = CreateSubTexture(Device, video.Width, video.Height); } if (video.UVWidth != uTexture.Width || video.UVHeight != uTexture.Height) { uTexture.Dispose(); uTexture = CreateSubTexture(Device, video.UVWidth, video.UVHeight); } if (video.UVWidth != vTexture.Width || video.UVHeight != vTexture.Height) { vTexture.Dispose(); vTexture = CreateSubTexture(Device, video.UVWidth, video.UVHeight); } Video = video; InitializeDav1dStream(); } } /// /// Starts playing back and decoding the loaded video. /// public void Play() { if (Video == null) { return; } if (State == VideoState.Playing) { return; } timer.Start(); State = VideoState.Playing; } /// /// Pauses playback and decoding of the currently playing video. /// public void Pause() { if (Video == null) { return; } if (State != VideoState.Playing) { return; } timer.Stop(); State = VideoState.Paused; } /// /// Stops and resets decoding of the currently playing video. /// public void Stop() { if (Video == null) { return; } if (State == VideoState.Stopped) { return; } timer.Stop(); timer.Reset(); lastTimestamp = 0; timeElapsed = 0; InitializeDav1dStream(); State = VideoState.Stopped; } /// /// Unloads the currently playing video. /// public void Unload() { Stop(); ResetTask?.Wait(); ResetSecondaryStreamTask?.Wait(); Video = null; } /// /// Renders the video data into RenderTexture. /// public void Render() { if (Video == null || State == VideoState.Stopped) { return; } timeElapsed += (timer.Elapsed.TotalMilliseconds - lastTimestamp) * PlaybackSpeed; lastTimestamp = timer.Elapsed.TotalMilliseconds; int thisFrame = ((int) (timeElapsed / (1000.0 / Video.FramesPerSecond))); if (thisFrame > currentFrame) { if (CurrentStream.FrameDataUpdated) { UpdateRenderTexture(); CurrentStream.FrameDataUpdated = false; } currentFrame = thisFrame; ReadNextFrameTask = Task.Run(CurrentStream.ReadNextFrame); ReadNextFrameTask.ContinueWith(HandleTaskException, TaskContinuationOptions.OnlyOnFaulted); } if (CurrentStream.Ended) { timer.Stop(); timer.Reset(); ResetTask = Task.Run(CurrentStream.Reset); ResetTask.ContinueWith(HandleTaskException, TaskContinuationOptions.OnlyOnFaulted); if (Loop) { // Start over on the next stream! CurrentStream = (CurrentStream == Video.StreamA) ? Video.StreamB : Video.StreamA; currentFrame = -1; timer.Start(); } else { State = VideoState.Stopped; } } } private void UpdateRenderTexture() { lock (CurrentStream) { ResetTask?.Wait(); var commandBuffer = Device.AcquireCommandBuffer(); var ySpan = new Span((void*) CurrentStream.yDataHandle, (int) CurrentStream.yDataLength); var uSpan = new Span((void*) CurrentStream.uDataHandle, (int) CurrentStream.uvDataLength); var vSpan = new Span((void*) CurrentStream.vDataHandle, (int) CurrentStream.uvDataLength); if (TransferBuffer == null || TransferBuffer.Size < ySpan.Length + uSpan.Length + vSpan.Length) { TransferBuffer?.Dispose(); TransferBuffer = new TransferBuffer(Device, TransferUsage.Texture, (uint) (ySpan.Length + uSpan.Length + vSpan.Length)); } TransferBuffer.SetData(ySpan, 0, TransferOptions.Cycle); TransferBuffer.SetData(uSpan, (uint) ySpan.Length, TransferOptions.Unsafe); TransferBuffer.SetData(vSpan, (uint) (ySpan.Length + uSpan.Length), TransferOptions.Unsafe); commandBuffer.BeginCopyPass(); commandBuffer.UploadToTexture( TransferBuffer, yTexture, new BufferImageCopy { BufferOffset = 0, BufferStride = CurrentStream.yStride, BufferImageHeight = yTexture.Height }, WriteOptions.Cycle ); commandBuffer.UploadToTexture( TransferBuffer, uTexture, new BufferImageCopy{ BufferOffset = (uint) ySpan.Length, BufferStride = CurrentStream.uvStride, BufferImageHeight = uTexture.Height }, WriteOptions.Cycle ); commandBuffer.UploadToTexture( TransferBuffer, vTexture, new BufferImageCopy { BufferOffset = (uint) (ySpan.Length + uSpan.Length), BufferStride = CurrentStream.uvStride, BufferImageHeight = vTexture.Height }, WriteOptions.Cycle ); commandBuffer.EndCopyPass(); commandBuffer.BeginRenderPass( new ColorAttachmentInfo(RenderTexture, WriteOptions.Cycle, Color.Black) ); commandBuffer.BindGraphicsPipeline(Device.VideoPipeline); commandBuffer.BindFragmentSamplers( new TextureSamplerBinding(yTexture, LinearSampler), new TextureSamplerBinding(uTexture, LinearSampler), new TextureSamplerBinding(vTexture, LinearSampler) ); commandBuffer.DrawPrimitives(0, 1); commandBuffer.EndRenderPass(); Device.Submit(commandBuffer); } } private static Texture CreateRenderTexture(GraphicsDevice graphicsDevice, int width, int height) { return Texture.CreateTexture2D( graphicsDevice, (uint) width, (uint) height, TextureFormat.R8G8B8A8, TextureUsageFlags.ColorTarget | TextureUsageFlags.Sampler ); } private static Texture CreateSubTexture(GraphicsDevice graphicsDevice, int width, int height) { return Texture.CreateTexture2D( graphicsDevice, (uint) width, (uint) height, TextureFormat.R8, TextureUsageFlags.Sampler ); } private void InitializeDav1dStream() { ReadNextFrameTask?.Wait(); ResetTask = Task.Run(Video.StreamA.Reset); ResetTask.ContinueWith(HandleTaskException, TaskContinuationOptions.OnlyOnFaulted); ResetSecondaryStreamTask = Task.Run(Video.StreamB.Reset); ResetSecondaryStreamTask.ContinueWith(HandleTaskException, TaskContinuationOptions.OnlyOnFaulted); CurrentStream = Video.StreamA; currentFrame = -1; } private static void HandleTaskException(Task task) { if (task.Exception.InnerException is not TaskCanceledException) { throw task.Exception; } } protected override void Dispose(bool disposing) { if (!IsDisposed) { if (disposing) { Unload(); RenderTexture?.Dispose(); yTexture?.Dispose(); uTexture?.Dispose(); vTexture?.Dispose(); } } base.Dispose(disposing); } } }