From 2b773101c822fbad217b5df478011c5c9284f32d Mon Sep 17 00:00:00 2001 From: Caleb Cornett Date: Sat, 14 Jan 2023 23:34:55 -0500 Subject: [PATCH 1/3] Added shader cross-compiler using glslc and spirv-cross --- shadercompiler/Program.cs | 105 +++++++++++++++++++++++++++++++++ shadercompiler/refreshc.csproj | 9 +++ 2 files changed, 114 insertions(+) create mode 100644 shadercompiler/Program.cs create mode 100644 shadercompiler/refreshc.csproj diff --git a/shadercompiler/Program.cs b/shadercompiler/Program.cs new file mode 100644 index 0000000..ae2a780 --- /dev/null +++ b/shadercompiler/Program.cs @@ -0,0 +1,105 @@ +using System; +using System.IO; +using System.Diagnostics; + +partial class Program +{ + public static int Main(string[] args) + { + if (args.Length < 2) + { + Console.WriteLine("Usage: refreshc "); + return 1; + } + + string glslPath = args[0]; + string outputDir = args[1]; + + if (!ValidateArgs(glslPath, outputDir)) + { + return 1; + } + + string shaderName = Path.GetFileNameWithoutExtension(glslPath); + string shaderType = Path.GetExtension(glslPath); + string spirvPath = Path.Combine(outputDir, $"{shaderName}.spv"); + + if (CompileGlslToSpirv(glslPath, shaderName, spirvPath) != 0) + { + return 1; + } + + string hlslPath = Path.Combine(outputDir, $"{shaderName}.hlsl"); + if (TranslateSpirvToHlsl(spirvPath, hlslPath) != 0) + { + return 1; + } + + // FIXME: Is there a cross-platform way to compile HLSL to DXBC? + +#if PS5 + if (TranslateHlslToPS5(hlslPath, shaderName, shaderType, outputDir) != 0) + { + return 1; + } +#endif + + return 0; + } + + static bool ValidateArgs(string glslPath, string outputDir) + { + if (!File.Exists(glslPath)) + { + Console.WriteLine($"refreshc: glsl source file ({glslPath}) does not exist"); + return false; + } + + string ext = Path.GetExtension(glslPath); + if (ext != ".vert" && ext != ".frag" && ext != ".comp") + { + Console.WriteLine("refreshc: Expected glsl source file with extension '.vert', '.frag', or '.comp'"); + return false; + } + + if (!Directory.Exists(outputDir)) + { + Console.WriteLine($"refreshc: Output directory ({outputDir}) does not exist"); + return false; + } + + return true; + } + + static int CompileGlslToSpirv(string glslPath, string shaderName, string outputPath) + { + Process glslc = Process.Start( + "glslc", + $"{glslPath} -o {outputPath}" + ); + glslc.WaitForExit(); + if (glslc.ExitCode != 0) + { + Console.WriteLine($"refreshc: Could not compile GLSL code"); + return 1; + } + + return 0; + } + + static int TranslateSpirvToHlsl(string spirvPath, string outputPath) + { + Process spirvcross = Process.Start( + "spirv-cross", + $"{spirvPath} --hlsl --shader-model 50 --output {outputPath}" + ); + spirvcross.WaitForExit(); + if (spirvcross.ExitCode != 0) + { + Console.WriteLine($"refreshc: Could not translate SPIR-V to HLSL"); + return 1; + } + + return 0; + } +} diff --git a/shadercompiler/refreshc.csproj b/shadercompiler/refreshc.csproj new file mode 100644 index 0000000..cce8a4b --- /dev/null +++ b/shadercompiler/refreshc.csproj @@ -0,0 +1,9 @@ + + + + Exe + net7.0 + refreshc + + + -- 2.25.1 From 8a0d4472b4f1e4d9a55eb9335cf4be1cf603c4a6 Mon Sep 17 00:00:00 2001 From: Caleb Cornett Date: Sun, 15 Jan 2023 15:04:28 -0500 Subject: [PATCH 2/3] Add support for compiling all files in a directory --- shadercompiler/Program.cs | 89 ++++++++++++++++++++++++--------------- 1 file changed, 55 insertions(+), 34 deletions(-) diff --git a/shadercompiler/Program.cs b/shadercompiler/Program.cs index ae2a780..59d43fc 100644 --- a/shadercompiler/Program.cs +++ b/shadercompiler/Program.cs @@ -8,67 +8,88 @@ partial class Program { if (args.Length < 2) { - Console.WriteLine("Usage: refreshc "); + Console.WriteLine("Usage: refreshc "); return 1; } - string glslPath = args[0]; + string inputPath = args[0]; string outputDir = args[1]; - if (!ValidateArgs(glslPath, outputDir)) + if (!Directory.Exists(outputDir)) { + Console.WriteLine($"refreshc: Output directory ({outputDir}) does not exist"); return 1; } + if (Directory.Exists(inputPath)) + { + // Loop over and compile each file in the directory + string[] files = Directory.GetFiles(inputPath); + foreach (string file in files) + { + Console.WriteLine($"Compiling {file}"); + int res = CompileShader(file, outputDir); + if (res != 0) + { + return res; + } + } + } + else + { + if (!File.Exists(inputPath)) + { + Console.WriteLine($"refreshc: glsl source file or directory ({inputPath}) does not exist"); + return 1; + } + + int res = CompileShader(inputPath, outputDir); + if (res != 0) + { + return res; + } + } + + return 0; + } + + static int CompileShader(string glslPath, string outputDir) + { + int res = 0; string shaderName = Path.GetFileNameWithoutExtension(glslPath); string shaderType = Path.GetExtension(glslPath); - string spirvPath = Path.Combine(outputDir, $"{shaderName}.spv"); - if (CompileGlslToSpirv(glslPath, shaderName, spirvPath) != 0) + if (shaderType != ".vert" && shaderType != ".frag" && shaderType != ".comp") { + Console.WriteLine("refreshc: Expected glsl source file with extension '.vert', '.frag', or '.comp'"); return 1; } + string spirvPath = Path.Combine(outputDir, $"{shaderName}.spv"); + res = CompileGlslToSpirv(glslPath, shaderName, spirvPath); + if (res != 0) + { + return res; + } + string hlslPath = Path.Combine(outputDir, $"{shaderName}.hlsl"); - if (TranslateSpirvToHlsl(spirvPath, hlslPath) != 0) + res = TranslateSpirvToHlsl(spirvPath, hlslPath); + if (res != 0) { - return 1; + return res; } // FIXME: Is there a cross-platform way to compile HLSL to DXBC? #if PS5 - if (TranslateHlslToPS5(hlslPath, shaderName, shaderType, outputDir) != 0) + res = TranslateHlslToPS5(hlslPath, shaderName, shaderType, outputDir); + if (res != 0) { - return 1; + return res; } #endif - return 0; - } - - static bool ValidateArgs(string glslPath, string outputDir) - { - if (!File.Exists(glslPath)) - { - Console.WriteLine($"refreshc: glsl source file ({glslPath}) does not exist"); - return false; - } - - string ext = Path.GetExtension(glslPath); - if (ext != ".vert" && ext != ".frag" && ext != ".comp") - { - Console.WriteLine("refreshc: Expected glsl source file with extension '.vert', '.frag', or '.comp'"); - return false; - } - - if (!Directory.Exists(outputDir)) - { - Console.WriteLine($"refreshc: Output directory ({outputDir}) does not exist"); - return false; - } - - return true; + return res; } static int CompileGlslToSpirv(string glslPath, string shaderName, string outputPath) -- 2.25.1 From ec2a3a273bd4b2a47ca2020795427f7eb6263302 Mon Sep 17 00:00:00 2001 From: Caleb Cornett Date: Fri, 20 Jan 2023 15:04:01 -0500 Subject: [PATCH 3/3] Use custom blob format for combining shader code for multiple backends into a single file --- shadercompiler/Program.cs | 188 +++++++++++++++++++++++++++++++++----- src/Refresh.c | 44 ++++++++- 2 files changed, 208 insertions(+), 24 deletions(-) diff --git a/shadercompiler/Program.cs b/shadercompiler/Program.cs index 59d43fc..2a11626 100644 --- a/shadercompiler/Program.cs +++ b/shadercompiler/Program.cs @@ -4,20 +4,98 @@ using System.Diagnostics; partial class Program { + struct CompileShaderData + { + public string glslPath; + public string outputDir; + public bool preserveTemp; + public bool vulkan; + public bool d3d11; + public bool ps5; + } + + private static void DisplayHelpText() + { + Console.WriteLine("Usage: refreshc "); + Console.WriteLine("Options:"); + Console.WriteLine(" --vulkan Emit shader compatible with the Refresh Vulkan backend"); + Console.WriteLine(" --d3d11 Emit shader compatible with the Refresh D3D11 backend"); + Console.WriteLine(" --ps5 Emit shader compatible with the Refresh PS5 backend"); + Console.WriteLine(" --out dir Write output file(s) to the directory `dir`"); + Console.WriteLine(" --preserve-temp Do not delete the temp directory after compilation. Useful for debugging."); + } + public static int Main(string[] args) { - if (args.Length < 2) + if (args.Length == 0) { - Console.WriteLine("Usage: refreshc "); + DisplayHelpText(); return 1; } - string inputPath = args[0]; - string outputDir = args[1]; + CompileShaderData data = new CompileShaderData(); + string inputPath = null; - if (!Directory.Exists(outputDir)) + for (int i = 0; i < args.Length; i += 1) { - Console.WriteLine($"refreshc: Output directory ({outputDir}) does not exist"); + switch (args[i]) + { + case "--vulkan": + data.vulkan = true; + break; + + case "--d3d11": + data.d3d11 = true; + break; + + case "--ps5": + data.ps5 = true; + break; + + case "--out": + i += 1; + data.outputDir = args[i]; + break; + + case "--preserve-temp": + data.preserveTemp = true; + break; + + default: + if (inputPath == null) + { + inputPath = args[i]; + } + else + { + Console.WriteLine($"refreshc: Unknown parameter {args[i]}"); + return 1; + } + break; + } + } + + if (!data.vulkan && !data.d3d11 && !data.ps5) + { + Console.WriteLine($"refreshc: No Refresh platforms selected!"); + return 1; + } + +#if !PS5 + if (data.ps5) + { + Console.WriteLine($"refreshc: `PS5` must be defined in the to target the PS5 backend!"); + return 1; + } +#endif + + if (data.outputDir == null) + { + data.outputDir = Directory.GetCurrentDirectory(); + } + else if (!Directory.Exists(data.outputDir)) + { + Console.WriteLine($"refreshc: Output directory {data.outputDir} does not exist"); return 1; } @@ -28,7 +106,8 @@ partial class Program foreach (string file in files) { Console.WriteLine($"Compiling {file}"); - int res = CompileShader(file, outputDir); + data.glslPath = file; + int res = CompileShader(ref data); if (res != 0) { return res; @@ -43,7 +122,8 @@ partial class Program return 1; } - int res = CompileShader(inputPath, outputDir); + data.glslPath = inputPath; + int res = CompileShader(ref data); if (res != 0) { return res; @@ -53,11 +133,11 @@ partial class Program return 0; } - static int CompileShader(string glslPath, string outputDir) + static int CompileShader(ref CompileShaderData data) { int res = 0; - string shaderName = Path.GetFileNameWithoutExtension(glslPath); - string shaderType = Path.GetExtension(glslPath); + string shaderName = Path.GetFileNameWithoutExtension(data.glslPath); + string shaderType = Path.GetExtension(data.glslPath); if (shaderType != ".vert" && shaderType != ".frag" && shaderType != ".comp") { @@ -65,33 +145,95 @@ partial class Program return 1; } - string spirvPath = Path.Combine(outputDir, $"{shaderName}.spv"); - res = CompileGlslToSpirv(glslPath, shaderName, spirvPath); - if (res != 0) + // Create the temp directory, if needed + string tempDir = Path.Combine(Directory.GetCurrentDirectory(), "temp"); + if (!Directory.Exists(tempDir)) { - return res; + Directory.CreateDirectory(tempDir); } - string hlslPath = Path.Combine(outputDir, $"{shaderName}.hlsl"); - res = TranslateSpirvToHlsl(spirvPath, hlslPath); + // Compile to spirv + string spirvPath = Path.Combine(tempDir, $"{shaderName}.spv"); + res = CompileGlslToSpirv(data.glslPath, shaderName, spirvPath); if (res != 0) { - return res; + goto cleanup; } - // FIXME: Is there a cross-platform way to compile HLSL to DXBC? + if (data.d3d11 || data.ps5) + { + // Transpile to hlsl + string hlslPath = Path.Combine(tempDir, $"{shaderName}.hlsl"); + res = TranslateSpirvToHlsl(spirvPath, hlslPath); + if (res != 0) + { + goto cleanup; + } + + // FIXME: Is there a cross-platform way to compile HLSL to DXBC? #if PS5 - res = TranslateHlslToPS5(hlslPath, shaderName, shaderType, outputDir); - if (res != 0) - { - return res; + // Transpile to ps5, if requested + if (data.ps5) + { + res = TranslateHlslToPS5(hlslPath, shaderName, shaderType, tempDir); + if (res != 0) + { + goto cleanup; + } + } +#endif } + + // Create the output blob file + string outputFilepath = Path.Combine(data.outputDir, $"{shaderName}.refresh"); + using (FileStream fs = File.Create(outputFilepath)) + { + using (BinaryWriter writer = new BinaryWriter(fs)) + { + // Magic + writer.Write(new char[] { 'R', 'F', 'S', 'H'}); + + if (data.vulkan) + { + string inputPath = Path.Combine(tempDir, $"{shaderName}.spv"); + WriteShaderBlob(writer, inputPath, 1); + } + +#if PS5 + if (data.ps5) + { + string ext = GetPS5ShaderFileExtension(); + string inputPath = Path.Combine(tempDir, $"{shaderName}{ext}"); + WriteShaderBlob(writer, inputPath, 2); + } #endif + if (data.d3d11) + { + string inputPath = Path.Combine(tempDir, $"{shaderName}.hlsl"); + WriteShaderBlob(writer, inputPath, 3); + } + } + } + + cleanup: + // Clean up the temp directory + if (!data.preserveTemp) + { + Directory.Delete(tempDir, true); + } return res; } + static void WriteShaderBlob(BinaryWriter writer, string inputPath, byte backend) + { + byte[] shaderBlob = File.ReadAllBytes(inputPath); + writer.Write(backend); // Corresponds to Refresh_Backend + writer.Write(shaderBlob.Length); + writer.Write(shaderBlob); + } + static int CompileGlslToSpirv(string glslPath, string shaderName, string outputPath) { Process glslc = Process.Start( diff --git a/src/Refresh.c b/src/Refresh.c index c78b721..2f9c0bf 100644 --- a/src/Refresh.c +++ b/src/Refresh.c @@ -338,10 +338,52 @@ Refresh_ShaderModule* Refresh_CreateShaderModule( Refresh_Device *device, Refresh_ShaderModuleCreateInfo *shaderModuleCreateInfo ) { + Refresh_ShaderModuleCreateInfo driverSpecificCreateInfo = { 0, NULL }; + uint8_t *bytes; + uint32_t i, size; + NULL_RETURN_NULL(device); + + /* verify the magic number in the shader blob header */ + bytes = (uint8_t*) shaderModuleCreateInfo->byteCode; + if (bytes[0] != 'R' || bytes[1] != 'F' || bytes[2] != 'S' || bytes[3] != 'H') + { + Refresh_LogError("Cannot parse malformed Refresh shader blob!"); + return NULL; + } + + /* find the code for the selected backend */ + i = 4; + while (i < shaderModuleCreateInfo->codeSize) + { + size = *((uint32_t*) &bytes[i + 1]); + + if (bytes[i] == (uint8_t) selectedBackend) + { + driverSpecificCreateInfo.codeSize = size; + driverSpecificCreateInfo.byteCode = (uint32_t*) &bytes[i + 1 + sizeof(uint32_t)]; + break; + } + else + { + /* skip over the backend byte, the blob size, and the blob */ + i += 1 + sizeof(uint32_t) + size; + } + } + + /* verify the shader blob supports the selected backend */ + if (driverSpecificCreateInfo.byteCode == NULL) + { + Refresh_LogError( + "Cannot create shader module that does not contain shader code for the selected backend! " + "Recompile your shader and enable this backend." + ); + return NULL; + } + return device->CreateShaderModule( device->driverData, - shaderModuleCreateInfo + &driverSpecificCreateInfo ); } -- 2.25.1