Shader cross-compiler (#34)

This PR adds a shader cross-compiler program (written in C#) that uses glslc and spirv-cross to compile GLSL source files to a custom, Refresh-specific shader blob format that contains shader code for various backends.

The goal is that whenever you want to compile your shaders, you just run this program (either via `refreshc` or `dotnet run`), passing in the path to the GLSL source file. You can specify which backends you want to support via flags (`--vulkan`, for instance), and it will generate a .refresh blob file for Refresh to consume.

This can also be extended via C# defines and the `partial class` system. An example is the PS5 shader implementation, whose logic is squirreled away in the NDA'd repository but can be called in sequence with the rest of the shader cross-compile logic. This way, we don't have to run two separate programs to compile our shaders to all supported platforms.

Refresh handles all the parsing of the file format in Refresh_CreateShaderModule, so individual backends do not need to concern themselves with the particulars.

Co-authored-by: Caleb Cornett <caleb.cornett@outlook.com>
Reviewed-on: MoonsideGames/Refresh#34
Co-authored-by: TheSpydog <thespydog@noreply.example.org>
Co-committed-by: TheSpydog <thespydog@noreply.example.org>

shadercompiler/Program.cs

@@ -0,0 +1,268 @@
+using System;
+using System.IO;
+using System.Diagnostics;
+
+partial class Program
+{
+	struct CompileShaderData
+	{
+		public string glslPath;
+		public string outputDir;
+		public bool preserveTemp;
+		public bool vulkan;
+		public bool d3d11;
+		public bool ps5;
+	}
+
+	private static void DisplayHelpText()
+	{
+		Console.WriteLine("Usage: refreshc <path-to-glsl-source | directory-with-glsl-source-files>");
+		Console.WriteLine("Options:");
+		Console.WriteLine("  --vulkan           Emit shader compatible with the Refresh Vulkan backend");
+		Console.WriteLine("  --d3d11            Emit shader compatible with the Refresh D3D11 backend");
+		Console.WriteLine("  --ps5              Emit shader compatible with the Refresh PS5 backend");
+		Console.WriteLine("  --out dir          Write output file(s) to the directory `dir`");
+		Console.WriteLine("  --preserve-temp    Do not delete the temp directory after compilation. Useful for debugging.");
+	}
+
+	public static int Main(string[] args)
+	{
+		if (args.Length == 0)
+		{
+			DisplayHelpText();
+			return 1;
+		}
+
+		CompileShaderData data = new CompileShaderData();
+		string inputPath = null;
+
+		for (int i = 0; i < args.Length; i += 1)
+		{
+			switch (args[i])
+			{
+				case "--vulkan":
+					data.vulkan = true;
+					break;
+
+				case "--d3d11":
+					data.d3d11 = true;
+					break;
+
+				case "--ps5":
+					data.ps5 = true;
+					break;
+
+				case "--out":
+					i += 1;
+					data.outputDir = args[i];
+					break;
+
+				case "--preserve-temp":
+					data.preserveTemp = true;
+					break;
+
+				default:
+					if (inputPath == null)
+					{
+						inputPath = args[i];
+					}
+					else
+					{
+						Console.WriteLine($"refreshc: Unknown parameter {args[i]}");
+						return 1;
+					}
+					break;
+			}
+		}
+
+		if (!data.vulkan && !data.d3d11 && !data.ps5)
+		{
+			Console.WriteLine($"refreshc: No Refresh platforms selected!");
+			return 1;
+		}
+
+#if !PS5
+		if (data.ps5)
+		{
+			Console.WriteLine($"refreshc: `PS5` must be defined in the to target the PS5 backend!");
+			return 1;
+		}
+#endif
+
+		if (data.outputDir == null)
+		{
+			data.outputDir = Directory.GetCurrentDirectory();
+		}
+		else if (!Directory.Exists(data.outputDir))
+		{
+			Console.WriteLine($"refreshc: Output directory {data.outputDir} does not exist");
+			return 1;
+		}
+
+		if (Directory.Exists(inputPath))
+		{
+			// Loop over and compile each file in the directory
+			string[] files = Directory.GetFiles(inputPath);
+			foreach (string file in files)
+			{
+				Console.WriteLine($"Compiling {file}");
+				data.glslPath = file;
+				int res = CompileShader(ref data);
+				if (res != 0)
+				{
+					return res;
+				}
+			}
+		}
+		else
+		{
+			if (!File.Exists(inputPath))
+			{
+				Console.WriteLine($"refreshc: glsl source file or directory ({inputPath}) does not exist");
+				return 1;
+			}
+
+			data.glslPath = inputPath;
+			int res = CompileShader(ref data);
+			if (res != 0)
+			{
+				return res;
+			}
+		}
+
+		return 0;
+	}
+
+	static int CompileShader(ref CompileShaderData data)
+	{
+		int res = 0;
+		string shaderName = Path.GetFileNameWithoutExtension(data.glslPath);
+		string shaderType = Path.GetExtension(data.glslPath);
+
+		if (shaderType != ".vert" && shaderType != ".frag" && shaderType != ".comp")
+		{
+			Console.WriteLine("refreshc: Expected glsl source file with extension '.vert', '.frag', or '.comp'");
+			return 1;
+		}
+
+		// Create the temp directory, if needed
+		string tempDir = Path.Combine(Directory.GetCurrentDirectory(), "temp");
+		if (!Directory.Exists(tempDir))
+		{
+			Directory.CreateDirectory(tempDir);
+		}
+
+		// Compile to spirv
+		string spirvPath = Path.Combine(tempDir, $"{shaderName}.spv");
+		res = CompileGlslToSpirv(data.glslPath, shaderName, spirvPath);
+		if (res != 0)
+		{
+			goto cleanup;
+		}
+
+		if (data.d3d11 || data.ps5)
+		{
+			// Transpile to hlsl
+			string hlslPath = Path.Combine(tempDir, $"{shaderName}.hlsl");
+			res = TranslateSpirvToHlsl(spirvPath, hlslPath);
+			if (res != 0)
+			{
+				goto cleanup;
+			}
+
+			// FIXME: Is there a cross-platform way to compile HLSL to DXBC?
+
+#if PS5
+			// Transpile to ps5, if requested
+			if (data.ps5)
+			{
+				res = TranslateHlslToPS5(hlslPath, shaderName, shaderType, tempDir);
+				if (res != 0)
+				{
+					goto cleanup;
+				}
+			}
+#endif
+		}
+
+		// Create the output blob file
+		string outputFilepath = Path.Combine(data.outputDir, $"{shaderName}.refresh");
+		using (FileStream fs = File.Create(outputFilepath))
+		{
+			using (BinaryWriter writer = new BinaryWriter(fs))
+			{
+				// Magic
+				writer.Write(new char[] { 'R', 'F', 'S', 'H'});
+
+				if (data.vulkan)
+				{
+					string inputPath = Path.Combine(tempDir, $"{shaderName}.spv");
+					WriteShaderBlob(writer, inputPath, 1);
+				}
+
+#if PS5
+				if (data.ps5)
+				{
+					string ext = GetPS5ShaderFileExtension();
+					string inputPath = Path.Combine(tempDir, $"{shaderName}{ext}");
+					WriteShaderBlob(writer, inputPath, 2);
+				}
+#endif
+
+				if (data.d3d11)
+				{
+					string inputPath = Path.Combine(tempDir, $"{shaderName}.hlsl");
+					WriteShaderBlob(writer, inputPath, 3);
+				}
+			}
+		}
+
+	cleanup:
+		// Clean up the temp directory
+		if (!data.preserveTemp)
+		{
+			Directory.Delete(tempDir, true);
+		}
+		return res;
+	}
+
+	static void WriteShaderBlob(BinaryWriter writer, string inputPath, byte backend)
+	{
+		byte[] shaderBlob = File.ReadAllBytes(inputPath);
+		writer.Write(backend); // Corresponds to Refresh_Backend
+		writer.Write(shaderBlob.Length);
+		writer.Write(shaderBlob);
+	}
+
+	static int CompileGlslToSpirv(string glslPath, string shaderName, string outputPath)
+	{
+		Process glslc = Process.Start(
+			"glslc",
+			$"{glslPath} -o {outputPath}"
+		);
+		glslc.WaitForExit();
+		if (glslc.ExitCode != 0)
+		{
+			Console.WriteLine($"refreshc: Could not compile GLSL code");
+			return 1;
+		}
+
+		return 0;
+	}
+
+	static int TranslateSpirvToHlsl(string spirvPath, string outputPath)
+	{
+		Process spirvcross = Process.Start(
+			"spirv-cross",
+			$"{spirvPath} --hlsl --shader-model 50 --output {outputPath}"
+		);
+		spirvcross.WaitForExit();
+		if (spirvcross.ExitCode != 0)
+		{
+			Console.WriteLine($"refreshc: Could not translate SPIR-V to HLSL");
+			return 1;
+		}
+
+		return 0;
+	}
+}

shadercompiler/refreshc.csproj

@@ -0,0 +1,9 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net7.0</TargetFramework>
+    <TargetName>refreshc</TargetName>
+  </PropertyGroup>
+
+</Project>

src/Refresh.c

@@ -338,10 +338,52 @@ Refresh_ShaderModule* Refresh_CreateShaderModule(
 	Refresh_Device *device,
 	Refresh_ShaderModuleCreateInfo *shaderModuleCreateInfo
 ) {
+	Refresh_ShaderModuleCreateInfo driverSpecificCreateInfo = { 0, NULL };
+	uint8_t *bytes;
+	uint32_t i, size;
+
 	NULL_RETURN_NULL(device);
+
+	/* verify the magic number in the shader blob header */
+	bytes = (uint8_t*) shaderModuleCreateInfo->byteCode;
+	if (bytes[0] != 'R' || bytes[1] != 'F' || bytes[2] != 'S' || bytes[3] != 'H')
+	{
+		Refresh_LogError("Cannot parse malformed Refresh shader blob!");
+		return NULL;
+	}
+
+	/* find the code for the selected backend */
+	i = 4;
+	while (i < shaderModuleCreateInfo->codeSize)
+	{
+		size = *((uint32_t*) &bytes[i + 1]);
+
+		if (bytes[i] == (uint8_t) selectedBackend)
+		{
+			driverSpecificCreateInfo.codeSize = size;
+			driverSpecificCreateInfo.byteCode = (uint32_t*) &bytes[i + 1 + sizeof(uint32_t)];
+			break;
+		}
+		else
+		{
+			/* skip over the backend byte, the blob size, and the blob */
+			i += 1 + sizeof(uint32_t) + size;
+		}
+	}
+
+	/* verify the shader blob supports the selected backend */
+	if (driverSpecificCreateInfo.byteCode == NULL)
+	{
+		Refresh_LogError(
+			"Cannot create shader module that does not contain shader code for the selected backend! "
+			"Recompile your shader and enable this backend."
+		);
+		return NULL;
+	}
+
 	return device->CreateShaderModule(
 		device->driverData,
-		shaderModuleCreateInfo
+		&driverSpecificCreateInfo
 	);
 }