Use custom blob format for combining shader code for multiple backends into a single file

shadercompiler/Program.cs

@@ -4,20 +4,98 @@ using System.Diagnostics;
 
 partial class Program
 {
+	struct CompileShaderData
+	{
+		public string glslPath;
+		public string outputDir;
+		public bool preserveTemp;
+		public bool vulkan;
+		public bool d3d11;
+		public bool ps5;
+	}
+
+	private static void DisplayHelpText()
+	{
+		Console.WriteLine("Usage: refreshc <path-to-glsl-source | directory-with-glsl-source-files>");
+		Console.WriteLine("Options:");
+		Console.WriteLine("  --vulkan           Emit shader compatible with the Refresh Vulkan backend");
+		Console.WriteLine("  --d3d11            Emit shader compatible with the Refresh D3D11 backend");
+		Console.WriteLine("  --ps5              Emit shader compatible with the Refresh PS5 backend");
+		Console.WriteLine("  --out dir          Write output file(s) to the directory `dir`");
+		Console.WriteLine("  --preserve-temp    Do not delete the temp directory after compilation. Useful for debugging.");
+	}
+
 	public static int Main(string[] args)
 	{
-		if (args.Length < 2)
+		if (args.Length == 0)
 		{
-			Console.WriteLine("Usage: refreshc <path-to-glsl | path-to-input-directory> <output-directory>");
+			DisplayHelpText();
 			return 1;
 		}
 
-		string inputPath = args[0];
+		CompileShaderData data = new CompileShaderData();
-		string outputDir = args[1];
+		string inputPath = null;
 
-		if (!Directory.Exists(outputDir))
+		for (int i = 0; i < args.Length; i += 1)
 		{
-			Console.WriteLine($"refreshc: Output directory ({outputDir}) does not exist");
+			switch (args[i])
+			{
+				case "--vulkan":
+					data.vulkan = true;
+					break;
+
+				case "--d3d11":
+					data.d3d11 = true;
+					break;
+
+				case "--ps5":
+					data.ps5 = true;
+					break;
+
+				case "--out":
+					i += 1;
+					data.outputDir = args[i];
+					break;
+
+				case "--preserve-temp":
+					data.preserveTemp = true;
+					break;
+
+				default:
+					if (inputPath == null)
+					{
+						inputPath = args[i];
+					}
+					else
+					{
+						Console.WriteLine($"refreshc: Unknown parameter {args[i]}");
+						return 1;
+					}
+					break;
+			}
+		}
+
+		if (!data.vulkan && !data.d3d11 && !data.ps5)
+		{
+			Console.WriteLine($"refreshc: No Refresh platforms selected!");
+			return 1;
+		}
+
+#if !PS5
+		if (data.ps5)
+		{
+			Console.WriteLine($"refreshc: `PS5` must be defined in the to target the PS5 backend!");
+			return 1;
+		}
+#endif
+
+		if (data.outputDir == null)
+		{
+			data.outputDir = Directory.GetCurrentDirectory();
+		}
+		else if (!Directory.Exists(data.outputDir))
+		{
+			Console.WriteLine($"refreshc: Output directory {data.outputDir} does not exist");
 			return 1;
 		}
 
@@ -28,7 +106,8 @@ partial class Program
 			foreach (string file in files)
 			{
 				Console.WriteLine($"Compiling {file}");
-				int res = CompileShader(file, outputDir);
+				data.glslPath = file;
+				int res = CompileShader(ref data);
 				if (res != 0)
 				{
 					return res;
@@ -43,7 +122,8 @@ partial class Program
 				return 1;
 			}
 
-			int res = CompileShader(inputPath, outputDir);
+			data.glslPath = inputPath;
+			int res = CompileShader(ref data);
 			if (res != 0)
 			{
 				return res;
@@ -53,11 +133,11 @@ partial class Program
 		return 0;
 	}
 
-	static int CompileShader(string glslPath, string outputDir)
+	static int CompileShader(ref CompileShaderData data)
 	{
 		int res = 0;
-		string shaderName = Path.GetFileNameWithoutExtension(glslPath);
+		string shaderName = Path.GetFileNameWithoutExtension(data.glslPath);
-		string shaderType = Path.GetExtension(glslPath);
+		string shaderType = Path.GetExtension(data.glslPath);
 
 		if (shaderType != ".vert" && shaderType != ".frag" && shaderType != ".comp")
 		{
@@ -65,33 +145,95 @@ partial class Program
 			return 1;
 		}
 
-		string spirvPath = Path.Combine(outputDir, $"{shaderName}.spv");
+		// Create the temp directory, if needed
-		res = CompileGlslToSpirv(glslPath, shaderName, spirvPath);
+		string tempDir = Path.Combine(Directory.GetCurrentDirectory(), "temp");
-		if (res != 0)
+		if (!Directory.Exists(tempDir))
 		{
-			return res;
+			Directory.CreateDirectory(tempDir);
 		}
 
-		string hlslPath = Path.Combine(outputDir, $"{shaderName}.hlsl");
+		// Compile to spirv
+		string spirvPath = Path.Combine(tempDir, $"{shaderName}.spv");
+		res = CompileGlslToSpirv(data.glslPath, shaderName, spirvPath);
+		if (res != 0)
+		{
+			goto cleanup;
+		}
+
+		if (data.d3d11 || data.ps5)
+		{
+			// Transpile to hlsl
+			string hlslPath = Path.Combine(tempDir, $"{shaderName}.hlsl");
 			res = TranslateSpirvToHlsl(spirvPath, hlslPath);
 			if (res != 0)
 			{
-			return res;
+				goto cleanup;
 			}
 
 			// FIXME: Is there a cross-platform way to compile HLSL to DXBC?
 
 #if PS5
-		res = TranslateHlslToPS5(hlslPath, shaderName, shaderType, outputDir);
+			// Transpile to ps5, if requested
+			if (data.ps5)
+			{
+				res = TranslateHlslToPS5(hlslPath, shaderName, shaderType, tempDir);
 				if (res != 0)
 				{
-			return res;
+					goto cleanup;
+				}
+			}
+#endif
+		}
+
+		// Create the output blob file
+		string outputFilepath = Path.Combine(data.outputDir, $"{shaderName}.refresh");
+		using (FileStream fs = File.Create(outputFilepath))
+		{
+			using (BinaryWriter writer = new BinaryWriter(fs))
+			{
+				// Magic
+				writer.Write(new char[] { 'R', 'F', 'S', 'H'});
+
+				if (data.vulkan)
+				{
+					string inputPath = Path.Combine(tempDir, $"{shaderName}.spv");
+					WriteShaderBlob(writer, inputPath, 1);
+				}
+
+#if PS5
+				if (data.ps5)
+				{
+					string ext = GetPS5ShaderFileExtension();
+					string inputPath = Path.Combine(tempDir, $"{shaderName}{ext}");
+					WriteShaderBlob(writer, inputPath, 2);
 				}
 #endif
 
+				if (data.d3d11)
+				{
+					string inputPath = Path.Combine(tempDir, $"{shaderName}.hlsl");
+					WriteShaderBlob(writer, inputPath, 3);
+				}
+			}
+		}
+
+	cleanup:
+		// Clean up the temp directory
+		if (!data.preserveTemp)
+		{
+			Directory.Delete(tempDir, true);
+		}
 		return res;
 	}
 
+	static void WriteShaderBlob(BinaryWriter writer, string inputPath, byte backend)
+	{
+		byte[] shaderBlob = File.ReadAllBytes(inputPath);
+		writer.Write(backend); // Corresponds to Refresh_Backend
+		writer.Write(shaderBlob.Length);
+		writer.Write(shaderBlob);
+	}
+
 	static int CompileGlslToSpirv(string glslPath, string shaderName, string outputPath)
 	{
 		Process glslc = Process.Start(

src/Refresh.c

@@ -338,10 +338,52 @@ Refresh_ShaderModule* Refresh_CreateShaderModule(
 	Refresh_Device *device,
 	Refresh_ShaderModuleCreateInfo *shaderModuleCreateInfo
 ) {
+	Refresh_ShaderModuleCreateInfo driverSpecificCreateInfo = { 0, NULL };
+	uint8_t *bytes;
+	uint32_t i, size;
+
 	NULL_RETURN_NULL(device);
+
+	/* verify the magic number in the shader blob header */
+	bytes = (uint8_t*) shaderModuleCreateInfo->byteCode;
+	if (bytes[0] != 'R' || bytes[1] != 'F' || bytes[2] != 'S' || bytes[3] != 'H')
+	{
+		Refresh_LogError("Cannot parse malformed Refresh shader blob!");
+		return NULL;
+	}
+
+	/* find the code for the selected backend */
+	i = 4;
+	while (i < shaderModuleCreateInfo->codeSize)
+	{
+		size = *((uint32_t*) &bytes[i + 1]);
+
+		if (bytes[i] == (uint8_t) selectedBackend)
+		{
+			driverSpecificCreateInfo.codeSize = size;
+			driverSpecificCreateInfo.byteCode = (uint32_t*) &bytes[i + 1 + sizeof(uint32_t)];
+			break;
+		}
+		else
+		{
+			/* skip over the backend byte, the blob size, and the blob */
+			i += 1 + sizeof(uint32_t) + size;
+		}
+	}
+
+	/* verify the shader blob supports the selected backend */
+	if (driverSpecificCreateInfo.byteCode == NULL)
+	{
+		Refresh_LogError(
+			"Cannot create shader module that does not contain shader code for the selected backend! "
+			"Recompile your shader and enable this backend."
+		);
+		return NULL;
+	}
+
 	return device->CreateShaderModule(
 		device->driverData,
-		shaderModuleCreateInfo
+		&driverSpecificCreateInfo
 	);
 }