Shader cross-compiler (#34)
continuous-integration/drone/push Build is passing Details

This PR adds a shader cross-compiler program (written in C#) that uses glslc and spirv-cross to compile GLSL source files to a custom, Refresh-specific shader blob format that contains shader code for various backends.

The goal is that whenever you want to compile your shaders, you just run this program (either via `refreshc` or `dotnet run`), passing in the path to the GLSL source file. You can specify which backends you want to support via flags (`--vulkan`, for instance), and it will generate a .refresh blob file for Refresh to consume.

This can also be extended via C# defines and the `partial class` system. An example is the PS5 shader implementation, whose logic is squirreled away in the NDA'd repository but can be called in sequence with the rest of the shader cross-compile logic. This way, we don't have to run two separate programs to compile our shaders to all supported platforms.

Refresh handles all the parsing of the file format in Refresh_CreateShaderModule, so individual backends do not need to concern themselves with the particulars.

Co-authored-by: Caleb Cornett <caleb.cornett@outlook.com>
Reviewed-on: #34
Co-authored-by: TheSpydog <thespydog@noreply.example.org>
Co-committed-by: TheSpydog <thespydog@noreply.example.org>
pull/35/head
TheSpydog 2023-01-20 23:19:12 +00:00 committed by cosmonaut
parent f7250ab12a
commit 05900bee14
3 changed files with 320 additions and 1 deletions

268
shadercompiler/Program.cs Normal file
View File

@ -0,0 +1,268 @@
using System;
using System.IO;
using System.Diagnostics;
partial class Program
{
struct CompileShaderData
{
public string glslPath;
public string outputDir;
public bool preserveTemp;
public bool vulkan;
public bool d3d11;
public bool ps5;
}
private static void DisplayHelpText()
{
Console.WriteLine("Usage: refreshc <path-to-glsl-source | directory-with-glsl-source-files>");
Console.WriteLine("Options:");
Console.WriteLine(" --vulkan Emit shader compatible with the Refresh Vulkan backend");
Console.WriteLine(" --d3d11 Emit shader compatible with the Refresh D3D11 backend");
Console.WriteLine(" --ps5 Emit shader compatible with the Refresh PS5 backend");
Console.WriteLine(" --out dir Write output file(s) to the directory `dir`");
Console.WriteLine(" --preserve-temp Do not delete the temp directory after compilation. Useful for debugging.");
}
public static int Main(string[] args)
{
if (args.Length == 0)
{
DisplayHelpText();
return 1;
}
CompileShaderData data = new CompileShaderData();
string inputPath = null;
for (int i = 0; i < args.Length; i += 1)
{
switch (args[i])
{
case "--vulkan":
data.vulkan = true;
break;
case "--d3d11":
data.d3d11 = true;
break;
case "--ps5":
data.ps5 = true;
break;
case "--out":
i += 1;
data.outputDir = args[i];
break;
case "--preserve-temp":
data.preserveTemp = true;
break;
default:
if (inputPath == null)
{
inputPath = args[i];
}
else
{
Console.WriteLine($"refreshc: Unknown parameter {args[i]}");
return 1;
}
break;
}
}
if (!data.vulkan && !data.d3d11 && !data.ps5)
{
Console.WriteLine($"refreshc: No Refresh platforms selected!");
return 1;
}
#if !PS5
if (data.ps5)
{
Console.WriteLine($"refreshc: `PS5` must be defined in the to target the PS5 backend!");
return 1;
}
#endif
if (data.outputDir == null)
{
data.outputDir = Directory.GetCurrentDirectory();
}
else if (!Directory.Exists(data.outputDir))
{
Console.WriteLine($"refreshc: Output directory {data.outputDir} does not exist");
return 1;
}
if (Directory.Exists(inputPath))
{
// Loop over and compile each file in the directory
string[] files = Directory.GetFiles(inputPath);
foreach (string file in files)
{
Console.WriteLine($"Compiling {file}");
data.glslPath = file;
int res = CompileShader(ref data);
if (res != 0)
{
return res;
}
}
}
else
{
if (!File.Exists(inputPath))
{
Console.WriteLine($"refreshc: glsl source file or directory ({inputPath}) does not exist");
return 1;
}
data.glslPath = inputPath;
int res = CompileShader(ref data);
if (res != 0)
{
return res;
}
}
return 0;
}
static int CompileShader(ref CompileShaderData data)
{
int res = 0;
string shaderName = Path.GetFileNameWithoutExtension(data.glslPath);
string shaderType = Path.GetExtension(data.glslPath);
if (shaderType != ".vert" && shaderType != ".frag" && shaderType != ".comp")
{
Console.WriteLine("refreshc: Expected glsl source file with extension '.vert', '.frag', or '.comp'");
return 1;
}
// Create the temp directory, if needed
string tempDir = Path.Combine(Directory.GetCurrentDirectory(), "temp");
if (!Directory.Exists(tempDir))
{
Directory.CreateDirectory(tempDir);
}
// Compile to spirv
string spirvPath = Path.Combine(tempDir, $"{shaderName}.spv");
res = CompileGlslToSpirv(data.glslPath, shaderName, spirvPath);
if (res != 0)
{
goto cleanup;
}
if (data.d3d11 || data.ps5)
{
// Transpile to hlsl
string hlslPath = Path.Combine(tempDir, $"{shaderName}.hlsl");
res = TranslateSpirvToHlsl(spirvPath, hlslPath);
if (res != 0)
{
goto cleanup;
}
// FIXME: Is there a cross-platform way to compile HLSL to DXBC?
#if PS5
// Transpile to ps5, if requested
if (data.ps5)
{
res = TranslateHlslToPS5(hlslPath, shaderName, shaderType, tempDir);
if (res != 0)
{
goto cleanup;
}
}
#endif
}
// Create the output blob file
string outputFilepath = Path.Combine(data.outputDir, $"{shaderName}.refresh");
using (FileStream fs = File.Create(outputFilepath))
{
using (BinaryWriter writer = new BinaryWriter(fs))
{
// Magic
writer.Write(new char[] { 'R', 'F', 'S', 'H'});
if (data.vulkan)
{
string inputPath = Path.Combine(tempDir, $"{shaderName}.spv");
WriteShaderBlob(writer, inputPath, 1);
}
#if PS5
if (data.ps5)
{
string ext = GetPS5ShaderFileExtension();
string inputPath = Path.Combine(tempDir, $"{shaderName}{ext}");
WriteShaderBlob(writer, inputPath, 2);
}
#endif
if (data.d3d11)
{
string inputPath = Path.Combine(tempDir, $"{shaderName}.hlsl");
WriteShaderBlob(writer, inputPath, 3);
}
}
}
cleanup:
// Clean up the temp directory
if (!data.preserveTemp)
{
Directory.Delete(tempDir, true);
}
return res;
}
static void WriteShaderBlob(BinaryWriter writer, string inputPath, byte backend)
{
byte[] shaderBlob = File.ReadAllBytes(inputPath);
writer.Write(backend); // Corresponds to Refresh_Backend
writer.Write(shaderBlob.Length);
writer.Write(shaderBlob);
}
static int CompileGlslToSpirv(string glslPath, string shaderName, string outputPath)
{
Process glslc = Process.Start(
"glslc",
$"{glslPath} -o {outputPath}"
);
glslc.WaitForExit();
if (glslc.ExitCode != 0)
{
Console.WriteLine($"refreshc: Could not compile GLSL code");
return 1;
}
return 0;
}
static int TranslateSpirvToHlsl(string spirvPath, string outputPath)
{
Process spirvcross = Process.Start(
"spirv-cross",
$"{spirvPath} --hlsl --shader-model 50 --output {outputPath}"
);
spirvcross.WaitForExit();
if (spirvcross.ExitCode != 0)
{
Console.WriteLine($"refreshc: Could not translate SPIR-V to HLSL");
return 1;
}
return 0;
}
}

View File

@ -0,0 +1,9 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net7.0</TargetFramework>
<TargetName>refreshc</TargetName>
</PropertyGroup>
</Project>

View File

@ -338,10 +338,52 @@ Refresh_ShaderModule* Refresh_CreateShaderModule(
Refresh_Device *device,
Refresh_ShaderModuleCreateInfo *shaderModuleCreateInfo
) {
Refresh_ShaderModuleCreateInfo driverSpecificCreateInfo = { 0, NULL };
uint8_t *bytes;
uint32_t i, size;
NULL_RETURN_NULL(device);
/* verify the magic number in the shader blob header */
bytes = (uint8_t*) shaderModuleCreateInfo->byteCode;
if (bytes[0] != 'R' || bytes[1] != 'F' || bytes[2] != 'S' || bytes[3] != 'H')
{
Refresh_LogError("Cannot parse malformed Refresh shader blob!");
return NULL;
}
/* find the code for the selected backend */
i = 4;
while (i < shaderModuleCreateInfo->codeSize)
{
size = *((uint32_t*) &bytes[i + 1]);
if (bytes[i] == (uint8_t) selectedBackend)
{
driverSpecificCreateInfo.codeSize = size;
driverSpecificCreateInfo.byteCode = (uint32_t*) &bytes[i + 1 + sizeof(uint32_t)];
break;
}
else
{
/* skip over the backend byte, the blob size, and the blob */
i += 1 + sizeof(uint32_t) + size;
}
}
/* verify the shader blob supports the selected backend */
if (driverSpecificCreateInfo.byteCode == NULL)
{
Refresh_LogError(
"Cannot create shader module that does not contain shader code for the selected backend! "
"Recompile your shader and enable this backend."
);
return NULL;
}
return device->CreateShaderModule(
device->driverData,
shaderModuleCreateInfo
&driverSpecificCreateInfo
);
}