using Ryujinx.Graphics.GAL; using Ryujinx.Graphics.Shader; using System; using System.IO; using System.Numerics; using System.Runtime.CompilerServices; namespace Ryujinx.Graphics.Gpu.Shader.DiskCache { /// /// On-disk shader cache storage for host code. /// class DiskCacheHostStorage { private const uint TocsMagic = (byte)'T' | ((byte)'O' << 8) | ((byte)'C' << 16) | ((byte)'S' << 24); private const uint TochMagic = (byte)'T' | ((byte)'O' << 8) | ((byte)'C' << 16) | ((byte)'H' << 24); private const uint ShdiMagic = (byte)'S' | ((byte)'H' << 8) | ((byte)'D' << 16) | ((byte)'I' << 24); private const uint BufdMagic = (byte)'B' | ((byte)'U' << 8) | ((byte)'F' << 16) | ((byte)'D' << 24); private const uint TexdMagic = (byte)'T' | ((byte)'E' << 8) | ((byte)'X' << 16) | ((byte)'D' << 24); private const ushort FileFormatVersionMajor = 1; private const ushort FileFormatVersionMinor = 1; private const uint FileFormatVersionPacked = ((uint)FileFormatVersionMajor << 16) | FileFormatVersionMinor; private const uint CodeGenVersion = 3478; private const string SharedTocFileName = "shared.toc"; private const string SharedDataFileName = "shared.data"; private readonly string _basePath; public bool CacheEnabled => !string.IsNullOrEmpty(_basePath); /// /// TOC (Table of contents) file header. /// private struct TocHeader { /// /// Magic value, for validation and identification. /// public uint Magic; /// /// File format version. /// public uint FormatVersion; /// /// Generated shader code version. /// public uint CodeGenVersion; /// /// Header padding. /// public uint Padding; /// /// Reserved space, to be used in the future. Write as zero. /// public ulong Reserved; /// /// Reserved space, to be used in the future. Write as zero. /// public ulong Reserved2; } /// /// Offset and size pair. /// private struct OffsetAndSize { /// /// Offset. /// public ulong Offset; /// /// Size. /// public uint Size; } /// /// Per-stage data entry. /// private struct DataEntryPerStage { /// /// Index of the guest code on the guest code cache TOC file. /// public int GuestCodeIndex; } /// /// Per-program data entry. /// private struct DataEntry { /// /// Bit mask where each bit set is a used shader stage. Should be zero for compute shaders. /// public uint StagesBitMask; } /// /// Per-stage shader information, returned by the translator. /// private struct DataShaderInfo { /// /// Total constant buffers used. /// public ushort CBuffersCount; /// /// Total storage buffers used. /// public ushort SBuffersCount; /// /// Total textures used. /// public ushort TexturesCount; /// /// Total images used. /// public ushort ImagesCount; /// /// Shader stage. /// public ShaderStage Stage; /// /// Indicates if the shader accesses the Instance ID built-in variable. /// public bool UsesInstanceId; /// /// Indicates if the shader modifies the Layer built-in variable. /// public bool UsesRtLayer; /// /// Bit mask with the clip distances written on the vertex stage. /// public byte ClipDistancesWritten; /// /// Bit mask of the render target components written by the fragment stage. /// public int FragmentOutputMap; } private readonly DiskCacheGuestStorage _guestStorage; /// /// Creates a disk cache host storage. /// /// Base path of the shader cache public DiskCacheHostStorage(string basePath) { _basePath = basePath; _guestStorage = new DiskCacheGuestStorage(basePath); if (CacheEnabled) { Directory.CreateDirectory(basePath); } } /// /// Gets the total of host programs on the cache. /// /// Host programs count public int GetProgramCount() { string tocFilePath = Path.Combine(_basePath, SharedTocFileName); if (!File.Exists(tocFilePath)) { return 0; } return (int)((new FileInfo(tocFilePath).Length - Unsafe.SizeOf()) / sizeof(ulong)); } /// /// Guest the name of the host program cache file, with extension. /// /// GPU context /// Name of the file, without extension private static string GetHostFileName(GpuContext context) { string apiName = context.Capabilities.Api.ToString().ToLowerInvariant(); string vendorName = RemoveInvalidCharacters(context.Capabilities.VendorName.ToLowerInvariant()); return $"{apiName}_{vendorName}"; } /// /// Removes invalid path characters and spaces from a file name. /// /// File name /// Filtered file name private static string RemoveInvalidCharacters(string fileName) { int indexOfSpace = fileName.IndexOf(' '); if (indexOfSpace >= 0) { fileName = fileName.Substring(0, indexOfSpace); } return string.Concat(fileName.Split(Path.GetInvalidFileNameChars(), StringSplitOptions.RemoveEmptyEntries)); } /// /// Gets the name of the TOC host file. /// /// GPU context /// File name private static string GetHostTocFileName(GpuContext context) { return GetHostFileName(context) + ".toc"; } /// /// Gets the name of the data host file. /// /// GPU context /// File name private static string GetHostDataFileName(GpuContext context) { return GetHostFileName(context) + ".data"; } /// /// Checks if a disk cache exists for the current application. /// /// True if a disk cache exists, false otherwise public bool CacheExists() { string tocFilePath = Path.Combine(_basePath, SharedTocFileName); string dataFilePath = Path.Combine(_basePath, SharedDataFileName); if (!File.Exists(tocFilePath) || !File.Exists(dataFilePath) || !_guestStorage.TocFileExists() || !_guestStorage.DataFileExists()) { return false; } return true; } /// /// Loads all shaders from the cache. /// /// GPU context /// Parallel disk cache loader public void LoadShaders(GpuContext context, ParallelDiskCacheLoader loader) { if (!CacheExists()) { return; } Stream hostTocFileStream = null; Stream hostDataFileStream = null; try { using var tocFileStream = DiskCacheCommon.OpenFile(_basePath, SharedTocFileName, writable: false); using var dataFileStream = DiskCacheCommon.OpenFile(_basePath, SharedDataFileName, writable: false); using var guestTocFileStream = _guestStorage.OpenTocFileStream(); using var guestDataFileStream = _guestStorage.OpenDataFileStream(); BinarySerializer tocReader = new BinarySerializer(tocFileStream); BinarySerializer dataReader = new BinarySerializer(dataFileStream); TocHeader header = new TocHeader(); if (!tocReader.TryRead(ref header) || header.Magic != TocsMagic) { throw new DiskCacheLoadException(DiskCacheLoadResult.FileCorruptedGeneric); } if (header.FormatVersion != FileFormatVersionPacked) { throw new DiskCacheLoadException(DiskCacheLoadResult.IncompatibleVersion); } bool loadHostCache = header.CodeGenVersion == CodeGenVersion; int programIndex = 0; DataEntry entry = new DataEntry(); while (tocFileStream.Position < tocFileStream.Length && loader.Active) { ulong dataOffset = 0; tocReader.Read(ref dataOffset); if ((ulong)dataOffset >= (ulong)dataFileStream.Length) { throw new DiskCacheLoadException(DiskCacheLoadResult.FileCorruptedGeneric); } dataFileStream.Seek((long)dataOffset, SeekOrigin.Begin); dataReader.BeginCompression(); dataReader.Read(ref entry); uint stagesBitMask = entry.StagesBitMask; if ((stagesBitMask & ~0x3fu) != 0) { throw new DiskCacheLoadException(DiskCacheLoadResult.FileCorruptedGeneric); } bool isCompute = stagesBitMask == 0; if (isCompute) { stagesBitMask = 1; } CachedShaderStage[] shaders = new CachedShaderStage[isCompute ? 1 : Constants.ShaderStages + 1]; DataEntryPerStage stageEntry = new DataEntryPerStage(); while (stagesBitMask != 0) { int stageIndex = BitOperations.TrailingZeroCount(stagesBitMask); dataReader.Read(ref stageEntry); ShaderProgramInfo info = stageIndex != 0 || isCompute ? ReadShaderProgramInfo(ref dataReader) : null; (byte[] guestCode, byte[] cb1Data) = _guestStorage.LoadShader( guestTocFileStream, guestDataFileStream, stageEntry.GuestCodeIndex); shaders[stageIndex] = new CachedShaderStage(info, guestCode, cb1Data); stagesBitMask &= ~(1u << stageIndex); } ShaderSpecializationState specState = ShaderSpecializationState.Read(ref dataReader); dataReader.EndCompression(); if (loadHostCache) { byte[] hostCode = ReadHostCode(context, ref hostTocFileStream, ref hostDataFileStream, programIndex); if (hostCode != null) { bool hasFragmentShader = shaders.Length > 5 && shaders[5] != null; int fragmentOutputMap = hasFragmentShader ? shaders[5].Info.FragmentOutputMap : -1; IProgram hostProgram = context.Renderer.LoadProgramBinary(hostCode, hasFragmentShader, new ShaderInfo(fragmentOutputMap)); CachedShaderProgram program = new CachedShaderProgram(hostProgram, specState, shaders); loader.QueueHostProgram(program, hostProgram, programIndex, isCompute); } else { loadHostCache = false; } } if (!loadHostCache) { loader.QueueGuestProgram(shaders, specState, programIndex, isCompute); } loader.CheckCompilation(); programIndex++; } } finally { _guestStorage.ClearMemoryCache(); hostTocFileStream?.Dispose(); hostDataFileStream?.Dispose(); } } /// /// Reads the host code for a given shader, if existent. /// /// GPU context /// Host TOC file stream, intialized if needed /// Host data file stream, initialized if needed /// Index of the program on the cache /// Host binary code, or null if not found private byte[] ReadHostCode(GpuContext context, ref Stream tocFileStream, ref Stream dataFileStream, int programIndex) { if (tocFileStream == null && dataFileStream == null) { string tocFilePath = Path.Combine(_basePath, GetHostTocFileName(context)); string dataFilePath = Path.Combine(_basePath, GetHostDataFileName(context)); if (!File.Exists(tocFilePath) || !File.Exists(dataFilePath)) { return null; } tocFileStream = DiskCacheCommon.OpenFile(_basePath, GetHostTocFileName(context), writable: false); dataFileStream = DiskCacheCommon.OpenFile(_basePath, GetHostDataFileName(context), writable: false); } int offset = Unsafe.SizeOf() + programIndex * Unsafe.SizeOf(); if (offset + Unsafe.SizeOf() > tocFileStream.Length) { return null; } if ((ulong)offset >= (ulong)dataFileStream.Length) { throw new DiskCacheLoadException(DiskCacheLoadResult.FileCorruptedGeneric); } tocFileStream.Seek(offset, SeekOrigin.Begin); BinarySerializer tocReader = new BinarySerializer(tocFileStream); OffsetAndSize offsetAndSize = new OffsetAndSize(); tocReader.Read(ref offsetAndSize); if (offsetAndSize.Offset >= (ulong)dataFileStream.Length) { throw new DiskCacheLoadException(DiskCacheLoadResult.FileCorruptedGeneric); } dataFileStream.Seek((long)offsetAndSize.Offset, SeekOrigin.Begin); byte[] hostCode = new byte[offsetAndSize.Size]; BinarySerializer.ReadCompressed(dataFileStream, hostCode); return hostCode; } /// /// Gets output streams for the disk cache, for faster batch writing. /// /// The GPU context, used to determine the host disk cache /// A collection of disk cache output streams public DiskCacheOutputStreams GetOutputStreams(GpuContext context) { var tocFileStream = DiskCacheCommon.OpenFile(_basePath, SharedTocFileName, writable: true); var dataFileStream = DiskCacheCommon.OpenFile(_basePath, SharedDataFileName, writable: true); var hostTocFileStream = DiskCacheCommon.OpenFile(_basePath, GetHostTocFileName(context), writable: true); var hostDataFileStream = DiskCacheCommon.OpenFile(_basePath, GetHostDataFileName(context), writable: true); return new DiskCacheOutputStreams(tocFileStream, dataFileStream, hostTocFileStream, hostDataFileStream); } /// /// Adds a shader to the cache. /// /// GPU context /// Cached program /// Optional host binary code /// Output streams to use public void AddShader(GpuContext context, CachedShaderProgram program, ReadOnlySpan hostCode, DiskCacheOutputStreams streams = null) { uint stagesBitMask = 0; for (int index = 0; index < program.Shaders.Length; index++) { var shader = program.Shaders[index]; if (shader == null || (shader.Info != null && shader.Info.Stage == ShaderStage.Compute)) { continue; } stagesBitMask |= 1u << index; } var tocFileStream = streams != null ? streams.TocFileStream : DiskCacheCommon.OpenFile(_basePath, SharedTocFileName, writable: true); var dataFileStream = streams != null ? streams.DataFileStream : DiskCacheCommon.OpenFile(_basePath, SharedDataFileName, writable: true); if (tocFileStream.Length == 0) { TocHeader header = new TocHeader(); CreateToc(tocFileStream, ref header, TocsMagic, CodeGenVersion); } tocFileStream.Seek(0, SeekOrigin.End); dataFileStream.Seek(0, SeekOrigin.End); BinarySerializer tocWriter = new BinarySerializer(tocFileStream); BinarySerializer dataWriter = new BinarySerializer(dataFileStream); ulong dataOffset = (ulong)dataFileStream.Position; tocWriter.Write(ref dataOffset); DataEntry entry = new DataEntry(); entry.StagesBitMask = stagesBitMask; dataWriter.BeginCompression(DiskCacheCommon.GetCompressionAlgorithm()); dataWriter.Write(ref entry); DataEntryPerStage stageEntry = new DataEntryPerStage(); for (int index = 0; index < program.Shaders.Length; index++) { var shader = program.Shaders[index]; if (shader == null) { continue; } stageEntry.GuestCodeIndex = _guestStorage.AddShader(shader.Code, shader.Cb1Data); dataWriter.Write(ref stageEntry); WriteShaderProgramInfo(ref dataWriter, shader.Info); } program.SpecializationState.Write(ref dataWriter); dataWriter.EndCompression(); if (streams == null) { tocFileStream.Dispose(); dataFileStream.Dispose(); } if (hostCode.IsEmpty) { return; } WriteHostCode(context, hostCode, -1, streams); } /// /// Clears all content from the guest cache files. /// public void ClearGuestCache() { _guestStorage.ClearCache(); } /// /// Clears all content from the shared cache files. /// /// GPU context public void ClearSharedCache() { using var tocFileStream = DiskCacheCommon.OpenFile(_basePath, SharedTocFileName, writable: true); using var dataFileStream = DiskCacheCommon.OpenFile(_basePath, SharedDataFileName, writable: true); tocFileStream.SetLength(0); dataFileStream.SetLength(0); } /// /// Deletes all content from the host cache files. /// /// GPU context public void ClearHostCache(GpuContext context) { using var tocFileStream = DiskCacheCommon.OpenFile(_basePath, GetHostTocFileName(context), writable: true); using var dataFileStream = DiskCacheCommon.OpenFile(_basePath, GetHostDataFileName(context), writable: true); tocFileStream.SetLength(0); dataFileStream.SetLength(0); } /// /// Adds a host binary shader to the host cache. /// /// /// This only modifies the host cache. The shader must already exist in the other caches. /// This method should only be used for rebuilding the host cache after a clear. /// /// GPU context /// Host binary code /// Index of the program in the cache public void AddHostShader(GpuContext context, ReadOnlySpan hostCode, int programIndex) { WriteHostCode(context, hostCode, programIndex); } /// /// Writes the host binary code on the host cache. /// /// GPU context /// Host binary code /// Index of the program in the cache /// Output streams to use private void WriteHostCode(GpuContext context, ReadOnlySpan hostCode, int programIndex, DiskCacheOutputStreams streams = null) { var tocFileStream = streams != null ? streams.HostTocFileStream : DiskCacheCommon.OpenFile(_basePath, GetHostTocFileName(context), writable: true); var dataFileStream = streams != null ? streams.HostDataFileStream : DiskCacheCommon.OpenFile(_basePath, GetHostDataFileName(context), writable: true); if (tocFileStream.Length == 0) { TocHeader header = new TocHeader(); CreateToc(tocFileStream, ref header, TochMagic, 0); } if (programIndex == -1) { tocFileStream.Seek(0, SeekOrigin.End); } else { tocFileStream.Seek(Unsafe.SizeOf() + (programIndex * Unsafe.SizeOf()), SeekOrigin.Begin); } dataFileStream.Seek(0, SeekOrigin.End); BinarySerializer tocWriter = new BinarySerializer(tocFileStream); OffsetAndSize offsetAndSize = new OffsetAndSize(); offsetAndSize.Offset = (ulong)dataFileStream.Position; offsetAndSize.Size = (uint)hostCode.Length; tocWriter.Write(ref offsetAndSize); BinarySerializer.WriteCompressed(dataFileStream, hostCode, DiskCacheCommon.GetCompressionAlgorithm()); if (streams == null) { tocFileStream.Dispose(); dataFileStream.Dispose(); } } /// /// Creates a TOC file for the host or shared cache. /// /// TOC file stream /// Set to the TOC file header /// Magic value to be written /// Shader codegen version, only valid for the host file private void CreateToc(Stream tocFileStream, ref TocHeader header, uint magic, uint codegenVersion) { BinarySerializer writer = new BinarySerializer(tocFileStream); header.Magic = magic; header.FormatVersion = FileFormatVersionPacked; header.CodeGenVersion = codegenVersion; header.Padding = 0; header.Reserved = 0; header.Reserved2 = 0; if (tocFileStream.Length > 0) { tocFileStream.Seek(0, SeekOrigin.Begin); tocFileStream.SetLength(0); } writer.Write(ref header); } /// /// Reads the shader program info from the cache. /// /// Cache data reader /// Shader program info private static ShaderProgramInfo ReadShaderProgramInfo(ref BinarySerializer dataReader) { DataShaderInfo dataInfo = new DataShaderInfo(); dataReader.ReadWithMagicAndSize(ref dataInfo, ShdiMagic); BufferDescriptor[] cBuffers = new BufferDescriptor[dataInfo.CBuffersCount]; BufferDescriptor[] sBuffers = new BufferDescriptor[dataInfo.SBuffersCount]; TextureDescriptor[] textures = new TextureDescriptor[dataInfo.TexturesCount]; TextureDescriptor[] images = new TextureDescriptor[dataInfo.ImagesCount]; for (int index = 0; index < dataInfo.CBuffersCount; index++) { dataReader.ReadWithMagicAndSize(ref cBuffers[index], BufdMagic); } for (int index = 0; index < dataInfo.SBuffersCount; index++) { dataReader.ReadWithMagicAndSize(ref sBuffers[index], BufdMagic); } for (int index = 0; index < dataInfo.TexturesCount; index++) { dataReader.ReadWithMagicAndSize(ref textures[index], TexdMagic); } for (int index = 0; index < dataInfo.ImagesCount; index++) { dataReader.ReadWithMagicAndSize(ref images[index], TexdMagic); } return new ShaderProgramInfo( cBuffers, sBuffers, textures, images, dataInfo.Stage, dataInfo.UsesInstanceId, dataInfo.UsesRtLayer, dataInfo.ClipDistancesWritten, dataInfo.FragmentOutputMap); } /// /// Writes the shader program info into the cache. /// /// Cache data writer /// Program info private static void WriteShaderProgramInfo(ref BinarySerializer dataWriter, ShaderProgramInfo info) { if (info == null) { return; } DataShaderInfo dataInfo = new DataShaderInfo(); dataInfo.CBuffersCount = (ushort)info.CBuffers.Count; dataInfo.SBuffersCount = (ushort)info.SBuffers.Count; dataInfo.TexturesCount = (ushort)info.Textures.Count; dataInfo.ImagesCount = (ushort)info.Images.Count; dataInfo.Stage = info.Stage; dataInfo.UsesInstanceId = info.UsesInstanceId; dataInfo.UsesRtLayer = info.UsesRtLayer; dataInfo.ClipDistancesWritten = info.ClipDistancesWritten; dataInfo.FragmentOutputMap = info.FragmentOutputMap; dataWriter.WriteWithMagicAndSize(ref dataInfo, ShdiMagic); for (int index = 0; index < info.CBuffers.Count; index++) { var entry = info.CBuffers[index]; dataWriter.WriteWithMagicAndSize(ref entry, BufdMagic); } for (int index = 0; index < info.SBuffers.Count; index++) { var entry = info.SBuffers[index]; dataWriter.WriteWithMagicAndSize(ref entry, BufdMagic); } for (int index = 0; index < info.Textures.Count; index++) { var entry = info.Textures[index]; dataWriter.WriteWithMagicAndSize(ref entry, TexdMagic); } for (int index = 0; index < info.Images.Count; index++) { var entry = info.Images[index]; dataWriter.WriteWithMagicAndSize(ref entry, TexdMagic); } } } }