From 7085bafa60272a0cbb707c5b4b96abcddec78bd0 Mon Sep 17 00:00:00 2001 From: LotP1 <68976644+LotP1@users.noreply.github.com> Date: Wed, 29 Jan 2025 03:36:58 +0100 Subject: [PATCH] PPTC Profiles (#370) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added functionality that allows ExeFS mods to compile to their own PPTC Profile and therefore store PTC data between sessions. The feature calculates the hash of the currently loaded ExeFS mods and stores the PPTC data in a profile that matches said hash, so you can have multiple ExeFS loadouts without causing issues. This includes different versions of the same mod as their hashes will be different. Using this PR should be seamless as the JIT Sparse PR already laid the groundwork for PPTC Profiles and this PR just allows ExeFS mods to load and store their own profiles besides the `default` profile. ❗❗❗ **WARNING!** ❗❗❗ **This will update your PPTC profile version, which means the PPTC profile will be invalidated if you try to run a PR/Build/Branch that does not include this change!** **This is only relevant for the default PPTC Profile, as any other profiles do not exist to older versions!** --- src/ARMeilleure/Translation/PTC/Ptc.cs | 56 ++++++++++-- .../Translation/PTC/PtcProfiler.cs | 90 ++++++++++++++++--- src/ARMeilleure/Translation/Translator.cs | 10 +++ .../HOS/ArmProcessContextFactory.cs | 5 +- src/Ryujinx.HLE/HOS/ModLoader.cs | 26 +++++- .../Extensions/FileSystemExtensions.cs | 10 +-- .../Loaders/Processes/ProcessLoader.cs | 1 + .../Loaders/Processes/ProcessLoaderHelper.cs | 3 + src/Ryujinx/Assets/locales.json | 75 ++++++++++++++++ .../UI/Controls/ApplicationContextMenu.axaml | 5 ++ .../Controls/ApplicationContextMenu.axaml.cs | 46 ++++++++++ 11 files changed, 299 insertions(+), 28 deletions(-) diff --git a/src/ARMeilleure/Translation/PTC/Ptc.cs b/src/ARMeilleure/Translation/PTC/Ptc.cs index b53fdd4df..d1ffda830 100644 --- a/src/ARMeilleure/Translation/PTC/Ptc.cs +++ b/src/ARMeilleure/Translation/PTC/Ptc.cs @@ -3,6 +3,7 @@ using ARMeilleure.CodeGen.Linking; using ARMeilleure.CodeGen.Unwinding; using ARMeilleure.Common; using ARMeilleure.Memory; +using ARMeilleure.State; using Ryujinx.Common; using Ryujinx.Common.Configuration; using Ryujinx.Common.Logging; @@ -30,8 +31,8 @@ namespace ARMeilleure.Translation.PTC { private const string OuterHeaderMagicString = "PTCohd\0\0"; private const string InnerHeaderMagicString = "PTCihd\0\0"; - - private const uint InternalVersion = 6998; //! To be incremented manually for each change to the ARMeilleure project. + + private const uint InternalVersion = 7007; //! To be incremented manually for each change to the ARMeilleure project. private const string ActualDir = "0"; private const string BackupDir = "1"; @@ -184,6 +185,36 @@ namespace ARMeilleure.Translation.PTC InitializeCarriers(); } + private bool ContainsBlacklistedFunctions() + { + List blacklist = Profiler.GetBlacklistedFunctions(); + bool containsBlacklistedFunctions = false; + _infosStream.Seek(0L, SeekOrigin.Begin); + bool foundBadFunction = false; + + for (int index = 0; index < GetEntriesCount(); index++) + { + InfoEntry infoEntry = DeserializeStructure(_infosStream); + foreach (ulong address in blacklist) + { + if (infoEntry.Address == address) + { + containsBlacklistedFunctions = true; + Logger.Warning?.Print(LogClass.Ptc, "PPTC cache invalidated: Found blacklisted functions in PPTC cache"); + foundBadFunction = true; + break; + } + } + + if (foundBadFunction) + { + break; + } + } + + return containsBlacklistedFunctions; + } + private void PreLoad() { string fileNameActual = $"{CachePathActual}.cache"; @@ -532,7 +563,7 @@ namespace ARMeilleure.Translation.PTC public void LoadTranslations(Translator translator) { - if (AreCarriersEmpty()) + if (AreCarriersEmpty() || ContainsBlacklistedFunctions()) { return; } @@ -835,10 +866,18 @@ namespace ARMeilleure.Translation.PTC while (profiledFuncsToTranslate.TryDequeue(out (ulong address, PtcProfiler.FuncProfile funcProfile) item)) { ulong address = item.address; + ExecutionMode executionMode = item.funcProfile.Mode; + bool highCq = item.funcProfile.HighCq; Debug.Assert(Profiler.IsAddressInStaticCodeRange(address)); - TranslatedFunction func = translator.Translate(address, item.funcProfile.Mode, item.funcProfile.HighCq); + TranslatedFunction func = translator.Translate(address, executionMode, highCq); + + if (func == null) + { + Profiler.UpdateEntry(address, executionMode, true, true); + continue; + } bool isAddressUnique = translator.Functions.TryAdd(address, func.GuestSize, func); @@ -885,7 +924,14 @@ namespace ARMeilleure.Translation.PTC PtcStateChanged?.Invoke(PtcLoadingState.Loaded, _translateCount, _translateTotalCount); - Logger.Info?.Print(LogClass.Ptc, $"{_translateCount} of {_translateTotalCount} functions translated | Thread count: {degreeOfParallelism} in {sw.Elapsed.TotalSeconds} s"); + if (_translateCount == _translateTotalCount) + { + Logger.Info?.Print(LogClass.Ptc, $"{_translateCount} of {_translateTotalCount} functions translated | Thread count: {degreeOfParallelism} in {sw.Elapsed.TotalSeconds} s"); + } + else + { + Logger.Info?.Print(LogClass.Ptc, $"{_translateCount} of {_translateTotalCount} functions translated | {_translateTotalCount - _translateCount} function{(_translateTotalCount - _translateCount != 1 ? "s" : "")} blacklisted | Thread count: {degreeOfParallelism} in {sw.Elapsed.TotalSeconds} s"); + } Thread preSaveThread = new(PreSave) { diff --git a/src/ARMeilleure/Translation/PTC/PtcProfiler.cs b/src/ARMeilleure/Translation/PTC/PtcProfiler.cs index 21987f72d..de0b78dbe 100644 --- a/src/ARMeilleure/Translation/PTC/PtcProfiler.cs +++ b/src/ARMeilleure/Translation/PTC/PtcProfiler.cs @@ -24,11 +24,12 @@ namespace ARMeilleure.Translation.PTC { private const string OuterHeaderMagicString = "Pohd\0\0\0\0"; - private const uint InternalVersion = 5518; //! Not to be incremented manually for each change to the ARMeilleure project. + private const uint InternalVersion = 7007; //! Not to be incremented manually for each change to the ARMeilleure project. - private static readonly uint[] _migrateInternalVersions = + private static readonly uint[] _migrateInternalVersions = [ - 1866 + 1866, + 5518, ]; private const int SaveInterval = 30; // Seconds. @@ -77,20 +78,30 @@ namespace ARMeilleure.Translation.PTC private void TimerElapsed(object _, ElapsedEventArgs __) => new Thread(PreSave) { Name = "Ptc.DiskWriter" }.Start(); - public void AddEntry(ulong address, ExecutionMode mode, bool highCq) + public void AddEntry(ulong address, ExecutionMode mode, bool highCq, bool blacklist = false) { if (IsAddressInStaticCodeRange(address)) { Debug.Assert(!highCq); - lock (_lock) + if (blacklist) { - ProfiledFuncs.TryAdd(address, new FuncProfile(mode, highCq: false)); + lock (_lock) + { + ProfiledFuncs[address] = new FuncProfile(mode, highCq: false, true); + } + } + else + { + lock (_lock) + { + ProfiledFuncs.TryAdd(address, new FuncProfile(mode, highCq: false, false)); + } } } } - public void UpdateEntry(ulong address, ExecutionMode mode, bool highCq) + public void UpdateEntry(ulong address, ExecutionMode mode, bool highCq, bool? blacklist = null) { if (IsAddressInStaticCodeRange(address)) { @@ -100,7 +111,7 @@ namespace ARMeilleure.Translation.PTC { Debug.Assert(ProfiledFuncs.ContainsKey(address)); - ProfiledFuncs[address] = new FuncProfile(mode, highCq: true); + ProfiledFuncs[address] = new FuncProfile(mode, highCq: true, blacklist ?? ProfiledFuncs[address].Blacklist); } } } @@ -116,7 +127,7 @@ namespace ARMeilleure.Translation.PTC foreach (KeyValuePair profiledFunc in ProfiledFuncs) { - if (!funcs.ContainsKey(profiledFunc.Key)) + if (!funcs.ContainsKey(profiledFunc.Key) && !profiledFunc.Value.Blacklist) { profiledFuncsToTranslate.Enqueue((profiledFunc.Key, profiledFunc.Value)); } @@ -131,6 +142,24 @@ namespace ARMeilleure.Translation.PTC ProfiledFuncs.TrimExcess(); } + public List GetBlacklistedFunctions() + { + List funcs = new List(); + + foreach (var profiledFunc in ProfiledFuncs) + { + if (profiledFunc.Value.Blacklist) + { + if (!funcs.Contains(profiledFunc.Key)) + { + funcs.Add(profiledFunc.Key); + } + } + } + + return funcs; + } + public void PreLoad() { _lastHash = default; @@ -221,13 +250,18 @@ namespace ARMeilleure.Translation.PTC return false; } + Func migrateEntryFunc = null; + switch (outerHeader.InfoFileVersion) { case InternalVersion: ProfiledFuncs = Deserialize(stream); break; case 1866: - ProfiledFuncs = Deserialize(stream, (address, profile) => (address + 0x500000UL, profile)); + migrateEntryFunc = (address, profile) => (address + 0x500000UL, profile); + goto case 5518; + case 5518: + ProfiledFuncs = DeserializeAddBlacklist(stream, migrateEntryFunc); break; default: Logger.Error?.Print(LogClass.Ptc, $"No migration path for {nameof(outerHeader.InfoFileVersion)} '{outerHeader.InfoFileVersion}'. Discarding cache."); @@ -257,6 +291,16 @@ namespace ARMeilleure.Translation.PTC return DeserializeDictionary(stream, DeserializeStructure); } + private static Dictionary DeserializeAddBlacklist(Stream stream, Func migrateEntryFunc = null) + { + if (migrateEntryFunc != null) + { + return DeserializeAndUpdateDictionary(stream, (Stream stream) => { return new FuncProfile(DeserializeStructure(stream)); }, migrateEntryFunc); + } + + return DeserializeDictionary(stream, (Stream stream) => { return new FuncProfile(DeserializeStructure(stream)); }); + } + private static ReadOnlySpan GetReadOnlySpan(MemoryStream memoryStream) { return new(memoryStream.GetBuffer(), (int)memoryStream.Position, (int)memoryStream.Length - (int)memoryStream.Position); @@ -388,13 +432,35 @@ namespace ARMeilleure.Translation.PTC } } - [StructLayout(LayoutKind.Sequential, Pack = 1/*, Size = 5*/)] + [StructLayout(LayoutKind.Sequential, Pack = 1/*, Size = 6*/)] public struct FuncProfile { public ExecutionMode Mode; public bool HighCq; + public bool Blacklist; - public FuncProfile(ExecutionMode mode, bool highCq) + public FuncProfile(ExecutionMode mode, bool highCq, bool blacklist) + { + Mode = mode; + HighCq = highCq; + Blacklist = blacklist; + } + + public FuncProfile(FuncProfilePreBlacklist fp) + { + Mode = fp.Mode; + HighCq = fp.HighCq; + Blacklist = false; + } + } + + [StructLayout(LayoutKind.Sequential, Pack = 1/*, Size = 5*/)] + public struct FuncProfilePreBlacklist + { + public ExecutionMode Mode; + public bool HighCq; + + public FuncProfilePreBlacklist(ExecutionMode mode, bool highCq) { Mode = mode; HighCq = highCq; diff --git a/src/ARMeilleure/Translation/Translator.cs b/src/ARMeilleure/Translation/Translator.cs index 0f18c8045..a50702add 100644 --- a/src/ARMeilleure/Translation/Translator.cs +++ b/src/ARMeilleure/Translation/Translator.cs @@ -249,6 +249,11 @@ namespace ARMeilleure.Translation ControlFlowGraph cfg = EmitAndGetCFG(context, blocks, out Range funcRange, out Counter counter); + if (cfg == null) + { + return null; + } + ulong funcSize = funcRange.End - funcRange.Start; Logger.EndPass(PassName.Translation, cfg); @@ -407,6 +412,11 @@ namespace ARMeilleure.Translation if (opCode.Instruction.Emitter != null) { opCode.Instruction.Emitter(context); + if (opCode.Instruction.Name == InstName.Und && blkIndex == 0) + { + range = new Range(rangeStart, rangeEnd); + return null; + } } else { diff --git a/src/Ryujinx.HLE/HOS/ArmProcessContextFactory.cs b/src/Ryujinx.HLE/HOS/ArmProcessContextFactory.cs index 95b6167f3..08d929bf0 100644 --- a/src/Ryujinx.HLE/HOS/ArmProcessContextFactory.cs +++ b/src/Ryujinx.HLE/HOS/ArmProcessContextFactory.cs @@ -20,6 +20,7 @@ namespace Ryujinx.HLE.HOS private readonly string _titleIdText; private readonly string _displayVersion; private readonly bool _diskCacheEnabled; + private readonly string _diskCacheSelector; private readonly ulong _codeAddress; private readonly ulong _codeSize; @@ -31,6 +32,7 @@ namespace Ryujinx.HLE.HOS string titleIdText, string displayVersion, bool diskCacheEnabled, + string diskCacheSelector, ulong codeAddress, ulong codeSize) { @@ -39,6 +41,7 @@ namespace Ryujinx.HLE.HOS _titleIdText = titleIdText; _displayVersion = displayVersion; _diskCacheEnabled = diskCacheEnabled; + _diskCacheSelector = diskCacheSelector; _codeAddress = codeAddress; _codeSize = codeSize; } @@ -114,7 +117,7 @@ namespace Ryujinx.HLE.HOS } } - DiskCacheLoadState = processContext.Initialize(_titleIdText, _displayVersion, _diskCacheEnabled, _codeAddress, _codeSize, "default"); //Ready for exefs profiles + DiskCacheLoadState = processContext.Initialize(_titleIdText, _displayVersion, _diskCacheEnabled, _codeAddress, _codeSize, _diskCacheSelector ?? "default"); return processContext; } diff --git a/src/Ryujinx.HLE/HOS/ModLoader.cs b/src/Ryujinx.HLE/HOS/ModLoader.cs index ff691914c..6c97e6fed 100644 --- a/src/Ryujinx.HLE/HOS/ModLoader.cs +++ b/src/Ryujinx.HLE/HOS/ModLoader.cs @@ -6,6 +6,7 @@ using LibHac.Loader; using LibHac.Tools.Fs; using LibHac.Tools.FsSystem; using LibHac.Tools.FsSystem.RomFs; +using LibHac.Util; using Ryujinx.Common.Configuration; using Ryujinx.Common.Logging; using Ryujinx.Common.Utilities; @@ -19,6 +20,7 @@ using System.Collections.Specialized; using System.Globalization; using System.IO; using System.Linq; +using System.Security.Cryptography; using LazyFile = Ryujinx.HLE.HOS.Services.Fs.FileSystemProxy.LazyFile; using Path = System.IO.Path; @@ -581,6 +583,7 @@ namespace Ryujinx.HLE.HOS public BitVector32 Stubs; public BitVector32 Replaces; public MetaLoader Npdm; + public string Hash; public bool Modified => (Stubs.Data | Replaces.Data) != 0; } @@ -591,8 +594,11 @@ namespace Ryujinx.HLE.HOS { Stubs = new BitVector32(), Replaces = new BitVector32(), + Hash = null, }; + string tempHash = string.Empty; + if (!_appMods.TryGetValue(applicationId, out ModCache mods) || mods.ExefsDirs.Count == 0) { return modLoadResult; @@ -628,8 +634,16 @@ namespace Ryujinx.HLE.HOS modLoadResult.Replaces[1 << i] = true; - nsos[i] = new NsoExecutable(nsoFile.OpenRead().AsStorage(), nsoName); - Logger.Info?.Print(LogClass.ModLoader, $"NSO '{nsoName}' replaced"); + using (FileStream stream = nsoFile.OpenRead()) + { + nsos[i] = new NsoExecutable(stream.AsStorage(), nsoName); + Logger.Info?.Print(LogClass.ModLoader, $"NSO '{nsoName}' replaced"); + using (MD5 md5 = MD5.Create()) + { + stream.Seek(0, SeekOrigin.Begin); + tempHash += BitConverter.ToString(md5.ComputeHash(stream)).Replace("-", "").ToLowerInvariant(); + } + } } modLoadResult.Stubs[1 << i] |= File.Exists(Path.Combine(mod.Path.FullName, nsoName + StubExtension)); @@ -661,6 +675,14 @@ namespace Ryujinx.HLE.HOS } } + if (!string.IsNullOrEmpty(tempHash)) + { + using (MD5 md5 = MD5.Create()) + { + modLoadResult.Hash += BitConverter.ToString(md5.ComputeHash(tempHash.ToBytes())).Replace("-", "").ToLowerInvariant(); + } + } + return modLoadResult; } diff --git a/src/Ryujinx.HLE/Loaders/Processes/Extensions/FileSystemExtensions.cs b/src/Ryujinx.HLE/Loaders/Processes/Extensions/FileSystemExtensions.cs index 5874636e7..01f65206f 100644 --- a/src/Ryujinx.HLE/Loaders/Processes/Extensions/FileSystemExtensions.cs +++ b/src/Ryujinx.HLE/Loaders/Processes/Extensions/FileSystemExtensions.cs @@ -84,13 +84,6 @@ namespace Ryujinx.HLE.Loaders.Processes.Extensions // Apply Nsos patches. device.Configuration.VirtualFileSystem.ModLoader.ApplyNsoPatches(programId, nsoExecutables); - // Don't use PTC if ExeFS files have been replaced. - bool enablePtc = device.System.EnablePtc && !modLoadResult.Modified; - if (!enablePtc) - { - Logger.Warning?.Print(LogClass.Ptc, "Detected unsupported ExeFs modifications. PTC disabled."); - } - string programName = string.Empty; if (!isHomebrew && programId > 0x010000000000FFFF) @@ -117,7 +110,8 @@ namespace Ryujinx.HLE.Loaders.Processes.Extensions device.System.KernelContext, metaLoader, nacpData, - enablePtc, + device.System.EnablePtc, + modLoadResult.Hash, true, programName, metaLoader.GetProgramId(), diff --git a/src/Ryujinx.HLE/Loaders/Processes/ProcessLoader.cs b/src/Ryujinx.HLE/Loaders/Processes/ProcessLoader.cs index 726b017b6..4c0866531 100644 --- a/src/Ryujinx.HLE/Loaders/Processes/ProcessLoader.cs +++ b/src/Ryujinx.HLE/Loaders/Processes/ProcessLoader.cs @@ -235,6 +235,7 @@ namespace Ryujinx.HLE.Loaders.Processes dummyExeFs.GetNpdm(), nacpData, diskCacheEnabled: false, + diskCacheSelector: null, allowCodeMemoryForJit: true, programName, programId, diff --git a/src/Ryujinx.HLE/Loaders/Processes/ProcessLoaderHelper.cs b/src/Ryujinx.HLE/Loaders/Processes/ProcessLoaderHelper.cs index b11057da2..badced1b9 100644 --- a/src/Ryujinx.HLE/Loaders/Processes/ProcessLoaderHelper.cs +++ b/src/Ryujinx.HLE/Loaders/Processes/ProcessLoaderHelper.cs @@ -186,6 +186,7 @@ namespace Ryujinx.HLE.Loaders.Processes string.Empty, string.Empty, false, + null, codeAddress, codeSize); @@ -226,6 +227,7 @@ namespace Ryujinx.HLE.Loaders.Processes MetaLoader metaLoader, BlitStruct applicationControlProperties, bool diskCacheEnabled, + string diskCacheSelector, bool allowCodeMemoryForJit, string name, ulong programId, @@ -379,6 +381,7 @@ namespace Ryujinx.HLE.Loaders.Processes $"{programId:x16}", displayVersion, diskCacheEnabled, + diskCacheSelector, codeStart, codeSize); diff --git a/src/Ryujinx/Assets/locales.json b/src/Ryujinx/Assets/locales.json index d96682956..ed1f8a1f0 100644 --- a/src/Ryujinx/Assets/locales.json +++ b/src/Ryujinx/Assets/locales.json @@ -2022,6 +2022,56 @@ "zh_TW": "下一次啟動遊戲時，觸發 PPTC 進行重建" } }, + { + "ID": "GameListContextMenuCacheManagementNukePptc", + "Translations": { + "ar_SA": "", + "de_DE": "", + "el_GR": "", + "en_US": "Purge PPTC cache", + "es_ES": "", + "fr_FR": "", + "he_IL": "", + "it_IT": "", + "ja_JP": "", + "ko_KR": "", + "no_NO": "", + "pl_PL": "", + "pt_BR": "", + "ru_RU": "", + "sv_SE": "", + "th_TH": "", + "tr_TR": "", + "uk_UA": "", + "zh_CN": "", + "zh_TW": "" + } + }, + { + "ID": "GameListContextMenuCacheManagementNukePptcToolTip", + "Translations": { + "ar_SA": "", + "de_DE": "", + "el_GR": "", + "en_US": "Deletes all PPTC cache files for the Application", + "es_ES": "", + "fr_FR": "", + "he_IL": "", + "it_IT": "", + "ja_JP": "", + "ko_KR": "", + "no_NO": "", + "pl_PL": "", + "pt_BR": "", + "ru_RU": "", + "sv_SE": "", + "th_TH": "", + "tr_TR": "", + "uk_UA": "", + "zh_CN": "", + "zh_TW": "" + } + }, { "ID": "GameListContextMenuCacheManagementPurgeShaderCache", "Translations": { @@ -12947,6 +12997,31 @@ "zh_TW": "在 {0} 清除 PPTC 快取時出錯: {1}" } }, + { + "ID": "DialogPPTCNukeMessage", + "Translations": { + "ar_SA": "", + "de_DE": "", + "el_GR": "", + "en_US": "You are about to purge all PPTC data from:\n\n{0}\n\nAre you sure you want to proceed?", + "es_ES": "", + "fr_FR": "", + "he_IL": "", + "it_IT": "", + "ja_JP": "", + "ko_KR": "", + "no_NO": "", + "pl_PL": "", + "pt_BR": "", + "ru_RU": "", + "sv_SE": "", + "th_TH": "", + "tr_TR": "", + "uk_UA": "", + "zh_CN": "", + "zh_TW": "" + } + }, { "ID": "DialogShaderDeletionMessage", "Translations": { diff --git a/src/Ryujinx/UI/Controls/ApplicationContextMenu.axaml b/src/Ryujinx/UI/Controls/ApplicationContextMenu.axaml index 9fed95aa7..475b26787 100644 --- a/src/Ryujinx/UI/Controls/ApplicationContextMenu.axaml +++ b/src/Ryujinx/UI/Controls/ApplicationContextMenu.axaml @@ -81,6 +81,11 @@ Header="{ext:Locale GameListContextMenuCacheManagementPurgePptc}" Icon="{ext:Icon mdi-refresh}" ToolTip.Tip="{ext:Locale GameListContextMenuCacheManagementPurgePptcToolTip}" /> + cacheFiles = new(); + + if (mainDir.Exists) + { + cacheFiles.AddRange(mainDir.EnumerateFiles("*.cache")); + cacheFiles.AddRange(mainDir.EnumerateFiles("*.info")); + } + + if (backupDir.Exists) + { + cacheFiles.AddRange(backupDir.EnumerateFiles("*.cache")); + cacheFiles.AddRange(mainDir.EnumerateFiles("*.info")); + } + + if (cacheFiles.Count > 0) + { + foreach (FileInfo file in cacheFiles) + { + try + { + file.Delete(); + } + catch (Exception ex) + { + await ContentDialogHelper.CreateErrorDialog(LocaleManager.Instance.UpdateAndGetDynamicValue(LocaleKeys.DialogPPTCDeletionErrorMessage, file.Name, ex)); + } + } + } + } + } + public async void PurgeShaderCache_Click(object sender, RoutedEventArgs args) { if (sender is not MenuItem { DataContext: MainWindowViewModel { SelectedApplication: not null } viewModel })