From 89791ba68dba70999643c5d876e9329b385c6e8a Mon Sep 17 00:00:00 2001 From: FICTURE7 Date: Mon, 19 Apr 2021 01:43:53 +0400 Subject: [PATCH] Add inlined on translation call counting (#2190) * Add EntryTable * Add on translation call counting * Add Counter * Add PPTC support * Make Counter a generic & use a 32-bit counter instead * Return false on overflow * Set PPTC version * Print more information about the rejit queue * Make Counter disposable * Remove Block.TailCall since it is not used anymore * Apply suggestions from code review Address gdkchan's feedback Co-authored-by: gdkchan * Fix more stale docs * Remove rejit requests queue logging * Make Counter finalizable Most certainly quite an odd use case. * Make EntryTable.TryAllocate set entry to default * Re-trigger CI * Dispose Counters before they hit the finalizer queue * Re-trigger CI Just for good measure... * Make EntryTable expandable * EntryTable is now expandable instead of being a fixed slab. * Remove EntryTable.TryAllocate * Remove Counter.TryCreate Address LDj3SNuD's feedback * Apply suggestions from code review Address LDj3SNuD's feedback Co-authored-by: LDj3SNuD <35856442+LDj3SNuD@users.noreply.github.com> * Remove useless return * POH approach, but the sequel * Revert "POH approach, but the sequel" This reverts commit 5f5abaa24735726ff2db367dc74f98055d4f4cba. The sequel got shelved * Add extra documentation Co-authored-by: gdkchan Co-authored-by: LDj3SNuD <35856442+LDj3SNuD@users.noreply.github.com> --- ARMeilleure/Common/Counter.cs | 99 +++++++++ ARMeilleure/Common/EntryTable.cs | 196 ++++++++++++++++++ ARMeilleure/Decoders/Block.cs | 3 +- .../Decoders/Optimizations/TailCallRemover.cs | 5 +- .../Instructions/InstEmitFlowHelper.cs | 6 +- ARMeilleure/Instructions/NativeInterface.cs | 19 +- .../OperandHelper.cs | 6 + ARMeilleure/Translation/ArmEmitterContext.cs | 19 +- ARMeilleure/Translation/Delegates.cs | 2 +- ARMeilleure/Translation/PTC/Ptc.cs | 51 ++++- ARMeilleure/Translation/TranslatedFunction.cs | 20 +- ARMeilleure/Translation/Translator.cs | 134 ++++++++---- 12 files changed, 468 insertions(+), 92 deletions(-) create mode 100644 ARMeilleure/Common/Counter.cs create mode 100644 ARMeilleure/Common/EntryTable.cs diff --git a/ARMeilleure/Common/Counter.cs b/ARMeilleure/Common/Counter.cs new file mode 100644 index 00000000..defb5aba --- /dev/null +++ b/ARMeilleure/Common/Counter.cs @@ -0,0 +1,99 @@ +using System; + +namespace ARMeilleure.Common +{ + /// + /// Represents a numeric counter which can be used for instrumentation of compiled code. + /// + /// Type of the counter + class Counter : IDisposable where T : unmanaged + { + private bool _disposed; + private readonly int _index; + private readonly EntryTable _countTable; + + /// + /// Initializes a new instance of the class from the specified + /// instance and index. + /// + /// instance + /// Index in the + /// is + /// is unsupported + public Counter(EntryTable countTable) + { + if (typeof(T) != typeof(byte) && typeof(T) != typeof(sbyte) && + typeof(T) != typeof(short) && typeof(T) != typeof(ushort) && + typeof(T) != typeof(int) && typeof(T) != typeof(uint) && + typeof(T) != typeof(long) && typeof(T) != typeof(ulong) && + typeof(T) != typeof(nint) && typeof(T) != typeof(nuint) && + typeof(T) != typeof(float) && typeof(T) != typeof(double)) + { + throw new ArgumentException("Counter does not support the specified type."); + } + + _countTable = countTable ?? throw new ArgumentNullException(nameof(countTable)); + _index = countTable.Allocate(); + } + + /// + /// Gets a reference to the value of the counter. + /// + /// instance was disposed + /// + /// This can refer to freed memory if the owning is disposed. + /// + public ref T Value + { + get + { + if (_disposed) + { + throw new ObjectDisposedException(null); + } + + return ref _countTable.GetValue(_index); + } + } + + /// + /// Releases all resources used by the instance. + /// + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + /// + /// Releases all unmanaged and optionally managed resources used by the instance. + /// + /// to dispose managed resources also; otherwise just unmanaged resouces + protected virtual void Dispose(bool disposing) + { + if (!_disposed) + { + try + { + // The index into the EntryTable is essentially an unmanaged resource since we allocate and free the + // resource ourselves. + _countTable.Free(_index); + } + catch (ObjectDisposedException) + { + // Can happen because _countTable may be disposed before the Counter instance. + } + + _disposed = true; + } + } + + /// + /// Frees resources used by the instance. + /// + ~Counter() + { + Dispose(false); + } + } +} diff --git a/ARMeilleure/Common/EntryTable.cs b/ARMeilleure/Common/EntryTable.cs new file mode 100644 index 00000000..a0ed7c8e --- /dev/null +++ b/ARMeilleure/Common/EntryTable.cs @@ -0,0 +1,196 @@ +using System; +using System.Collections.Generic; +using System.Numerics; +using System.Runtime.InteropServices; + +namespace ARMeilleure.Common +{ + /// + /// Represents an expandable table of the type , whose entries will remain at the same + /// address through out the table's lifetime. + /// + /// Type of the entry in the table + class EntryTable : IDisposable where TEntry : unmanaged + { + private bool _disposed; + private int _freeHint; + private readonly int _pageCapacity; // Number of entries per page. + private readonly int _pageLogCapacity; + private readonly Dictionary _pages; + private readonly BitMap _allocated; + + /// + /// Initializes a new instance of the class with the desired page size in + /// bytes. + /// + /// Desired page size in bytes + /// is less than 0 + /// 's size is zero + /// + /// The actual page size may be smaller or larger depending on the size of . + /// + public unsafe EntryTable(int pageSize = 4096) + { + if (pageSize < 0) + { + throw new ArgumentOutOfRangeException(nameof(pageSize), "Page size cannot be negative."); + } + + if (sizeof(TEntry) == 0) + { + throw new ArgumentException("Size of TEntry cannot be zero."); + } + + _allocated = new BitMap(); + _pages = new Dictionary(); + _pageLogCapacity = BitOperations.Log2((uint)(pageSize / sizeof(TEntry))); + _pageCapacity = 1 << _pageLogCapacity; + } + + /// + /// Allocates an entry in the . + /// + /// Index of entry allocated in the table + /// instance was disposed + public int Allocate() + { + if (_disposed) + { + throw new ObjectDisposedException(null); + } + + lock (_allocated) + { + if (_allocated.IsSet(_freeHint)) + { + _freeHint = _allocated.FindFirstUnset(); + } + + int index = _freeHint++; + var page = GetPage(index); + + _allocated.Set(index); + + GetValue(page, index) = default; + + return index; + } + } + + /// + /// Frees the entry at the specified . + /// + /// Index of entry to free + /// instance was disposed + public void Free(int index) + { + if (_disposed) + { + throw new ObjectDisposedException(null); + } + + lock (_allocated) + { + if (_allocated.IsSet(index)) + { + _allocated.Clear(index); + + _freeHint = index; + } + } + } + + /// + /// Gets a reference to the entry at the specified allocated . + /// + /// Index of the entry + /// Reference to the entry at the specified + /// instance was disposed + /// Entry at is not allocated + public ref TEntry GetValue(int index) + { + if (_disposed) + { + throw new ObjectDisposedException(null); + } + + lock (_allocated) + { + if (!_allocated.IsSet(index)) + { + throw new ArgumentException("Entry at the specified index was not allocated", nameof(index)); + } + + var page = GetPage(index); + + return ref GetValue(page, index); + } + } + + /// + /// Gets a reference to the entry at using the specified from the specified + /// . + /// + /// Page to use + /// Index to use + /// Reference to the entry + private ref TEntry GetValue(Span page, int index) + { + return ref page[index & (_pageCapacity - 1)]; + } + + /// + /// Gets the page for the specified . + /// + /// Index to use + /// Page for the specified + private unsafe Span GetPage(int index) + { + var pageIndex = (int)((uint)(index & ~(_pageCapacity - 1)) >> _pageLogCapacity); + + if (!_pages.TryGetValue(pageIndex, out IntPtr page)) + { + page = Marshal.AllocHGlobal(sizeof(TEntry) * _pageCapacity); + + _pages.Add(pageIndex, page); + } + + return new Span((void*)page, _pageCapacity); + } + + /// + /// Releases all resources used by the instance. + /// + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + /// + /// Releases all unmanaged and optionally managed resources used by the + /// instance. + /// + /// to dispose managed resources also; otherwise just unmanaged resouces + protected virtual void Dispose(bool disposing) + { + if (!_disposed) + { + foreach (var page in _pages.Values) + { + Marshal.FreeHGlobal(page); + } + + _disposed = true; + } + } + + /// + /// Frees resources used by the instance. + /// + ~EntryTable() + { + Dispose(false); + } + } +} diff --git a/ARMeilleure/Decoders/Block.cs b/ARMeilleure/Decoders/Block.cs index 9d380bca..f296d299 100644 --- a/ARMeilleure/Decoders/Block.cs +++ b/ARMeilleure/Decoders/Block.cs @@ -11,8 +11,7 @@ namespace ARMeilleure.Decoders public Block Next { get; set; } public Block Branch { get; set; } - public bool TailCall { get; set; } - public bool Exit { get; set; } + public bool Exit { get; set; } public List OpCodes { get; } diff --git a/ARMeilleure/Decoders/Optimizations/TailCallRemover.cs b/ARMeilleure/Decoders/Optimizations/TailCallRemover.cs index e64f9a54..17c17812 100644 --- a/ARMeilleure/Decoders/Optimizations/TailCallRemover.cs +++ b/ARMeilleure/Decoders/Optimizations/TailCallRemover.cs @@ -58,15 +58,14 @@ namespace ARMeilleure.Decoders.Optimizations return blocks.ToArray(); // Nothing to do here. } - // Mark branches outside of contiguous region as exit blocks. + // Mark branches whose target is outside of the contiguous region as an exit block. for (int i = startBlockIndex; i <= endBlockIndex; i++) { Block block = blocks[i]; if (block.Branch != null && (block.Branch.Address > endBlock.EndAddress || block.Branch.EndAddress < startBlock.Address)) { - block.Branch.Exit = true; - block.Branch.TailCall = true; + block.Branch.Exit = true; } } diff --git a/ARMeilleure/Instructions/InstEmitFlowHelper.cs b/ARMeilleure/Instructions/InstEmitFlowHelper.cs index 296e20a5..f995ffa1 100644 --- a/ARMeilleure/Instructions/InstEmitFlowHelper.cs +++ b/ARMeilleure/Instructions/InstEmitFlowHelper.cs @@ -200,7 +200,7 @@ namespace ARMeilleure.Instructions } } - public static void EmitTailContinue(ArmEmitterContext context, Operand address, bool allowRejit) + public static void EmitTailContinue(ArmEmitterContext context, Operand address) { // Left option here as it may be useful if we need to return to managed rather than tail call in future. // (eg. for debug) @@ -218,9 +218,7 @@ namespace ARMeilleure.Instructions { context.StoreToContext(); - Operand fallbackAddr = context.Call(typeof(NativeInterface).GetMethod(allowRejit - ? nameof(NativeInterface.GetFunctionAddress) - : nameof(NativeInterface.GetFunctionAddressWithoutRejit)), address); + Operand fallbackAddr = context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFunctionAddress)), address); EmitNativeCall(context, fallbackAddr, isJump: true); } diff --git a/ARMeilleure/Instructions/NativeInterface.cs b/ARMeilleure/Instructions/NativeInterface.cs index b8b7ff0e..fa17d334 100644 --- a/ARMeilleure/Instructions/NativeInterface.cs +++ b/ARMeilleure/Instructions/NativeInterface.cs @@ -220,6 +220,11 @@ namespace ARMeilleure.Instructions } #endregion + public static void EnqueueForRejit(ulong address) + { + Context.Translator.EnqueueForRejit(address, GetContext().ExecutionMode); + } + public static void SignalMemoryTracking(ulong address, ulong size, bool write) { GetMemoryManager().SignalMemoryTracking(address, size, write); @@ -232,24 +237,14 @@ namespace ARMeilleure.Instructions public static ulong GetFunctionAddress(ulong address) { - return GetFunctionAddressWithHint(address, true); - } - - public static ulong GetFunctionAddressWithoutRejit(ulong address) - { - return GetFunctionAddressWithHint(address, false); - } - - private static ulong GetFunctionAddressWithHint(ulong address, bool hintRejit) - { - TranslatedFunction function = Context.Translator.GetOrTranslate(address, GetContext().ExecutionMode, hintRejit); + TranslatedFunction function = Context.Translator.GetOrTranslate(address, GetContext().ExecutionMode); return (ulong)function.FuncPtr.ToInt64(); } public static ulong GetIndirectFunctionAddress(ulong address, ulong entryAddress) { - TranslatedFunction function = Context.Translator.GetOrTranslate(address, GetContext().ExecutionMode, hintRejit: true); + TranslatedFunction function = Context.Translator.GetOrTranslate(address, GetContext().ExecutionMode); ulong ptr = (ulong)function.FuncPtr.ToInt64(); diff --git a/ARMeilleure/IntermediateRepresentation/OperandHelper.cs b/ARMeilleure/IntermediateRepresentation/OperandHelper.cs index 26d66478..1b748f6a 100644 --- a/ARMeilleure/IntermediateRepresentation/OperandHelper.cs +++ b/ARMeilleure/IntermediateRepresentation/OperandHelper.cs @@ -1,4 +1,5 @@ using ARMeilleure.Common; +using System.Runtime.CompilerServices; namespace ARMeilleure.IntermediateRepresentation { @@ -34,6 +35,11 @@ namespace ARMeilleure.IntermediateRepresentation return Operand().With(value); } + public static unsafe Operand Const(ref T reference, int? index = null) + { + return Operand().With((long)Unsafe.AsPointer(ref reference), index != null, index); + } + public static Operand ConstF(float value) { return Operand().With(value); diff --git a/ARMeilleure/Translation/ArmEmitterContext.cs b/ARMeilleure/Translation/ArmEmitterContext.cs index 8f153192..ad44b0cf 100644 --- a/ARMeilleure/Translation/ArmEmitterContext.cs +++ b/ARMeilleure/Translation/ArmEmitterContext.cs @@ -1,3 +1,4 @@ +using ARMeilleure.Common; using ARMeilleure.Decoders; using ARMeilleure.Instructions; using ARMeilleure.IntermediateRepresentation; @@ -41,18 +42,26 @@ namespace ARMeilleure.Translation public IMemoryManager Memory { get; } public JumpTable JumpTable { get; } + public EntryTable CountTable { get; } public ulong EntryAddress { get; } public bool HighCq { get; } public Aarch32Mode Mode { get; } - public ArmEmitterContext(IMemoryManager memory, JumpTable jumpTable, ulong entryAddress, bool highCq, Aarch32Mode mode) + public ArmEmitterContext( + IMemoryManager memory, + JumpTable jumpTable, + EntryTable countTable, + ulong entryAddress, + bool highCq, + Aarch32Mode mode) { - Memory = memory; - JumpTable = jumpTable; + Memory = memory; + JumpTable = jumpTable; + CountTable = countTable; EntryAddress = entryAddress; - HighCq = highCq; - Mode = mode; + HighCq = highCq; + Mode = mode; _labels = new Dictionary(); } diff --git a/ARMeilleure/Translation/Delegates.cs b/ARMeilleure/Translation/Delegates.cs index 5ad71872..a561d265 100644 --- a/ARMeilleure/Translation/Delegates.cs +++ b/ARMeilleure/Translation/Delegates.cs @@ -103,6 +103,7 @@ namespace ARMeilleure.Translation SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.Break))); SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.CheckSynchronization))); + SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.EnqueueForRejit))); SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetCntfrqEl0))); SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetCntpctEl0))); SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetCntvctEl0))); @@ -113,7 +114,6 @@ namespace ARMeilleure.Translation SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFpscr))); // A32 only. SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFpsr))); SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFunctionAddress))); - SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFunctionAddressWithoutRejit))); SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetIndirectFunctionAddress))); SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetTpidr))); SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetTpidr32))); // A32 only. diff --git a/ARMeilleure/Translation/PTC/Ptc.cs b/ARMeilleure/Translation/PTC/Ptc.cs index 32e0e7e8..55a0f4d0 100644 --- a/ARMeilleure/Translation/PTC/Ptc.cs +++ b/ARMeilleure/Translation/PTC/Ptc.cs @@ -1,6 +1,7 @@ using ARMeilleure.CodeGen; using ARMeilleure.CodeGen.Unwinding; using ARMeilleure.CodeGen.X86; +using ARMeilleure.Common; using ARMeilleure.Memory; using ARMeilleure.Translation.Cache; using Ryujinx.Common; @@ -27,7 +28,7 @@ namespace ARMeilleure.Translation.PTC private const string OuterHeaderMagicString = "PTCohd\0\0"; private const string InnerHeaderMagicString = "PTCihd\0\0"; - private const uint InternalVersion = 2169; //! To be incremented manually for each change to the ARMeilleure project. + private const uint InternalVersion = 2190; //! To be incremented manually for each change to the ARMeilleure project. private const string ActualDir = "0"; private const string BackupDir = "1"; @@ -38,6 +39,7 @@ namespace ARMeilleure.Translation.PTC internal const int PageTablePointerIndex = -1; // Must be a negative value. internal const int JumpPointerIndex = -2; // Must be a negative value. internal const int DynamicPointerIndex = -3; // Must be a negative value. + internal const int CountTableIndex = -4; // Must be a negative value. private const byte FillingByte = 0x00; private const CompressionLevel SaveCompressionLevel = CompressionLevel.Fastest; @@ -538,7 +540,11 @@ namespace ARMeilleure.Translation.PTC } } - internal static void LoadTranslations(ConcurrentDictionary funcs, IMemoryManager memory, JumpTable jumpTable) + internal static void LoadTranslations( + ConcurrentDictionary funcs, + IMemoryManager memory, + JumpTable jumpTable, + EntryTable countTable) { if (AreCarriersEmpty()) { @@ -567,16 +573,18 @@ namespace ARMeilleure.Translation.PTC { byte[] code = ReadCode(index, infoEntry.CodeLength); + Counter callCounter = null; + if (infoEntry.RelocEntriesCount != 0) { RelocEntry[] relocEntries = GetRelocEntries(relocsReader, infoEntry.RelocEntriesCount); - PatchCode(code.AsSpan(), relocEntries, memory.PageTablePointer, jumpTable); + PatchCode(code, relocEntries, memory.PageTablePointer, jumpTable, countTable, out callCounter); } UnwindInfo unwindInfo = ReadUnwindInfo(unwindInfosReader); - TranslatedFunction func = FastTranslate(code, infoEntry.GuestSize, unwindInfo, infoEntry.HighCq); + TranslatedFunction func = FastTranslate(code, callCounter, infoEntry.GuestSize, unwindInfo, infoEntry.HighCq); bool isAddressUnique = funcs.TryAdd(infoEntry.Address, func); @@ -670,8 +678,16 @@ namespace ARMeilleure.Translation.PTC return relocEntries; } - private static void PatchCode(Span code, RelocEntry[] relocEntries, IntPtr pageTablePointer, JumpTable jumpTable) + private static void PatchCode( + Span code, + RelocEntry[] relocEntries, + IntPtr pageTablePointer, + JumpTable jumpTable, + EntryTable countTable, + out Counter callCounter) { + callCounter = null; + foreach (RelocEntry relocEntry in relocEntries) { ulong imm; @@ -688,6 +704,12 @@ namespace ARMeilleure.Translation.PTC { imm = (ulong)jumpTable.DynamicPointer.ToInt64(); } + else if (relocEntry.Index == CountTableIndex) + { + callCounter = new Counter(countTable); + + unsafe { imm = (ulong)Unsafe.AsPointer(ref callCounter.Value); } + } else if (Delegates.TryGetDelegateFuncPtrByIndex(relocEntry.Index, out IntPtr funcPtr)) { imm = (ulong)funcPtr.ToInt64(); @@ -722,7 +744,12 @@ namespace ARMeilleure.Translation.PTC return new UnwindInfo(pushEntries, prologueSize); } - private static TranslatedFunction FastTranslate(byte[] code, ulong guestSize, UnwindInfo unwindInfo, bool highCq) + private static TranslatedFunction FastTranslate( + byte[] code, + Counter callCounter, + ulong guestSize, + UnwindInfo unwindInfo, + bool highCq) { CompiledFunction cFunc = new CompiledFunction(code, unwindInfo); @@ -730,7 +757,7 @@ namespace ARMeilleure.Translation.PTC GuestFunction gFunc = Marshal.GetDelegateForFunctionPointer(codePtr); - TranslatedFunction tFunc = new TranslatedFunction(gFunc, guestSize, highCq); + TranslatedFunction tFunc = new TranslatedFunction(gFunc, callCounter, guestSize, highCq); return tFunc; } @@ -771,7 +798,11 @@ namespace ARMeilleure.Translation.PTC } } - internal static void MakeAndSaveTranslations(ConcurrentDictionary funcs, IMemoryManager memory, JumpTable jumpTable) + internal static void MakeAndSaveTranslations( + ConcurrentDictionary funcs, + IMemoryManager memory, + JumpTable jumpTable, + EntryTable countTable) { var profiledFuncsToTranslate = PtcProfiler.GetProfiledFuncsToTranslate(funcs); @@ -813,7 +844,7 @@ namespace ARMeilleure.Translation.PTC Debug.Assert(PtcProfiler.IsAddressInStaticCodeRange(address)); - TranslatedFunction func = Translator.Translate(memory, jumpTable, address, item.mode, item.highCq); + TranslatedFunction func = Translator.Translate(memory, jumpTable, countTable, address, item.mode, item.highCq); bool isAddressUnique = funcs.TryAdd(address, func); @@ -1058,4 +1089,4 @@ namespace ARMeilleure.Translation.PTC } } } -} \ No newline at end of file +} diff --git a/ARMeilleure/Translation/TranslatedFunction.cs b/ARMeilleure/Translation/TranslatedFunction.cs index 8b075928..04dd769c 100644 --- a/ARMeilleure/Translation/TranslatedFunction.cs +++ b/ARMeilleure/Translation/TranslatedFunction.cs @@ -1,24 +1,22 @@ +using ARMeilleure.Common; using System; using System.Runtime.InteropServices; -using System.Threading; namespace ARMeilleure.Translation { class TranslatedFunction { - private const int MinCallsForRejit = 100; - private readonly GuestFunction _func; // Ensure that this delegate will not be garbage collected. - private int _callCount; - + public Counter CallCounter { get; } public ulong GuestSize { get; } public bool HighCq { get; } public IntPtr FuncPtr { get; } - public TranslatedFunction(GuestFunction func, ulong guestSize, bool highCq) + public TranslatedFunction(GuestFunction func, Counter callCounter, ulong guestSize, bool highCq) { _func = func; + CallCounter = callCounter; GuestSize = guestSize; HighCq = highCq; FuncPtr = Marshal.GetFunctionPointerForDelegate(func); @@ -28,15 +26,5 @@ namespace ARMeilleure.Translation { return _func(context.NativeContextPtr); } - - public bool ShouldRejit() - { - return !HighCq && Interlocked.Increment(ref _callCount) == MinCallsForRejit; - } - - public void ResetCallCount() - { - Interlocked.Exchange(ref _callCount, 0); - } } } \ No newline at end of file diff --git a/ARMeilleure/Translation/Translator.cs b/ARMeilleure/Translation/Translator.cs index 73a321fd..9f88f17a 100644 --- a/ARMeilleure/Translation/Translator.cs +++ b/ARMeilleure/Translation/Translator.cs @@ -1,3 +1,4 @@ +using ARMeilleure.Common; using ARMeilleure.Decoders; using ARMeilleure.Diagnostics; using ARMeilleure.Instructions; @@ -10,7 +11,6 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; -using System.Linq; using System.Runtime; using System.Threading; @@ -22,23 +22,27 @@ namespace ARMeilleure.Translation { public class Translator { + private const int CountTableCapacity = 4 * 1024 * 1024; + private readonly IJitMemoryAllocator _allocator; private readonly IMemoryManager _memory; private readonly ConcurrentDictionary _funcs; - private readonly ConcurrentQueue> _oldFuncs; + private readonly ConcurrentQueue> _oldFuncs; + private readonly ConcurrentDictionary _backgroundSet; private readonly ConcurrentStack _backgroundStack; private readonly AutoResetEvent _backgroundTranslatorEvent; private readonly ReaderWriterLock _backgroundTranslatorLock; private JumpTable _jumpTable; internal JumpTable JumpTable => _jumpTable; + internal EntryTable CountTable { get; } private volatile int _threadCount; // FIXME: Remove this once the init logic of the emulator will be redone. - public static ManualResetEvent IsReadyForTranslation = new ManualResetEvent(false); + public static readonly ManualResetEvent IsReadyForTranslation = new(false); public Translator(IJitMemoryAllocator allocator, IMemoryManager memory) { @@ -46,12 +50,15 @@ namespace ARMeilleure.Translation _memory = memory; _funcs = new ConcurrentDictionary(); - _oldFuncs = new ConcurrentQueue>(); + _oldFuncs = new ConcurrentQueue>(); + _backgroundSet = new ConcurrentDictionary(); _backgroundStack = new ConcurrentStack(); _backgroundTranslatorEvent = new AutoResetEvent(false); _backgroundTranslatorLock = new ReaderWriterLock(); + CountTable = new EntryTable(); + JitCache.Initialize(allocator); DirectCallStubs.InitializeStubs(); @@ -63,9 +70,16 @@ namespace ARMeilleure.Translation { _backgroundTranslatorLock.AcquireReaderLock(Timeout.Infinite); - if (_backgroundStack.TryPop(out RejitRequest request)) + if (_backgroundStack.TryPop(out RejitRequest request) && + _backgroundSet.TryRemove(request.Address, out _)) { - TranslatedFunction func = Translate(_memory, _jumpTable, request.Address, request.Mode, highCq: true); + TranslatedFunction func = Translate( + _memory, + _jumpTable, + CountTable, + request.Address, + request.Mode, + highCq: true); _funcs.AddOrUpdate(request.Address, func, (key, oldFunc) => { @@ -89,7 +103,8 @@ namespace ARMeilleure.Translation } } - _backgroundTranslatorEvent.Set(); // Wake up any other background translator threads, to encourage them to exit. + // Wake up any other background translator threads, to encourage them to exit. + _backgroundTranslatorEvent.Set(); } public void Execute(State.ExecutionContext context, ulong address) @@ -104,18 +119,21 @@ namespace ARMeilleure.Translation if (Ptc.State == PtcState.Enabled) { Debug.Assert(_funcs.Count == 0); - Ptc.LoadTranslations(_funcs, _memory, _jumpTable); - Ptc.MakeAndSaveTranslations(_funcs, _memory, _jumpTable); + Ptc.LoadTranslations(_funcs, _memory, _jumpTable, CountTable); + Ptc.MakeAndSaveTranslations(_funcs, _memory, _jumpTable, CountTable); } PtcProfiler.Start(); Ptc.Disable(); - // Simple heuristic, should be user configurable in future. (1 for 4 core/ht or less, 2 for 6 core+ht etc). - // All threads are normal priority except from the last, which just fills as much of the last core as the os lets it with a low priority. - // If we only have one rejit thread, it should be normal priority as highCq code is performance critical. - // TODO: Use physical cores rather than logical. This only really makes sense for processors with hyperthreading. Requires OS specific code. + // Simple heuristic, should be user configurable in future. (1 for 4 core/ht or less, 2 for 6 core + ht + // etc). All threads are normal priority except from the last, which just fills as much of the last core + // as the os lets it with a low priority. If we only have one rejit thread, it should be normal priority + // as highCq code is performance critical. + // + // TODO: Use physical cores rather than logical. This only really makes sense for processors with + // hyperthreading. Requires OS specific code. int unboundedThreadCount = Math.Max(1, (Environment.ProcessorCount - 6) / 3); int threadCount = Math.Min(4, unboundedThreadCount); @@ -156,6 +174,8 @@ namespace ARMeilleure.Translation _jumpTable.Dispose(); _jumpTable = null; + CountTable.Dispose(); + GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce; } } @@ -173,11 +193,11 @@ namespace ARMeilleure.Translation return nextAddr; } - internal TranslatedFunction GetOrTranslate(ulong address, ExecutionMode mode, bool hintRejit = false) + internal TranslatedFunction GetOrTranslate(ulong address, ExecutionMode mode) { if (!_funcs.TryGetValue(address, out TranslatedFunction func)) { - func = Translate(_memory, _jumpTable, address, mode, highCq: false); + func = Translate(_memory, _jumpTable, CountTable, address, mode, highCq: false); TranslatedFunction getFunc = _funcs.GetOrAdd(address, func); @@ -193,18 +213,18 @@ namespace ARMeilleure.Translation } } - if (hintRejit && func.ShouldRejit()) - { - _backgroundStack.Push(new RejitRequest(address, mode)); - _backgroundTranslatorEvent.Set(); - } - return func; } - internal static TranslatedFunction Translate(IMemoryManager memory, JumpTable jumpTable, ulong address, ExecutionMode mode, bool highCq) + internal static TranslatedFunction Translate( + IMemoryManager memory, + JumpTable jumpTable, + EntryTable countTable, + ulong address, + ExecutionMode mode, + bool highCq) { - ArmEmitterContext context = new ArmEmitterContext(memory, jumpTable, address, highCq, Aarch32Mode.User); + var context = new ArmEmitterContext(memory, jumpTable, countTable, address, highCq, Aarch32Mode.User); Logger.StartPass(PassName.Decoding); @@ -216,6 +236,13 @@ namespace ARMeilleure.Translation Logger.StartPass(PassName.Translation); + Counter counter = null; + + if (!context.HighCq) + { + EmitRejitCheck(context, out counter); + } + EmitSynchronization(context); if (blocks[0].Address != address) @@ -258,7 +285,7 @@ namespace ARMeilleure.Translation Ptc.WriteInfoCodeRelocUnwindInfo(address, funcSize, highCq, ptcInfo); } - return new TranslatedFunction(func, funcSize, highCq); + return new TranslatedFunction(func, counter, funcSize, highCq); } internal static void PreparePool(int groupId = 0) @@ -320,7 +347,7 @@ namespace ARMeilleure.Translation if (block.Exit) { - InstEmitFlowHelper.EmitTailContinue(context, Const(block.Address), block.TailCall); + InstEmitFlowHelper.EmitTailContinue(context, Const(block.Address)); } else { @@ -368,29 +395,43 @@ namespace ARMeilleure.Translation return context.GetControlFlowGraph(); } + internal static void EmitRejitCheck(ArmEmitterContext context, out Counter counter) + { + const int MinsCallForRejit = 100; + + counter = new Counter(context.CountTable); + + Operand lblEnd = Label(); + + Operand address = Const(ref counter.Value, Ptc.CountTableIndex); + Operand curCount = context.Load(OperandType.I32, address); + Operand count = context.Add(curCount, Const(1)); + context.Store(address, count); + context.BranchIf(lblEnd, curCount, Const(MinsCallForRejit), Comparison.NotEqual, BasicBlockFrequency.Cold); + + context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.EnqueueForRejit)), Const(context.EntryAddress)); + + context.MarkLabel(lblEnd); + } + internal static void EmitSynchronization(EmitterContext context) { long countOffs = NativeContext.GetCounterOffset(); - Operand countAddr = context.Add(context.LoadArgument(OperandType.I64, 0), Const(countOffs)); - - Operand count = context.Load(OperandType.I32, countAddr); - Operand lblNonZero = Label(); - Operand lblExit = Label(); + Operand lblExit = Label(); + Operand countAddr = context.Add(context.LoadArgument(OperandType.I64, 0), Const(countOffs)); + Operand count = context.Load(OperandType.I32, countAddr); context.BranchIfTrue(lblNonZero, count, BasicBlockFrequency.Cold); Operand running = context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.CheckSynchronization))); - context.BranchIfTrue(lblExit, running, BasicBlockFrequency.Cold); context.Return(Const(0L)); context.MarkLabel(lblNonZero); - count = context.Subtract(count, Const(1)); - context.Store(countAddr, count); context.MarkLabel(lblExit); @@ -404,9 +445,18 @@ namespace ARMeilleure.Translation // TODO: Completely remove functions overlapping the specified range from the cache. } + internal void EnqueueForRejit(ulong guestAddress, ExecutionMode mode) + { + if (_backgroundSet.TryAdd(guestAddress, null)) + { + _backgroundStack.Push(new RejitRequest(guestAddress, mode)); + _backgroundTranslatorEvent.Set(); + } + } + private void EnqueueForDeletion(ulong guestAddress, TranslatedFunction func) { - _oldFuncs.Enqueue(new KeyValuePair(guestAddress, func.FuncPtr)); + _oldFuncs.Enqueue(new(guestAddress, func)); } private void ClearJitCache() @@ -414,16 +464,20 @@ namespace ARMeilleure.Translation // Ensure no attempt will be made to compile new functions due to rejit. ClearRejitQueue(allowRequeue: false); - foreach (var kv in _funcs) + foreach (var func in _funcs.Values) { - JitCache.Unmap(kv.Value.FuncPtr); + JitCache.Unmap(func.FuncPtr); + + func.CallCounter?.Dispose(); } _funcs.Clear(); while (_oldFuncs.TryDequeue(out var kv)) { - JitCache.Unmap(kv.Value); + JitCache.Unmap(kv.Value.FuncPtr); + + kv.Value.CallCounter?.Dispose(); } } @@ -435,10 +489,12 @@ namespace ARMeilleure.Translation { while (_backgroundStack.TryPop(out var request)) { - if (_funcs.TryGetValue(request.Address, out var func)) + if (_funcs.TryGetValue(request.Address, out var func) && func.CallCounter != null) { - func.ResetCallCount(); + Volatile.Write(ref func.CallCounter.Value, 0); } + + _backgroundSet.TryRemove(request.Address, out _); } } else