aboutsummaryrefslogtreecommitdiff
path: root/ARMeilleure/Instructions/InstEmitMemoryExHelper.cs
diff options
context:
space:
mode:
authorgdkchan <gab.dark.100@gmail.com>2020-07-30 11:29:28 -0300
committerGitHub <noreply@github.com>2020-07-30 11:29:28 -0300
commit9878fc2d3cf4c64f56c44c2a5de013acb6bcbade (patch)
tree8f5e5cde68fec213ab61dbee0e121448f0970ca2 /ARMeilleure/Instructions/InstEmitMemoryExHelper.cs
parent57bb0abda3dc277dc7575250fdb080edb83abcbc (diff)
Implement inline memory load/store exclusive and ordered (#1413)
* Implement inline memory load/store exclusive * Fix missing REX prefix on 8-bits CMPXCHG * Increment PTC version due to bugfix * Remove redundant memory checks * Address PR feedback * Increment PPTC version
Diffstat (limited to 'ARMeilleure/Instructions/InstEmitMemoryExHelper.cs')
-rw-r--r--ARMeilleure/Instructions/InstEmitMemoryExHelper.cs185
1 files changed, 139 insertions, 46 deletions
diff --git a/ARMeilleure/Instructions/InstEmitMemoryExHelper.cs b/ARMeilleure/Instructions/InstEmitMemoryExHelper.cs
index 059b9b6a..a22cd235 100644
--- a/ARMeilleure/Instructions/InstEmitMemoryExHelper.cs
+++ b/ARMeilleure/Instructions/InstEmitMemoryExHelper.cs
@@ -1,87 +1,180 @@
using ARMeilleure.IntermediateRepresentation;
+using ARMeilleure.State;
using ARMeilleure.Translation;
-using System.Reflection;
+
+using static ARMeilleure.Instructions.InstEmitHelper;
+using static ARMeilleure.IntermediateRepresentation.OperandHelper;
namespace ARMeilleure.Instructions
{
static class InstEmitMemoryExHelper
{
- public static Operand EmitLoadExclusive(
- ArmEmitterContext context,
- Operand address,
- bool exclusive,
- int size)
- {
- MethodInfo info = null;
+ private const int ErgSizeLog2 = 4;
+ public static Operand EmitLoadExclusive(ArmEmitterContext context, Operand address, bool exclusive, int size)
+ {
if (exclusive)
{
- switch (size)
+ Operand value;
+
+ if (size == 4)
{
- case 0: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.ReadByteExclusive)); break;
- case 1: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.ReadUInt16Exclusive)); break;
- case 2: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.ReadUInt32Exclusive)); break;
- case 3: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.ReadUInt64Exclusive)); break;
- case 4: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.ReadVector128Exclusive)); break;
+ Operand isUnalignedAddr = InstEmitMemoryHelper.EmitAddressCheck(context, address, size);
+
+ Operand lblFastPath = Label();
+
+ context.BranchIfFalse(lblFastPath, isUnalignedAddr);
+
+ // The call is not expected to return (it should throw).
+ context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.ThrowInvalidMemoryAccess)), address);
+
+ context.MarkLabel(lblFastPath);
+
+ // Only 128-bit CAS is guaranteed to have a atomic load.
+ Operand physAddr = InstEmitMemoryHelper.EmitPtPointerLoad(context, address, null, write: false);
+
+ Operand zero = context.VectorZero();
+
+ value = context.CompareAndSwap(physAddr, zero, zero);
}
+ else
+ {
+ value = InstEmitMemoryHelper.EmitReadIntAligned(context, address, size);
+ }
+
+ Operand arg0 = context.LoadArgument(OperandType.I64, 0);
+
+ Operand exAddrPtr = context.Add(arg0, Const((long)NativeContext.GetExclusiveAddressOffset()));
+ Operand exValuePtr = context.Add(arg0, Const((long)NativeContext.GetExclusiveValueOffset()));
+
+ context.Store(exAddrPtr, context.BitwiseAnd(address, Const(address.Type, GetExclusiveAddressMask())));
+ context.Store(exValuePtr, value);
+
+ return value;
}
else
{
- switch (size)
- {
- case 0: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.ReadByte)); break;
- case 1: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.ReadUInt16)); break;
- case 2: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.ReadUInt32)); break;
- case 3: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.ReadUInt64)); break;
- case 4: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.ReadVector128)); break;
- }
+ return InstEmitMemoryHelper.EmitReadIntAligned(context, address, size);
}
-
- return context.Call(info, address);
}
- public static Operand EmitStoreExclusive(
+ public static void EmitStoreExclusive(
ArmEmitterContext context,
Operand address,
Operand value,
bool exclusive,
- int size)
+ int size,
+ int rs,
+ bool a32)
{
if (size < 3)
{
value = context.ConvertI64ToI32(value);
}
- MethodInfo info = null;
-
if (exclusive)
{
- switch (size)
+ void SetRs(Operand value)
{
- case 0: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteByteExclusive)); break;
- case 1: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteUInt16Exclusive)); break;
- case 2: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteUInt32Exclusive)); break;
- case 3: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteUInt64Exclusive)); break;
- case 4: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteVector128Exclusive)); break;
+ if (a32)
+ {
+ SetIntA32(context, rs, value);
+ }
+ else
+ {
+ SetIntOrZR(context, rs, value);
+ }
}
- return context.Call(info, address, value);
- }
- else
- {
- switch (size)
+ Operand arg0 = context.LoadArgument(OperandType.I64, 0);
+
+ Operand exAddrPtr = context.Add(arg0, Const((long)NativeContext.GetExclusiveAddressOffset()));
+ Operand exAddr = context.Load(address.Type, exAddrPtr);
+
+ // STEP 1: Check if we have exclusive access to this memory region. If not, fail and skip store.
+ Operand maskedAddress = context.BitwiseAnd(address, Const(GetExclusiveAddressMask()));
+
+ Operand exFailed = context.ICompareNotEqual(exAddr, maskedAddress);
+
+ Operand lblExit = Label();
+
+ SetRs(exFailed);
+
+ context.BranchIfTrue(lblExit, exFailed);
+
+ // STEP 2: We have exclusive access, make sure that the address is valid.
+ Operand isUnalignedAddr = InstEmitMemoryHelper.EmitAddressCheck(context, address, size);
+
+ Operand lblFastPath = Label();
+
+ context.BranchIfFalse(lblFastPath, isUnalignedAddr);
+
+ // The call is not expected to return (it should throw).
+ context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.ThrowInvalidMemoryAccess)), address);
+
+ // STEP 3: We have exclusive access and the address is valid, attempt the store using CAS.
+ context.MarkLabel(lblFastPath);
+
+ Operand physAddr = InstEmitMemoryHelper.EmitPtPointerLoad(context, address, null, write: true);
+
+ Operand exValuePtr = context.Add(arg0, Const((long)NativeContext.GetExclusiveValueOffset()));
+ Operand exValue = size switch
{
- case 0: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteByte)); break;
- case 1: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteUInt16)); break;
- case 2: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteUInt32)); break;
- case 3: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteUInt64)); break;
- case 4: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteVector128)); break;
+ 0 => context.Load8(exValuePtr),
+ 1 => context.Load16(exValuePtr),
+ 2 => context.Load(OperandType.I32, exValuePtr),
+ 3 => context.Load(OperandType.I64, exValuePtr),
+ _ => context.Load(OperandType.V128, exValuePtr)
+ };
+
+ Operand currValue = size switch
+ {
+ 0 => context.CompareAndSwap8(physAddr, exValue, value),
+ 1 => context.CompareAndSwap16(physAddr, exValue, value),
+ _ => context.CompareAndSwap(physAddr, exValue, value)
+ };
+
+ // STEP 4: Check if we succeeded by comparing expected and in-memory values.
+ Operand storeFailed;
+
+ if (size == 4)
+ {
+ Operand currValueLow = context.VectorExtract(OperandType.I64, currValue, 0);
+ Operand currValueHigh = context.VectorExtract(OperandType.I64, currValue, 1);
+
+ Operand exValueLow = context.VectorExtract(OperandType.I64, exValue, 0);
+ Operand exValueHigh = context.VectorExtract(OperandType.I64, exValue, 1);
+
+ storeFailed = context.BitwiseOr(
+ context.ICompareNotEqual(currValueLow, exValueLow),
+ context.ICompareNotEqual(currValueHigh, exValueHigh));
+ }
+ else
+ {
+ storeFailed = context.ICompareNotEqual(currValue, exValue);
}
- context.Call(info, address, value);
+ SetRs(storeFailed);
- return null;
+ context.MarkLabel(lblExit);
+ }
+ else
+ {
+ InstEmitMemoryHelper.EmitWriteIntAligned(context, address, value, size);
}
}
+
+ public static void EmitClearExclusive(ArmEmitterContext context)
+ {
+ Operand arg0 = context.LoadArgument(OperandType.I64, 0);
+
+ Operand exAddrPtr = context.Add(arg0, Const((long)NativeContext.GetExclusiveAddressOffset()));
+
+ // We store ULONG max to force any exclusive address checks to fail,
+ // since this value is not aligned to the ERG mask.
+ context.Store(exAddrPtr, Const(ulong.MaxValue));
+ }
+
+ private static long GetExclusiveAddressMask() => ~((4L << ErgSizeLog2) - 1);
}
}
generated by cgit v1.2.3-70-g09d2 (git 2.48.1) at 2025-03-13 02:06:07 +0000