public CircularLinkedList()
{
head = null;
tail = null;
}
public LLNode()
{
this.Data = default(T);
this.Next = null;
}
public void LinkList_NullTail_ArgumentNullException()
{
LLNode head = new LLNode();
Assert.Throws <ArgumentNullException>(() => new LinkList(head, null));
}
public SortedLinkedList()
{
head = null;
}
private static void HandleLoadArgumentSystemVAbi(
CompilerContext cctx,
LLNode node,
Operand[] preservedArgs,
Operation operation)
{
Operand source = operation.GetSource(0);
Debug.Assert(source.Kind == OperandKind.Constant, "Non-constant LoadArgument source kind.");
int index = source.AsInt32();
int intCount = 0;
int vecCount = 0;
for (int cIndex = 0; cIndex < index; cIndex++)
{
OperandType argType = cctx.FuncArgTypes[cIndex];
if (argType.IsInteger())
{
intCount++;
}
else if (argType == OperandType.V128)
{
intCount += 2;
}
else
{
vecCount++;
}
}
bool passOnReg;
if (source.Type.IsInteger())
{
passOnReg = intCount < CallingConvention.GetIntArgumentsOnRegsCount();
}
else if (source.Type == OperandType.V128)
{
passOnReg = intCount + 1 < CallingConvention.GetIntArgumentsOnRegsCount();
}
else
{
passOnReg = vecCount < CallingConvention.GetVecArgumentsOnRegsCount();
}
if (passOnReg)
{
Operand dest = operation.Destination;
if (preservedArgs[index] == null)
{
if (dest.Type == OperandType.V128)
{
// V128 is a struct, we pass each half on a GPR if possible.
Operand pArg = Local(OperandType.V128);
Operand argLReg = Gpr(CallingConvention.GetIntArgumentRegister(intCount), OperandType.I64);
Operand argHReg = Gpr(CallingConvention.GetIntArgumentRegister(intCount + 1), OperandType.I64);
Operation copyL = new Operation(Instruction.VectorCreateScalar, pArg, argLReg);
Operation copyH = new Operation(Instruction.VectorInsert, pArg, pArg, argHReg, Const(1));
cctx.Cfg.Entry.Operations.AddFirst(copyH);
cctx.Cfg.Entry.Operations.AddFirst(copyL);
preservedArgs[index] = pArg;
}
else
{
Operand pArg = Local(dest.Type);
Operand argReg = dest.Type.IsInteger()
? Gpr(CallingConvention.GetIntArgumentRegister(intCount), dest.Type)
: Xmm(CallingConvention.GetVecArgumentRegister(vecCount), dest.Type);
Operation copyOp = new Operation(Instruction.Copy, pArg, argReg);
cctx.Cfg.Entry.Operations.AddFirst(copyOp);
preservedArgs[index] = pArg;
}
}
Operation argCopyOp = new Operation(Instruction.Copy, dest, preservedArgs[index]);
node.List.AddBefore(node, argCopyOp);
Delete(node, operation);
}
else
{
// TODO: Pass on stack.
}
}
public void LinkList_NullHead_ArgumentNullException()
{
LLNode tail = new LLNode();
Assert.Throws <ArgumentNullException>(() => new LinkList(null, tail));
}
private static LLNode HandleCallSystemVAbi(LLNode node, Operation operation)
{
Operand dest = operation.Destination;
LinkedList <Node> nodes = node.List;
List <Operand> sources = new List <Operand>();
sources.Add(operation.GetSource(0));
int argsCount = operation.SourcesCount - 1;
int intMax = CallingConvention.GetIntArgumentsOnRegsCount();
int vecMax = CallingConvention.GetVecArgumentsOnRegsCount();
int intCount = 0;
int vecCount = 0;
int stackOffset = 0;
for (int index = 0; index < argsCount; index++)
{
Operand source = operation.GetSource(index + 1);
bool passOnReg;
if (source.Type.IsInteger())
{
passOnReg = intCount < intMax;
}
else if (source.Type == OperandType.V128)
{
passOnReg = intCount + 1 < intMax;
}
else
{
passOnReg = vecCount < vecMax;
}
if (source.Type == OperandType.V128 && passOnReg)
{
// V128 is a struct, we pass each half on a GPR if possible.
Operand argReg = Gpr(CallingConvention.GetIntArgumentRegister(intCount++), OperandType.I64);
Operand argReg2 = Gpr(CallingConvention.GetIntArgumentRegister(intCount++), OperandType.I64);
nodes.AddBefore(node, new Operation(Instruction.VectorExtract, argReg, source, Const(0)));
nodes.AddBefore(node, new Operation(Instruction.VectorExtract, argReg2, source, Const(1)));
continue;
}
if (passOnReg)
{
Operand argReg = source.Type.IsInteger()
? Gpr(CallingConvention.GetIntArgumentRegister(intCount++), source.Type)
: Xmm(CallingConvention.GetVecArgumentRegister(vecCount++), source.Type);
Operation copyOp = new Operation(Instruction.Copy, argReg, source);
HandleConstantCopy(nodes.AddBefore(node, copyOp), copyOp);
sources.Add(argReg);
}
else
{
Operand offset = new Operand(stackOffset);
Operation spillOp = new Operation(Instruction.SpillArg, null, offset, source);
HandleConstantCopy(nodes.AddBefore(node, spillOp), spillOp);
stackOffset += source.Type.GetSizeInBytes();
}
}
if (dest != null)
{
if (dest.Type == OperandType.V128)
{
Operand retLReg = Gpr(CallingConvention.GetIntReturnRegister(), OperandType.I64);
Operand retHReg = Gpr(CallingConvention.GetIntReturnRegisterHigh(), OperandType.I64);
node = nodes.AddAfter(node, new Operation(Instruction.VectorCreateScalar, dest, retLReg));
node = nodes.AddAfter(node, new Operation(Instruction.VectorInsert, dest, dest, retHReg, Const(1)));
operation.Destination = null;
}
else
{
Operand retReg = dest.Type.IsInteger()
? Gpr(CallingConvention.GetIntReturnRegister(), dest.Type)
: Xmm(CallingConvention.GetVecReturnRegister(), dest.Type);
Operation copyOp = new Operation(Instruction.Copy, dest, retReg);
node = nodes.AddAfter(node, copyOp);
operation.Destination = retReg;
}
}
operation.SetSources(sources.ToArray());
return(node);
}
public abstract bool DetectAndRemoveLoop(LLNode linkedListNode);
private static LLNode HandleSameDestSrc1Copy(LLNode node, Operation operation)
{
if (operation.Destination == null || operation.SourcesCount == 0)
{
return(node);
}
Instruction inst = operation.Instruction;
Operand dest = operation.Destination;
Operand src1 = operation.GetSource(0);
LinkedList <Node> nodes = node.List;
// The multiply instruction (that maps to IMUL) is somewhat special, it has
// a three operand form where the second source is a immediate value.
bool threeOperandForm = inst == Instruction.Multiply && operation.GetSource(1).Kind == OperandKind.Constant;
if (IsSameOperandDestSrc1(operation) && src1.Kind == OperandKind.LocalVariable && !threeOperandForm)
{
bool useNewLocal = false;
for (int srcIndex = 1; srcIndex < operation.SourcesCount; srcIndex++)
{
if (operation.GetSource(srcIndex) == dest)
{
useNewLocal = true;
break;
}
}
if (useNewLocal)
{
// Dest is being used as some source already, we need to use a new
// local to store the temporary value, otherwise the value on dest
// local would be overwritten.
Operand temp = Local(dest.Type);
nodes.AddBefore(node, new Operation(Instruction.Copy, temp, src1));
operation.SetSource(0, temp);
node = nodes.AddAfter(node, new Operation(Instruction.Copy, dest, temp));
operation.Destination = temp;
}
else
{
nodes.AddBefore(node, new Operation(Instruction.Copy, dest, src1));
operation.SetSource(0, dest);
}
}
else if (inst == Instruction.ConditionalSelect)
{
Operand src2 = operation.GetSource(1);
Operand src3 = operation.GetSource(2);
if (src1 == dest || src2 == dest)
{
Operand temp = Local(dest.Type);
nodes.AddBefore(node, new Operation(Instruction.Copy, temp, src3));
operation.SetSource(2, temp);
node = nodes.AddAfter(node, new Operation(Instruction.Copy, dest, temp));
operation.Destination = temp;
}
else
{
nodes.AddBefore(node, new Operation(Instruction.Copy, dest, src3));
operation.SetSource(2, dest);
}
}
return(node);
}
private static LLNode HandleCallWindowsAbi(StackAllocator stackAlloc, LLNode node, Operation operation)
{
Operand dest = operation.Destination;
LinkedList <Node> nodes = node.List;
// Handle struct arguments.
int retArgs = 0;
int stackAllocOffset = 0;
int AllocateOnStack(int size)
{
// We assume that the stack allocator is initially empty (TotalSize = 0).
// Taking that into account, we can reuse the space allocated for other
// calls by keeping track of our own allocated size (stackAllocOffset).
// If the space allocated is not big enough, then we just expand it.
int offset = stackAllocOffset;
if (stackAllocOffset + size > stackAlloc.TotalSize)
{
stackAlloc.Allocate((stackAllocOffset + size) - stackAlloc.TotalSize);
}
stackAllocOffset += size;
return(offset);
}
Operand arg0Reg = null;
if (dest != null && dest.Type == OperandType.V128)
{
int stackOffset = AllocateOnStack(dest.Type.GetSizeInBytes());
arg0Reg = Gpr(CallingConvention.GetIntArgumentRegister(0), OperandType.I64);
Operation allocOp = new Operation(Instruction.StackAlloc, arg0Reg, Const(stackOffset));
nodes.AddBefore(node, allocOp);
retArgs = 1;
}
int argsCount = operation.SourcesCount - 1;
int maxArgs = CallingConvention.GetArgumentsOnRegsCount() - retArgs;
if (argsCount > maxArgs)
{
argsCount = maxArgs;
}
Operand[] sources = new Operand[1 + retArgs + argsCount];
sources[0] = operation.GetSource(0);
if (arg0Reg != null)
{
sources[1] = arg0Reg;
}
for (int index = 1; index < operation.SourcesCount; index++)
{
Operand source = operation.GetSource(index);
if (source.Type == OperandType.V128)
{
Operand stackAddr = Local(OperandType.I64);
int stackOffset = AllocateOnStack(source.Type.GetSizeInBytes());
nodes.AddBefore(node, new Operation(Instruction.StackAlloc, stackAddr, Const(stackOffset)));
Operation storeOp = new Operation(Instruction.Store, null, stackAddr, source);
HandleConstantCopy(nodes.AddBefore(node, storeOp), storeOp);
operation.SetSource(index, stackAddr);
}
}
// Handle arguments passed on registers.
for (int index = 0; index < argsCount; index++)
{
Operand source = operation.GetSource(index + 1);
Operand argReg;
int argIndex = index + retArgs;
if (source.Type.IsInteger())
{
argReg = Gpr(CallingConvention.GetIntArgumentRegister(argIndex), source.Type);
}
else
{
argReg = Xmm(CallingConvention.GetVecArgumentRegister(argIndex), source.Type);
}
Operation copyOp = new Operation(Instruction.Copy, argReg, source);
HandleConstantCopy(nodes.AddBefore(node, copyOp), copyOp);
sources[1 + retArgs + index] = argReg;
}
// The remaining arguments (those that are not passed on registers)
// should be passed on the stack, we write them to the stack with "SpillArg".
for (int index = argsCount; index < operation.SourcesCount - 1; index++)
{
Operand source = operation.GetSource(index + 1);
Operand offset = new Operand((index + retArgs) * 8);
Operation spillOp = new Operation(Instruction.SpillArg, null, offset, source);
HandleConstantCopy(nodes.AddBefore(node, spillOp), spillOp);
}
if (dest != null)
{
if (dest.Type == OperandType.V128)
{
Operand retValueAddr = Local(OperandType.I64);
nodes.AddBefore(node, new Operation(Instruction.Copy, retValueAddr, arg0Reg));
Operation loadOp = new Operation(Instruction.Load, dest, retValueAddr);
node = nodes.AddAfter(node, loadOp);
operation.Destination = null;
}
else
{
Operand retReg = dest.Type.IsInteger()
? Gpr(CallingConvention.GetIntReturnRegister(), dest.Type)
: Xmm(CallingConvention.GetVecReturnRegister(), dest.Type);
Operation copyOp = new Operation(Instruction.Copy, dest, retReg);
node = nodes.AddAfter(node, copyOp);
operation.Destination = retReg;
}
}
operation.SetSources(sources);
return(node);
}
private static LLNode HandleFixedRegisterCopy(LLNode node, Operation operation)
{
Operand dest = operation.Destination;
LinkedList <Node> nodes = node.List;
switch (operation.Instruction)
{
case Instruction.CompareAndSwap128:
{
// Handle the many restrictions of the compare and exchange (16 bytes) instruction:
// - The expected value should be in RDX:RAX.
// - The new value to be written should be in RCX:RBX.
// - The value at the memory location is loaded to RDX:RAX.
void SplitOperand(Operand source, Operand lr, Operand hr)
{
nodes.AddBefore(node, new Operation(Instruction.VectorExtract, lr, source, Const(0)));
nodes.AddBefore(node, new Operation(Instruction.VectorExtract, hr, source, Const(1)));
}
Operand rax = Gpr(X86Register.Rax, OperandType.I64);
Operand rbx = Gpr(X86Register.Rbx, OperandType.I64);
Operand rcx = Gpr(X86Register.Rcx, OperandType.I64);
Operand rdx = Gpr(X86Register.Rdx, OperandType.I64);
SplitOperand(operation.GetSource(1), rax, rdx);
SplitOperand(operation.GetSource(2), rbx, rcx);
node = nodes.AddAfter(node, new Operation(Instruction.VectorCreateScalar, dest, rax));
node = nodes.AddAfter(node, new Operation(Instruction.VectorInsert, dest, dest, rdx, Const(1)));
operation.SetDestinations(new Operand[] { rdx, rax });
operation.SetSources(new Operand[] { operation.GetSource(0), rdx, rax, rcx, rbx });
break;
}
case Instruction.CpuId:
{
// Handle the many restrictions of the CPU Id instruction:
// - EAX controls the information returned by this instruction.
// - When EAX is 1, feature information is returned.
// - The information is written to registers EAX, EBX, ECX and EDX.
Debug.Assert(dest.Type == OperandType.I64);
Operand eax = Gpr(X86Register.Rax, OperandType.I32);
Operand ebx = Gpr(X86Register.Rbx, OperandType.I32);
Operand ecx = Gpr(X86Register.Rcx, OperandType.I32);
Operand edx = Gpr(X86Register.Rdx, OperandType.I32);
// Value 0x01 = Version, family and feature information.
nodes.AddBefore(node, new Operation(Instruction.Copy, eax, Const(1)));
// Copy results to the destination register.
// The values are split into 2 32-bits registers, we merge them
// into a single 64-bits register.
Operand rcx = Gpr(X86Register.Rcx, OperandType.I64);
node = nodes.AddAfter(node, new Operation(Instruction.ZeroExtend32, dest, edx));
node = nodes.AddAfter(node, new Operation(Instruction.ShiftLeft, dest, dest, Const(32)));
node = nodes.AddAfter(node, new Operation(Instruction.BitwiseOr, dest, dest, rcx));
operation.SetDestinations(new Operand[] { eax, ebx, ecx, edx });
operation.SetSources(new Operand[] { eax });
break;
}
case Instruction.Divide:
case Instruction.DivideUI:
{
// Handle the many restrictions of the division instructions:
// - The dividend is always in RDX:RAX.
// - The result is always in RAX.
// - Additionally it also writes the remainder in RDX.
if (dest.Type.IsInteger())
{
Operand src1 = operation.GetSource(0);
Operand rax = Gpr(X86Register.Rax, src1.Type);
Operand rdx = Gpr(X86Register.Rdx, src1.Type);
nodes.AddBefore(node, new Operation(Instruction.Copy, rax, src1));
nodes.AddBefore(node, new Operation(Instruction.Clobber, rdx));
node = nodes.AddAfter(node, new Operation(Instruction.Copy, dest, rax));
operation.SetDestinations(new Operand[] { rdx, rax });
operation.SetSources(new Operand[] { rdx, rax, operation.GetSource(1) });
operation.Destination = rax;
}
break;
}
case Instruction.Extended:
{
IntrinsicOperation intrinOp = (IntrinsicOperation)operation;
// PBLENDVB last operand is always implied to be XMM0 when VEX is not supported.
if (intrinOp.Intrinsic == Intrinsic.X86Pblendvb && !HardwareCapabilities.SupportsVexEncoding)
{
Operand xmm0 = Xmm(X86Register.Xmm0, OperandType.V128);
nodes.AddBefore(node, new Operation(Instruction.Copy, xmm0, operation.GetSource(2)));
operation.SetSource(2, xmm0);
}
break;
}
case Instruction.Multiply64HighSI:
case Instruction.Multiply64HighUI:
{
// Handle the many restrictions of the i64 * i64 = i128 multiply instructions:
// - The multiplicand is always in RAX.
// - The lower 64-bits of the result is always in RAX.
// - The higher 64-bits of the result is always in RDX.
Operand src1 = operation.GetSource(0);
Operand rax = Gpr(X86Register.Rax, src1.Type);
Operand rdx = Gpr(X86Register.Rdx, src1.Type);
nodes.AddBefore(node, new Operation(Instruction.Copy, rax, src1));
operation.SetSource(0, rax);
node = nodes.AddAfter(node, new Operation(Instruction.Copy, dest, rdx));
operation.SetDestinations(new Operand[] { rdx, rax });
break;
}
case Instruction.RotateRight:
case Instruction.ShiftLeft:
case Instruction.ShiftRightSI:
case Instruction.ShiftRightUI:
{
// The shift register is always implied to be CL (low 8-bits of RCX or ECX).
if (operation.GetSource(1).Kind == OperandKind.LocalVariable)
{
Operand rcx = Gpr(X86Register.Rcx, OperandType.I32);
nodes.AddBefore(node, new Operation(Instruction.Copy, rcx, operation.GetSource(1)));
operation.SetSource(1, rcx);
}
break;
}
}
return(node);
}
public static void RunPass(CompilerContext cctx, StackAllocator stackAlloc, out int maxCallArgs)
{
maxCallArgs = -1;
CallConvName callConv = CallingConvention.GetCurrentCallConv();
Operand[] preservedArgs = new Operand[CallingConvention.GetArgumentsOnRegsCount()];
foreach (BasicBlock block in cctx.Cfg.Blocks)
{
LLNode nextNode;
for (LLNode node = block.Operations.First; node != null; node = nextNode)
{
nextNode = node.Next;
if (!(node.Value is Operation operation))
{
continue;
}
HandleConstantCopy(node, operation);
HandleSameDestSrc1Copy(node, operation);
HandleFixedRegisterCopy(node, operation);
switch (operation.Instruction)
{
case Instruction.Call:
// Get the maximum number of arguments used on a call.
// On windows, when a struct is returned from the call,
// we also need to pass the pointer where the struct
// should be written on the first argument.
int argsCount = operation.SourcesCount - 1;
if (operation.Destination != null && operation.Destination.Type == OperandType.V128)
{
argsCount++;
}
if (maxCallArgs < argsCount)
{
maxCallArgs = argsCount;
}
// Copy values to registers expected by the function
// being called, as mandated by the ABI.
if (callConv == CallConvName.Windows)
{
node = HandleCallWindowsAbi(stackAlloc, node, operation);
}
else /* if (callConv == CallConvName.SystemV) */
{
node = HandleCallSystemVAbi(node, operation);
}
break;
case Instruction.ConvertToFPUI:
HandleConvertToFPUI(node, operation);
break;
case Instruction.LoadArgument:
if (callConv == CallConvName.Windows)
{
HandleLoadArgumentWindowsAbi(cctx, node, preservedArgs, operation);
}
else /* if (callConv == CallConvName.SystemV) */
{
HandleLoadArgumentSystemVAbi(cctx, node, preservedArgs, operation);
}
break;
case Instruction.Negate:
if (!operation.GetSource(0).Type.IsInteger())
{
node = HandleNegate(node, operation);
}
break;
case Instruction.Return:
if (callConv == CallConvName.Windows)
{
HandleReturnWindowsAbi(cctx, node, preservedArgs, operation);
}
else /* if (callConv == CallConvName.SystemV) */
{
HandleReturnSystemVAbi(node, operation);
}
break;
case Instruction.VectorInsert8:
if (!HardwareCapabilities.SupportsSse41)
{
node = HandleVectorInsert8(node, operation);
}
break;
}
}
}
}
private static void HandleConstantCopy(LLNode node, Operation operation)
{
if (operation.SourcesCount == 0 || IsIntrinsic(operation.Instruction))
{
return;
}
Instruction inst = operation.Instruction;
Operand src1 = operation.GetSource(0);
Operand src2;
if (src1.Kind == OperandKind.Constant)
{
if (!src1.Type.IsInteger())
{
// Handle non-integer types (FP32, FP64 and V128).
// For instructions without an immediate operand, we do the following:
// - Insert a copy with the constant value (as integer) to a GPR.
// - Insert a copy from the GPR to a XMM register.
// - Replace the constant use with the XMM register.
src1 = AddXmmCopy(node, src1);
operation.SetSource(0, src1);
}
else if (!HasConstSrc1(inst))
{
// Handle integer types.
// Most ALU instructions accepts a 32-bits immediate on the second operand.
// We need to ensure the following:
// - If the constant is on operand 1, we need to move it.
// -- But first, we try to swap operand 1 and 2 if the instruction is commutative.
// -- Doing so may allow us to encode the constant as operand 2 and avoid a copy.
// - If the constant is on operand 2, we check if the instruction supports it,
// if not, we also add a copy. 64-bits constants are usually not supported.
if (IsCommutative(inst))
{
src2 = operation.GetSource(1);
Operand temp = src1;
src1 = src2;
src2 = temp;
operation.SetSource(0, src1);
operation.SetSource(1, src2);
}
if (src1.Kind == OperandKind.Constant)
{
src1 = AddCopy(node, src1);
operation.SetSource(0, src1);
}
}
}
if (operation.SourcesCount < 2)
{
return;
}
src2 = operation.GetSource(1);
if (src2.Kind == OperandKind.Constant)
{
if (!src2.Type.IsInteger())
{
src2 = AddXmmCopy(node, src2);
operation.SetSource(1, src2);
}
else if (!HasConstSrc2(inst) || IsLongConst(src2))
{
src2 = AddCopy(node, src2);
operation.SetSource(1, src2);
}
}
}
public LLNode(T data)
{
this.Data = data;
this.Next = null;
}
private static void HandleLoadArgumentWindowsAbi(
CompilerContext cctx,
LLNode node,
Operand[] preservedArgs,
Operation operation)
{
Operand source = operation.GetSource(0);
Debug.Assert(source.Kind == OperandKind.Constant, "Non-constant LoadArgument source kind.");
int retArgs = cctx.FuncReturnType == OperandType.V128 ? 1 : 0;
int index = source.AsInt32() + retArgs;
if (index < CallingConvention.GetArgumentsOnRegsCount())
{
Operand dest = operation.Destination;
if (preservedArgs[index] == null)
{
Operand argReg, pArg;
if (dest.Type.IsInteger())
{
argReg = Gpr(CallingConvention.GetIntArgumentRegister(index), dest.Type);
pArg = Local(dest.Type);
}
else if (dest.Type == OperandType.V128)
{
argReg = Gpr(CallingConvention.GetIntArgumentRegister(index), OperandType.I64);
pArg = Local(OperandType.I64);
}
else
{
argReg = Xmm(CallingConvention.GetVecArgumentRegister(index), dest.Type);
pArg = Local(dest.Type);
}
Operation copyOp = new Operation(Instruction.Copy, pArg, argReg);
cctx.Cfg.Entry.Operations.AddFirst(copyOp);
preservedArgs[index] = pArg;
}
Operation argCopyOp = new Operation(dest.Type == OperandType.V128
? Instruction.Load
: Instruction.Copy, dest, preservedArgs[index]);
node.List.AddBefore(node, argCopyOp);
Delete(node, operation);
}
else
{
// TODO: Pass on stack.
}
}
public LL()
{
head = tail = curr = prev = null;
length = 0;
}
public SinglyLinkedList()
{
head = null;
}
