/// <summary cref="IBackendCodeGenerator.GenerateCode(LoadFieldAddress)"/>
public void GenerateCode(LoadFieldAddress value)
{
var source = LoadPrimitive(value.Source);
var fieldOffset = value.StructureType.GetOffset(
value.FieldSpan.Access);
if (fieldOffset != 0)
{
var targetRegister = AllocatePlatformRegister(
value,
out RegisterDescription _);
using var command = BeginCommand(
PTXInstructions.GetArithmeticOperation(
BinaryArithmeticKind.Add,
Backend.PointerArithmeticType,
false));
command.AppendArgument(targetRegister);
command.AppendArgument(source);
command.AppendConstant(fieldOffset);
}
else
{
Alias(value, value.Source);
}
}
/// <summary>
/// Creates a set of instructions to realize a generic lea operation.
/// </summary>
/// <param name="sourceType">The source address type (pointer or view).</param>
/// <param name="elementIndex">The current element index (the offset).</param>
/// <param name="targetAddressRegister">The allocated target pointer register to write to.</param>
/// <param name="address">The source address.</param>
private void MakeLoadElementAddress(
AddressSpaceType sourceType,
PrimitiveRegister elementIndex,
PrimitiveRegister targetAddressRegister,
PrimitiveRegister address)
{
var elementSize = ABI.GetSizeOf(sourceType.ElementType);
var offsetRegister = AllocatePlatformRegister(out RegisterDescription _);
using (var command = BeginCommand(
PTXInstructions.GetLEAMulOperation(ABI.PointerArithmeticType)))
{
command.AppendArgument(offsetRegister);
command.AppendArgument(elementIndex);
command.AppendConstant(elementSize);
}
using (var command = BeginCommand(
PTXInstructions.GetArithmeticOperation(
BinaryArithmeticKind.Add,
ABI.PointerArithmeticType,
false)))
{
command.AppendArgument(targetAddressRegister);
command.AppendArgument(address);
command.AppendArgument(offsetRegister);
}
FreeRegister(offsetRegister);
}
/// <summary cref="IValueVisitor.Visit(GenericAtomic)"/>
public void Visit(GenericAtomic atomic)
{
var target = LoadPrimitive(atomic.Target);
var value = LoadPrimitive(atomic.Value);
var requiresResult = atomic.Uses.HasAny || atomic.Kind == AtomicKind.Exchange;
var atomicOperation = PTXInstructions.GetAtomicOperation(
atomic.Kind,
requiresResult);
var suffix = PTXInstructions.GetAtomicOperationSuffix(
atomic.Kind,
atomic.ArithmeticBasicValueType);
var targetRegister = requiresResult ? AllocatePrimitive(atomic) : default;
using (var command = BeginCommand(atomicOperation))
{
command.AppendNonLocalAddressSpace(
(atomic.Target.Type as AddressSpaceType).AddressSpace);
command.AppendSuffix(suffix);
if (requiresResult)
{
command.AppendArgument(targetRegister);
}
command.AppendArgumentValue(target);
command.AppendArgument(value);
}
}
/// <summary cref="IValueVisitor.Visit(PredicateBarrier)"/>
public void Visit(PredicateBarrier barrier)
{
var targetRegister = AllocatePrimitive(barrier);
var sourcePredicate = LoadPrimitive(barrier.Predicate);
switch (barrier.Kind)
{
case PredicateBarrierKind.And:
case PredicateBarrierKind.Or:
using (var command = BeginCommand(
PTXInstructions.GetPredicateBarrier(barrier.Kind)))
{
command.AppendArgument(targetRegister);
command.AppendConstant(0);
command.AppendArgument(sourcePredicate);
}
break;
case PredicateBarrierKind.PopCount:
using (var command = BeginCommand(
PTXInstructions.GetPredicateBarrier(barrier.Kind)))
{
command.AppendArgument(targetRegister);
command.AppendConstant(0);
command.AppendArgument(sourcePredicate);
}
break;
default:
throw new InvalidCodeGenerationException();
}
}
/// <summary cref="IBackendCodeGenerator.GenerateCode(LoadElementAddress)"/>
public void GenerateCode(LoadElementAddress value)
{
var elementIndex = LoadPrimitive(value.ElementIndex);
var targetAddressRegister = AllocatePlatformRegister(
value,
out RegisterDescription _);
Debug.Assert(value.IsPointerAccess, "Invalid pointer access");
var address = LoadPrimitive(value.Source);
var sourceType = value.Source.Type as AddressSpaceType;
var elementSize = sourceType.ElementType.Size;
var offsetRegister = AllocatePlatformRegister(out RegisterDescription _);
using (var command = BeginCommand(
PTXInstructions.GetLEAMulOperation(Backend.PointerArithmeticType)))
{
command.AppendArgument(offsetRegister);
command.AppendArgument(elementIndex);
command.AppendConstant(elementSize);
}
using (var command = BeginCommand(
PTXInstructions.GetArithmeticOperation(
BinaryArithmeticKind.Add,
Backend.PointerArithmeticType,
false)))
{
command.AppendArgument(targetAddressRegister);
command.AppendArgument(address);
command.AppendArgument(offsetRegister);
}
FreeRegister(offsetRegister);
}
/// <summary cref="IBackendCodeGenerator.GenerateCode(LoadElementAddress)"/>
public void GenerateCode(LoadElementAddress value)
{
var elementIndex = LoadPrimitive(value.Offset);
var targetAddressRegister = AllocatePlatformRegister(
value,
out RegisterDescription _);
Debug.Assert(value.IsPointerAccess, "Invalid pointer access");
var address = LoadPrimitive(value.Source);
var sourceType = value.Source.Type as AddressSpaceType;
var elementSize = sourceType.ElementType.Size;
if (value.Is32BitAccess)
{
// Perform two efficient operations TODO
var offsetRegister = AllocatePlatformRegister(out RegisterDescription _);
using (var command = BeginCommand(
PTXInstructions.GetLEAMulOperation(Backend.PointerArithmeticType)))
{
command.AppendArgument(offsetRegister);
command.AppendArgument(elementIndex);
command.AppendConstant(elementSize);
}
using (var command = BeginCommand(
PTXInstructions.GetArithmeticOperation(
BinaryArithmeticKind.Add,
Backend.PointerArithmeticType,
Backend.Capabilities,
false)))
{
command.AppendArgument(targetAddressRegister);
command.AppendArgument(address);
command.AppendArgument(offsetRegister);
}
FreeRegister(offsetRegister);
}
else
{
// Use an efficient MAD instruction to compute the effective address
using var command = BeginCommand(
PTXInstructions.GetArithmeticOperation(
TernaryArithmeticKind.MultiplyAdd,
Backend.PointerArithmeticType));
command.AppendArgument(targetAddressRegister);
command.AppendArgument(elementIndex);
command.AppendConstant(elementSize);
command.AppendArgument(address);
}
}
/// <summary cref="IBackendCodeGenerator.GenerateCode(UnaryArithmeticValue)"/>
public void GenerateCode(UnaryArithmeticValue value)
{
var argument = LoadPrimitive(value.Value);
var targetRegister = AllocateHardware(value);
using var command = BeginCommand(
PTXInstructions.GetArithmeticOperation(
value.Kind,
value.ArithmeticBasicValueType,
FastMath));
command.AppendArgument(targetRegister);
command.AppendArgument(argument);
}
/// <summary cref="IBackendCodeGenerator.GenerateCode(ConvertValue)"/>
public void GenerateCode(ConvertValue value)
{
var sourceValue = LoadPrimitive(value.Value);
var convertOperation = PTXInstructions.GetConvertOperation(
value.SourceType,
value.TargetType);
var targetRegister = AllocateHardware(value);
using var command = BeginCommand(convertOperation);
command.AppendArgument(targetRegister);
command.AppendArgument(sourceValue);
}
/// <summary cref="IValueVisitor.Visit(Predicate)"/>
public void Visit(Predicate predicate)
{
var condition = LoadPrimitive(predicate.Condition);
var trueValue = Load(predicate.TrueValue);
var falseValue = Load(predicate.FalseValue);
var targetRegister = Allocate(predicate);
EmitComplexCommand(
PTXInstructions.GetSelectValueOperation(predicate.BasicValueType),
new PredicateEmitter(condition),
targetRegister,
trueValue,
falseValue);
}
/// <summary cref="IBackendCodeGenerator.GenerateCode(BinaryArithmeticValue)"/>
public void GenerateCode(BinaryArithmeticValue value)
{
var left = LoadPrimitive(value.Left);
var right = LoadPrimitive(value.Right);
var targetRegister = Allocate(value, left.Description);
using var command = BeginCommand(
PTXInstructions.GetArithmeticOperation(
value.Kind,
value.ArithmeticBasicValueType,
FastMath));
command.AppendArgument(targetRegister);
command.AppendArgument(left);
command.AppendArgument(right);
}
/// <summary cref="IValueVisitor.Visit(CompareValue)"/>
public void Visit(CompareValue value)
{
var left = LoadPrimitive(value.Left);
var right = LoadPrimitive(value.Right);
var targetRegister = AllocatePrimitive(value);
if (left.Kind == PTXRegisterKind.Predicate)
{
// Predicate registers require a special treatment
using (var command = BeginCommand(
PTXInstructions.GetArithmeticOperation(
BinaryArithmeticKind.Xor,
ArithmeticBasicValueType.UInt1,
false)))
{
command.AppendArgument(targetRegister);
command.AppendArgument(left);
command.AppendArgument(right);
}
if (value.Kind == CompareKind.Equal)
{
using (var command = BeginCommand(
PTXInstructions.GetArithmeticOperation(
UnaryArithmeticKind.Not,
ArithmeticBasicValueType.UInt1,
false)))
{
command.AppendArgument(targetRegister);
command.AppendArgument(targetRegister);
}
}
}
else
{
using (var command = BeginCommand(
PTXInstructions.GetCompareOperation(
value.Kind,
value.CompareType)))
{
command.AppendArgument(targetRegister);
command.AppendArgument(left);
command.AppendArgument(right);
}
}
}
/// <summary cref="IBackendCodeGenerator.GenerateCode(Barrier)"/>
public void GenerateCode(Barrier barrier)
{
using var command = BeginCommand(PTXInstructions.GetBarrier(barrier.Kind));
switch (barrier.Kind)
{
case BarrierKind.WarpLevel:
command.AppendConstant(
PTXInstructions.AllThreadsInAWarpMemberMask);
break;
case BarrierKind.GroupLevel:
command.AppendConstant(0);
break;
default:
throw new InvalidCodeGenerationException();
}
}
/// <summary cref="IBackendCodeGenerator.GenerateCode(TernaryArithmeticValue)"/>
public void GenerateCode(TernaryArithmeticValue value)
{
var first = LoadPrimitive(value.First);
var second = LoadPrimitive(value.Second);
var third = LoadPrimitive(value.Third);
var targetRegister = Allocate(value, first.Description);
using var command = BeginCommand(
PTXInstructions.GetArithmeticOperation(
value.Kind,
value.ArithmeticBasicValueType));
command.AppendArgument(targetRegister);
command.AppendArgument(first);
command.AppendArgument(second);
command.AppendArgument(third);
}
/// <summary>
/// Creates an address-space cast conversion.
/// </summary>
/// <param name="sourceRegister">The source register.</param>
/// <param name="targetRegister">The target register.</param>
/// <param name="sourceAddressSpace">The source address space.</param>
/// <param name="targetAddressSpace">The target address space.</param>
private void CreateAddressSpaceCast(
PrimitiveRegister sourceRegister,
HardwareRegister targetRegister,
MemoryAddressSpace sourceAddressSpace,
MemoryAddressSpace targetAddressSpace)
{
var toGeneric = targetAddressSpace == MemoryAddressSpace.Generic;
var addressSpaceOperation = PTXInstructions.GetAddressSpaceCast(toGeneric);
var addressSpaceOperationSuffix =
PTXInstructions.GetAddressSpaceCastSuffix(Backend);
using var command = BeginCommand(addressSpaceOperation);
command.AppendAddressSpace(
toGeneric ? sourceAddressSpace : targetAddressSpace);
command.AppendSuffix(addressSpaceOperationSuffix);
command.AppendArgument(targetRegister);
command.AppendArgument(sourceRegister);
}
/// <summary cref="IValueVisitor.Visit(ViewCast)"/>
public void Visit(ViewCast value)
{
var source = LoadAs <ViewImplementationRegister>(value.Value);
var pointer = source.Pointer;
var length = source.Length;
var sourceElementSize = ABI.GetSizeOf(value.SourceElementType);
var targetElementSize = ABI.GetSizeOf(value.TargetElementType);
// var newLength = length * sourceElementSize / targetElementSize;
var lengthTimesSourceElementSize = AllocateRegister(length.Description);
var newLength = AllocateRegister(length.Description);
using (var command = BeginCommand(
PTXInstructions.GetArithmeticOperation(
BinaryArithmeticKind.Mul,
ArithmeticBasicValueType.Int32,
FastMath)))
{
command.AppendArgument(lengthTimesSourceElementSize);
command.AppendArgument(length);
command.AppendConstant(sourceElementSize);
}
using (var command = BeginCommand(
PTXInstructions.GetArithmeticOperation(
BinaryArithmeticKind.Div,
ArithmeticBasicValueType.Int32,
FastMath)))
{
command.AppendArgument(newLength);
command.AppendArgument(lengthTimesSourceElementSize);
command.AppendConstant(targetElementSize);
}
var newView = new ViewImplementationRegister(
value.Type as ViewType,
pointer,
newLength);
Bind(value, newView);
FreeRegister(lengthTimesSourceElementSize);
}
/// <summary cref="IBackendCodeGenerator.GenerateCode(AddressSpaceCast)"/>
public void GenerateCode(AddressSpaceCast value)
{
var sourceType = value.SourceType as AddressSpaceType;
var targetAdressRegister = AllocatePlatformRegister(
value,
out RegisterDescription _);
Debug.Assert(value.IsPointerCast, "Invalid pointer access");
var address = LoadPrimitive(value.Value);
var toGeneric = value.TargetAddressSpace == MemoryAddressSpace.Generic;
var addressSpaceOperation = PTXInstructions.GetAddressSpaceCast(toGeneric);
var addressSpaceOperationSuffix =
PTXInstructions.GetAddressSpaceCastSuffix(Backend);
using var command = BeginCommand(addressSpaceOperation);
command.AppendAddressSpace(
toGeneric ? sourceType.AddressSpace : value.TargetAddressSpace);
command.AppendSuffix(addressSpaceOperationSuffix);
command.AppendArgument(targetAdressRegister);
command.AppendArgument(address);
}
/// <summary cref="IBackendCodeGenerator.GenerateCode(DynamicMemoryLengthValue)"/>
public void GenerateCode(DynamicMemoryLengthValue value)
{
if (value.AddressSpace != MemoryAddressSpace.Shared)
{
throw new InvalidCodeGenerationException();
}
// Load the dynamic memory size (in bytes) from the PTX special register
// and divide by the size in bytes of the array element.
var lengthRegister = AllocateHardware(value);
var dynamicMemorySizeRegister = MoveFromIntrinsicRegister(
PTXRegisterKind.DynamicSharedMemorySize);
using var command = BeginCommand(
PTXInstructions.GetArithmeticOperation(
BinaryArithmeticKind.Div,
ArithmeticBasicValueType.UInt32,
Backend.Capabilities,
false));
command.AppendArgument(lengthRegister);
command.AppendArgument(dynamicMemorySizeRegister);
command.AppendConstant(value.ElementType.Size);
}
/// <summary cref="IValueVisitor.Visit(AddressSpaceCast)"/>
public void Visit(AddressSpaceCast value)
{
var sourceType = value.SourceType as AddressSpaceType;
var targetAdressRegister = AllocatePlatformRegister(value, out RegisterDescription _);
PrimitiveRegister address;
if (value.IsPointerCast)
{
address = LoadPrimitive(value.Value);
}
else
{
var viewSource = LoadAs <ViewImplementationRegister>(value.Value);
address = viewSource.Pointer;
// Reuse the existing length register since we don't modify the result
var viewTarget = new ViewImplementationRegister(
value.Type as ViewType,
targetAdressRegister,
viewSource.Length);
Bind(value, viewTarget);
}
var toGeneric = value.TargetAddressSpace == MemoryAddressSpace.Generic;
var addressSpaceOperation = PTXInstructions.GetAddressSpaceCast(toGeneric);
var addressSpaceOperationSuffix = PTXInstructions.GetAddressSpaceCastSuffix(ABI);
using (var command = BeginCommand(addressSpaceOperation))
{
command.AppendAddressSpace(
toGeneric ? sourceType.AddressSpace : value.TargetAddressSpace);
command.AppendSuffix(addressSpaceOperationSuffix);
command.AppendArgument(targetAdressRegister);
command.AppendArgument(address);
}
}
/// <summary cref="IBackendCodeGenerator.GenerateCode(IfPredicate)"/>
public void GenerateCode(IfPredicate predicate)
{
var condition = LoadPrimitive(predicate.Condition);
var trueValue = Load(predicate.TrueValue);
var falseValue = Load(predicate.FalseValue);
var targetRegister = Allocate(predicate);
if (predicate.BasicValueType == BasicValueType.Int1)
{
// We need a specific sequence of instructions for predicate registers
var conditionRegister = EnsureHardwareRegister(condition);
using (var statement1 = BeginMove(
new PredicateConfiguration(conditionRegister, true)))
{
statement1.AppendSuffix(BasicValueType.Int1);
statement1.AppendArgument(targetRegister as PrimitiveRegister);
statement1.AppendArgument(trueValue as PrimitiveRegister);
}
using var statement2 = BeginMove(
new PredicateConfiguration(conditionRegister, false));
statement2.AppendSuffix(BasicValueType.Int1);
statement2.AppendArgument(targetRegister as PrimitiveRegister);
statement2.AppendArgument(falseValue as PrimitiveRegister);
}
else
{
EmitComplexCommand(
PTXInstructions.GetSelectValueOperation(predicate.BasicValueType),
new PredicateEmitter(condition),
targetRegister,
trueValue,
falseValue);
}
}
/// <summary cref="IValueVisitor.Visit(LoadFieldAddress)"/>
public void Visit(LoadFieldAddress value)
{
var source = LoadPrimitive(value.Source);
var fieldOffset = ABI.GetOffsetOf(value.StructureType, value.FieldIndex);
if (fieldOffset != 0)
{
var targetRegister = AllocatePlatformRegister(value, out RegisterDescription _);
using (var command = BeginCommand(
PTXInstructions.GetArithmeticOperation(
BinaryArithmeticKind.Add,
ABI.PointerArithmeticType,
false)))
{
command.AppendArgument(targetRegister);
command.AppendArgument(source);
command.AppendConstant(fieldOffset);
}
}
else
{
Alias(value, value.Source);
}
}
/// <summary> /// Appends the given vector length suffix. /// </summary> /// <param name="vectorLength">The vector length.</param> public void AppendVectorSuffix(int vectorLength) => AppendSuffix(PTXInstructions.GetVectorOperationSuffix(vectorLength));
/// <summary cref="IValueVisitor.Visit(SwitchBranch)"/>
public void Visit(SwitchBranch branch)
{
var idx = LoadPrimitive(branch.Condition);
using (var lowerBoundsScope = new PredicateScope(this))
{
// Emit less than
var lessThanCommand = PTXInstructions.GetCompareOperation(
CompareKind.LessThan,
ArithmeticBasicValueType.Int32);
using (var command = BeginCommand(
lessThanCommand))
{
command.AppendArgument(lowerBoundsScope.PredicateRegister);
command.AppendArgument(idx);
command.AppendConstant(0);
}
using (var upperBoundsScope = new PredicateScope(this))
{
using (var command = BeginCommand(
PTXInstructions.BranchIndexRangeComparison))
{
command.AppendArgument(upperBoundsScope.PredicateRegister);
command.AppendArgument(idx);
command.AppendConstant(branch.NumCasesWithoutDefault);
command.AppendArgument(lowerBoundsScope.PredicateRegister);
}
using (var command = BeginCommand(
PTXInstructions.BranchOperation,
new PredicateConfiguration(upperBoundsScope.PredicateRegister, true)))
{
var defaultTarget = blockLookup[branch.DefaultBlock];
command.AppendLabel(defaultTarget);
}
}
}
var targetLabel = DeclareLabel();
MarkLabel(targetLabel);
Builder.Append('\t');
Builder.Append(PTXInstructions.BranchTargetsDeclaration);
Builder.Append(' ');
for (int i = 0, e = branch.NumCasesWithoutDefault; i < e; ++i)
{
var caseTarget = branch.GetCaseTarget(i);
var caseLabel = blockLookup[caseTarget];
Builder.Append(caseLabel);
if (i + 1 < e)
{
Builder.Append(", ");
}
}
Builder.AppendLine(";");
using (var command = BeginCommand(
PTXInstructions.BranchIndexOperation))
{
command.AppendArgument(idx);
command.AppendLabel(targetLabel);
}
}
/// <summary cref="IValueVisitor.Visit(MemoryBarrier)"/>
public void Visit(MemoryBarrier barrier)
{
var command = PTXInstructions.GetMemoryBarrier(barrier.Kind);
Command(command);
}
/// <summary> /// Gets the actual select command. /// </summary> public string AdjustCommand(string command, PrimitiveRegister[] registers) => PTXInstructions.GetSelectValueOperation(registers[0].BasicValueType);
/// <summary cref="IBackendCodeGenerator.GenerateCode(MemoryBarrier)"/>
public void GenerateCode(MemoryBarrier barrier)
{
var command = PTXInstructions.GetMemoryBarrier(barrier.Kind);
Command(command);
}
