/// <summary>
/// Lowers set field operations into separate SSA values.
/// </summary>
private static void Lower(
RewriterContext context,
TypeLowering <ViewType> _,
AddressSpaceCast value)
{
var builder = context.Builder;
var location = value.Location;
var pointer = builder.CreateGetField(
location,
value.Value,
new FieldSpan(0));
var length = builder.CreateGetField(
location,
value.Value,
new FieldSpan(1));
var newPointer = builder.CreateAddressSpaceCast(
location,
pointer,
value.TargetAddressSpace);
var newInstance = builder.CreateDynamicStructure(
location,
newPointer,
length);
context.ReplaceAndRemove(value, newInstance);
}
/// <summary>
/// Lowers set field operations into separate SSA values.
/// </summary>
private static void Lower(
RewriterContext context,
TypeLowering <ViewType> _,
SubViewValue value)
{
var builder = context.Builder;
var location = value.Location;
var pointer = builder.CreateGetField(
location,
value.Source,
new FieldSpan(0));
var newPointer = builder.CreateLoadElementAddress(
location,
pointer,
value.Offset);
var length = value.Length;
if (length.BasicValueType != BasicValueType.Int64)
{
length = builder.CreateConvertToInt64(
value.Location,
length);
}
var subView = builder.CreateDynamicStructure(
location,
newPointer,
length);
context.ReplaceAndRemove(value, subView);
}
/// <summary>
/// Lowers an as-aligned-view operation.
/// </summary>
private static void Lower(
RewriterContext context,
TypeLowering <ViewType> typeLowering,
AsAligned value)
{
var builder = context.Builder;
var location = value.Location;
// Extract basic view information from the converted structure
var pointer = builder.CreateGetField(
location,
value.Source,
new FieldSpan(0));
var length = builder.CreateGetField(
location,
value.Source,
new FieldSpan(1));
// Ensure that the underyling pointer is aligned
var aligned = builder.CreateAsAligned(
location,
pointer,
value.AlignmentInBytes);
// Create a new wrapped instance
var newInstance = builder.CreateDynamicStructure(
location,
aligned,
length);
context.ReplaceAndRemove(value, newInstance);
}
/// <summary>
/// Lowers a primitive type.
/// </summary>
/// <typeparam name="TValue">The value type.</typeparam>
/// <typeparam name="TLoweringImplementation">
/// The implementation type.
/// </typeparam>
/// <param name="context">The current rewriter context.</param>
/// <param name="sourceValue">The source value to get the values from.</param>
/// <param name="variable">The source variable.</param>
/// <returns>The lowered thread value.</returns>
private static Value LowerPrimitive <TValue, TLoweringImplementation>(
RewriterContext context,
TValue sourceValue,
Value variable)
where TValue : ThreadValue
where TLoweringImplementation : struct, ILoweringImplementation <TValue>
{
var builder = context.Builder;
var primitiveType = variable.Type as PrimitiveType;
Value value = variable;
if (primitiveType.BasicValueType < BasicValueType.Int32)
{
value = builder.CreateConvert(
sourceValue.Location,
value,
builder.GetPrimitiveType(BasicValueType.Int32));
}
TLoweringImplementation loweringImplementation = default;
var result = loweringImplementation.Lower(
builder,
sourceValue,
value);
if (primitiveType.BasicValueType < BasicValueType.Int32)
{
result = builder.CreateConvert(
sourceValue.Location,
result,
variable.Type);
}
return(result);
}
/// <summary>
/// Lowers leae nodes to linear <see cref="LoadElementAddress"/> values.
/// </summary>
private static void Lower(
RewriterContext context,
TypeLowering <ArrayType> typeLowering,
LoadArrayElementAddress value)
{
var builder = context.Builder;
var location = value.Location;
Value array = value.ArrayValue;
// Get and validate the array type
var structureType = array.Type.As <StructureType>(location);
// Compute linear address based on the .Net array layouts
Value elementIndex = builder.CreatePrimitiveValue(location, 0L);
// (((0 * Width + x) * Height) + y) * Depth + z...
for (int i = 0, e = structureType.NumFields - 1; i < e; ++i)
{
Value length = builder.CreateGetField(
location,
array,
new FieldAccess(1 + i));
// Create a debug assertion to check for out-of-bounds accesses
builder.CreateDebugAssert(
location,
builder.CreateCompare(
location,
value.Dimensions[i],
length,
CompareKind.LessThan),
builder.CreatePrimitiveValue(
location,
$"{i}-th array index out of range"));
// Update index computation
elementIndex = builder.CreateArithmetic(
location,
elementIndex,
length,
BinaryArithmeticKind.Mul);
elementIndex = builder.CreateArithmetic(
location,
elementIndex,
value.Dimensions[i],
BinaryArithmeticKind.Add);
}
// Extract the actual view field from the structure and compute the
// appropriate target element address.
var view = GetViewFromArray(builder, location, array);
var lea = builder.CreateLoadElementAddress(
location,
view,
elementIndex);
context.ReplaceAndRemove(value, lea);
}
/// <summary>
/// Specializes accelerator-specific values.
/// </summary>
private static void Specialize(
RewriterContext context,
Value value,
int constant)
{
var newValue = context.Builder.CreatePrimitiveValue(
value.Location,
constant);
context.ReplaceAndRemove(value, newValue);
}
/// <summary>
/// Specializes warp size values.
/// </summary>
private static void Specialize(
RewriterContext context,
AcceleratorSpecializer specializer,
WarpSizeValue value)
{
if (!specializer.WarpSize.HasValue)
{
return;
}
Specialize(context, value, specializer.WarpSize.Value);
}
/// <summary>
/// Lowers null values with nested types.
/// </summary>
protected static void Lower(
RewriterContext context,
TypeLowering <TType> typeConverter,
NullValue value)
{
var targetType = typeConverter.ConvertType(value);
var newValue = context.Builder.CreateNull(
value.Location,
targetType);
context.ReplaceAndRemove(value, newValue);
}
/// <summary>
/// Specializes native pointer casts.
/// </summary>
private static void Specialize(
RewriterContext context,
AcceleratorSpecializer specializer,
IntAsPointerCast value)
{
var convert = context.Builder.CreateConvert(
value.Location,
value.Value,
specializer.IntPointerType);
context.ReplaceAndRemove(value, convert);
}
/// <summary>
/// Lowers set field operations into separate SSA values.
/// </summary>
private static void Lower(
RewriterContext context,
TypeLowering <ViewType> _,
GetViewLength value)
{
var length = context.Builder.CreateGetField(
value.Location,
value.View,
new FieldSpan(1));
context.ReplaceAndRemove(value, length);
}
/// <summary>
/// Lowers set field operations into separate SSA values.
/// </summary>
private static void Lower(
RewriterContext context,
TypeLowering <ViewType> _,
NewView value)
{
var viewInstance = context.Builder.CreateDynamicStructure(
value.Location,
value.Pointer,
value.Length);
context.ReplaceAndRemove(value, viewInstance);
}
/// <summary>
/// Specializes warp size values.
/// </summary>
private static void Specialize(
RewriterContext context,
SpecializerData data,
WarpSizeValue value)
{
var warpSizeValue = data.WarpSize;
if (!warpSizeValue.HasValue)
{
return;
}
Specialize(context, value, warpSizeValue.Value);
}
/// <summary>
/// Removes or collects IO operations.
/// </summary>
private static void Specialize(
RewriterContext context,
SpecializerData data,
WriteToOutput value)
{
if (data.EnableIOOperations)
{
data.ToImplement.Add(value);
}
else
{
context.Remove(value);
}
}
/// <summary>
/// Specializes debug operations via the instance method
/// <see cref="Specialize(in RewriterContext, IRContext, DebugAssertOperation)"/>
/// of the parent <paramref name="data"/> instance.
/// </summary>
private static void Specialize(
RewriterContext context,
SpecializerData data,
DebugAssertOperation value)
{
if (data.EnableAssertions)
{
data.Specializer.Specialize(context, data.Context, value);
}
else
{
context.Remove(value);
}
}
/// <summary>
/// Lowers LFA operations into an adapted version.
/// </summary>
protected static void Lower(
RewriterContext context,
TypeLowering <TType> typeConverter,
LoadFieldAddress lfa)
{
// Compute the new span
var span = typeConverter.ComputeSpan(lfa, lfa.FieldSpan);
var newValue = context.Builder.CreateLoadFieldAddress(
lfa.Location,
lfa.Source,
span);
context.ReplaceAndRemove(lfa, newValue);
}
/// <summary>
/// Lowers pointer cast values into their appropriate counter parts.
/// </summary>
protected static void Lower(
RewriterContext context,
TypeLowering <TType> typeConverter,
PointerCast cast)
{
// Compute the cast type
var newType = typeConverter.ConvertType(cast);
var newCast = context.Builder.CreatePointerCast(
cast.Location,
cast.Value,
newType);
context.ReplaceAndRemove(cast, newCast);
}
/// <summary>
/// Lowers alloca values into their appropriate counter parts.
/// </summary>
protected static void Lower(
RewriterContext context,
TypeLowering <TType> typeConverter,
Alloca alloca)
{
// Compute the alloca type
var newType = typeConverter.ConvertType(alloca);
var newAlloca = context.Builder.CreateAlloca(
alloca.Location,
newType,
alloca.AddressSpace);
context.ReplaceAndRemove(alloca, newAlloca);
}
/// <summary>
/// Lowers set field operations into separate SSA values.
/// </summary>
private static void Lower(
RewriterContext context,
TypeLowering <ViewType> typeLowering,
ViewCast value)
{
var builder = context.Builder;
var location = value.Location;
var pointer = builder.CreateGetField(
location,
value.Value,
new FieldSpan(0));
var length = builder.CreateGetField(
location,
value.Value,
new FieldSpan(1));
// New pointer
var newPointer = builder.CreatePointerCast(
location,
pointer,
value.TargetElementType);
// Compute new length:
// newLength = length * sourceElementSize / targetElementSize;
var sourceElementType = (typeLowering[value] as ViewType).ElementType;
var sourceElementSize = builder.CreateLongSizeOf(
location,
sourceElementType);
var targetElementSize = builder.CreateLongSizeOf(
location,
value.TargetElementType);
var newLength = builder.CreateArithmetic(
location,
builder.CreateArithmetic(
location,
length,
sourceElementSize,
BinaryArithmeticKind.Mul),
targetElementSize, BinaryArithmeticKind.Div);
var newInstance = builder.CreateDynamicStructure(
location,
newPointer,
newLength);
context.ReplaceAndRemove(value, newInstance);
}
private static void Lower(
RewriterContext context,
TypeLowering <ArrayType> typeLowering,
GetArrayLength value)
{
var builder = context.Builder;
var location = value.Location;
Value length;
if (value.IsFullLength)
{
// Extract the view from the underlying array-implementation structure
var view = GetViewFromArray(builder, location, value.ArrayValue);
length = builder.CreateGetViewLength(location, view);
}
else
{
// Check whether the dimension is a compile-time known constant
Value dimension = value.Dimension;
if (!(dimension is PrimitiveValue primitiveValue))
{
throw location.GetNotSupportedException(
ErrorMessages.NotSupportNonConstArrayDimension,
dimension.ToString());
}
// Check whether the field index is out of range
var structType = value.ArrayValue.Type.As <StructureType>(location);
int index = primitiveValue.Int32Value;
if (index < 0 || index >= structType.NumFields - 1)
{
throw location.GetNotSupportedException(
ErrorMessages.NotSupportNonConstArrayDimension,
index.ToString());
}
// BaseView + dimension offset
length = builder.CreateGetField(
location,
value.ArrayValue,
new FieldAccess(1 + index));
}
// Replace the actual length information
context.ReplaceAndRemove(value, length);
}
/// <summary>
/// Lowers new array values.
/// </summary>
private static void Lower(
RewriterContext context,
TypeLowering <ArrayType> typeLowering,
ArrayValue value)
{
var builder = context.Builder;
var location = value.Location;
int currentPosition = builder.InsertPosition;
// Allocate a memory array in the entry block
var methodBuilder = builder.MethodBuilder;
var entryBlock = methodBuilder[methodBuilder.EntryBlock];
entryBlock.InsertPosition = 0;
var arrayLength = entryBlock.ComputeArrayLength(
location,
value.Extent);
var newArray = entryBlock.CreateStaticAllocaArray(
location,
arrayLength,
value.ArrayType.ElementType,
MemoryAddressSpace.Local);
// Create resulting structure in current block
builder.InsertPosition = currentPosition;
var instance = builder.CreateDynamicStructure(
location,
typeLowering.GetNumFields(value.ArrayType));
// Insert pointer field
instance.Add(newArray);
// Insert all dimensions
for (int i = 0, e = value.ArrayType.Dimensions; i < e; ++i)
{
instance.Add(builder.CreateGetField(
location,
value.Extent,
new FieldSpan(i)));
}
var newStructure = instance.Seal();
context.ReplaceAndRemove(value, newStructure);
}
/// <summary>
/// Lowers set field operations into separate SSA values.
/// </summary>
private static void Lower(
RewriterContext context,
TypeLowering <ViewType> _,
LoadElementAddress value)
{
var builder = context.Builder;
var location = value.Location;
var pointer = builder.CreateGetField(
location,
value.Source,
new FieldSpan(0));
var newLea = builder.CreateLoadElementAddress(
location,
pointer,
value.Offset);
context.ReplaceAndRemove(value, newLea);
}
/// <summary>
/// Lowers address-computation values.
/// </summary>
private static void Lower(
RewriterContext context,
TypeLowering <ArrayType> _,
LoadElementAddress value)
{
var linearAddress = GetLinearAddress(
context,
value.Source,
value.ElementIndex,
out var ptr);
var newLea = context.Builder.CreateLoadElementAddress(
value.Location,
ptr,
linearAddress);
context.ReplaceAndRemove(value, newLea);
}
/// <summary>
/// Lowers set field operations into separate SSA values.
/// </summary>
private static void Lower(
RewriterContext context,
TypeLowering <ViewType> _,
GetViewLength value)
{
var builder = context.Builder;
var length = builder.CreateGetField(
value.Location,
value.View,
new FieldSpan(1));
// Convert to a 32bit length value
if (value.Is32BitProperty)
{
length = builder.CreateConvertToInt32(
value.Location,
length);
}
context.ReplaceAndRemove(value, length);
}
/// <summary>
/// Lower array to view casts to direct references to the underyling view.
/// </summary>
private static void Lower(
RewriterContext context,
TypeLowering <ArrayType> typeLowering,
ArrayToViewCast value)
{
var builder = context.Builder;
var location = value.Location;
// Get the view from the array implementation structure
Value array = value.Value;
var view = GetViewFromArray(builder, location, array);
// Cast this view into the generic address space to match all invariants
view = builder.CreateAddressSpaceCast(
location,
view,
MemoryAddressSpace.Generic);
// Replace the current cast with the actual view pointing to the elements
context.ReplaceAndRemove(value, view);
}
/// <summary>
/// Specializes int to native pointer casts.
/// </summary>
private static void Specialize(
RewriterContext context,
SpecializerData data,
IntAsPointerCast value)
{
// Convert from int -> native int type -> pointer
var builder = context.Builder;
// int -> native int type
var convertToNativeInt = builder.CreateConvert(
value.Location,
value.Value,
data.IntPointerType);
// native int type -> pointer
var convert = builder.CreateIntAsPointerCast(
value.Location,
convertToNativeInt);
context.ReplaceAndRemove(value, convert);
}
/// <summary>
/// Lowers set field operations into separate SSA values.
/// </summary>
protected static void Lower(
RewriterContext context,
TypeLowering <TType> typeConverter,
SetField setValue)
{
var builder = context.Builder;
var location = setValue.Location;
// Compute the new base index
var span = typeConverter.ComputeSpan(setValue, setValue.FieldSpan);
// Check whether we have to insert multiple elements
Value targetValue = setValue.ObjectValue;
if (typeConverter[setValue] is TType)
{
for (int i = 0; i < span.Span; ++i)
{
var viewField = builder.CreateGetField(
location,
setValue.Value,
new FieldSpan(i));
targetValue = builder.CreateSetField(
location,
targetValue,
new FieldSpan(span.Index + i),
viewField);
}
}
else
{
// Simple field access
targetValue = builder.CreateSetField(
location,
targetValue,
span,
setValue.Value);
}
context.ReplaceAndRemove(setValue, targetValue);
}
/// <summary>
/// Lowers get element values.
/// </summary>
private static void Lower(
RewriterContext context,
TypeLowering <ArrayType> _,
GetArrayElement value)
{
var linearAddress = GetLinearAddress(
context,
value.ObjectValue,
value.Index,
out var ptr);
var builder = context.Builder;
var address = builder.CreateLoadElementAddress(
value.Location,
ptr,
linearAddress);
var newLoad = builder.CreateLoad(
value.Location,
address);
context.ReplaceAndRemove(value, newLoad);
}
/// <summary>
/// Specializes native pointer to int casts.
/// </summary>
private static void Specialize(
RewriterContext context,
SpecializerData data,
PointerAsIntCast value)
{
// Convert from ptr -> native int type -> desired int type
var builder = context.Builder;
// ptr -> native int type
var convertToNativeType = builder.CreatePointerAsIntCast(
value.Location,
value.Value,
data.IntPointerType.BasicValueType);
// native int type -> desired int type
var convert = builder.CreateConvert(
value.Location,
convertToNativeType,
value.TargetType);
context.ReplaceAndRemove(value, convert);
}
/// <summary>
/// Lowers set field operations into separate SSA values.
/// </summary>
protected static void Lower(
RewriterContext context,
TypeLowering <TType> typeConverter,
GetField getValue)
{
var builder = context.Builder;
var location = getValue.Location;
// Compute the new base index
var span = typeConverter.ComputeSpan(getValue, getValue.FieldSpan);
// Check whether we have to extract a nested type implementation
Value newValue;
if (typeConverter[getValue] is TType)
{
// We have to extract multiple elements from this structure
var instance = builder.CreateDynamicStructure(location, span.Span);
for (int i = 0; i < span.Span; ++i)
{
var viewField = builder.CreateGetField(
location,
getValue.ObjectValue,
new FieldSpan(span.Index + i));
instance.Add(viewField);
}
newValue = instance.Seal();
}
else
{
// Simple field access
newValue = builder.CreateGetField(
location,
getValue.ObjectValue,
span);
}
context.ReplaceAndRemove(getValue, newValue);
}
private static Value GetLinearAddress(
RewriterContext context,
Value array,
Value index,
out Value ptr)
{
var builder = context.Builder;
ptr = builder.CreateGetField(
array.Location,
array,
new FieldSpan(0));
var extent = builder.CreateGetField(
array.Location,
array,
new FieldSpan(1));
return(builder.ComputeArrayAddress(
index.Location,
index,
extent,
ArrayTypeLowering.DimensionOffset));
}