/// <summary>
/// Lowers phi values.
/// </summary>
private static void Lower(
SSARewriterContext <FieldRef> context,
LoweringData data,
PhiValue phi)
{
foreach (var(fieldType, fieldAccess) in phi.Type as StructureType)
{
// Build a new phi which might become dead in the future
var phiBuilder = context.Builder.CreatePhi(
phi.Location,
fieldType);
// Register the lowered phi
data.AddPhi(new LoweredPhi(
phi,
fieldAccess,
phiBuilder));
// Bind the new phi value
context.SetValue(
context.Block,
new FieldRef(phi, fieldAccess),
phiBuilder.PhiValue);
}
context.Remove(phi);
}
/// <summary>
/// Lowers a thread value.
/// </summary>
private static void LowerThreadValue <TValue, TLoweringImplementation>(
SSARewriterContext <FieldRef> context,
StructureType structureType,
TValue value)
where TValue : ThreadValue
where TLoweringImplementation :
LowerThreadIntrinsics.ILoweringImplementation <TValue>
{
// We require a single input
var variable = AssembleStructure(
context,
structureType,
value.Variable);
// Build a new thread value using the assembled structure
TLoweringImplementation implementation = default;
var newValue = implementation.Lower(
context.Builder,
value,
variable).ResolveAs <TValue>();
// Disassemble the resulting structure value
DisassembleStructure(context, structureType, newValue);
// Replace old value with new value
context.ReplaceAndRemove(value, newValue);
}
/// <summary>
/// Lowers get field operations into separate SSA values.
/// </summary>
private static void Lower(
SSARewriterContext <FieldRef> context,
LoweringData _,
GetField getField)
{
if (getField.Type is StructureType structureType)
{
foreach (var(_, fieldAccess) in structureType)
{
// Get the source value
var value = context.GetValue(
context.Block,
new FieldRef(
getField.ObjectValue,
new FieldSpan(
getField.FieldSpan.Index + fieldAccess.Index)));
// Bind the mapped SSA value
context.SetValue(
context.Block,
new FieldRef(getField, fieldAccess),
value);
}
context.Remove(getField);
}
else
{
var getFieldValue = context.GetValue(
context.Block,
new FieldRef(getField.ObjectValue, getField.FieldSpan));
context.ReplaceAndRemove(getField, getFieldValue);
}
}
/// <summary>
/// Lowers method calls involving structure types.
/// </summary>
private static void Lower(
SSARewriterContext <FieldRef> context,
LoweringData _,
MethodCall call)
{
// Check for structure arguments that need to be rebuilt
var callBuilder = context.Builder.CreateCall(call.Location, call.Target);
foreach (Value argument in call)
{
Value newArgument = argument;
if (argument.Type is StructureType argumentType)
{
newArgument = AssembleStructure(
context,
argumentType,
argument);
}
callBuilder.Add(newArgument);
}
// Create new call node
var newCall = callBuilder.Seal();
context.ReplaceAndRemove(call, newCall);
// Convert the return value
if (call.Type is StructureType callType)
{
DisassembleStructure(
context,
callType,
newCall);
}
}
/// <summary>
/// Lowers structure store operations into distinct stores for each field.
/// </summary>
private static void Lower(
SSARewriterContext <FieldRef> context,
LoweringData _,
Store store)
{
var builder = context.Builder;
foreach (var(_, fieldAccess) in store.Value.Type as StructureType)
{
// Update target address
var address = builder.CreateLoadFieldAddress(
store.Location,
store.Target,
new FieldSpan(fieldAccess));
// Load the currently registered SSA value and store it
var value = context.GetValue(
context.Block,
new FieldRef(store.Value, fieldAccess));
var loweredStore = builder.CreateStore(
store.Location,
address,
value);
context.MarkConverted(loweredStore);
}
context.Remove(store);
}
private static void Lower(
SSARewriterContext <FieldRef> context,
LoweringData _,
Predicate predicate)
{
foreach (var(_, fieldAccess) in predicate.Type as StructureType)
{
// Build a new if predicate which might become dead in the future
var trueValue = context.GetValue(
context.Block,
new FieldRef(predicate.TrueValue, fieldAccess));
var falseValue = context.GetValue(
context.Block,
new FieldRef(predicate.FalseValue, fieldAccess));
var newPredicate = context.Builder.CreatePredicate(
predicate.Location,
predicate.Condition,
trueValue,
falseValue);
// Bind the new if predicate
context.SetValue(
context.Block,
new FieldRef(predicate, fieldAccess),
newPredicate);
}
}
/// <summary>
/// Converts a load node into an SSA value.
/// </summary>
private static void Convert <TConstructionData>(
SSARewriterContext <Value> context,
TConstructionData data,
LoadElementAddress loadElementAddress)
where TConstructionData : IConstructionData
{
if (!data.TryGetConverted(loadElementAddress.Source, out var leaRef))
{
return;
}
// Get the primitive constant field offset
loadElementAddress.Assert(loadElementAddress.IsViewAccess);
var fieldOffset = loadElementAddress.Offset.ResolveAs <PrimitiveValue>();
loadElementAddress.AssertNotNull(fieldOffset);
// Map the field index + 1 to skip the initial array length
int fieldIndex = fieldOffset.Int32Value + 1;
data.AddConverted(
loadElementAddress,
leaRef.Access(new FieldAccess(fieldIndex)));
context.Remove(loadElementAddress);
}
/// <summary>
/// Lowers structure load operations into distinct loads for each field.
/// </summary>
private static void Lower(
SSARewriterContext <FieldRef> context,
LoweringData _,
Load load)
{
var builder = context.Builder;
foreach (var(_, fieldAccess) in load.Type as StructureType)
{
// Update source address
var address = builder.CreateLoadFieldAddress(
load.Location,
load.Source,
new FieldSpan(fieldAccess));
// Load value and store its reference in the current block
var loweredLoad = builder.CreateLoad(load.Location, address);
context.SetValue(
context.Block,
new FieldRef(load, fieldAccess),
loweredLoad);
context.MarkConverted(loweredLoad);
}
context.Remove(load);
}
/// <summary>
/// Converts a store node into an SSA value.
/// </summary>
private static void Convert(
SSARewriterContext <Value> context,
ConstructionData data,
Store store)
{
if (!data.TryGetConverted(store.Target, out var storeRef))
{
return;
}
Value ssaValue = store.Value;
if (!storeRef.IsDirect)
{
ssaValue = context.GetValue(context.Block, storeRef.Source);
ssaValue = context.Builder.CreateSetField(
store.Location,
ssaValue,
storeRef.FieldSpan,
store.Value);
}
context.SetValue(context.Block, storeRef.Source, ssaValue);
context.Remove(store);
}
/// <summary>
/// Keeps structure load operations.
/// </summary>
private static void Keep(
SSARewriterContext <FieldRef> context,
LoweringData _,
Load load) =>
DisassembleStructure(
context,
load.Type as StructureType,
load);
/// <summary>
/// Lowers parameter values containing structure values.
/// </summary>
private static void Lower(
SSARewriterContext <FieldRef> context,
LoweringData _,
Parameter value) =>
DisassembleStructure(
context,
value.Type as StructureType,
value);
/// <summary>
/// Lowers generic values containing structure types that cannot be rewritten.
/// </summary>
private static void Keep(
SSARewriterContext <FieldRef> context,
LoweringData _,
Value value) =>
DisassembleStructure(
context,
value.Type.As <StructureType>(value),
value);
/// <summary>
/// Builds a new structure value during lowering of a source value.
/// </summary>
private static Value AssembleStructure(
SSARewriterContext <FieldRef> context,
StructureType structureType,
Value value) =>
context.AssembleStructure(
structureType,
value,
(ctx, source, fieldAccess) =>
// Load the currently registered SSA value
ctx.GetValue(ctx.Block, new FieldRef(source, fieldAccess)));
/// <summary>
/// Lowers sub-warp shuffles.
/// </summary>
private static void Lower(
SSARewriterContext <FieldRef> context,
LoweringData _,
SubWarpShuffle value) =>
LowerThreadValue <
SubWarpShuffle,
LowerThreadIntrinsics.SubWarpShuffleLowering>(
context,
value.Type as StructureType,
value);
/// <summary>
/// Lowers warp shuffles.
/// </summary>
private static void Lower(
SSARewriterContext <FieldRef> context,
LoweringData _,
Broadcast value) =>
LowerThreadValue <
Broadcast,
LowerThreadIntrinsics.BroadcastLowering>(
context,
value.Type as StructureType,
value);
/// <summary>
/// Registers all structure values in the current SSA builder.
/// </summary>
private static void DisassembleStructure(
SSARewriterContext <FieldRef> context,
StructureType structureType,
Value value) =>
context.DisassembleStructure(
structureType,
value,
(ctx, source, getField, fieldAccess) =>
ctx.SetValue(
ctx.Block,
new FieldRef(value, fieldAccess),
getField));
/// <summary>
/// Converts an address-space cast into an SSA binding.
/// </summary>
private static void Convert(
SSARewriterContext <Value> context,
ConstructionData data,
AddressSpaceCast addressSpaceCast)
{
if (!data.TryGetConverted(addressSpaceCast.Value, out var castRef))
{
return;
}
data.AddConverted(addressSpaceCast, castRef);
context.Remove(addressSpaceCast);
}
/// <summary>
/// Converts an address-space cast into an SSA binding.
/// </summary>
protected static void Convert <TConstructionData>(
SSARewriterContext <Value> context,
TConstructionData data,
AddressSpaceCast addressSpaceCast)
where TConstructionData : IConstructionData
{
if (!data.TryGetConverted(addressSpaceCast.Value, out var castRef))
{
return;
}
data.AddConverted(addressSpaceCast, castRef);
context.Remove(addressSpaceCast);
}
/// <summary>
/// Converts a new view into an SSA value.
/// </summary>
private static void Convert <TConstructionData>(
SSARewriterContext <Value> context,
TConstructionData data,
NewView newView)
where TConstructionData : IConstructionData
{
if (!data.TryGetConverted(newView.Pointer, out var newViewRef))
{
return;
}
data.AddConverted(newView, newViewRef);
context.Remove(newView);
}
protected static void ConvertAlloca <TConstructionData>(
SSARewriterContext <Value> context,
TConstructionData data,
Alloca alloca,
Value initValue)
where TConstructionData : IConstructionData
{
alloca.Assert(data.ContainsAlloca(alloca));
// Bind the init value and remove the allocation from the block
context.SetValue(context.Block, alloca, initValue);
data.AddConverted(alloca, new FieldRef(alloca));
context.Remove(alloca);
}
/// <summary>
/// Converts a field-address operation into an SSA binding.
/// </summary>
private static void Convert(
SSARewriterContext <Value> context,
ConstructionData data,
LoadFieldAddress loadFieldAddress)
{
if (!data.TryGetConverted(loadFieldAddress.Source, out var fieldRef))
{
return;
}
data.AddConverted(
loadFieldAddress,
fieldRef.Access(loadFieldAddress.FieldSpan));
context.Remove(loadFieldAddress);
}
/// <summary>
/// Keeps structure store operations.
/// </summary>
private static void Keep(
SSARewriterContext <FieldRef> context,
LoweringData _,
Store store)
{
var newValue = AssembleStructure(
context,
store.Value.Type as StructureType,
store.Value);
var newStore = context.Builder.CreateStore(
store.Location,
store.Target,
newValue);
context.ReplaceAndRemove(store, newStore);
}
/// <summary>
/// Converts an alloca node to its initial SSA value.
/// </summary>
private static void Convert(
SSARewriterContext <Value> context,
ConstructionData data,
Alloca alloca)
{
if (!data.ContainsAlloca(alloca))
{
return;
}
var initValue = context.Builder.CreateNull(
alloca.Location,
alloca.AllocaType);
ConvertAlloca(context, data, alloca, initValue);
}
/// <summary>
/// Converts a field-address operation into an SSA binding.
/// </summary>
protected static void Convert <TConstructionData>(
SSARewriterContext <Value> context,
TConstructionData data,
LoadFieldAddress loadFieldAddress)
where TConstructionData : IConstructionData
{
if (!data.TryGetConverted(loadFieldAddress.Source, out var fieldRef))
{
return;
}
data.AddConverted(
loadFieldAddress,
fieldRef.Access(loadFieldAddress.FieldSpan));
context.Remove(loadFieldAddress);
}
/// <summary>
/// Lowers structure values into separate SSA values.
/// </summary>
private static void Lower(
SSARewriterContext <FieldRef> context,
LoweringData _,
StructureValue structureValue)
{
foreach (var(_, fieldAccess) in structureValue.StructureType)
{
// Build the new target value
var value = structureValue[fieldAccess.Index];
// Bind the new SSA value
context.SetValue(
context.Block,
new FieldRef(structureValue, fieldAccess),
value);
}
}
/// <summary>
/// Converts an alloca node to its initial SSA value.
/// </summary>
private static void Convert <TConstructionData>(
SSARewriterContext <Value> context,
TConstructionData data,
Alloca alloca)
where TConstructionData : IConstructionData
{
if (!data.ContainsAlloca(alloca))
{
return;
}
alloca.Assert(!alloca.IsSimpleAllocation);
// Get the builder and the associated array length value
var builder = context.Builder;
var arrayLengthValue = alloca.ArrayLength.ResolveAs <PrimitiveValue>();
alloca.AssertNotNull(arrayLengthValue);
int arrayLength = arrayLengthValue.Int32Value;
// Create a structure with the appropriate number of fields that correspond
// to the current array length
var allocaTypeBuilder = builder.CreateStructureType(arrayLength + 1);
// Append array length
allocaTypeBuilder.Add(builder.GetPrimitiveType(BasicValueType.Int32));
// Append all virtual fields
for (int i = 0; i < arrayLength; ++i)
{
allocaTypeBuilder.Add(alloca.AllocaType);
}
var allocationType = allocaTypeBuilder.Seal();
// Initialize the structure value
var initValue = builder.CreateNull(alloca.Location, allocationType);
// ... and set the array length
initValue = builder.CreateSetField(
alloca.Location,
initValue,
new FieldSpan(new FieldAccess(0)),
builder.CreateConvertToInt32(alloca.Location, arrayLengthValue));
ConvertAlloca(context, data, alloca, initValue);
}
/// <summary>
/// Converts an alloca node to its initial SSA value.
/// </summary>
private static void Convert(
SSARewriterContext <Value> context,
ConstructionData data,
Alloca alloca)
{
if (!data.ContainsAlloca(alloca))
{
return;
}
var initValue = context.Builder.CreateNull(
alloca.Location,
alloca.AllocaType);
context.SetValue(context.Block, alloca, initValue);
data.AddConverted(alloca, new FieldRef(alloca));
context.Remove(alloca);
}
/// <summary>
/// Lowers null values into separate SSA values.
/// </summary>
private static void Lower(
SSARewriterContext <FieldRef> context,
LoweringData _,
NullValue nullValue)
{
foreach (var(fieldType, fieldAccess) in nullValue.Type as StructureType)
{
// Build the new target value
var value = context.Builder.CreateNull(
nullValue.Location,
fieldType);
context.MarkConverted(value);
// Bind the new SSA value
context.SetValue(
context.Block,
new FieldRef(nullValue, fieldAccess),
value);
}
}
/// <summary>
/// Lowers return terminators returning structure values.
/// </summary>
private static void Lower(
SSARewriterContext <FieldRef> context,
LoweringData _,
ReturnTerminator value)
{
// Move the block builder to a valid insert position in the source block
var blockBuilder = context.GetMethodBuilder()[value.BasicBlock];
blockBuilder.SetupInsertPositionToEnd();
// Assemble return value
Value returnValue = value.ReturnValue;
var returnType = returnValue.Type.As <StructureType>(value);
var newReturnValue = AssembleStructure(
context.SpecializeBuilder(blockBuilder),
returnType,
returnValue);
// Replace return terminator with a new terminator
context.Builder.Terminator = context.Builder.CreateReturn(
value.Location,
newReturnValue);
}
/// <summary>
/// Converts a load node into an SSA value.
/// </summary>
private static void Convert(
SSARewriterContext <Value> context,
ConstructionData data,
Load load)
{
if (data.TryGetConverted(load.Source, out var loadRef))
{
var ssaValue = context.GetValue(context.Block, loadRef.Source);
if (!loadRef.IsDirect)
{
ssaValue = context.Builder.CreateGetField(
load.Location,
ssaValue,
loadRef.FieldSpan);
}
context.ReplaceAndRemove(load, ssaValue);
}
else if (!load.Uses.HasAny)
{
context.Remove(load);
}
}
