/// <summary>
/// Return the <see cref="StructuralInfoModule"/> for this Graph without any fusing
/// </summary>
/// <typeparam name="TShape">TBD</typeparam>
/// <typeparam name="TMat"></typeparam>
/// <param name="graph"></param>
/// <param name="attributes"></param>
/// <returns></returns>
public static StructuralInfoModule StructuralInfo <TShape, TMat>(IGraph <TShape, TMat> graph, Attributes attributes) where TShape : Shape
{
var structuralInfo = new BuildStructuralInfo();
// First perform normalization by descending the module tree and recording
// information in the BuildStructuralInfo instance.
try
{
var materializedValue = Descend <TMat>(graph.Module, Attributes.None, structuralInfo,
structuralInfo.CreateGroup(0), 0);
// Then create a copy of the original Shape with the new copied ports.
var shape = graph.Shape.CopyFromPorts(structuralInfo.NewInlets(graph.Shape.Inlets),
structuralInfo.NewOutlets(graph.Shape.Outlets));
// Extract the full topological information from the builder
return(structuralInfo.ToInfo(shape, materializedValue.Select(pair => Tuple.Create(pair.Key, pair.Value)).ToList(), attributes));
}
catch (Exception)
{
if (IsDebug)
{
structuralInfo.Dump();
}
throw;
}
}
/// <summary>
/// Fuse everything that is not forbidden via AsyncBoundary attribute.
/// </summary>
/// <typeparam name="TShape">TBD</typeparam>
/// <typeparam name="TMat">TBD</typeparam>
/// <param name="graph">TBD</param>
/// <returns>TBD</returns>
public static Streams.Fusing.FusedGraph <TShape, TMat> Aggressive <TShape, TMat>(IGraph <TShape, TMat> graph)
where TShape : Shape
{
if (graph is Streams.Fusing.FusedGraph <TShape, TMat> fusedGraph)
{
return(fusedGraph);
}
var structInfo = new BuildStructuralInfo();
// First perform normalization by descending the module tree and recording information in the BuildStructuralInfo instance.
LinkedList <KeyValuePair <IModule, IMaterializedValueNode> > materializedValue;
try
{
materializedValue = Descend <TMat>(graph.Module, Attributes.None, structInfo,
structInfo.CreateGroup(0), 0);
}
catch
{
if (IsDebug)
{
structInfo.Dump();
}
throw;
}
// Then create a copy of the original Shape with the new copied ports.
var shape = graph.Shape.CopyFromPorts(structInfo.NewInlets(graph.Shape.Inlets),
structInfo.NewOutlets(graph.Shape.Outlets));
// Extract the full topological information from the builder before removing assembly-internal (fused) wirings in the next step.
var info = structInfo.ToInfo(shape,
materializedValue.Select(pair => Tuple.Create(pair.Key, pair.Value)).ToList());
// Perform the fusing of `structInfo.groups` into GraphModules (leaving them as they are for non - fusable modules).
structInfo.RemoveInternalWires();
structInfo.BreakUpGroupsByDispatcher();
var modules = Fuse(structInfo);
// Now we have everything ready for a FusedModule.
var module = new FusedModule(
modules,
shape,
ImmutableDictionary.CreateRange(structInfo.Downstreams),
ImmutableDictionary.CreateRange(structInfo.Upstreams),
materializedValue.First().Value,
Attributes.None,
info);
if (StreamLayout.IsDebug)
{
StreamLayout.Validate(module);
}
if (IsDebug)
{
Console.WriteLine(module.ToString());
}
return(new Streams.Fusing.FusedGraph <TShape, TMat>(module, (TShape)shape));
}
/// <summary>
/// This is a normalization step for the graph that also collects the needed
/// information for later fusing. The goal is to transform an arbitrarily deep
/// module tree into one that has exactly two levels: all direct submodules are
/// CopiedModules where each contains exactly one atomic module. This way all
/// modules have their own identity and all necessary port copies have been
/// made. The upstreams/downstreams in the BuildStructuralInfo are rewritten
/// to point to the shapes of the copied modules.
///
/// The materialized value computation is rewritten as well in that all
/// leaf nodes point to the copied modules and all nested computations are
/// "inlined", resulting in only one big computation tree for the whole
/// normalized overall module. The contained MaterializedValueSource stages
/// are also rewritten to point to the copied MaterializedValueNodes. This
/// correspondence is then used during materialization to trigger these sources
/// when "their" node has received its value.
/// </summary>
private static LinkedList <KeyValuePair <IModule, IMaterializedValueNode> > Descend <T>(
IModule module,
Attributes inheritedAttributes,
BuildStructuralInfo structInfo,
ISet <IModule> openGroup,
int indent)
{
var isAsync = module is GraphStageModule || module is GraphModule
? module.Attributes.Contains(Attributes.AsyncBoundary.Instance)
: module.IsAtomic || module.Attributes.Contains(Attributes.AsyncBoundary.Instance);
if (IsDebug)
{
Log(indent,
$"entering {module.GetType().Name} (hash={module.GetHashCode()}, async={isAsync}, name={module.Attributes.GetNameLifted()}, dispatcher={GetDispatcher(module)})");
}
var localGroup = isAsync ? structInfo.CreateGroup(indent) : openGroup;
if (module.IsAtomic)
{
GraphModule graphModule;
if ((graphModule = module as GraphModule) != null)
{
if (!isAsync)
{
if (IsDebug)
{
Log(indent,
$"dissolving graph module {module.ToString().Replace("\n", $"\n{string.Empty.PadLeft(indent*2)}")}");
}
// need to dissolve previously fused GraphStages to allow further fusion
var attributes = inheritedAttributes.And(module.Attributes);
return(new LinkedList <KeyValuePair <IModule, IMaterializedValueNode> >(
graphModule.MaterializedValueIds.SelectMany(
sub => Descend <T>(sub, attributes, structInfo, localGroup, indent + 1))));
}
else
{
/*
* Importing a GraphModule that has an AsyncBoundary attribute is a little more work:
*
* - we need to copy all the CopiedModules that are in matValIDs
* - we need to rewrite the corresponding MaterializedValueNodes
* - we need to match up the new (copied) GraphModule shape with the individual Shape copies
* - we need to register the contained modules but take care to not include the internal
* wirings into the final result, see also `struct.removeInternalWires()`
*/
if (IsDebug)
{
Log(indent,
$"graph module {module.ToString().Replace("\n", $"\n{string.Empty.PadLeft(indent*2)}")}");
}
var oldShape = graphModule.Shape;
var mvids = graphModule.MaterializedValueIds;
// storing the old Shape in arrays for in-place updating as we clone the contained GraphStages
var oldIns = oldShape.Inlets.ToArray();
var oldOuts = oldShape.Outlets.ToArray();
var newIds = mvids.OfType <CopiedModule>().Select(x =>
{
var newShape = x.Shape.DeepCopy();
IModule copy = new CopiedModule(newShape, x.Attributes, x.CopyOf);
// rewrite shape: first the inlets
var oldIn = x.Shape.Inlets.GetEnumerator();
var newIn = newShape.Inlets.GetEnumerator();
while (oldIn.MoveNext() && newIn.MoveNext())
{
var idx = Array.IndexOf(oldIns, oldIn.Current);
if (idx >= 0)
{
oldIns[idx] = newIn.Current;
}
}
// ... then the outlets
var oldOut = x.Shape.Outlets.GetEnumerator();
var newOut = newShape.Outlets.GetEnumerator();
while (oldOut.MoveNext() && newOut.MoveNext())
{
var idx = Array.IndexOf(oldOuts, oldOut.Current);
if (idx >= 0)
{
oldOuts[idx] = newOut.Current;
}
}
// need to add the module so that the structural (internal) wirings can be rewritten as well
// but these modules must not be added to any of the groups
structInfo.AddModule(copy, new HashSet <IModule>(), inheritedAttributes, indent, x.Shape);
structInfo.RegisterInternals(newShape, indent);
return(copy);
}).ToArray();
var newGraphModule = new GraphModule(graphModule.Assembly,
oldShape.CopyFromPorts(oldIns.ToImmutableArray(), oldOuts.ToImmutableArray()),
graphModule.Attributes, newIds);
// make sure to add all the port mappings from old GraphModule Shape to new shape
structInfo.AddModule(newGraphModule, localGroup, inheritedAttributes, indent, oldShape);
// now compute the list of all materialized value computation updates
var result = new LinkedList <KeyValuePair <IModule, IMaterializedValueNode> >(
mvids.Select(
(t, i) =>
new KeyValuePair <IModule, IMaterializedValueNode>(t,
new Atomic(newIds[i]))));
result.AddLast(new KeyValuePair <IModule, IMaterializedValueNode>(module,
new Atomic(newGraphModule)));
return(result);
}
}
else
{
if (IsDebug)
{
Log(indent, $"atomic module {module}");
}
var result = new LinkedList <KeyValuePair <IModule, IMaterializedValueNode> >();
result.AddFirst(
new KeyValuePair <IModule, IMaterializedValueNode>(module,
structInfo.AddModule(module, localGroup, inheritedAttributes, indent))
);
return(result);
}
}
else
{
var allAttributes = inheritedAttributes.And(module.Attributes);
if (module is CopiedModule copied)
{
var result = Descend <T>(copied.CopyOf, allAttributes, structInfo, localGroup, indent + 1);
if (result.Count == 0)
{
throw new IllegalStateException("Descend returned empty result from CopiedModule");
}
result.AddFirst(new KeyValuePair <IModule, IMaterializedValueNode>(copied, result.First.Value.Value));
structInfo.Rewire(copied.CopyOf.Shape, copied.Shape, indent);
return(result);
}
else
{
// we need to keep track of all MaterializedValueSource nodes that get pushed into the current
// computation context (i.e. that need the same value).
structInfo.EnterMaterializationContext();
// now descend into submodules and collect their computations (plus updates to `struct`)
var subMatBuilder = ImmutableDictionary.CreateBuilder <IModule, IMaterializedValueNode>();
foreach (var sub in module.SubModules)
{
var res = Descend <T>(sub, allAttributes, structInfo, localGroup, indent + 1);
foreach (var r in res)
{
// key may already be in builder, we overwrite
subMatBuilder[r.Key] = r.Value;
}
}
var subMat = subMatBuilder.ToImmutable();
if (IsDebug)
{
Log(indent,
$"subMat\n {string.Empty.PadLeft(indent*2)}{string.Join("\n " + string.Empty.PadLeft(indent*2), subMat.Select(p => $"{p.Key.GetType().Name}[{p.Key.GetHashCode()}] -> {p.Value}"))}");
}
// we need to remove all wirings that this module copied from nested modules so that we don’t do wirings twice
var oldDownstreams =
(module as FusedModule)?.Info.Downstreams.ToImmutableHashSet()
?? module.Downstreams.ToImmutableHashSet();
var down = module.SubModules.Aggregate(oldDownstreams, (set, m) => set.Except(m.Downstreams));
foreach (var entry in down)
{
structInfo.Wire(entry.Key, entry.Value, indent);
}
// now rewrite the materialized value computation based on the copied modules and their computation nodes
var matNodeMapping = new Dictionary <IMaterializedValueNode, IMaterializedValueNode>();
var newMat = RewriteMaterializer(subMat, module.MaterializedValueComputation, matNodeMapping);
// and finally rewire all MaterializedValueSources to their new computation nodes
var materializedSources = structInfo.ExitMaterializationContext();
foreach (var c in materializedSources)
{
if (IsDebug)
{
Log(indent, $"materialized value source: {structInfo.Hash(c)}");
}
var ms = (IMaterializedValueSource)((GraphStageModule)c.CopyOf).Stage;
IMaterializedValueNode mapped;
if (ms.Computation is Atomic atomic)
{
mapped = subMat[atomic.Module];
}
else if (ms.Computation == StreamLayout.Ignore.Instance)
{
mapped = ms.Computation;
}
else
{
mapped = matNodeMapping[ms.Computation];
}
var outputType = ms.Outlet.GetType().GetGenericArguments().First();
var materializedValueSourceType = typeof(MaterializedValueSource <>).MakeGenericType(outputType);
var newSrc = (IMaterializedValueSource)Activator.CreateInstance(materializedValueSourceType, mapped, ms.Outlet);
var replacement = new CopiedModule(c.Shape, c.Attributes, newSrc.Module);
structInfo.Replace(c, replacement, localGroup);
}
// the result for each level is the materialized value computation
var result = new LinkedList <KeyValuePair <IModule, IMaterializedValueNode> >();
result.AddFirst(new KeyValuePair <IModule, IMaterializedValueNode>(module, newMat));
return(result);
}
}
}