/// <summary> /// Return the <see cref="StructuralInfoModule"/> for this Graph without any fusing /// </summary> /// <typeparam name="TShape">TBD</typeparam> /// <typeparam name="TMat"></typeparam> /// <param name="graph"></param> /// <param name="attributes"></param> /// <returns></returns> public static StructuralInfoModule StructuralInfo <TShape, TMat>(IGraph <TShape, TMat> graph, Attributes attributes) where TShape : Shape { var structuralInfo = new BuildStructuralInfo(); // First perform normalization by descending the module tree and recording // information in the BuildStructuralInfo instance. try { var materializedValue = Descend <TMat>(graph.Module, Attributes.None, structuralInfo, structuralInfo.CreateGroup(0), 0); // Then create a copy of the original Shape with the new copied ports. var shape = graph.Shape.CopyFromPorts(structuralInfo.NewInlets(graph.Shape.Inlets), structuralInfo.NewOutlets(graph.Shape.Outlets)); // Extract the full topological information from the builder return(structuralInfo.ToInfo(shape, materializedValue.Select(pair => Tuple.Create(pair.Key, pair.Value)).ToList(), attributes)); } catch (Exception) { if (IsDebug) { structuralInfo.Dump(); } throw; } }
/// <summary> /// Fuse everything that is not forbidden via AsyncBoundary attribute. /// </summary> /// <typeparam name="TShape">TBD</typeparam> /// <typeparam name="TMat">TBD</typeparam> /// <param name="graph">TBD</param> /// <returns>TBD</returns> public static Streams.Fusing.FusedGraph <TShape, TMat> Aggressive <TShape, TMat>(IGraph <TShape, TMat> graph) where TShape : Shape { if (graph is Streams.Fusing.FusedGraph <TShape, TMat> fusedGraph) { return(fusedGraph); } var structInfo = new BuildStructuralInfo(); // First perform normalization by descending the module tree and recording information in the BuildStructuralInfo instance. LinkedList <KeyValuePair <IModule, IMaterializedValueNode> > materializedValue; try { materializedValue = Descend <TMat>(graph.Module, Attributes.None, structInfo, structInfo.CreateGroup(0), 0); } catch { if (IsDebug) { structInfo.Dump(); } throw; } // Then create a copy of the original Shape with the new copied ports. var shape = graph.Shape.CopyFromPorts(structInfo.NewInlets(graph.Shape.Inlets), structInfo.NewOutlets(graph.Shape.Outlets)); // Extract the full topological information from the builder before removing assembly-internal (fused) wirings in the next step. var info = structInfo.ToInfo(shape, materializedValue.Select(pair => Tuple.Create(pair.Key, pair.Value)).ToList()); // Perform the fusing of `structInfo.groups` into GraphModules (leaving them as they are for non - fusable modules). structInfo.RemoveInternalWires(); structInfo.BreakUpGroupsByDispatcher(); var modules = Fuse(structInfo); // Now we have everything ready for a FusedModule. var module = new FusedModule( modules, shape, ImmutableDictionary.CreateRange(structInfo.Downstreams), ImmutableDictionary.CreateRange(structInfo.Upstreams), materializedValue.First().Value, Attributes.None, info); if (StreamLayout.IsDebug) { StreamLayout.Validate(module); } if (IsDebug) { Console.WriteLine(module.ToString()); } return(new Streams.Fusing.FusedGraph <TShape, TMat>(module, (TShape)shape)); }
/// <summary> /// This is a normalization step for the graph that also collects the needed /// information for later fusing. The goal is to transform an arbitrarily deep /// module tree into one that has exactly two levels: all direct submodules are /// CopiedModules where each contains exactly one atomic module. This way all /// modules have their own identity and all necessary port copies have been /// made. The upstreams/downstreams in the BuildStructuralInfo are rewritten /// to point to the shapes of the copied modules. /// /// The materialized value computation is rewritten as well in that all /// leaf nodes point to the copied modules and all nested computations are /// "inlined", resulting in only one big computation tree for the whole /// normalized overall module. The contained MaterializedValueSource stages /// are also rewritten to point to the copied MaterializedValueNodes. This /// correspondence is then used during materialization to trigger these sources /// when "their" node has received its value. /// </summary> private static LinkedList <KeyValuePair <IModule, IMaterializedValueNode> > Descend <T>( IModule module, Attributes inheritedAttributes, BuildStructuralInfo structInfo, ISet <IModule> openGroup, int indent) { var isAsync = module is GraphStageModule || module is GraphModule ? module.Attributes.Contains(Attributes.AsyncBoundary.Instance) : module.IsAtomic || module.Attributes.Contains(Attributes.AsyncBoundary.Instance); if (IsDebug) { Log(indent, $"entering {module.GetType().Name} (hash={module.GetHashCode()}, async={isAsync}, name={module.Attributes.GetNameLifted()}, dispatcher={GetDispatcher(module)})"); } var localGroup = isAsync ? structInfo.CreateGroup(indent) : openGroup; if (module.IsAtomic) { GraphModule graphModule; if ((graphModule = module as GraphModule) != null) { if (!isAsync) { if (IsDebug) { Log(indent, $"dissolving graph module {module.ToString().Replace("\n", $"\n{string.Empty.PadLeft(indent*2)}")}"); } // need to dissolve previously fused GraphStages to allow further fusion var attributes = inheritedAttributes.And(module.Attributes); return(new LinkedList <KeyValuePair <IModule, IMaterializedValueNode> >( graphModule.MaterializedValueIds.SelectMany( sub => Descend <T>(sub, attributes, structInfo, localGroup, indent + 1)))); } else { /* * Importing a GraphModule that has an AsyncBoundary attribute is a little more work: * * - we need to copy all the CopiedModules that are in matValIDs * - we need to rewrite the corresponding MaterializedValueNodes * - we need to match up the new (copied) GraphModule shape with the individual Shape copies * - we need to register the contained modules but take care to not include the internal * wirings into the final result, see also `struct.removeInternalWires()` */ if (IsDebug) { Log(indent, $"graph module {module.ToString().Replace("\n", $"\n{string.Empty.PadLeft(indent*2)}")}"); } var oldShape = graphModule.Shape; var mvids = graphModule.MaterializedValueIds; // storing the old Shape in arrays for in-place updating as we clone the contained GraphStages var oldIns = oldShape.Inlets.ToArray(); var oldOuts = oldShape.Outlets.ToArray(); var newIds = mvids.OfType <CopiedModule>().Select(x => { var newShape = x.Shape.DeepCopy(); IModule copy = new CopiedModule(newShape, x.Attributes, x.CopyOf); // rewrite shape: first the inlets var oldIn = x.Shape.Inlets.GetEnumerator(); var newIn = newShape.Inlets.GetEnumerator(); while (oldIn.MoveNext() && newIn.MoveNext()) { var idx = Array.IndexOf(oldIns, oldIn.Current); if (idx >= 0) { oldIns[idx] = newIn.Current; } } // ... then the outlets var oldOut = x.Shape.Outlets.GetEnumerator(); var newOut = newShape.Outlets.GetEnumerator(); while (oldOut.MoveNext() && newOut.MoveNext()) { var idx = Array.IndexOf(oldOuts, oldOut.Current); if (idx >= 0) { oldOuts[idx] = newOut.Current; } } // need to add the module so that the structural (internal) wirings can be rewritten as well // but these modules must not be added to any of the groups structInfo.AddModule(copy, new HashSet <IModule>(), inheritedAttributes, indent, x.Shape); structInfo.RegisterInternals(newShape, indent); return(copy); }).ToArray(); var newGraphModule = new GraphModule(graphModule.Assembly, oldShape.CopyFromPorts(oldIns.ToImmutableArray(), oldOuts.ToImmutableArray()), graphModule.Attributes, newIds); // make sure to add all the port mappings from old GraphModule Shape to new shape structInfo.AddModule(newGraphModule, localGroup, inheritedAttributes, indent, oldShape); // now compute the list of all materialized value computation updates var result = new LinkedList <KeyValuePair <IModule, IMaterializedValueNode> >( mvids.Select( (t, i) => new KeyValuePair <IModule, IMaterializedValueNode>(t, new Atomic(newIds[i])))); result.AddLast(new KeyValuePair <IModule, IMaterializedValueNode>(module, new Atomic(newGraphModule))); return(result); } } else { if (IsDebug) { Log(indent, $"atomic module {module}"); } var result = new LinkedList <KeyValuePair <IModule, IMaterializedValueNode> >(); result.AddFirst( new KeyValuePair <IModule, IMaterializedValueNode>(module, structInfo.AddModule(module, localGroup, inheritedAttributes, indent)) ); return(result); } } else { var allAttributes = inheritedAttributes.And(module.Attributes); if (module is CopiedModule copied) { var result = Descend <T>(copied.CopyOf, allAttributes, structInfo, localGroup, indent + 1); if (result.Count == 0) { throw new IllegalStateException("Descend returned empty result from CopiedModule"); } result.AddFirst(new KeyValuePair <IModule, IMaterializedValueNode>(copied, result.First.Value.Value)); structInfo.Rewire(copied.CopyOf.Shape, copied.Shape, indent); return(result); } else { // we need to keep track of all MaterializedValueSource nodes that get pushed into the current // computation context (i.e. that need the same value). structInfo.EnterMaterializationContext(); // now descend into submodules and collect their computations (plus updates to `struct`) var subMatBuilder = ImmutableDictionary.CreateBuilder <IModule, IMaterializedValueNode>(); foreach (var sub in module.SubModules) { var res = Descend <T>(sub, allAttributes, structInfo, localGroup, indent + 1); foreach (var r in res) { // key may already be in builder, we overwrite subMatBuilder[r.Key] = r.Value; } } var subMat = subMatBuilder.ToImmutable(); if (IsDebug) { Log(indent, $"subMat\n {string.Empty.PadLeft(indent*2)}{string.Join("\n " + string.Empty.PadLeft(indent*2), subMat.Select(p => $"{p.Key.GetType().Name}[{p.Key.GetHashCode()}] -> {p.Value}"))}"); } // we need to remove all wirings that this module copied from nested modules so that we don’t do wirings twice var oldDownstreams = (module as FusedModule)?.Info.Downstreams.ToImmutableHashSet() ?? module.Downstreams.ToImmutableHashSet(); var down = module.SubModules.Aggregate(oldDownstreams, (set, m) => set.Except(m.Downstreams)); foreach (var entry in down) { structInfo.Wire(entry.Key, entry.Value, indent); } // now rewrite the materialized value computation based on the copied modules and their computation nodes var matNodeMapping = new Dictionary <IMaterializedValueNode, IMaterializedValueNode>(); var newMat = RewriteMaterializer(subMat, module.MaterializedValueComputation, matNodeMapping); // and finally rewire all MaterializedValueSources to their new computation nodes var materializedSources = structInfo.ExitMaterializationContext(); foreach (var c in materializedSources) { if (IsDebug) { Log(indent, $"materialized value source: {structInfo.Hash(c)}"); } var ms = (IMaterializedValueSource)((GraphStageModule)c.CopyOf).Stage; IMaterializedValueNode mapped; if (ms.Computation is Atomic atomic) { mapped = subMat[atomic.Module]; } else if (ms.Computation == StreamLayout.Ignore.Instance) { mapped = ms.Computation; } else { mapped = matNodeMapping[ms.Computation]; } var outputType = ms.Outlet.GetType().GetGenericArguments().First(); var materializedValueSourceType = typeof(MaterializedValueSource <>).MakeGenericType(outputType); var newSrc = (IMaterializedValueSource)Activator.CreateInstance(materializedValueSourceType, mapped, ms.Outlet); var replacement = new CopiedModule(c.Shape, c.Attributes, newSrc.Module); structInfo.Replace(c, replacement, localGroup); } // the result for each level is the materialized value computation var result = new LinkedList <KeyValuePair <IModule, IMaterializedValueNode> >(); result.AddFirst(new KeyValuePair <IModule, IMaterializedValueNode>(module, newMat)); return(result); } } }