private static void ReverseComponents(string name, MatchingDictionary outputsDict, List <WorkflowComponent> workflowComponents, out List <WorkflowComponent> matchedComponents, out List <WorkflowComponent> reversedComponents) { matchedComponents = new List <WorkflowComponent>(); reversedComponents = new List <WorkflowComponent>(); foreach (WorkflowComponent component in workflowComponents) { WorkflowComponent componentToAdd = component; // 2.1 Check if the inputs correspont to the default ones, if not create workflow for reversal if (component.IsReversed(outputsDict[component.Id])) { // 2.2 Classify into inputs and outputs HashSet <string> componentOuts = outputsDict[component.Id]; (List <Data> outpts, List <Data> inpts) = component.GetAllData().Classify(d => componentOuts.Contains(d.Id)); if (component is Model model) //(modelObjects[nrow] is cModel) { componentToAdd = model.Reverse(inpts, outpts); } else if (component is Workflow workflow) { componentToAdd = ScheduleWorkflowGlobal($"{name}#Global#{workflow.Id}", "", inpts, outpts, workflow.Components, GlobalReversalMode.NoReversedModels); } reversedComponents.Add(componentToAdd); } matchedComponents.Add(componentToAdd); } }
public static Workflow ScheduleWorkflow(string name, string description, List <Data> inputs, List <Data> outputs, List <WorkflowComponent> components, MatchingDictionary matching, ReversalMode mode) { if (mode == ReversalMode.Global || mode == ReversalMode.Legacy) { throw new ArgumentException($"Only reversals modes: '{ReversalMode.GroupNonReversibleOnly}' and '{ReversalMode.GroupBoth}' " + $"are valid to schedule a workflow from a matching", nameof(mode)); } List <WorkflowComponent> workflowComponents = components.GetAllComponents(); var allData = inputs.Concat(outputs).ToList(); // 2. Determine which models are reversed, and create reversed Workflows (Model or Global) to cater for them ReverseComponents(name, matching, workflowComponents, out List <WorkflowComponent> matchedComponents, out List <WorkflowComponent> reversedComponents); // 3. Cluster reversed components joint by non-reversible variables (e.g. arrays) List <WorkflowComponent> reversedClusteredComponents = ClusterReversedModel(name, allData, matchedComponents, reversedComponents, mode, components); // 4. Cluster Strongly Connected Components to get an acyclic graph List <WorkflowComponent> clusteredComponents = ClusterStronglyConnectedComponents(name, allData, reversedClusteredComponents); // 5. Schedule the acyclic graph to get the final order of the components List <WorkflowComponent> scheduledComponents = ScheduleAcyclic(allData, clusteredComponents); return(new Workflow(name, description, inputs, outputs, components, scheduledComponents, false, mode.ToString()) { DependencyAnalysis = new GraphBasedDependencyAnalysis(matchedComponents) }); }
public MatchingDictionary(MatchingDictionary matchingDictionary) { foreach (KeyValuePair <string, HashSet <string> > kvp in matchingDictionary) { this[kvp.Key] = kvp.Value; } validComponentsHash = new HashSet <string>(matchingDictionary.validComponentsHash); }
public static Workflow ScheduleWorkflowGlobal(string name, string description, List <Data> inputs, List <Data> outputs, List <WorkflowComponent> components, GlobalReversalMode mode) { List <WorkflowComponent> workflowComponents = components.GetAllComponents(); var allData = inputs.Concat(outputs).ToList(); // 0. Samity checks, check is not more than 1 output if (mode == GlobalReversalMode.NoReversedModels) { if (!AllSanityCheck(workflowComponents)) { mode = GlobalReversalMode.ReverseModelsWhenReversibleVariables; } } else if (mode == GlobalReversalMode.Global) { if (!AllSanityCheck(workflowComponents)) { throw new ArgumentException($"A default workflow does not exist. Therefore the gloabl workflow cannot be created"); } } if (mode == GlobalReversalMode.ReverseModelsWhenReversibleVariables) { NonReversibleSanityCheck(workflowComponents); } // 1. Find Model-Variable matching. Which variables are inputs, and wich are outputs for each model MatchingDictionary outputsDict = MatchModelsWithVariablesGlobal(allData, components, mode); // 2. Determine which models are reversed, and create reversed Workflows (Model or Global) to cater for them ReverseComponents(name, outputsDict, workflowComponents, out List <WorkflowComponent> matchedComponents, out List <WorkflowComponent> reversedComponents); // 4. Cluster Strongly Connected Components to get an acyclic graph List <WorkflowComponent> clusteredComponents = ClusterStronglyConnectedComponents(name, allData, matchedComponents); // 5. Schedule the acyclic graph to get the final order of the components List <WorkflowComponent> scheduledComponents = ScheduleAcyclic(allData, clusteredComponents); Workflow workflow = null; if (clusteredComponents.Count < matchedComponents.Count) { workflow = ScheduleWorkflowSCC(name.Split(':').First(), description, inputs, outputs, workflowComponents, matchedComponents); } else { workflow = new WorkflowGlobal(name, description, inputs, outputs, components, scheduledComponents, DeafaultSolvers(), mode != GlobalReversalMode.Global, mode.ToString()); } workflow.DependencyAnalysis = new GraphBasedDependencyAnalysis(matchedComponents); return(workflow); }
public static Workflow ScheduleWorkflow(string name, string description, List <Data> inputs, List <Data> outputs, List <WorkflowComponent> components, ReversalMode mode) { // Redirect to right scheduler method if (mode == ReversalMode.Global) { return(ScheduleWorkflowGlobal(name, description, inputs, outputs, components, GlobalReversalMode.Global)); } else if (mode == ReversalMode.Legacy) { return(LibishScheduler.ScheduleWorkflow(name, description, inputs, outputs, components)); } List <WorkflowComponent> workflowComponents = components.GetAllComponents(); var allData = inputs.Concat(outputs).ToList(); // 0. Sanity checks, check array is not more than 1 output NonReversibleSanityCheck(workflowComponents); // 1. Find Model-Variable matching. Which variables are inputs, and wich are outputs for each model MatchingDictionary outputsDict = MatchModelsWithVariables(inputs, allData, workflowComponents); // 2. Determine which models are reversed, and create reversed Workflows (Model or Global) to cater for them ReverseComponents(name, outputsDict, workflowComponents, out List <WorkflowComponent> matchedComponents, out List <WorkflowComponent> reversedComponents); // 3. Cluster reversed components joint by non-reversible variables (e.g. arrays) List <WorkflowComponent> reversedClusteredComponents = ClusterReversedModel(name, allData, matchedComponents, reversedComponents, mode, components); // 4. Cluster Strongly Connected Components to get an acyclic graph List <WorkflowComponent> clusteredComponents = ClusterStronglyConnectedComponents(name, allData, reversedClusteredComponents); // 5. Schedule the acyclic graph to get the final order of the components List <WorkflowComponent> scheduledComponents = ScheduleAcyclic(allData, clusteredComponents); return(new Workflow(name, description, inputs, outputs, components, scheduledComponents, false, mode.ToString()) { DependencyAnalysis = new GraphBasedDependencyAnalysis(matchedComponents) }); }
private static (MatchingDictionary completeMatching, MaximumMatchings2 matches) FindAllMatches(List <Data> inputs, List <Data> allData, List <WorkflowComponent> workflowComponents) { var inputsHash = new HashSet <string>(inputs.Select(d => d.Id)); var outputsHash = new HashSet <string>(); // 1.0. Check for non-reversible variables that are either only input or output var nonReversibleData = allData.Where(d => !(d is DoubleData)).ToList(); Dictionary <string, IOStatus> nonReversibleStatus = workflowComponents.GetDataStatus(nonReversibleData); //var collisionDictionary = new Dictionary<string, char>(); //foreach (var component in workflowComponents) //{ // foreach (var input in component.ModelDataInputs) // { // if (!(input is DoubleData)) // { // string inName = input.Id; // if (collisionDictionary.ContainsKey(inName)) // collisionDictionary[inName] = 'b'; // else // collisionDictionary[inName] = 'i'; // } // } // foreach (var output in component.ModelDataOutputs) // { // if (!(output is DoubleData)) // { // string outName = output.Id; // if (collisionDictionary.ContainsKey(outName)) // collisionDictionary[outName] = 'b'; // else // collisionDictionary[outName] = 'o'; // } // } //} foreach (string data in nonReversibleStatus.Keys) { if (nonReversibleStatus[data] == IOStatus.Input) { inputsHash.Add(data); } else if (nonReversibleStatus[data] == IOStatus.Output) { outputsHash.Add(data); } } var bipartite = new BipartiteGraph(); Primes.Reset(); var outputsDict = new MatchingDictionary(workflowComponents.Select(c => c.Id)); // 1.1 Add Nodes for the Variables // Filter-out the selected inputs and outputs, as they don't belong to the bipartite graph foreach (Data data in allData.Where(d => !inputsHash.Contains(d.Id) && !outputsHash.Contains(d.Id))) { bipartite.AddNode(data.Id, GraphNode.Type.Type1); } // 1.2 Add Nodes for the Models, and edges between Variables and Nodes foreach (WorkflowComponent component in workflowComponents) { int uniqueNonReversibleOutputs = 0; outputsDict.Add(component.Id, new HashSet <string>()); bipartite.AddNode(CS(component.Id), GraphNode.Type.Type2); GraphNode modelNode = bipartite.GetNode(CS(component.Id)); // Filter-out the selected inputs, as they don't belong to the bipartite graph foreach (Data data in component.ModelDataInputs.Where(d => !inputsHash.Contains(d.Id))) { if (outputsHash.Contains(data.Id)) { // Shouldn't arrive here uniqueNonReversibleOutputs++; outputsDict[component.Id].Add(data.Id); } else { bipartite.AddDirectedEdge(data.Id, modelNode, Primes.Next(), 1); } } // Filter-out the selected inputs, as they don't belong to the bipartite graph foreach (Data data in component.ModelDataOutputs.Where(d => !inputsHash.Contains(d.Id))) { if (outputsHash.Contains(data.Id)) { uniqueNonReversibleOutputs++; outputsDict[component.Id].Add(data.Id); } else { bipartite.AddDirectedEdge(data.Id, modelNode, Primes.Next(), 0); } } // How many output the model need if (component.ModelDataOutputs.Count - uniqueNonReversibleOutputs > 1) { bipartite.modelAndItsOutputs.Add(CS(component.Id), component.ModelDataOutputs.Count - uniqueNonReversibleOutputs); //signle output models are not stored } else if (component.ModelDataOutputs.Count - uniqueNonReversibleOutputs == 0) { bipartite.Remove(CS(component.Id)); } } // 1.3 Associate matchings to each model, maping models to set of outputs var matches = new MaximumMatchings2(bipartite, getAllMatchings: true); return(outputsDict, matches); }
private static MatchingDictionary MatchModelsWithVariablesGlobal(List <Data> allData, List <WorkflowComponent> components, GlobalReversalMode mode) { if (mode == GlobalReversalMode.NoReversedModels || mode == GlobalReversalMode.Global) { List <WorkflowComponent> originalComponents = components.GetAllComponents(); Dictionary <string, IOStatus> status = originalComponents.GetInputsOutputsStatus(allData, out List <Data> inputs, out List <Data> outputs); foreach (string key in status.Keys) { if (status[key] == IOStatus.Conflict) { throw new ArgumentException($"The following varibale \"{key}\" is of non-reversible type and output of more than one model. Therefore the workflow cannot be created"); } } var inputsHash = new HashSet <string>(inputs.Select(d => d.Id)); var bipartite = new BipartiteGraph(); Primes.Reset(); var completeMatching = new MatchingDictionary(originalComponents.Select(c => c.Id)); // 1.1 Add Nodes for the Variables // Filter-out the selected inputs and outputs, as they don't belong to the bipartite graph foreach (Data data in allData.Where(d => !inputsHash.Contains(d.Id))) { bipartite.AddNode(data.Id, GraphNode.Type.Type1); } // 1.2 Add Nodes for the Models, and edges between Variables and Nodes foreach (WorkflowComponent component in originalComponents) { completeMatching.Add(component.Id, new HashSet <string>()); bipartite.AddNode(CS(component.Id), GraphNode.Type.Type2); GraphNode modelNode = bipartite.GetNode(CS(component.Id)); // Filter-out the selected inputs, as they don't belong to the bipartite graph foreach (Data data in component.ModelDataInputs.Where(d => !inputsHash.Contains(d.Id))) { bipartite.AddDirectedEdge(data.Id, modelNode, Primes.Next(), 1); } // Filter-out the selected inputs, as they don't belong to the bipartite graph foreach (Data data in component.ModelDataOutputs.Where(d => !inputsHash.Contains(d.Id))) { bipartite.AddDirectedEdge(data.Id, modelNode, Primes.Next(), 0); } // How many output the model need if (component.ModelDataOutputs.Count > 1) { bipartite.modelAndItsOutputs.Add(CS(component.Id), component.ModelDataOutputs.Count); //signle output models are not stored } } // 1.3 Associate matchings to each model, maping models to set of outputs var matches = new MaximumMatchings2(bipartite, true); if (matches.OverConstrainedModels.Count > 0) { throw new ArgumentException($"The inputs conbination is not valid as the following models are overconstraint:\r\n\t" + matches.OverConstrainedModels.Aggregate((total, last) => total += "\r\n\t" + last) + "\r\n"); } Matching matching = matches.OrderedFilteredMaximumMatchings.FirstOrDefault() ?? throw new NullReferenceException("NO suitable matchings were found"); completeMatching.CompleteWithMatching(matching); return(completeMatching); } else { Dictionary <string, IOStatus> status = components.GetInputsOutputsStatus(allData, out List <Data> inputs, out List <Data> outputs); var inputsHash = new HashSet <string>(inputs.Select(d => d.Id)); var outputsHash = new HashSet <string>(); List <WorkflowComponent> originalComponents = components.GetAllComponents(); Dictionary <string, IOStatus> originalStatus = originalComponents.GetDataStatus(allData); var nonReversibleData = allData.Where(d => !(d is DoubleData)).ToList(); Dictionary <string, IOStatus> nonReversibleStatus = components.GetDataStatus(nonReversibleData); foreach (string data in nonReversibleStatus.Keys) { if (nonReversibleStatus[data] == IOStatus.Input) { inputsHash.Add(data); // Should be already there } else if (nonReversibleStatus[data] == IOStatus.Output || nonReversibleStatus[data] == IOStatus.Both) { outputsHash.Add(data); } } var reversedInputsHash = new HashSet <string>(); var reversedOutputsHash = new HashSet <string>(); foreach (WorkflowComponent component in components) { if (component is IReversableWorkflow rw) { foreach (Data data in rw.ReversedInputs) { reversedInputsHash.Add(data.Id); } foreach (Data data in rw.ReversedOutputs) { reversedOutputsHash.Add(data.Id); } IEnumerable <Data> NonReversableReversedInputs = rw.ReversedInputs.Where(d => !(d is DoubleData)); IEnumerable <Data> NonReversableReversedOutputs = rw.ReversedOutputs.Where(d => !(d is DoubleData)); // If the model id the upper end of a reversal throug non-reversible variables. int difference = NonReversableReversedOutputs.Count() - NonReversableReversedInputs.Count(); while (difference > 0) { // Assign as many reversible variable to the inputs as reversals end in the component IEnumerable <Data> reversableReversedInputs = rw.ReversedInputs.Where(d => d is DoubleData); foreach (Data data in reversableReversedInputs.Take(difference)) { inputsHash.Add(data.Id); } } } } // Relax one input per reversal -> lowerEndOfReversal might have more elements than non-reversible reversal, but those should be already not in inputHash IEnumerable <string> lowerEndOfReversals = reversedOutputsHash.Except(reversedInputsHash); inputsHash.ExceptWith(lowerEndOfReversals); var bipartite = new BipartiteGraph(); Primes.Reset(); var completeMatching = new MatchingDictionary(originalComponents.Select(c => c.Id)); // 1.1 Add Nodes for the Variables // Filter-out the selected inputs and outputs, as they don't belong to the bipartite graph foreach (Data data in allData.Where(d => !inputsHash.Contains(d.Id) && !outputsHash.Contains(d.Id))) { bipartite.AddNode(data.Id, GraphNode.Type.Type1); } // 1.2 Add Nodes for the Models, and edges between Variables and Nodes foreach (WorkflowComponent component in originalComponents) { completeMatching.Add(component.Id, new HashSet <string>()); // if the component has all its inputs and outputs determined do not add to the graph bool addModel = component.ModelDataOutputs.Count > component.ModelDataOutputs.Where(d => inputsHash.Count > -1 && outputsHash.Contains(d.Id)).Count(); if (addModel) { bipartite.AddNode(component.Id, GraphNode.Type.Type2); } GraphNode modelNode = bipartite.GetNode(component.Id); int uniqueNonReversibleOutputs = 0; // Filter-out the selected inputs, as they don't belong to the bipartite graph foreach (Data data in component.ModelDataInputs.Where(d => !inputsHash.Contains(d.Id) && !outputsHash.Contains(d.Id))) { bipartite.AddDirectedEdge(data.Id, modelNode, Primes.Next(), 1); } // Filter-out the selected inputs, as they don't belong to the bipartite graph foreach (Data data in component.ModelDataOutputs.Where(d => !inputsHash.Contains(d.Id))) { if (outputsHash.Contains(data.Id)) { uniqueNonReversibleOutputs++; completeMatching[component.Id].Add(data.Id); } else { bipartite.AddDirectedEdge(data.Id, modelNode, Primes.Next(), 0); } } // How many output the model need if (component.ModelDataOutputs.Count - uniqueNonReversibleOutputs > 1 && addModel) { bipartite.modelAndItsOutputs.Add(component.Id, component.ModelDataOutputs.Count - uniqueNonReversibleOutputs); //signle output models are not stored } } // 1.3 Associate matchings to each model, maping models to set of outputs var matches = new MaximumMatchings2(bipartite, true); if (matches.OverConstrainedModels.Count > 0) { throw new ArgumentException($"The inputs conbination is not valid as the following models are overconstraint:\r\n\t" + matches.OverConstrainedModels.Aggregate((total, last) => total += "\r\n\t" + last) + "\r\n"); } Matching matching = matches.OrderedFilteredMaximumMatchings.FirstOrDefault() ?? throw new NullReferenceException("NO suitable matchings were found"); completeMatching.CompleteWithMatching(matching); return(completeMatching); } }