public void emite() { string saida = ((IStringInstance)Input_data.Instance).Value; IIteratorInstance <IKVPair <IInteger, IPageNode> > output = (IIteratorInstance <IKVPair <IInteger, IPageNode> >)Output_data.Instance; double X = 0.0; string[] lines = saida.Split(new char[] { System.Environment.NewLine[0] }); string[] pr = lines[0].Split(' '); if (pr [0].Equals("X")) { X = double.Parse(pr [1]) / dic.Count; } else { System.Console.WriteLine("Error of index to X"); } lines [0] = ""; foreach (string line in lines) { if (!line.Trim().Equals("")) { string[] prank = line.Split(' '); //if (b && !prank [0].Equals ("X")) { IKVPairInstance <IInteger, IPageNode> kvpair = dic [int.Parse(prank [0])]; IPageNodeInstance no = (IPageNodeInstance)kvpair.Value; no.PgrankInstance = double.Parse(prank [1]) + X; output.put(kvpair); //} } } output.finish(); }
public override void main() { Trace.WriteLine(WorldComm.Rank + ": STARTING BREAK IN LINE (SPLIT FUNCTION)"); IStringInstance input_data_instance = (IStringInstance)Input_data.Instance; IIteratorInstance <IKVPair <IInteger, IString> > output_data_instance = (IIteratorInstance <IKVPair <IInteger, IString> >)Output_data.Instance; string s = input_data_instance.Value; string[] lines = s.Split(new char[] { System.Environment.NewLine[0] }); int line_counter = 0; foreach (string line in lines) { if (!line.Trim().Equals("")) { Trace.WriteLine(WorldComm.Rank + ": LINE = " + line); IKVPairInstance <IInteger, IString> line_pair = (IKVPairInstance <IInteger, IString>)Output_data.createItem(); ((IIntegerInstance)line_pair.Key).Value = line_counter++; ((IStringInstance)line_pair.Value).Value = line; output_data_instance.put(line_pair); } } Trace.WriteLine(WorldComm.Rank + ": FINISH BREAK IN LINES #1 !!! " + output_data_instance.GetHashCode()); output_data_instance.finish(); Trace.WriteLine(WorldComm.Rank + ": FINISH BREAK IN LINES #2 !!!"); }
public override void main() { IIteratorInstance <IKVPair <IMK, IMV> > input_instance = (IIteratorInstance <IKVPair <IMK, IMV> >)Input.Instance; IIteratorInstance <IKVPair <OMK, OMV> > output_instance = (IIteratorInstance <IKVPair <OMK, OMV> >)Output.Instance; object bin_object = null; int count = 0; while (input_instance.fetch_next(out bin_object)) { Trace.WriteLine(Rank + ": LOOP MAPPER !!!" + (count++)); IKVPairInstance <IMK, IMV> bin = (IKVPairInstance <IMK, IMV>)bin_object; Trace.WriteLine(Rank + ": " + (Map_value.Instance == null) + " $$$$ " + bin.Value.GetType()); Map_key.Instance = bin.Key; Map_value.Instance = bin.Value; Map_function.go(); } Trace.WriteLine(Rank + ": FINISH MAPPER !!!"); output_instance.finish(); }
public override void main() { IIteratorInstance <IKVPair <IVertex, IDataTriangle> > output_pairs = (IIteratorInstance <IKVPair <IVertex, IDataTriangle> >)Output_pairs.Instance; IIteratorInstance <IKVPair <IVertex, IDataTriangle> > input_pairs = (IIteratorInstance <IKVPair <IVertex, IDataTriangle> >)Input_pairs.Instance; object pair; // For 3 iterations: startup_push, gust0, gust1 while ((iterate_num++) < 2) { while (iterate_pairs.fetch_next(out pair)) { input_pairs.put(pair); } input_pairs.finish(); } input_pairs.finish(); // Finally, all pairs are sent to the output. while (iterate_pairs.fetch_next(out pair)) { output_pairs.put(pair); } output_pairs.finish(); }
public override void main() { Trace.WriteLine(Rank + ": START COMBINE FUNCTION #1"); IIteratorInstance <ORV> input_data_instance = (IIteratorInstance <ORV>)Input_data.Instance; IKVPairInstance <IInteger, IIterator <IPathInfo> > output_data_instance = (IKVPairInstance <IInteger, IIterator <IPathInfo> >)Output_data.Instance; Trace.WriteLine(Rank + ": START COMBINE FUNCTION #2"); IIntegerInstance done_flag = (IIntegerInstance)output_data_instance.Key; IIteratorInstance <IPathInfo> output_distances = (IIteratorInstance <IPathInfo>)output_data_instance.Value; Trace.WriteLine(Rank + ": START COMBINE FUNCTION #3"); bool done = true; object o; while (input_data_instance.fetch_next(out o)) { IKVPairInstance <IInteger, IIterator <IPathInfo> > kv = (IKVPairInstance <IInteger, IIterator <IPathInfo> >)o; IIntegerInstance k = (IIntegerInstance)kv.Key; Trace.WriteLine(Rank + ": LOOP #1 COMBINE FUNCTION " + k.Value); output_distances.putAll((IIteratorInstance <IPathInfo>)kv.Value); done = done && k.Value == 1; } output_distances.finish(); done_flag.Value = done ? 1 : 0; Trace.WriteLine(Rank + ": END COMBINE FUNCTION " + done); }
public override void main() { Trace.WriteLine(Rank + ": START SPLIT PATH INFO 1"); IIteratorInstance <IPathInfo> input_data_instance = (IIteratorInstance <IPathInfo>)Input_data.Instance; IIteratorInstance <IKVPair <IInteger, IPathInfo> > output_data_instance = (IIteratorInstance <IKVPair <IInteger, IPathInfo> >)Output_data.Instance; Trace.WriteLine(Rank + ": START SPLIT PATH INFO 2"); object path_info; while (input_data_instance.fetch_next(out path_info)) { Trace.WriteLine(Rank + ": LOOP SPLIT PATH INFO " + path_info.GetType()); IPathInfoInstance path_info_instance = (IPathInfoInstance)path_info; IKVPairInstance <IInteger, IPathInfo> path_info_output = (IKVPairInstance <IInteger, IPathInfo>)Output_data.createItem(); ((IIntegerInstance)path_info_output.Key).Value = ((Info)path_info_instance.Value).vertex; path_info_output.Value = path_info_instance; Trace.WriteLine(Rank + ": vertex = " + path_info_output.Key); output_data_instance.put(path_info_output); } output_data_instance.finish(); Trace.Write(Rank + ": STOP SPLIT PATH INFO "); }
/* Recebimento de ORVs das unidades source */ public void receiveCombineORVs() { int listenFinishedObject = 0; Trace.WriteLine(WorldComm.Rank + ": START COMBINER TARGET !!!"); object orv; MPI.CompletedStatus status; while (listenFinishedObject < size_reducers) { Trace.WriteLine(WorldComm.Rank + ": BEGIN RECEIVE COMBINER TARGET to "); comm.Receive <object>(MPI.Unsafe.MPI_ANY_SOURCE, MPI.Unsafe.MPI_ANY_TAG, out orv, out status); Trace.WriteLine(WorldComm.Rank + ": END RECEIVE COMBINER TARGET to " + (status.Source)); if (status.Tag == TAG_COMBINER_ORV_FINISH) { listenFinishedObject = listenFinishedObject + 1; } else { combine_input_data_instance.put(orv); } } combine_input_data_instance.finish(); Trace.WriteLine(WorldComm.Rank + ": FINISH COMBINER TARGET !!!"); }
public override void main() { /* 1. Ler pares chave (OMK) e valores (OMV) de Input. * 2. Para cada par, atribuir a Key e Values e chamar Reduce_function.go(); * 3. Pegar o resultado de Reduction_function.go() de Output_reduce (ORV) * e colocar no iterator Output. */ IIteratorInstance <IKVPair <OMK, IIterator <OMV> > > input_instance = (IIteratorInstance <IKVPair <OMK, IIterator <OMV> > >)Input.Instance; IIteratorInstance <ORV> output_instance = (IIteratorInstance <ORV>)Output.Instance; //long t0 = (long)(DateTime.UtcNow - (new DateTime (1970, 1, 1, 0, 0, 0, DateTimeKind.Utc))).TotalMilliseconds; object kvpair_object; while (input_instance.fetch_next(out kvpair_object)) { IKVPairInstance <OMK, IIterator <OMV> > kvpair = (IKVPairInstance <OMK, IIterator <OMV> >)kvpair_object; Input_reduce.Instance = kvpair; Reduce_function.go(); active++; active = active - Reduce_function.Active; } string candidates_buffer = sincronize(); while (active > 0) { active = 0; string[] lines = candidates_buffer.Split(new char[] { System.Environment.NewLine[0] }); foreach (string line in lines) { if (!line.Trim().Equals("")) { string[] values = line.Split(' '); Reduce_function.step(int.Parse(values[0]), double.Parse(values[1])); active++; active = active - Reduce_function.Active; } } candidates_buffer = sincronize(); } Reduce_function.apply(); output_instance.put(Output_reduce.Instance); output_instance.finish(); //long t1 = (long)(DateTime.UtcNow - (new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc))).TotalMilliseconds; //writeFile ("./logTIME-REDUCE", "TIME: " + (t1 - t0) + " ms."+System.Environment.NewLine+"BSP"); }
public override void main() { Console.WriteLine("################################*IAplicacaoImpl Start ###############################"); IDoubleInstance tDouble = (IDoubleInstance)T_double.Instance; IKVPairInstance <IString, IDouble> tKVPair = (IKVPairInstance <IString, IDouble>)T_kvpair.Instance; //.newInstance (); IIntegerInstance tInteger = (IIntegerInstance)T_integer.Instance; IIteratorInstance <IInteger> tIterator = (IIteratorInstance <IInteger>)T_iterator.Instance; //.newInstance (); IStringInstance tString = (IStringInstance)T_string.Instance; tDouble.Value = 0.8; tInteger.Value = 8; tString.Value = "Tipo String"; ((IStringInstance)tKVPair.Key).Value = tString.Value; ((IDoubleInstance)tKVPair.Value).Value = tDouble.Value; tIterator.put(tInteger); tIterator.finish(); object o; int count = 0; while (tIterator.fetch_next(out o)) { IIntegerInstance oI = (IIntegerInstance)o; Console.WriteLine("Item Iterator " + (count++) + ": " + oI.Value); } Console.WriteLine("tDouble.Value=" + tDouble.Value + " : " + "tInteger.Value=" + tInteger.Value + " : " + "tString.Value=" + tString.Value); Console.WriteLine("tKVPair.Key=" + ((IStringInstance)tKVPair.Key).Value + " : " + "tKVPair.Value=" + ((IDoubleInstance)tKVPair.Value).Value); IPageNodeInstance instance = (IPageNodeInstance)T_aplicacao_node.Instance; ((IIntegerInstance)instance.IdInstance).Value = 8; IIntegerInstance item = (IIntegerInstance)T_aplicacao_node.Edge_node.createItem(); item.Value = 7; instance.NeighborsInstance.put(item); instance.NeighborsInstance.put(((IIntegerInstance)instance.IdInstance)); instance.NeighborsInstance.finish(); while (instance.NeighborsInstance.fetch_next(out o)) { IIntegerInstance II = (IIntegerInstance)o; Console.WriteLine("%%%%%%%%%%%%%%%%%%%%%%%%%%%% Valor no Iterator=" + II.Value); } Console.WriteLine("%%%%%%%%%%%%%%%%%%%%%%%%%%%% Resultado Id_node=" + ((IIntegerInstance)instance.IdInstance).Value + " PGRank=" + instance.PgrankInstance.Value); Console.WriteLine("################################ IAplicacaoImpl End ###############################"); }
public void gust0() //emite saídas { IIteratorInstance <IKVPair <IInteger, IDataSSSP> > output_value_instance = (IIteratorInstance <IKVPair <IInteger, IDataSSSP> >)Output.Instance; bool any_emite = false; foreach (bool any in emite) { any_emite = any_emite || any; } if (!any_emite && halt_sum == 0) { output_value_instance.finish(); // TerminatedFunctionSSSP é avisado com um finish(), preparando-se para a emissão definitiva de saída IKVPairInstance <IInteger, IDataSSSP> ITEM = (IKVPairInstance <IInteger, IDataSSSP>)Output.createItem(); ((IIntegerInstance)ITEM.Key).Value = this.partid; ((IDataSSSPInstance)ITEM.Value).Path_size = messages [((IIntegerInstance)ITEM.Key).Value]; ((IDataSSSPInstance)ITEM.Value).Halt = 0; output_value_instance.put(ITEM); } else { for (int i = 0; i < partition_size; i++) { IKVPairInstance <IInteger, IDataSSSP> ITEM = (IKVPairInstance <IInteger, IDataSSSP>)Output.createItem(); ((IIntegerInstance)ITEM.Key).Value = i; if (partition_own [i] || !emite [i]) { ((IDataSSSPInstance)ITEM.Value).Path_size = new Dictionary <int, float> (); } else { ((IDataSSSPInstance)ITEM.Value).Path_size = messages [i]; } ((IDataSSSPInstance)ITEM.Value).Halt = any_emite ? 1 : 0; output_value_instance.put(ITEM); } } emite = new bool[partition_size]; halt_sum = 0; }
public override void main() { IIteratorInstance <ORV> input_data_instance = (IIteratorInstance <ORV>)Input_data.Instance; IIteratorInstance <ORV> output_data_instance = (IIteratorInstance <ORV>)Output_data.Instance; Trace.WriteLine(WorldComm.Rank + ": START COMBINE FUNCTION !!! "); object item_object; int count = 0; while (input_data_instance.fetch_next(out item_object)) { Trace.WriteLine(WorldComm.Rank + ": COMBINE_FUNCTION_LOOP 1 " + (count++) + " " + item_object.GetType()); output_data_instance.put(item_object); } Trace.WriteLine(WorldComm.Rank + ": OUT LOOP COMBINE FUNCTION !!!"); output_data_instance.finish(); Trace.WriteLine(WorldComm.Rank + ": FINISH COMBINE FUNCTION !!!"); }
private void graph_creator() { IIteratorInstance <IKVPair <IInteger, IIterator <GIF> > > input_instance_gif = (IIteratorInstance <IKVPair <IInteger, IIterator <GIF> > >)Collect_graph.Client; IIteratorInstance <IKVPair <IInteger, GIF> > output_instance_gif = (IIteratorInstance <IKVPair <IInteger, GIF> >)Output_gif.Instance; Feed_graph.Server = output_instance_gif; IActionFuture sync_perform; bool end_iteration = false; while (!end_iteration) // take next chunk ... { Task_gusty.invoke(ITaskPortAdvance.READ_CHUNK); Task_gusty.invoke(ITaskPortAdvance.PERFORM, out sync_perform); IKVPairInstance <IInteger, IIterator <GIF> > kvpair = null; object kvpair_object; object o; if (!input_instance_gif.has_next()) { end_iteration = true; } while (input_instance_gif.fetch_next(out kvpair_object)) { kvpair = (IKVPairInstance <IInteger, IIterator <GIF> >)kvpair_object; Graph_values.Instance = kvpair; Gusty_function.graph_creator(); } sync_perform.wait(); } //IActionFuture gusty_chunk_ready; //Task_gusty.invoke (ITaskPortAdvance.CHUNK_READY, out gusty_chunk_ready); //*** output_instance_gif.finish(); //gusty_chunk_ready.wait (); }
public override void main() { Console.WriteLine(this.Rank + ": TERMINATE FUNCTION 1"); IIteratorInstance <IKVPair <OKey, OValue> > output_pairs = (IIteratorInstance <IKVPair <OKey, OValue> >)Output_pairs.Instance; IIteratorInstance <IKVPair <IKey, IValue> > input_pairs = (IIteratorInstance <IKVPair <IKey, IValue> >)Input_pairs.Instance; Console.WriteLine(this.Rank + ": TERMINATE FUNCTION 2"); input_pairs.finish(); // All pairs are sent to the output, i.e. the algorithm is not iterative. object pair; while (iterate_pairs.fetch_next(out pair)) { output_pairs.put(pair); } output_pairs.finish(); Console.WriteLine(this.Rank + ": TERMINATE FUNCTION 3"); }
public override void main() { int count = 0; // 1. recebe os bins enviados pelo gerente (unidade source), // através do MPI, e os insere no Target_data. Trace.WriteLine(WorldComm.Rank + ": STARTING SCATTER SPLIT DATA TARGET"); MPI.CompletedStatus status; int source_rank = this.UnitRanks["source"][0]; object bin_key; object bin_value; IIteratorInstance <IKVPair <IMK, IMV> > target_data_instance = (IIteratorInstance <IKVPair <IMK, IMV> >)Target_data.Instance; Trace.WriteLine(WorldComm.Rank + ": BEGIN RECEIVE BIN KEY from " + source_rank + "count=" + (count++)); comm.Receive <object> (source_rank, MPI.Unsafe.MPI_ANY_TAG, out bin_key, out status); Trace.WriteLine(WorldComm.Rank + ": END RECEIVE BIN KEY from " + source_rank); while (status.Tag != TAG_SPLITTER_IMK_FINISH) { Trace.WriteLine(WorldComm.Rank + ": BEGIN RECEIVE BIN VALUE from " + source_rank); comm.Receive <object> (source_rank, TAG_SPLITTER_IMV, out bin_value, out status); Trace.WriteLine(WorldComm.Rank + ": END RECEIVE BIN VALUE from " + source_rank); IKVPairInstance <IMK, IMV> pair = (IKVPairInstance <IMK, IMV>)Target_data.createItem(); pair.Key = bin_key; pair.Value = bin_value; target_data_instance.put(pair); Trace.WriteLine(WorldComm.Rank + ": BEGIN RECEIVE BIN KEY from " + source_rank + "count=" + (count++)); comm.Receive <object> (source_rank, MPI.Unsafe.MPI_ANY_TAG, out bin_key, out status); Trace.WriteLine(WorldComm.Rank + ": END RECEIVE BIN KEY from " + source_rank); } Trace.WriteLine(WorldComm.Rank + ": FINISH ALL BIN KEYs #1 !!!"); target_data_instance.finish(); Trace.WriteLine(WorldComm.Rank + ": FINISH ALL BIN KEYs #2 !!!"); }
public override void main() { int di = int.MaxValue; int dmin = int.MaxValue; int done = 1; Trace.WriteLine(Rank + ": START REDUCE FUNCTION"); IKVPairInstance <IInteger, IIterator <IPathInfo> > input_instance = (IKVPairInstance <IInteger, IIterator <IPathInfo> >)Input_values.Instance; IIntegerInstance k = (IIntegerInstance)input_instance.Key; IIteratorInstance <IPathInfo> v = (IIteratorInstance <IPathInfo>)input_instance.Value; int k_int = k.Value; if (!neighbours.ContainsKey(k_int)) { neighbours [k_int] = new Dictionary <int, int> (); } object o; while (v.fetch_next(out o)) { IPathInfoInstance item = (IPathInfoInstance)o; Info item_info = (Info)item.Value; Trace.WriteLine(Rank + ": REDUCE FUNCTION LOOP " + item_info); switch (item_info.info_type) { case Info.PATH_INFO_TYPE_DISTANCE_TRIAL: DistanceInfo distance_item_info_1 = (DistanceInfo)item_info; int tmp = distance_item_info_1.distance; dmin = (int)min(dmin, tmp); break; case Info.PATH_INFO_TYPE_DISTANCE_PARTIAL: DistanceInfo distance_item_info_2 = (DistanceInfo)item_info; di = distance_item_info_2.distance; break; case Info.PATH_INFO_TYPE_EDGE: EdgeInfo edge_item_info = (EdgeInfo)item_info; IDictionary <int, int> output_neibours = neighbours[k_int]; int n = edge_item_info.vertex_other; int d = 0; if (!output_neibours.TryGetValue(n, out d)) { output_neibours[n] = edge_item_info.weight; } else if (edge_item_info.weight < d) { output_neibours[n] = edge_item_info.weight; } break; } } Trace.WriteLine(Rank + ": REDUCE FUNCTION AFTER LOOP #1 "); IKVPairInstance <IInteger, IIterator <IPathInfo> > orv = (IKVPairInstance <IInteger, IIterator <IPathInfo> >)Output_value.newInstance(); IIteratorInstance <IPathInfo> buffer = (IIteratorInstance <IPathInfo>)orv.Value; Trace.WriteLine(Rank + ": REDUCE FUNCTION OUT LOOP #2 "); //IPathInfo path_info = buffer.createItem(); dmin = (int)min(dmin, di); if (dmin != di) { foreach (KeyValuePair <int, int> kv in neighbours[k_int]) { //IPathInfoInstance path_info_instance = (IPathInfoInstance) path_info.newInstance (); IPathInfoInstance path_info_instance = (IPathInfoInstance)buffer.createItem(); DistanceInfo distance_info = new DistanceInfo(); distance_info.info_type = Info.PATH_INFO_TYPE_DISTANCE_TRIAL; distance_info.vertex = kv.Key; distance_info.distance = kv.Value + dmin; path_info_instance.Value = distance_info; buffer.put(path_info_instance); Trace.WriteLine(Rank + ": REDUCE FUNCTION OTHER LOOP " + distance_info); } done = 0; } ((IIntegerInstance)orv.Key).Value = done; IPathInfoInstance path_info_instance_2 = (IPathInfoInstance)buffer.createItem(); DistanceInfo distance_info_2 = new DistanceInfo(); distance_info_2.info_type = Info.PATH_INFO_TYPE_DISTANCE_PARTIAL; distance_info_2.vertex = k.Value; distance_info_2.distance = dmin; path_info_instance_2.Value = distance_info_2; buffer.put(path_info_instance_2); Trace.WriteLine(Rank + ": REDUCE FUNCTION OTHER LOOP OUT " + distance_info_2); buffer.finish(); Trace.WriteLine(Rank + ": FINISH REDUCE FUNCTION"); }
private void send_to_mappers() { Console.WriteLine(this.Rank + ": IJoinGustyCollector 1"); IIteratorInstance <IKVPair <OKey, OValue> > output_instance = (IIteratorInstance <IKVPair <OKey, OValue> >)Output_pairs.Instance; IIteratorInstance <IKVPair <IKey, IValue> > input_instance = (IIteratorInstance <IKVPair <IKey, IValue> >)Input_pairs.Instance; object bin_object = null; // DETERMINE COMMUNICATION TARGETs Tuple <int, int> sink_ref = new Tuple <int, int> (this.FacetIndexes [FACET_SINK] [0], 0); IDictionary <int, Tuple <int, int> > unit_ref = new Dictionary <int, Tuple <int, int> > (); int m_size = 0; foreach (int i in this.FacetIndexes[FACET_MAP]) { int nr0 = m_size; m_size += this.UnitSizeInFacet [i] ["gusty_feeder"]; for (int k = 0, j = nr0; j < m_size; k++, j++) { unit_ref [j] = new Tuple <int, int> (i /*,0 index of MAP_FEEDER*/, k); } } Console.WriteLine(this.Rank + ": IJoinGustyCollector 2"); IActionFuture sync_perform; bool end_computation = false; while (!end_computation) // new iteration { Console.WriteLine(this.Rank + ": IJoinGustyCollector LOOP"); end_computation = true; //Console.WriteLine (this.Rank + ": IJoinGustyCollector ITERATE 1"); //Task_binding_split_next.invoke (ITaskPortAdvance.READ_CHUNK); //**** //Task_binding_split_next.invoke (ITaskPortAdvance.PERFORM, out sync_perform); //Console.WriteLine (this.Rank + ": IJoinGustyCollector ITERATE 2"); // SEND TO SINK IList <IKVPairInstance <OKey, OValue> > buffer_sink = new List <IKVPairInstance <OKey, OValue> >(); end_computation = !input_instance.has_next(); Thread thread_send_output = new Thread(new ThreadStart(delegate { int count1 = 0; while (output_instance.fetch_next(out bin_object)) { IKVPairInstance <OKey, OValue> item = (IKVPairInstance <OKey, OValue>)bin_object; buffer_sink.Add(item); if (count1 % CHUNK_SIZE == 0) { // PERFORM Console.WriteLine(this.Rank + ": IJoinGustyCollector SINK SEND CHUNK 3-1 count=" + count1); Split_channel.Send(buffer_sink, sink_ref, TAG_SPLIT_NEW_CHUNK); Console.WriteLine(this.Rank + ": IJoinGustyCollector SINK SEND CHUNK 3-2 count=" + count1); buffer_sink.Clear(); } count1++; } if (buffer_sink.Count > 0) { Console.WriteLine(this.Rank + ": IJoinGustyCollector SINK SEND CHUNK 3-3 count=" + count1); Split_channel.Send(buffer_sink, sink_ref, TAG_SPLIT_NEW_CHUNK); Console.WriteLine(this.Rank + ": IJoinGustyCollector SINK SEND CHUNK 3-4 count=" + count1); } })); thread_send_output.Start(); // SEND BACK TO MAPPER (new iteration) Bin_function_iterate.NumberOfPartitions = m_size; IList <IKVPairInstance <OKey, OValue> >[] buffer = new IList <IKVPairInstance <OKey, OValue> > [m_size]; for (int i = 0; i < m_size; i++) { buffer [i] = new List <IKVPairInstance <OKey, OValue> > (); } Console.WriteLine(this.Rank + ": IJoinGustyCollector ITERATE 3 end_computation=" + end_computation); int count = 0; while (input_instance.fetch_next(out bin_object)) { Console.WriteLine(this.Rank + ": IJoinGustyCollector ITERATE LOOP 4-1 count=" + count); IKVPairInstance <OKey, OValue> item = (IKVPairInstance <OKey, OValue>)bin_object; this.Input_key_iterate.Instance = item.Key; Bin_function_iterate.go(); int index = ((IIntegerInstance)this.Output_key_iterate.Instance).Value; buffer [index].Add(item); if (count % CHUNK_SIZE == 0) { Task_binding_split_next.invoke(ITaskPortAdvance.READ_CHUNK); Task_binding_split_next.invoke(ITaskPortAdvance.PERFORM, out sync_perform); for (int i = 0; i < m_size; i++) { Console.WriteLine("SPLITTER REDUCE COLLECTOR - Sending chunk of " + buffer[i].Count + " elements"); Split_channel.Send(buffer [i], unit_ref [i], TAG_SPLIT_NEW_CHUNK); buffer [i].Clear(); } sync_perform.wait(); Task_binding_split_next.invoke(ITaskPortAdvance.CHUNK_READY); Console.WriteLine(this.Rank + ": IJoinGustyCollector ITERATE 3-5"); } Console.WriteLine(this.Rank + ": IJoinGustyCollector ITERATE LOOP 4-2 count=" + count); count++; } Console.WriteLine(this.Rank + ": IJoinGustyCollector ITERATE 5"); Task_binding_split_next.invoke(ITaskPortAdvance.READ_CHUNK); Task_binding_split_next.invoke(ITaskPortAdvance.PERFORM, out sync_perform); // SEND REMAINING PAIRS AND CLOSES THE CHUNK LIST Console.WriteLine(this.Rank + ": IJoinGustyCollector ITERATE 6-1"); for (int i = 0; i < m_size; i++) { Split_channel.Send(buffer [i], unit_ref [i], TAG_SPLIT_END_CHUNK); } sync_perform.wait(); Console.WriteLine(this.Rank + ": IJoinGustyCollector ITERATE 7-1"); Task_binding_split_next.invoke(ITaskPortAdvance.CHUNK_READY); Console.WriteLine(this.Rank + ": IJoinGustyCollector ITERATE 7-2"); thread_send_output.Join(); //sync_perform.wait (); Console.WriteLine(this.Rank + ": IJoinGustyCollector ITERATE 7-3"); } Console.WriteLine(this.Rank + ": IJoinGustyCollector END COMPUTATION "); input_instance.finish(); output_instance.finish(); Split_channel.Send(new List <IKVPairInstance <OKey, OValue> >(), sink_ref, TAG_SPLIT_END_CHUNK); }
private void readPair_OMK_OMVs() { Console.WriteLine(this.Rank + ": REDUCE 1"); IIteratorInstance <IKVPair <TKey, IIterator <TValue> > > input_instance = (IIteratorInstance <IKVPair <TKey, IIterator <TValue> > >)Collect_pairs.Client; IIteratorInstance <IKVPair <OKey, OValue> > output_instance = (IIteratorInstance <IKVPair <OKey, OValue> >)Output.Instance; Feed_pairs.Server = output_instance; IActionFuture sync_perform; // TODO: Dividir em chunks a saa de pares (OKey,OValue) Console.WriteLine(this.Rank + ": REDUCER 2"); // object gusty_lock = new object (); bool end_computation = false; while (!end_computation) // new iteration { IDictionary <object, object> cont_dict = new Dictionary <object, object> (); Console.WriteLine(this.Rank + ": REDUCER LOOP"); end_computation = true; bool end_iteration = false; while (!end_iteration) // take next chunk ... { Console.WriteLine(this.Rank + ": REDUCER ITERATE 1"); Task_gusty.invoke(ITaskPortAdvance.READ_CHUNK); Task_gusty.invoke(ITaskPortAdvance.PERFORM, out sync_perform); Console.WriteLine(this.Rank + ": REDUCER ITERATE 2"); IKVPairInstance <TKey, IIterator <TValue> > kvpair = null; object kvpair_object; if (!input_instance.has_next()) { end_iteration = true; } else { end_computation = false; } int count = 0; while (input_instance.fetch_next(out kvpair_object)) { Console.WriteLine(this.Rank + ": REDUCER ITERATE INNER LOOP 3 count=" + count); kvpair = (IKVPairInstance <TKey, IIterator <TValue> >)kvpair_object; object acc_value; if (!cont_dict.TryGetValue(kvpair.Key, out acc_value)) { cont_dict[kvpair.Key] = new object(); } else { ((IDataInstance)Continue_value.Instance).ObjValue = acc_value; } Input_values.Instance = kvpair; Gusty_function.go(); cont_dict [kvpair.Key] = ((IDataInstance)((IKVPairInstance <OKey, OValue>)Output_value.Instance).Value).ObjValue; count++; } Console.WriteLine(this.Rank + ": REDUCER ITERATE 4 count=" + count); sync_perform.wait(); Console.WriteLine(this.Rank + ": REDUCER ITERATE 5"); } Console.WriteLine(this.Rank + ": REDUCER ITERATE END ITERATION"); int chunk_counter = 1; IActionFuture gusty_chunk_ready; Task_gusty.invoke(ITaskPortAdvance.CHUNK_READY, out gusty_chunk_ready); //*** foreach (KeyValuePair <object, object> output_pair in cont_dict) { IKVPairInstance <OKey, OValue> new_pair = (IKVPairInstance <OKey, OValue>)Output_value.newInstance(); new_pair.Key = output_pair.Key; new_pair.Value = output_pair.Value; output_instance.put(new_pair); } output_instance.finish(); gusty_chunk_ready.wait(); Console.WriteLine(this.Rank + ": REDUCER ITERATE FINISH"); } Console.WriteLine(this.Rank + ": REDUCER FINISH ... "); }
public override void main() { Trace.WriteLine(Rank + ": PARTITIONER (FETCH VALUES TARGET) - START "); MPI.CompletedStatus status; IKVPairInstance <OMK, OMV> kv; int senders_size = this.UnitSize ["mapper"]; IIteratorInstance <IKVPair <OMK, IIterator <OMV> > > reduce_job_instance = (IIteratorInstance <IKVPair <OMK, IIterator <OMV> > >)Reduce_job.Instance; IDictionary <object, IIteratorInstance <OMV> > kv_cache = new Dictionary <object, IIteratorInstance <OMV> >(); int finished_senders = 0; Trace.WriteLine(Rank + ": PARTITIONER (FETCH VALUES TARGET) BEGIN RECEIVE 1 !"); comm.Receive <IKVPairInstance <OMK, OMV> >(MPI.Unsafe.MPI_ANY_SOURCE, MPI.Unsafe.MPI_ANY_TAG, out kv, out status); Trace.WriteLine(Rank + ": PARTITIONER (FETCH VALUES TARGET) END RECEIVE 1 ! tag=" + status.Tag + ", source=" + status.Source); int count = 0; bool last_finished = false; if (status.Tag == TAG_FETCHVALUES_OMV_FINISH) { finished_senders++; last_finished = true; } while (finished_senders < senders_size) { if (!last_finished) { IIteratorInstance <OMV> iterator = null; if (!kv_cache.ContainsKey(kv.Key)) { Trace.WriteLine(Rank + ": PARTITIONER (FETCH VALUES TARGET) LOOP RECV 1 !" + (count++) + " " + kv.Key.GetType()); iterator = Reduce_job_values_factory.newIteratorInstance(); kv_cache.Add(kv.Key, iterator); IKVPairInstance <OMK, IIterator <OMV> > item = (IKVPairInstance <OMK, IIterator <OMV> >)Reduce_job.createItem(); item.Key = kv.Key; item.Value = iterator; reduce_job_instance.put(item); } else { Trace.WriteLine(Rank + ": PARTITIONER (FETCH VALUES TARGET) LOOP RECV 2 !" + (count++)); kv_cache.TryGetValue(kv.Key, out iterator); } iterator.put(kv.Value); } else { last_finished = false; Trace.WriteLine(Rank + ": PARTITIONER (FETCH VALUES TARGET) - FINISH DETECTED "); } Trace.WriteLine(Rank + ": PARTITIONER (FETCH VALUES TARGET) BEGIN RECEIVE n !"); comm.Receive <IKVPairInstance <OMK, OMV> >(MPI.Unsafe.MPI_ANY_SOURCE, MPI.Unsafe.MPI_ANY_TAG, out kv, out status); Trace.WriteLine(Rank + ": PARTITIONER (FETCH VALUES TARGET) END RECEIVE n ! tag=" + status.Tag + ", source=" + status.Source); if (status.Tag == TAG_FETCHVALUES_OMV_FINISH) { finished_senders++; last_finished = true; } } foreach (KeyValuePair <object, IIteratorInstance <OMV> > kv_item in kv_cache) { kv_item.Value.finish(); } reduce_job_instance.finish(); Trace.WriteLine(Rank + ": PARTITIONER (FETCH VALUES TARGET) - FINISH "); }
private void createPageNodes(string fileContent) { IIteratorInstance <IKVPair <IInteger, IPageNode> > output = (IIteratorInstance <IKVPair <IInteger, IPageNode> >)Output_data.Instance; IDictionary <int, IDictionary <int, IPageNodeInstance> > dictionary = new Dictionary <int, IDictionary <int, IPageNodeInstance> >(); IList <IKVPairInstance <IInteger, IPageNode> > PAGENODES = new List <IKVPairInstance <IInteger, IPageNode> >(); string[] lines = fileContent.Split(new char[] { System.Environment.NewLine[0] }); foreach (string line in lines) { if (!line.Trim().Equals("")) { IPageNodeInstance V, W, temp = null; IDictionary <int, IPageNodeInstance> referenceV, referenceW = null; int[] KEY = new int[2]; string[] vwID = line.Split(' '); for (int k = 0; k < 2; k++) { KEY [k] = int.Parse(vwID [k]); } if (!dictionary.TryGetValue(KEY [0], out referenceV)) { IKVPairInstance <IInteger, IPageNode> kvpair = (IKVPairInstance <IInteger, IPageNode>)Output_data.createItem(); V = (IPageNodeInstance)kvpair.Value; V.IdInstance = KEY [0]; ((IIntegerInstance)kvpair.Key).Value = V.IdInstance; referenceV = new Dictionary <int, IPageNodeInstance> (); dictionary [KEY [0]] = referenceV; referenceV [KEY [0]] = V; PAGENODES.Add(kvpair); } if (!dictionary.TryGetValue(KEY [1], out referenceW)) { IKVPairInstance <IInteger, IPageNode> kvpair = (IKVPairInstance <IInteger, IPageNode>)Output_data.createItem(); W = (IPageNodeInstance)kvpair.Value; W.IdInstance = KEY [1]; ((IIntegerInstance)kvpair.Key).Value = W.IdInstance; referenceW = new Dictionary <int, IPageNodeInstance> (); dictionary [KEY [1]] = referenceW; referenceW [KEY [1]] = W; PAGENODES.Add(kvpair); } if (!referenceV.TryGetValue(KEY [1], out temp)) { V = referenceV [KEY [0]]; W = referenceW [KEY [1]]; referenceV [KEY [1]] = W; V.NeighborsInstance.Add(W.IdInstance); } } } IEnumerator <IKVPairInstance <IInteger, IPageNode> > iterator = PAGENODES.GetEnumerator(); while (iterator.MoveNext()) { IKVPairInstance <IInteger, IPageNode> kvpair = iterator.Current; IIntegerInstance k = (IIntegerInstance)kvpair.Key; output.put(kvpair); dic [k.Value] = kvpair; } output.finish(); }
public void receive_write_gif(int senders_size, IDictionary <int, Tuple <int, int> > unit_ref) { IIteratorInstance <IKVPair <IInteger, IIterator <GIF> > > output_instance_gifs = (IIteratorInstance <IKVPair <IInteger, IIterator <GIF> > >)Output_gifs.Instance; Feed_graph.Server = output_instance_gifs; bool[] finished_stream = new bool[senders_size]; for (int i = 0; i < senders_size; i++) { finished_stream [i] = false; } int count_finished_streams = 0; while (count_finished_streams < senders_size) // take next chunk ... { IActionFuture sync_perform; Task_binding_shuffle.invoke(ITaskPortAdvance.READ_CHUNK); Task_binding_shuffle.invoke(ITaskPortAdvance.PERFORM, out sync_perform); IDictionary <object, IIteratorInstance <GIF> > kv_cache = new Dictionary <object, IIteratorInstance <GIF> > (); // PERFORM for (int i = 0; i < senders_size; i++) { if (!finished_stream [i]) { IList <IKVPairInstance <IInteger, GIF> > buffer; CompletedStatus status; Step_channel.Receive(unit_ref [i], MPI.Communicator.anyTag, out buffer, out status); foreach (IKVPairInstance <IInteger, GIF> kv in buffer) { IIteratorInstance <GIF> iterator = null; if (!kv_cache.ContainsKey(kv.Key)) { iterator = Value_factory_gif.newIteratorInstance(); kv_cache.Add(kv.Key, iterator); IKVPairInstance <IInteger, IIterator <GIF> > item = (IKVPairInstance <IInteger, IIterator <GIF> >)Output_gifs.createItem(); item.Key = kv.Key; item.Value = iterator; output_instance_gifs.put(item); } else { kv_cache.TryGetValue(kv.Key, out iterator); } iterator.put(kv.Value); } if (status.Tag == TAG_SHUFFLE_OMV_END_CHUNK) { count_finished_streams++; finished_stream [i] = true; } } } output_instance_gifs.finish(); foreach (IIteratorInstance <GIF> iterator in kv_cache.Values) { iterator.finish(); } sync_perform.wait(); // CHUNK_READY Task_binding_shuffle.invoke(ITaskPortAdvance.CHUNK_READY); //**** } output_instance_gifs.finish(); output_instance_gifs.finish(); }
public override void main() { Console.WriteLine(this.Rank + ": SHUFFLER REDUCE COLLECTOR START"); // IIteratorInstance<IKVPair<TKey,IIterator<TValue>>> output_instance = (IIteratorInstance<IKVPair<TKey,IIterator<TValue>>>) Output.Instance; // Feed_pairs.Server = output_instance; // DETERMINE COMMUNICATION SOURCEs IDictionary <int, Tuple <int, int> > unit_ref = new Dictionary <int, Tuple <int, int> > (); int nf = this.FacetMultiplicity [FACET_MAP]; int senders_size = 0; foreach (int i in this.FacetIndexes[FACET_MAP]) { int nr0 = senders_size; senders_size += this.UnitSizeInFacet [i] ["gusty_collector"]; for (int k = 0, j = nr0; j < senders_size; k++, j++) { unit_ref [j] = new Tuple <int, int> (i /*,0 INDEX OF gusty_collector*/, k); } } receive_write_gif(senders_size, unit_ref); IIteratorInstance <IKVPair <TKey, IIterator <TValue> > > output_instance = (IIteratorInstance <IKVPair <TKey, IIterator <TValue> > >)Output.Instance; Feed_pairs.Server = output_instance; bool end_computation = false; while (!end_computation) // next iteration { bool[] finished_stream = new bool[senders_size]; for (int i = 0; i < senders_size; i++) { finished_stream [i] = false; } int count_finished_streams = 0; end_computation = true; while (count_finished_streams < senders_size) // take next chunk ... { Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...1"); IActionFuture sync_perform; Task_binding_shuffle.invoke(ITaskPortAdvance.READ_CHUNK); Task_binding_shuffle.invoke(ITaskPortAdvance.PERFORM, out sync_perform); Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...2"); IDictionary <object, IIteratorInstance <TValue> > kv_cache = new Dictionary <object, IIteratorInstance <TValue> > (); // PERFORM for (int i = 0; i < senders_size; i++) { if (!finished_stream[i]) { Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...3 i=" + i); IList <IKVPairInstance <TKey, TValue> > buffer; CompletedStatus status; Step_channel.Receive(unit_ref[i], MPI.Communicator.anyTag, out buffer, out status); Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...4"); foreach (IKVPairInstance <TKey, TValue> kv in buffer) { IIteratorInstance <TValue> iterator = null; if (!kv_cache.ContainsKey(kv.Key)) { iterator = Value_factory.newIteratorInstance(); kv_cache.Add(kv.Key, iterator); IKVPairInstance <TKey, IIterator <TValue> > item = (IKVPairInstance <TKey, IIterator <TValue> >)Output.createItem(); item.Key = kv.Key; item.Value = iterator; output_instance.put(item); } else { kv_cache.TryGetValue(kv.Key, out iterator); } iterator.put(kv.Value); } Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...5"); if (status.Tag == TAG_SHUFFLE_OMV_END_CHUNK) { count_finished_streams++; finished_stream [i] = true; } else { end_computation = false; } Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...6"); } } output_instance.finish(); foreach (IIteratorInstance <TValue> iterator in kv_cache.Values) { iterator.finish(); } Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...7-1"); sync_perform.wait(); Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...7-2"); // CHUNK_READY Task_binding_shuffle.invoke(ITaskPortAdvance.CHUNK_READY); //**** Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...8"); } output_instance.finish(); Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...9 - FINISH"); } }