public void read_send_graph(int gusty_size, IDictionary <int, Tuple <int, int> > unit_ref) { IIteratorInstance <IKVPair <IInteger, GIF> > input_instance_graph = (IIteratorInstance <IKVPair <IInteger, GIF> >)Collect_graph.Client; object bin_object = null; IActionFuture sync_perform; Partition_function_gif.NumberOfPartitions = gusty_size; bool end_iteration_gif = false; bool set_table = true; while (!end_iteration_gif) { Task_binding_shuffle.invoke(ITaskPortAdvance.READ_CHUNK); Task_binding_shuffle.invoke(ITaskPortAdvance.PERFORM, out sync_perform); IList <IKVPairInstance <IInteger, GIF> >[] buffer_gif = new IList <IKVPairInstance <IInteger, GIF> > [gusty_size]; for (int i = 0; i < gusty_size; i++) { buffer_gif [i] = new List <IKVPairInstance <IInteger, GIF> > (); } IKVPairInstance <IInteger, GIF> item = null; if (!input_instance_graph.has_next()) { end_iteration_gif = true; } int count = 0; while (input_instance_graph.fetch_next(out bin_object)) { item = (IKVPairInstance <IInteger, GIF>)bin_object; this.Input_key_gif.Instance = item.Value; //item.Key; Partition_function_gif.go(); int index = ((IIntegerInstance)this.Output_key_gif.Instance).Value; buffer_gif [index].Add(item); Console.WriteLine(this.GlobalRank + ": SHUFFLER MAP Collector Graph count=" + (count++)); } // PERFORM for (int i = 0; i < gusty_size; i++) { Step_channel.Send(buffer_gif [i], unit_ref [i], end_iteration_gif ? TAG_SHUFFLE_OMV_END_CHUNK : TAG_SHUFFLE_OMV_NEW_CHUNK); } sync_perform.wait(); Task_binding_shuffle.invoke(ITaskPortAdvance.CHUNK_READY); } Partition_function.PartitionTABLE = Partition_function_gif.PartitionTABLE; }
public void receive_write_gif(int senders_size, IDictionary <int, Tuple <int, int> > unit_ref) { IIteratorInstance <IKVPair <IInteger, IIterator <GIF> > > output_instance_gifs = (IIteratorInstance <IKVPair <IInteger, IIterator <GIF> > >)Output_gifs.Instance; Feed_graph.Server = output_instance_gifs; bool[] finished_stream = new bool[senders_size]; for (int i = 0; i < senders_size; i++) { finished_stream [i] = false; } int count_finished_streams = 0; while (count_finished_streams < senders_size) // take next chunk ... { IActionFuture sync_perform; Task_binding_shuffle.invoke(ITaskPortAdvance.READ_CHUNK); Task_binding_shuffle.invoke(ITaskPortAdvance.PERFORM, out sync_perform); IDictionary <object, IIteratorInstance <GIF> > kv_cache = new Dictionary <object, IIteratorInstance <GIF> > (); // PERFORM for (int i = 0; i < senders_size; i++) { if (!finished_stream [i]) { IList <IKVPairInstance <IInteger, GIF> > buffer; CompletedStatus status; Step_channel.Receive(unit_ref [i], MPI.Communicator.anyTag, out buffer, out status); foreach (IKVPairInstance <IInteger, GIF> kv in buffer) { IIteratorInstance <GIF> iterator = null; if (!kv_cache.ContainsKey(kv.Key)) { iterator = Value_factory_gif.newIteratorInstance(); kv_cache.Add(kv.Key, iterator); IKVPairInstance <IInteger, IIterator <GIF> > item = (IKVPairInstance <IInteger, IIterator <GIF> >)Output_gifs.createItem(); item.Key = kv.Key; item.Value = iterator; output_instance_gifs.put(item); } else { kv_cache.TryGetValue(kv.Key, out iterator); } iterator.put(kv.Value); } if (status.Tag == TAG_SHUFFLE_OMV_END_CHUNK) { count_finished_streams++; finished_stream [i] = true; } } } output_instance_gifs.finish(); foreach (IIteratorInstance <GIF> iterator in kv_cache.Values) { iterator.finish(); } sync_perform.wait(); // CHUNK_READY Task_binding_shuffle.invoke(ITaskPortAdvance.CHUNK_READY); //**** } output_instance_gifs.finish(); output_instance_gifs.finish(); }
public override void main() { Console.WriteLine(this.Rank + ": SHUFFLER REDUCE COLLECTOR START"); // IIteratorInstance<IKVPair<TKey,IIterator<TValue>>> output_instance = (IIteratorInstance<IKVPair<TKey,IIterator<TValue>>>) Output.Instance; // Feed_pairs.Server = output_instance; // DETERMINE COMMUNICATION SOURCEs IDictionary <int, Tuple <int, int> > unit_ref = new Dictionary <int, Tuple <int, int> > (); int nf = this.FacetMultiplicity [FACET_MAP]; int senders_size = 0; foreach (int i in this.FacetIndexes[FACET_MAP]) { int nr0 = senders_size; senders_size += this.UnitSizeInFacet [i] ["gusty_collector"]; for (int k = 0, j = nr0; j < senders_size; k++, j++) { unit_ref [j] = new Tuple <int, int> (i /*,0 INDEX OF gusty_collector*/, k); } } receive_write_gif(senders_size, unit_ref); IIteratorInstance <IKVPair <TKey, IIterator <TValue> > > output_instance = (IIteratorInstance <IKVPair <TKey, IIterator <TValue> > >)Output.Instance; Feed_pairs.Server = output_instance; bool end_computation = false; while (!end_computation) // next iteration { bool[] finished_stream = new bool[senders_size]; for (int i = 0; i < senders_size; i++) { finished_stream [i] = false; } int count_finished_streams = 0; end_computation = true; while (count_finished_streams < senders_size) // take next chunk ... { Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...1"); IActionFuture sync_perform; Task_binding_shuffle.invoke(ITaskPortAdvance.READ_CHUNK); Task_binding_shuffle.invoke(ITaskPortAdvance.PERFORM, out sync_perform); Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...2"); IDictionary <object, IIteratorInstance <TValue> > kv_cache = new Dictionary <object, IIteratorInstance <TValue> > (); // PERFORM for (int i = 0; i < senders_size; i++) { if (!finished_stream[i]) { Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...3 i=" + i); IList <IKVPairInstance <TKey, TValue> > buffer; CompletedStatus status; Step_channel.Receive(unit_ref[i], MPI.Communicator.anyTag, out buffer, out status); Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...4"); foreach (IKVPairInstance <TKey, TValue> kv in buffer) { IIteratorInstance <TValue> iterator = null; if (!kv_cache.ContainsKey(kv.Key)) { iterator = Value_factory.newIteratorInstance(); kv_cache.Add(kv.Key, iterator); IKVPairInstance <TKey, IIterator <TValue> > item = (IKVPairInstance <TKey, IIterator <TValue> >)Output.createItem(); item.Key = kv.Key; item.Value = iterator; output_instance.put(item); } else { kv_cache.TryGetValue(kv.Key, out iterator); } iterator.put(kv.Value); } Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...5"); if (status.Tag == TAG_SHUFFLE_OMV_END_CHUNK) { count_finished_streams++; finished_stream [i] = true; } else { end_computation = false; } Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...6"); } } output_instance.finish(); foreach (IIteratorInstance <TValue> iterator in kv_cache.Values) { iterator.finish(); } Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...7-1"); sync_perform.wait(); Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...7-2"); // CHUNK_READY Task_binding_shuffle.invoke(ITaskPortAdvance.CHUNK_READY); //**** Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...8"); } output_instance.finish(); Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...9 - FINISH"); } }
public override void main() { Console.WriteLine(this.GlobalRank + ": STARTING SHUFFLER MAP ...1"); // IIteratorInstance<IKVPair<TKey,TValue>> input_instance = (IIteratorInstance<IKVPair<TKey,TValue>>) Collect_pairs.Client; object bin_object = null; IActionFuture sync_perform; // DETERMINE COMMMUNICATION TARGETs IDictionary <int, Tuple <int, int> > unit_ref = new Dictionary <int, Tuple <int, int> > (); int nf = this.FacetMultiplicity [FACET_REDUCE]; int gusty_size = 0; Console.WriteLine(this.GlobalRank + ": STARTING SHUFFLER MAP ...2"); foreach (int i in this.FacetIndexes[FACET_REDUCE]) { int nr0 = gusty_size; Console.WriteLine(this.GlobalRank + ": STARTING SHUFFLER MAP ...3 - BEGIN 1 - i=" + i); foreach (KeyValuePair <int, IDictionary <string, int> > ttt in this.UnitSizeInFacet) { Console.WriteLine(this.GlobalRank + ": STARTING SHUFFLER MAP ...3 - " + (ttt.Value == null)); foreach (KeyValuePair <string, int> tttt in ttt.Value) { Console.WriteLine(this.GlobalRank + ": STARTING SHUFFLER MAP ...3 --- " + ttt.Key + " / " + tttt.Key + " / " + tttt.Value); } } gusty_size += this.UnitSizeInFacet [i] ["gusty_feeder"]; Console.WriteLine(this.GlobalRank + ": STARTING SHUFFLER MAP ...3 - BEGIN 2 - " + i); for (int k = 0, j = nr0; j < gusty_size; k++, j++) { unit_ref [j] = new Tuple <int, int> (i /*,0 INDEX OF gusty_feeder*/, k); } Console.WriteLine(this.GlobalRank + ": STARTING SHUFFLER MAP ...3 - END - " + i); } read_send_graph(gusty_size, unit_ref); IIteratorInstance <IKVPair <TKey, TValue> > input_instance = (IIteratorInstance <IKVPair <TKey, TValue> >)Collect_pairs.Client; Partition_function.NumberOfPartitions = gusty_size; bool end_computation = false; while (!end_computation) // next iteration { end_computation = true; bool end_iteration = false; while (!end_iteration) // take next chunk ... { Console.WriteLine(this.GlobalRank + ": SHUFFLER MAP ...4"); Task_binding_shuffle.invoke(ITaskPortAdvance.READ_CHUNK); Task_binding_shuffle.invoke(ITaskPortAdvance.PERFORM, out sync_perform); Console.WriteLine(this.GlobalRank + ": SHUFFLER MAP ...5"); IList <IKVPairInstance <TKey, TValue> >[] buffer = new IList <IKVPairInstance <TKey, TValue> > [gusty_size]; for (int i = 0; i < gusty_size; i++) { buffer [i] = new List <IKVPairInstance <TKey, TValue> > (); } Console.WriteLine(this.GlobalRank + ": SHUFFLER MAP ...6"); IKVPairInstance <TKey, TValue> item = null; if (!input_instance.has_next()) { end_iteration = true; } else { end_computation = false; } int count = 0; while (input_instance.fetch_next(out bin_object)) { Console.WriteLine(this.GlobalRank + ": SHUFFLER MAP ...6 count=" + count); item = (IKVPairInstance <TKey, TValue>)bin_object; this.Input_key.Instance = item.Key; Partition_function.go(); int index = ((IIntegerInstance)this.Output_key.Instance).Value; buffer [index].Add(item); Console.WriteLine(this.GlobalRank + ": SHUFFLER MAP ...7 count=" + count); count++; } Console.WriteLine(this.GlobalRank + ": SHUFFLER MAP ...9"); for (int i = 0; i < buffer.Length; i++) { Console.WriteLine("SHUFFLER MAP ...9 - buffer[" + i + "]=" + buffer[i].Count); } // PERFORM for (int i = 0; i < gusty_size; i++) { Step_channel.Send(buffer [i], unit_ref [i], end_iteration ? TAG_SHUFFLE_OMV_END_CHUNK : TAG_SHUFFLE_OMV_NEW_CHUNK); } Console.WriteLine(this.GlobalRank + ": SHUFFLER MAP ...10"); sync_perform.wait(); Console.WriteLine(this.GlobalRank + ": SHUFFLER MAP ...11"); Task_binding_shuffle.invoke(ITaskPortAdvance.CHUNK_READY); //**** Console.WriteLine(this.GlobalRank + ": SHUFFLER MAP ...12"); } Console.WriteLine(this.GlobalRank + ": SHUFFLER MAP ...13"); } Console.WriteLine(this.GlobalRank + ": SHUFFLER MAP ...14"); }
public override void main() { Console.WriteLine(this.ThisFacetInstance + "/" + this.Rank + ": WORKFLOW 1"); Task_binding_data.TraceFlag = Task_map.TraceFlag = Task_gusty.TraceFlag = Task_binding_shuffle.TraceFlag = Task_binding_split_first.TraceFlag = Task_binding_split_next.TraceFlag = true; Task_binding_data.TraceFlag = Task_binding_shuffle.TraceFlag = Task_binding_split_first.TraceFlag = Task_binding_split_next.TraceFlag = true; IActionFutureSet future_iteration = null; Console.WriteLine(this.ThisFacetInstance + "/" + this.Rank + ": WORKFLOW 2-1"); IActionFuture future_split_first_chunk_ready = null; Task_binding_split_first.invoke(ITaskPortAdvance.CHUNK_READY, out future_split_first_chunk_ready); int action_id_split_first_chunk_ready = future_split_first_chunk_ready.GetHashCode(); future_iteration = future_split_first_chunk_ready.createSet(); Console.WriteLine(this.ThisFacetInstance + "/" + this.Rank + ": WORKFLOW 2-2"); IActionFuture future_split_next_chunk_ready = null; Task_binding_split_next.invoke(ITaskPortAdvance.CHUNK_READY, out future_split_next_chunk_ready); int action_id_split_next_chunk_ready = future_split_next_chunk_ready.GetHashCode(); future_iteration.addAction(future_split_next_chunk_ready); Console.WriteLine(this.ThisFacetInstance + "/" + this.Rank + ": WORKFLOW 3"); IActionFuture future_map_chunk_ready = null; Task_map.invoke(ITaskPortAdvance.CHUNK_READY, out future_map_chunk_ready); int action_id_map_chunk_ready = future_map_chunk_ready.GetHashCode(); future_iteration.addAction(future_map_chunk_ready); Console.WriteLine(this.ThisFacetInstance + "/" + this.Rank + ": WORKFLOW 4"); IActionFuture future_shuffle_chunk_ready = null; Task_binding_shuffle.invoke(ITaskPortAdvance.CHUNK_READY, out future_shuffle_chunk_ready); int action_id_shuffle_chunk_ready = future_shuffle_chunk_ready.GetHashCode(); future_iteration.addAction(future_shuffle_chunk_ready); Console.WriteLine(this.ThisFacetInstance + "/" + this.Rank + ": WORKFLOW 4"); IActionFuture future_gusty_chunk_ready = null; Task_gusty.invoke(ITaskPortAdvance.CHUNK_READY, out future_gusty_chunk_ready); int action_id_gusty_chunk_ready = future_gusty_chunk_ready.GetHashCode(); future_iteration.addAction(future_gusty_chunk_ready); Console.WriteLine(this.ThisFacetInstance + "/" + this.Rank + ": WORKFLOW 5"); IActionFuture future_data_terminate = null; Task_binding_data.invoke(ITaskPortData.TERMINATE, out future_data_terminate); future_iteration.addAction(future_data_terminate); int action_id_data_terminate = future_data_terminate.GetHashCode(); Console.WriteLine(this.ThisFacetInstance + "/" + this.Rank + ": WORKFLOW 6-1"); Task_binding_data.invoke(ITaskPortData.READ_SOURCE); Console.WriteLine(this.ThisFacetInstance + "/" + this.Rank + ": WORKFLOW 6-2"); Task_binding_split_first.invoke(ITaskPortAdvance.READ_CHUNK); Task_binding_split_first.invoke(ITaskPortAdvance.PERFORM); Console.WriteLine(this.ThisFacetInstance + "/" + this.Rank + ": WORKFLOW 7"); IList <Thread> bag_of_tasks = new List <Thread> (); bool terminate = false; do { Console.WriteLine(this.ThisFacetInstance + "/" + this.Rank + ": WORKFLOW 8 - LOOP"); IActionFuture action = future_iteration.waitAny(); int action_id = action.GetHashCode(); if (action_id == action_id_split_first_chunk_ready) { Thread t1 = new Thread((ThreadStart) delegate() { Console.WriteLine("INVOKE MAPPER READ_CHUNK - BEFORE"); Task_map.invoke(ITaskPortAdvance.READ_CHUNK); Console.WriteLine("INVOKE MAPPER READ_CHUNK - AFTER"); IActionFuture future_map_perform = null; Thread thread_map_perform = Task_map.invoke(ITaskPortAdvance.PERFORM, map_perform, out future_map_perform); Console.WriteLine("END INVOKE SPLITTER CHUNK_READY"); }); bag_of_tasks.Add(t1); t1.Start(); Thread t2 = new Thread((ThreadStart) delegate() { Console.WriteLine("INVOKE SPLITTER READ_CHUNK"); Task_binding_split_first.invoke(ITaskPortAdvance.READ_CHUNK); Console.WriteLine("INVOKE SPLITTER PERFORM"); Task_binding_split_first.invoke(ITaskPortAdvance.PERFORM); Console.WriteLine("INVOKE SPLITTER READ_CHUNK/PERFORM - AFTER"); IActionFuture future_split_chunk_ready_ = null; Task_binding_split_first.invoke(ITaskPortAdvance.CHUNK_READY, out future_split_chunk_ready_); action_id_split_first_chunk_ready = future_split_chunk_ready_.GetHashCode(); future_iteration.addAction(future_split_chunk_ready_); }); Console.WriteLine("THREAD LAUNCHED 1"); bag_of_tasks.Add(t2); t2.Start(); } else if (action_id == action_id_map_chunk_ready) { Task_map.invoke(ITaskPortAdvance.CHUNK_READY, out future_map_chunk_ready); action_id_map_chunk_ready = future_map_chunk_ready.GetHashCode(); future_iteration.addAction(future_map_chunk_ready); Thread t = new Thread((ThreadStart) delegate() { Console.WriteLine("INVOKE SHUFFLER READ_CHUNK - BEFORE"); // 110 executados (o 48 completou nos pares, mas não progrediu aqui (????), motivo do erro. Task_binding_shuffle.invoke(ITaskPortAdvance.READ_CHUNK); // Console.WriteLine("INVOKE SHUFFLER READ_CHUNK - AFTER"); // 47 completados IActionFuture future_shuffle_perform = null; Thread thread_shuffle_perform = Task_binding_shuffle.invoke(ITaskPortAdvance.PERFORM, shuffle_perform, out future_shuffle_perform); Console.WriteLine("END INVOKE MAPPER CHUNK_READY"); }); t.Start(); bag_of_tasks.Add(t); Console.WriteLine("THREAD LAUNCHED 2"); } else if (action_id == action_id_shuffle_chunk_ready) { Task_binding_shuffle.invoke(ITaskPortAdvance.CHUNK_READY, out future_shuffle_chunk_ready); action_id_shuffle_chunk_ready = future_shuffle_chunk_ready.GetHashCode(); future_iteration.addAction(future_shuffle_chunk_ready); Thread t = new Thread((ThreadStart) delegate() { Console.WriteLine("INVOKE REDUCER READ_CHUNK - BEFORE"); Task_gusty.invoke(ITaskPortAdvance.READ_CHUNK); // **** Console.WriteLine("INVOKE REDUCER READ_CHUNK - AFTER"); IActionFuture future_gusty_perform = null; Thread thread_gusty_perform = Task_gusty.invoke(ITaskPortAdvance.PERFORM, gusty_perform, out future_gusty_perform); Console.WriteLine("END INVOKE SHUFFLER CHUNK_READY"); }); t.Start(); bag_of_tasks.Add(t); Console.WriteLine("THREAD LAUNCHED 3"); } else if (action_id == action_id_gusty_chunk_ready) { Task_gusty.invoke(ITaskPortAdvance.CHUNK_READY, out future_gusty_chunk_ready); action_id_gusty_chunk_ready = future_gusty_chunk_ready.GetHashCode(); future_iteration.addAction(future_gusty_chunk_ready); Thread t = new Thread((ThreadStart) delegate() { Console.WriteLine("INVOKE SPLITTER NEXT READ_CHUNK - BEFORE"); Task_binding_split_next.invoke(ITaskPortAdvance.READ_CHUNK); // **** Console.WriteLine("INVOKE SPLITTER NEXT READ_CHUNK - AFTER"); IActionFuture future_split_perform = null; Thread thread_split_perform = Task_binding_split_next.invoke(ITaskPortAdvance.PERFORM, split_perform, out future_split_perform); Console.WriteLine("END INVOKE REDUCER CHUNK_READY"); }); t.Start(); bag_of_tasks.Add(t); Console.WriteLine("THREAD LAUNCHED 4"); } else if (action_id == action_id_split_next_chunk_ready) { Thread t1 = new Thread((ThreadStart) delegate() { Console.WriteLine("INVOKE MAP READ_CHUNK NEXT - BEFORE"); Task_map.invoke(ITaskPortAdvance.READ_CHUNK); // Console.WriteLine("INVOKE MAP READ_CHUNK NEXT - AFTER"); IActionFuture future_map_perform = null; Thread thread_map_perform = Task_map.invoke(ITaskPortAdvance.PERFORM, map_perform, out future_map_perform); Console.WriteLine("END INVOKE SPLIT NEXT CHUNK_READY"); }); bag_of_tasks.Add(t1); t1.Start(); Thread t2 = new Thread((ThreadStart) delegate() { Console.WriteLine("INVOKE SPLITTER NEXT READ_CHUNK"); Task_binding_split_next.invoke(ITaskPortAdvance.READ_CHUNK); Console.WriteLine("INVOKE SPLITTER NEXT PERFORM"); Task_binding_split_next.invoke(ITaskPortAdvance.PERFORM); Console.WriteLine("INVOKE SPLITTER NEXT READ_CHUNK/PERFORM - AFTER"); IActionFuture future_split_next_chunk_ready_ = null; Task_binding_split_next.invoke(ITaskPortAdvance.CHUNK_READY, out future_split_next_chunk_ready_); action_id_split_next_chunk_ready = future_split_next_chunk_ready_.GetHashCode(); future_iteration.addAction(future_split_next_chunk_ready_); }); Console.WriteLine("THREAD LAUNCHED 1"); bag_of_tasks.Add(t2); t2.Start(); Console.WriteLine("THREAD LAUNCHED 5"); } else if (action_id == action_id_data_terminate) { Thread t = new Thread((ThreadStart) delegate() { Console.WriteLine("INVOKE DATA WRITE_SINK - BEFORE"); Task_binding_data.invoke(ITaskPortData.WRITE_SINK); Console.WriteLine("INVOKE DATA WRITE_SINK - AFTER"); terminate = true; }); t.Start(); bag_of_tasks.Add(t); Console.WriteLine("THREAD LAUNCHED 6"); } else { Console.WriteLine("UNEXPECTED ERROR: ACTION FUTURE NOT RECOGNIZED ! "); } } while (!terminate); Console.WriteLine("WORKFLOW FINISHED ! "); foreach (Thread t in bag_of_tasks) { t.Join(); } }