public void read_send_graph(int gusty_size, IDictionary <int, Tuple <int, int> > unit_ref) { IIteratorInstance <IKVPair <IInteger, GIF> > input_instance_graph = (IIteratorInstance <IKVPair <IInteger, GIF> >)Collect_graph.Client; object bin_object = null; IActionFuture sync_perform; Partition_function_gif.NumberOfPartitions = gusty_size; bool end_iteration_gif = false; bool set_table = true; while (!end_iteration_gif) { Task_binding_shuffle.invoke(ITaskPortAdvance.READ_CHUNK); Task_binding_shuffle.invoke(ITaskPortAdvance.PERFORM, out sync_perform); IList <IKVPairInstance <IInteger, GIF> >[] buffer_gif = new IList <IKVPairInstance <IInteger, GIF> > [gusty_size]; for (int i = 0; i < gusty_size; i++) { buffer_gif [i] = new List <IKVPairInstance <IInteger, GIF> > (); } IKVPairInstance <IInteger, GIF> item = null; if (!input_instance_graph.has_next()) { end_iteration_gif = true; } int count = 0; while (input_instance_graph.fetch_next(out bin_object)) { item = (IKVPairInstance <IInteger, GIF>)bin_object; this.Input_key_gif.Instance = item.Value; //item.Key; Partition_function_gif.go(); int index = ((IIntegerInstance)this.Output_key_gif.Instance).Value; buffer_gif [index].Add(item); Console.WriteLine(this.GlobalRank + ": SHUFFLER MAP Collector Graph count=" + (count++)); } // PERFORM for (int i = 0; i < gusty_size; i++) { Step_channel.Send(buffer_gif [i], unit_ref [i], end_iteration_gif ? TAG_SHUFFLE_OMV_END_CHUNK : TAG_SHUFFLE_OMV_NEW_CHUNK); } sync_perform.wait(); Task_binding_shuffle.invoke(ITaskPortAdvance.CHUNK_READY); } Partition_function.PartitionTABLE = Partition_function_gif.PartitionTABLE; }
private void graph_creator() { IIteratorInstance <IKVPair <IInteger, IIterator <GIF> > > input_instance_gif = (IIteratorInstance <IKVPair <IInteger, IIterator <GIF> > >)Collect_graph.Client; IIteratorInstance <IKVPair <IInteger, GIF> > output_instance_gif = (IIteratorInstance <IKVPair <IInteger, GIF> >)Output_gif.Instance; Feed_graph.Server = output_instance_gif; IActionFuture sync_perform; bool end_iteration = false; while (!end_iteration) // take next chunk ... { Task_gusty.invoke(ITaskPortAdvance.READ_CHUNK); Task_gusty.invoke(ITaskPortAdvance.PERFORM, out sync_perform); IKVPairInstance <IInteger, IIterator <GIF> > kvpair = null; object kvpair_object; object o; if (!input_instance_gif.has_next()) { end_iteration = true; } while (input_instance_gif.fetch_next(out kvpair_object)) { kvpair = (IKVPairInstance <IInteger, IIterator <GIF> >)kvpair_object; Graph_values.Instance = kvpair; Gusty_function.graph_creator(); } sync_perform.wait(); } //IActionFuture gusty_chunk_ready; //Task_gusty.invoke (ITaskPortAdvance.CHUNK_READY, out gusty_chunk_ready); //*** output_instance_gif.finish(); //gusty_chunk_ready.wait (); }
private void send_to_mappers() { Console.WriteLine(this.Rank + ": IJoinGustyCollector 1"); IIteratorInstance <IKVPair <OKey, OValue> > output_instance = (IIteratorInstance <IKVPair <OKey, OValue> >)Output_pairs.Instance; IIteratorInstance <IKVPair <IKey, IValue> > input_instance = (IIteratorInstance <IKVPair <IKey, IValue> >)Input_pairs.Instance; object bin_object = null; // DETERMINE COMMUNICATION TARGETs Tuple <int, int> sink_ref = new Tuple <int, int> (this.FacetIndexes [FACET_SINK] [0], 0); IDictionary <int, Tuple <int, int> > unit_ref = new Dictionary <int, Tuple <int, int> > (); int m_size = 0; foreach (int i in this.FacetIndexes[FACET_MAP]) { int nr0 = m_size; m_size += this.UnitSizeInFacet [i] ["gusty_feeder"]; for (int k = 0, j = nr0; j < m_size; k++, j++) { unit_ref [j] = new Tuple <int, int> (i /*,0 index of MAP_FEEDER*/, k); } } Console.WriteLine(this.Rank + ": IJoinGustyCollector 2"); IActionFuture sync_perform; bool end_computation = false; while (!end_computation) // new iteration { Console.WriteLine(this.Rank + ": IJoinGustyCollector LOOP"); end_computation = true; //Console.WriteLine (this.Rank + ": IJoinGustyCollector ITERATE 1"); //Task_binding_split_next.invoke (ITaskPortAdvance.READ_CHUNK); //**** //Task_binding_split_next.invoke (ITaskPortAdvance.PERFORM, out sync_perform); //Console.WriteLine (this.Rank + ": IJoinGustyCollector ITERATE 2"); // SEND TO SINK IList <IKVPairInstance <OKey, OValue> > buffer_sink = new List <IKVPairInstance <OKey, OValue> >(); end_computation = !input_instance.has_next(); Thread thread_send_output = new Thread(new ThreadStart(delegate { int count1 = 0; while (output_instance.fetch_next(out bin_object)) { IKVPairInstance <OKey, OValue> item = (IKVPairInstance <OKey, OValue>)bin_object; buffer_sink.Add(item); if (count1 % CHUNK_SIZE == 0) { // PERFORM Console.WriteLine(this.Rank + ": IJoinGustyCollector SINK SEND CHUNK 3-1 count=" + count1); Split_channel.Send(buffer_sink, sink_ref, TAG_SPLIT_NEW_CHUNK); Console.WriteLine(this.Rank + ": IJoinGustyCollector SINK SEND CHUNK 3-2 count=" + count1); buffer_sink.Clear(); } count1++; } if (buffer_sink.Count > 0) { Console.WriteLine(this.Rank + ": IJoinGustyCollector SINK SEND CHUNK 3-3 count=" + count1); Split_channel.Send(buffer_sink, sink_ref, TAG_SPLIT_NEW_CHUNK); Console.WriteLine(this.Rank + ": IJoinGustyCollector SINK SEND CHUNK 3-4 count=" + count1); } })); thread_send_output.Start(); // SEND BACK TO MAPPER (new iteration) Bin_function_iterate.NumberOfPartitions = m_size; IList <IKVPairInstance <OKey, OValue> >[] buffer = new IList <IKVPairInstance <OKey, OValue> > [m_size]; for (int i = 0; i < m_size; i++) { buffer [i] = new List <IKVPairInstance <OKey, OValue> > (); } Console.WriteLine(this.Rank + ": IJoinGustyCollector ITERATE 3 end_computation=" + end_computation); int count = 0; while (input_instance.fetch_next(out bin_object)) { Console.WriteLine(this.Rank + ": IJoinGustyCollector ITERATE LOOP 4-1 count=" + count); IKVPairInstance <OKey, OValue> item = (IKVPairInstance <OKey, OValue>)bin_object; this.Input_key_iterate.Instance = item.Key; Bin_function_iterate.go(); int index = ((IIntegerInstance)this.Output_key_iterate.Instance).Value; buffer [index].Add(item); if (count % CHUNK_SIZE == 0) { Task_binding_split_next.invoke(ITaskPortAdvance.READ_CHUNK); Task_binding_split_next.invoke(ITaskPortAdvance.PERFORM, out sync_perform); for (int i = 0; i < m_size; i++) { Console.WriteLine("SPLITTER REDUCE COLLECTOR - Sending chunk of " + buffer[i].Count + " elements"); Split_channel.Send(buffer [i], unit_ref [i], TAG_SPLIT_NEW_CHUNK); buffer [i].Clear(); } sync_perform.wait(); Task_binding_split_next.invoke(ITaskPortAdvance.CHUNK_READY); Console.WriteLine(this.Rank + ": IJoinGustyCollector ITERATE 3-5"); } Console.WriteLine(this.Rank + ": IJoinGustyCollector ITERATE LOOP 4-2 count=" + count); count++; } Console.WriteLine(this.Rank + ": IJoinGustyCollector ITERATE 5"); Task_binding_split_next.invoke(ITaskPortAdvance.READ_CHUNK); Task_binding_split_next.invoke(ITaskPortAdvance.PERFORM, out sync_perform); // SEND REMAINING PAIRS AND CLOSES THE CHUNK LIST Console.WriteLine(this.Rank + ": IJoinGustyCollector ITERATE 6-1"); for (int i = 0; i < m_size; i++) { Split_channel.Send(buffer [i], unit_ref [i], TAG_SPLIT_END_CHUNK); } sync_perform.wait(); Console.WriteLine(this.Rank + ": IJoinGustyCollector ITERATE 7-1"); Task_binding_split_next.invoke(ITaskPortAdvance.CHUNK_READY); Console.WriteLine(this.Rank + ": IJoinGustyCollector ITERATE 7-2"); thread_send_output.Join(); //sync_perform.wait (); Console.WriteLine(this.Rank + ": IJoinGustyCollector ITERATE 7-3"); } Console.WriteLine(this.Rank + ": IJoinGustyCollector END COMPUTATION "); input_instance.finish(); output_instance.finish(); Split_channel.Send(new List <IKVPairInstance <OKey, OValue> >(), sink_ref, TAG_SPLIT_END_CHUNK); }
private void readPair_OMK_OMVs() { Console.WriteLine(this.Rank + ": REDUCE 1"); IIteratorInstance <IKVPair <TKey, IIterator <TValue> > > input_instance = (IIteratorInstance <IKVPair <TKey, IIterator <TValue> > >)Collect_pairs.Client; IIteratorInstance <IKVPair <OKey, OValue> > output_instance = (IIteratorInstance <IKVPair <OKey, OValue> >)Output.Instance; Feed_pairs.Server = output_instance; IActionFuture sync_perform; // TODO: Dividir em chunks a saa de pares (OKey,OValue) Console.WriteLine(this.Rank + ": REDUCER 2"); // object gusty_lock = new object (); bool end_computation = false; while (!end_computation) // new iteration { IDictionary <object, object> cont_dict = new Dictionary <object, object> (); Console.WriteLine(this.Rank + ": REDUCER LOOP"); end_computation = true; bool end_iteration = false; while (!end_iteration) // take next chunk ... { Console.WriteLine(this.Rank + ": REDUCER ITERATE 1"); Task_gusty.invoke(ITaskPortAdvance.READ_CHUNK); Task_gusty.invoke(ITaskPortAdvance.PERFORM, out sync_perform); Console.WriteLine(this.Rank + ": REDUCER ITERATE 2"); IKVPairInstance <TKey, IIterator <TValue> > kvpair = null; object kvpair_object; if (!input_instance.has_next()) { end_iteration = true; } else { end_computation = false; } int count = 0; while (input_instance.fetch_next(out kvpair_object)) { Console.WriteLine(this.Rank + ": REDUCER ITERATE INNER LOOP 3 count=" + count); kvpair = (IKVPairInstance <TKey, IIterator <TValue> >)kvpair_object; object acc_value; if (!cont_dict.TryGetValue(kvpair.Key, out acc_value)) { cont_dict[kvpair.Key] = new object(); } else { ((IDataInstance)Continue_value.Instance).ObjValue = acc_value; } Input_values.Instance = kvpair; Gusty_function.go(); cont_dict [kvpair.Key] = ((IDataInstance)((IKVPairInstance <OKey, OValue>)Output_value.Instance).Value).ObjValue; count++; } Console.WriteLine(this.Rank + ": REDUCER ITERATE 4 count=" + count); sync_perform.wait(); Console.WriteLine(this.Rank + ": REDUCER ITERATE 5"); } Console.WriteLine(this.Rank + ": REDUCER ITERATE END ITERATION"); int chunk_counter = 1; IActionFuture gusty_chunk_ready; Task_gusty.invoke(ITaskPortAdvance.CHUNK_READY, out gusty_chunk_ready); //*** foreach (KeyValuePair <object, object> output_pair in cont_dict) { IKVPairInstance <OKey, OValue> new_pair = (IKVPairInstance <OKey, OValue>)Output_value.newInstance(); new_pair.Key = output_pair.Key; new_pair.Value = output_pair.Value; output_instance.put(new_pair); } output_instance.finish(); gusty_chunk_ready.wait(); Console.WriteLine(this.Rank + ": REDUCER ITERATE FINISH"); } Console.WriteLine(this.Rank + ": REDUCER FINISH ... "); }
public override void main() { Console.WriteLine(this.GlobalRank + ": STARTING SHUFFLER MAP ...1"); // IIteratorInstance<IKVPair<TKey,TValue>> input_instance = (IIteratorInstance<IKVPair<TKey,TValue>>) Collect_pairs.Client; object bin_object = null; IActionFuture sync_perform; // DETERMINE COMMMUNICATION TARGETs IDictionary <int, Tuple <int, int> > unit_ref = new Dictionary <int, Tuple <int, int> > (); int nf = this.FacetMultiplicity [FACET_REDUCE]; int gusty_size = 0; Console.WriteLine(this.GlobalRank + ": STARTING SHUFFLER MAP ...2"); foreach (int i in this.FacetIndexes[FACET_REDUCE]) { int nr0 = gusty_size; Console.WriteLine(this.GlobalRank + ": STARTING SHUFFLER MAP ...3 - BEGIN 1 - i=" + i); foreach (KeyValuePair <int, IDictionary <string, int> > ttt in this.UnitSizeInFacet) { Console.WriteLine(this.GlobalRank + ": STARTING SHUFFLER MAP ...3 - " + (ttt.Value == null)); foreach (KeyValuePair <string, int> tttt in ttt.Value) { Console.WriteLine(this.GlobalRank + ": STARTING SHUFFLER MAP ...3 --- " + ttt.Key + " / " + tttt.Key + " / " + tttt.Value); } } gusty_size += this.UnitSizeInFacet [i] ["gusty_feeder"]; Console.WriteLine(this.GlobalRank + ": STARTING SHUFFLER MAP ...3 - BEGIN 2 - " + i); for (int k = 0, j = nr0; j < gusty_size; k++, j++) { unit_ref [j] = new Tuple <int, int> (i /*,0 INDEX OF gusty_feeder*/, k); } Console.WriteLine(this.GlobalRank + ": STARTING SHUFFLER MAP ...3 - END - " + i); } read_send_graph(gusty_size, unit_ref); IIteratorInstance <IKVPair <TKey, TValue> > input_instance = (IIteratorInstance <IKVPair <TKey, TValue> >)Collect_pairs.Client; Partition_function.NumberOfPartitions = gusty_size; bool end_computation = false; while (!end_computation) // next iteration { end_computation = true; bool end_iteration = false; while (!end_iteration) // take next chunk ... { Console.WriteLine(this.GlobalRank + ": SHUFFLER MAP ...4"); Task_binding_shuffle.invoke(ITaskPortAdvance.READ_CHUNK); Task_binding_shuffle.invoke(ITaskPortAdvance.PERFORM, out sync_perform); Console.WriteLine(this.GlobalRank + ": SHUFFLER MAP ...5"); IList <IKVPairInstance <TKey, TValue> >[] buffer = new IList <IKVPairInstance <TKey, TValue> > [gusty_size]; for (int i = 0; i < gusty_size; i++) { buffer [i] = new List <IKVPairInstance <TKey, TValue> > (); } Console.WriteLine(this.GlobalRank + ": SHUFFLER MAP ...6"); IKVPairInstance <TKey, TValue> item = null; if (!input_instance.has_next()) { end_iteration = true; } else { end_computation = false; } int count = 0; while (input_instance.fetch_next(out bin_object)) { Console.WriteLine(this.GlobalRank + ": SHUFFLER MAP ...6 count=" + count); item = (IKVPairInstance <TKey, TValue>)bin_object; this.Input_key.Instance = item.Key; Partition_function.go(); int index = ((IIntegerInstance)this.Output_key.Instance).Value; buffer [index].Add(item); Console.WriteLine(this.GlobalRank + ": SHUFFLER MAP ...7 count=" + count); count++; } Console.WriteLine(this.GlobalRank + ": SHUFFLER MAP ...9"); for (int i = 0; i < buffer.Length; i++) { Console.WriteLine("SHUFFLER MAP ...9 - buffer[" + i + "]=" + buffer[i].Count); } // PERFORM for (int i = 0; i < gusty_size; i++) { Step_channel.Send(buffer [i], unit_ref [i], end_iteration ? TAG_SHUFFLE_OMV_END_CHUNK : TAG_SHUFFLE_OMV_NEW_CHUNK); } Console.WriteLine(this.GlobalRank + ": SHUFFLER MAP ...10"); sync_perform.wait(); Console.WriteLine(this.GlobalRank + ": SHUFFLER MAP ...11"); Task_binding_shuffle.invoke(ITaskPortAdvance.CHUNK_READY); //**** Console.WriteLine(this.GlobalRank + ": SHUFFLER MAP ...12"); } Console.WriteLine(this.GlobalRank + ": SHUFFLER MAP ...13"); } Console.WriteLine(this.GlobalRank + ": SHUFFLER MAP ...14"); }