public void iterate() { IList <IKVPairInstance <IKey, IValue> > buffer; object buffer_obj; CompletedStatus status; Console.WriteLine(this.GlobalRank + ": STARTING SPLITTER ITERATE...1"); output_instance = (IIteratorInstance <IKVPair <IKey, IIterator <IValue> > >)Output.Instance; Feed_pairs.Server = output_instance; Console.WriteLine(this.GlobalRank + ": STARTING SPLITTER ITERATE...2"); IDictionary <int, Tuple <int, int> > unit_ref = new Dictionary <int, Tuple <int, int> > (); Console.WriteLine(this.GlobalRank + ": STARTING SPLITTER ...3"); // RECEIVE PAIRS FROM THE REDUCERS (next iterations) int senders_size = 0; foreach (int i in this.FacetIndexes[FACET_REDUCE]) { Console.WriteLine(this.GlobalRank + ": STARTING SPLITTER ...4 -- i=" + i); int nr0 = senders_size; senders_size += this.UnitSizeInFacet [i] ["gusty_collector"]; for (int k = 0, j = nr0; j < senders_size; k++, j++) { unit_ref [j] = new Tuple <int, int> (i, k); } } bool end_computation = false; while (!end_computation) { end_computation = true; int count_finished_streams = 0; while (count_finished_streams < senders_size) { IActionFuture sync_perform; Console.WriteLine(this.Rank + ": SPLITTER MAP FEEDER NEXT 1"); Task_binding_split_next.invoke(ITaskPortAdvance.READ_CHUNK); Task_binding_split_next.invoke(ITaskPortAdvance.PERFORM, out sync_perform); IDictionary <object, IIteratorInstance <IValue> > kv_cache = new Dictionary <object, IIteratorInstance <IValue> > (); Console.WriteLine(this.Rank + ": SPLITTER MAP FEEDER NEXT 2"); for (int i = 0; i < senders_size; i++) { Console.WriteLine(this.Rank + ": SPLITTER MAP FEEDER NEXT LOOP - before receive"); Split_channel.Receive(unit_ref [i], MPI.Communicator.anyTag, out buffer_obj, out status); Console.WriteLine(this.Rank + ": CHUNK PAIRS RECEIVED !!! from gusty_collector of index " + i + " / count_finished_streams=" + count_finished_streams); if (status.Tag == TAG_SPLIT_END_CHUNK) { count_finished_streams++; } else { end_computation = false; } try { buffer = (IList <IKVPairInstance <IKey, IValue> >)buffer_obj; //foreach (IKVPairInstance<IKey,IValue> kv in buffer) // output_instance.put (kv); foreach (IKVPairInstance <IKey, IValue> kv in buffer) { IIteratorInstance <IValue> iterator = null; if (!kv_cache.ContainsKey(kv.Key)) { iterator = Value_factory.newIteratorInstance(); kv_cache.Add(kv.Key, iterator); IKVPairInstance <IKey, IIterator <IValue> > item = (IKVPairInstance <IKey, IIterator <IValue> >)Output.createItem(); item.Key = kv.Key; item.Value = iterator; output_instance.put(item); } else { kv_cache.TryGetValue(kv.Key, out iterator); } iterator.put(kv.Value); } } catch (InvalidCastException e) { Console.WriteLine("SPLITTER MAPPER: incompatible input !"); count_finished_streams++; } } output_instance.finish(); foreach (IIteratorInstance <IValue> iterator in kv_cache.Values) { iterator.finish(); } Console.WriteLine(this.Rank + ": SPLITTER MAP FEEDER NEXT 3"); sync_perform.wait(); Console.WriteLine(this.Rank + ": SPLITTER MAP FEEDER NEXT 4"); // CHUNK_READY Task_binding_split_next.invoke(ITaskPortAdvance.CHUNK_READY); Console.WriteLine(this.Rank + ": SPLITTER MAP FEEDER NEXT 5"); } } Console.WriteLine(this.Rank + ": SPLITTER MAP FEEDER FINISH 1 !"); Task_binding_data.invoke(ITaskPortData.TERMINATE); Task_binding_data.invoke(ITaskPortData.WRITE_SINK); Console.WriteLine(this.Rank + ": SPLITTER MAP FEEDER FINISH 2 !"); }
public override void main() { Console.WriteLine(this.Rank + ": SHUFFLER REDUCE COLLECTOR START"); // IIteratorInstance<IKVPair<TKey,IIterator<TValue>>> output_instance = (IIteratorInstance<IKVPair<TKey,IIterator<TValue>>>) Output.Instance; // Feed_pairs.Server = output_instance; // DETERMINE COMMUNICATION SOURCEs IDictionary <int, Tuple <int, int> > unit_ref = new Dictionary <int, Tuple <int, int> > (); int nf = this.FacetMultiplicity [FACET_MAP]; int senders_size = 0; foreach (int i in this.FacetIndexes[FACET_MAP]) { int nr0 = senders_size; senders_size += this.UnitSizeInFacet [i] ["gusty_collector"]; for (int k = 0, j = nr0; j < senders_size; k++, j++) { unit_ref [j] = new Tuple <int, int> (i /*,0 INDEX OF gusty_collector*/, k); } } receive_write_gif(senders_size, unit_ref); IIteratorInstance <IKVPair <TKey, IIterator <TValue> > > output_instance = (IIteratorInstance <IKVPair <TKey, IIterator <TValue> > >)Output.Instance; Feed_pairs.Server = output_instance; bool end_computation = false; while (!end_computation) // next iteration { bool[] finished_stream = new bool[senders_size]; for (int i = 0; i < senders_size; i++) { finished_stream [i] = false; } int count_finished_streams = 0; end_computation = true; while (count_finished_streams < senders_size) // take next chunk ... { Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...1"); IActionFuture sync_perform; Task_binding_shuffle.invoke(ITaskPortAdvance.READ_CHUNK); Task_binding_shuffle.invoke(ITaskPortAdvance.PERFORM, out sync_perform); Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...2"); IDictionary <object, IIteratorInstance <TValue> > kv_cache = new Dictionary <object, IIteratorInstance <TValue> > (); // PERFORM for (int i = 0; i < senders_size; i++) { if (!finished_stream[i]) { Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...3 i=" + i); IList <IKVPairInstance <TKey, TValue> > buffer; CompletedStatus status; Step_channel.Receive(unit_ref[i], MPI.Communicator.anyTag, out buffer, out status); Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...4"); foreach (IKVPairInstance <TKey, TValue> kv in buffer) { IIteratorInstance <TValue> iterator = null; if (!kv_cache.ContainsKey(kv.Key)) { iterator = Value_factory.newIteratorInstance(); kv_cache.Add(kv.Key, iterator); IKVPairInstance <TKey, IIterator <TValue> > item = (IKVPairInstance <TKey, IIterator <TValue> >)Output.createItem(); item.Key = kv.Key; item.Value = iterator; output_instance.put(item); } else { kv_cache.TryGetValue(kv.Key, out iterator); } iterator.put(kv.Value); } Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...5"); if (status.Tag == TAG_SHUFFLE_OMV_END_CHUNK) { count_finished_streams++; finished_stream [i] = true; } else { end_computation = false; } Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...6"); } } output_instance.finish(); foreach (IIteratorInstance <TValue> iterator in kv_cache.Values) { iterator.finish(); } Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...7-1"); sync_perform.wait(); Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...7-2"); // CHUNK_READY Task_binding_shuffle.invoke(ITaskPortAdvance.CHUNK_READY); //**** Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...8"); } output_instance.finish(); Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...9 - FINISH"); } }