//CONFIG public void readSource() { Console.WriteLine(this.GlobalRank + ": STARTING SPLITTER READSOURCE...1"); output_instance_gifs = (IIteratorInstance <IKVPair <IInteger, IIterator <GIF> > >)Output_gifs.Instance; Feed_graph.Server = output_instance_gifs; Console.WriteLine(this.GlobalRank + ": STARTING SPLITTER READSOURCE...2"); // RECEIVE PAIR FROM THE SOURCE (1st iteration) Tuple <int, int> unit_ref_source = new Tuple <int, int> (this.FacetIndexes [FACET_SOURCE] [0], 0); // Thread[] threads_receive = new Thread[senders_size]; // for (int i = 0; i < senders_size; i++) // { // threads_receive [i] = new Thread ((ParameterizedThreadStart)delegate(object unit_ref_obj) { // Tuple<int,int> unit_ref_i = (Tuple<int,int>)unit_ref_obj; // receive_pairs_iteration (unit_ref_i); // }); // } // TODO: READ_SOURCE é necessário ? Não no map feeder. Tirar fatia de Task_binding data ... Task_binding_data.TraceFlag = true; Task_binding_data.invoke(ITaskPortData.READ_SOURCE); // Do nothing ... IList <IKVPairInstance <IInteger, GIF> > buffer; object buffer_obj; CompletedStatus status; do { IActionFuture sync_perform; Console.WriteLine(this.Rank + ": SPLITTER MAP FEEDER - BEFORE READ_CHUNK"); Task_binding_split_first.invoke(ITaskPortAdvance.READ_CHUNK); //**** Task_binding_split_first.invoke(ITaskPortAdvance.PERFORM, out sync_perform); IDictionary <object, IIteratorInstance <GIF> > kv_cache = new Dictionary <object, IIteratorInstance <GIF> > (); Console.WriteLine(this.Rank + ": SPLITTER MAP FEEDER !!! PERFORM OK !"); Split_channel.Receive(unit_ref_source, MPI.Communicator.anyTag, out buffer, out status); Console.WriteLine(this.Rank + ": CHUNK PAIRS RECEIVED !!! from source buffer.Count=" + buffer.Count); //foreach (IKVPairInstance<IInteger,GIF> kv in buffer) // output_instance_gif.put (kv); foreach (IKVPairInstance <IInteger, GIF> kv in buffer) { IIteratorInstance <GIF> iterator = null; if (!kv_cache.ContainsKey(kv.Key)) { iterator = Value_factory_gif.newIteratorInstance(); kv_cache.Add(kv.Key, iterator); IKVPairInstance <IInteger, IIterator <GIF> > item = (IKVPairInstance <IInteger, IIterator <GIF> >)Output_gifs.createItem(); item.Key = kv.Key; item.Value = iterator; output_instance_gifs.put(item); } else { kv_cache.TryGetValue(kv.Key, out iterator); } iterator.put(kv.Value); } output_instance_gifs.finish(); foreach (IIteratorInstance <IValue> iterator in kv_cache.Values) { iterator.finish(); } sync_perform.wait(); // CHUNK_READY Console.WriteLine(this.Rank + ": SPLITTER MAP FEEDER 1"); Task_binding_split_first.invoke(ITaskPortAdvance.CHUNK_READY); Console.WriteLine(this.Rank + ": SPLITTER MAP FEEDER 2"); } while (status.Tag != TAG_SPLIT_END_CHUNK); output_instance_gifs.finish(); Console.WriteLine(this.Rank + ": FINISH READING CHUNKS OF SOURCE"); }
public void iterate() { IList <IKVPairInstance <IKey, IValue> > buffer; object buffer_obj; CompletedStatus status; Console.WriteLine(this.GlobalRank + ": STARTING SPLITTER ITERATE...1"); output_instance = (IIteratorInstance <IKVPair <IKey, IIterator <IValue> > >)Output.Instance; Feed_pairs.Server = output_instance; Console.WriteLine(this.GlobalRank + ": STARTING SPLITTER ITERATE...2"); IDictionary <int, Tuple <int, int> > unit_ref = new Dictionary <int, Tuple <int, int> > (); Console.WriteLine(this.GlobalRank + ": STARTING SPLITTER ...3"); // RECEIVE PAIRS FROM THE REDUCERS (next iterations) int senders_size = 0; foreach (int i in this.FacetIndexes[FACET_REDUCE]) { Console.WriteLine(this.GlobalRank + ": STARTING SPLITTER ...4 -- i=" + i); int nr0 = senders_size; senders_size += this.UnitSizeInFacet [i] ["gusty_collector"]; for (int k = 0, j = nr0; j < senders_size; k++, j++) { unit_ref [j] = new Tuple <int, int> (i, k); } } bool end_computation = false; while (!end_computation) { end_computation = true; int count_finished_streams = 0; while (count_finished_streams < senders_size) { IActionFuture sync_perform; Console.WriteLine(this.Rank + ": SPLITTER MAP FEEDER NEXT 1"); Task_binding_split_next.invoke(ITaskPortAdvance.READ_CHUNK); Task_binding_split_next.invoke(ITaskPortAdvance.PERFORM, out sync_perform); IDictionary <object, IIteratorInstance <IValue> > kv_cache = new Dictionary <object, IIteratorInstance <IValue> > (); Console.WriteLine(this.Rank + ": SPLITTER MAP FEEDER NEXT 2"); for (int i = 0; i < senders_size; i++) { Console.WriteLine(this.Rank + ": SPLITTER MAP FEEDER NEXT LOOP - before receive"); Split_channel.Receive(unit_ref [i], MPI.Communicator.anyTag, out buffer_obj, out status); Console.WriteLine(this.Rank + ": CHUNK PAIRS RECEIVED !!! from gusty_collector of index " + i + " / count_finished_streams=" + count_finished_streams); if (status.Tag == TAG_SPLIT_END_CHUNK) { count_finished_streams++; } else { end_computation = false; } try { buffer = (IList <IKVPairInstance <IKey, IValue> >)buffer_obj; //foreach (IKVPairInstance<IKey,IValue> kv in buffer) // output_instance.put (kv); foreach (IKVPairInstance <IKey, IValue> kv in buffer) { IIteratorInstance <IValue> iterator = null; if (!kv_cache.ContainsKey(kv.Key)) { iterator = Value_factory.newIteratorInstance(); kv_cache.Add(kv.Key, iterator); IKVPairInstance <IKey, IIterator <IValue> > item = (IKVPairInstance <IKey, IIterator <IValue> >)Output.createItem(); item.Key = kv.Key; item.Value = iterator; output_instance.put(item); } else { kv_cache.TryGetValue(kv.Key, out iterator); } iterator.put(kv.Value); } } catch (InvalidCastException e) { Console.WriteLine("SPLITTER MAPPER: incompatible input !"); count_finished_streams++; } } output_instance.finish(); foreach (IIteratorInstance <IValue> iterator in kv_cache.Values) { iterator.finish(); } Console.WriteLine(this.Rank + ": SPLITTER MAP FEEDER NEXT 3"); sync_perform.wait(); Console.WriteLine(this.Rank + ": SPLITTER MAP FEEDER NEXT 4"); // CHUNK_READY Task_binding_split_next.invoke(ITaskPortAdvance.CHUNK_READY); Console.WriteLine(this.Rank + ": SPLITTER MAP FEEDER NEXT 5"); } } Console.WriteLine(this.Rank + ": SPLITTER MAP FEEDER FINISH 1 !"); Task_binding_data.invoke(ITaskPortData.TERMINATE); Task_binding_data.invoke(ITaskPortData.WRITE_SINK); Console.WriteLine(this.Rank + ": SPLITTER MAP FEEDER FINISH 2 !"); }
public override void main() { Console.WriteLine(this.Rank + ": SPLITTER 1 "); IPortTypeIterator input_instance = (IPortTypeIterator)Source.Client; // TODO: será que READ_SOURCE é necessária ??? Task_binding_data.TraceFlag = true; Task_binding_data.invoke(ITaskPortData.READ_SOURCE); // CALCULATE SubGraph TARGET int nfr = this.FacetMultiplicity [FACET_REDUCE]; int r_size = 0; foreach (int i in this.FacetIndexes[FACET_REDUCE]) { r_size += this.UnitSizeInFacet [i] ["gusty_collector"]; } Console.WriteLine(this.Rank + ": SPLITTER 2 "); Source.startReadSource(r_size); // subgrafos divididos em GIF, com destino final nos r_size redutores Console.WriteLine(this.Rank + ": SPLITTER 3 "); object bin_object = null; bool set_table_partition = true; // CALCULATE TARGETs IDictionary <int, Tuple <int, int> > unit_ref = new Dictionary <int, Tuple <int, int> > (); int nf = this.FacetMultiplicity [FACET_MAP]; int m_size = 0; foreach (int i in this.FacetIndexes[FACET_MAP]) { int nr0 = m_size; m_size += this.UnitSizeInFacet [i] ["gusty_feeder"]; for (int k = 0, j = nr0; j < m_size; k++, j++) { unit_ref [j] = new Tuple <int, int> (i /*, 0 INDEX OF gusty_feeder */, k); } } Console.WriteLine(this.Rank + ": SPLITTER 4 "); Task_binding_split_first.TraceFlag = true; Split_channel.TraceFlag = true; Thread t_output = new Thread(new ThreadStart(delegate { Task_binding_data.invoke(ITaskPortData.TERMINATE); Task_binding_data.invoke(ITaskPortData.WRITE_SINK); })); t_output.Start(); Console.WriteLine(this.Rank + ": SPLITTER 5 "); bool end_iteration = false; while (!end_iteration) // take next chunk { IActionFuture sync_perform; Console.WriteLine(this.Rank + ": SPLITTER READ SOURCE - BEFORE READ_CHUNK " + m_size); /* All the pairs will be read from the source in a single chunk */ Task_binding_split_first.invoke(ITaskPortAdvance.READ_CHUNK); //**** Task_binding_split_first.invoke(ITaskPortAdvance.PERFORM, out sync_perform); IList <IKVPairInstance <IInteger, GIF> >[] buffer = new IList <IKVPairInstance <IInteger, GIF> > [m_size]; for (int i = 0; i < m_size; i++) { buffer [i] = new List <IKVPairInstance <IInteger, GIF> > (); } Console.WriteLine(this.Rank + ": BEGIN READING CHUNKS and distributing to MAPPERS m_size=" + m_size); if (!input_instance.has_next()) { end_iteration = true; } Bin_function_gif.NumberOfPartitions = m_size; while (input_instance.fetch_next(out bin_object)) { IKVPairInstance <IInteger, GIF> item = (IKVPairInstance <IInteger, GIF>)bin_object; this.Input_key_gif.Instance = item.Value; //item.Key; Bin_function_gif.go(); int index = ((IIntegerInstance)this.Output_key_gif.Instance).Value; buffer [index].Add(item); } Console.WriteLine(this.Rank + ": END READING CHUNKS and distributing to MAPPERS"); for (int i = 0; i < m_size; i++) { Console.WriteLine("buffer[" + i + "].Count = " + buffer [i].Count); } Console.WriteLine(this.Rank + ": BEGIN SENDING CHUNKS to the MAPPERS "); // PERFORM for (int i = 0; i < m_size; i++) { Split_channel.Send(buffer [i], unit_ref [i], end_iteration ? TAG_SPLIT_END_CHUNK : TAG_SPLIT_NEW_CHUNK); } sync_perform.wait(); Console.WriteLine(this.Rank + ": END SENDING CHUNKS to the MAPPERS "); Task_binding_split_first.invoke(ITaskPortAdvance.CHUNK_READY); } Console.WriteLine(this.Rank + ": FINISH SPLITTER READ SOURCE "); t_output.Join(); Console.WriteLine(this.Rank + ": FINISH SPLITTER READ SOURCE - INVOKE TERMINATE"); }
public override void main() { Console.WriteLine(this.ThisFacetInstance + "/" + this.Rank + ": WORKFLOW 1"); Task_binding_data.TraceFlag = Task_gusty0.TraceFlag = Task_gusty1.TraceFlag = Task_binding_step1.TraceFlag = Task_binding_split_first.TraceFlag = Task_binding_split_next.TraceFlag = true; Task_binding_data.TraceFlag = Task_binding_step1.TraceFlag = Task_binding_split_first.TraceFlag = Task_binding_split_next.TraceFlag = true; IActionFutureSet future_iteration = null; Console.WriteLine(this.ThisFacetInstance + "/" + this.Rank + ": WORKFLOW 2-1"); IActionFuture future_split_first_chunk_ready = null; Task_binding_split_first.invoke(ITaskPortAdvance.CHUNK_READY, out future_split_first_chunk_ready); int action_id_split_first_chunk_ready = future_split_first_chunk_ready.GetHashCode(); future_iteration = future_split_first_chunk_ready.createSet(); Console.WriteLine(this.ThisFacetInstance + "/" + this.Rank + ": WORKFLOW 2-2"); IActionFuture future_split_next_chunk_ready = null; Task_binding_split_next.invoke(ITaskPortAdvance.CHUNK_READY, out future_split_next_chunk_ready); int action_id_split_next_chunk_ready = future_split_next_chunk_ready.GetHashCode(); future_iteration.addAction(future_split_next_chunk_ready); Console.WriteLine(this.ThisFacetInstance + "/" + this.Rank + ": WORKFLOW 3"); IActionFuture future_gusty0_chunk_ready = null; Task_gusty0.invoke(ITaskPortAdvance.CHUNK_READY, out future_gusty0_chunk_ready); int action_id_gusty0_chunk_ready = future_gusty0_chunk_ready.GetHashCode(); future_iteration.addAction(future_gusty0_chunk_ready); Console.WriteLine(this.ThisFacetInstance + "/" + this.Rank + ": WORKFLOW 4"); IActionFuture future_step1_chunk_ready = null; Task_binding_step1.invoke(ITaskPortAdvance.CHUNK_READY, out future_step1_chunk_ready); int action_id_step1_chunk_ready = future_step1_chunk_ready.GetHashCode(); future_iteration.addAction(future_step1_chunk_ready); Console.WriteLine(this.ThisFacetInstance + "/" + this.Rank + ": WORKFLOW 4"); IActionFuture future_gusty1_chunk_ready = null; Task_gusty1.invoke(ITaskPortAdvance.CHUNK_READY, out future_gusty1_chunk_ready); int action_id_gusty1_chunk_ready = future_gusty1_chunk_ready.GetHashCode(); future_iteration.addAction(future_gusty1_chunk_ready); Console.WriteLine(this.ThisFacetInstance + "/" + this.Rank + ": WORKFLOW 5"); IActionFuture future_data_terminate = null; Task_binding_data.invoke(ITaskPortData.TERMINATE, out future_data_terminate); future_iteration.addAction(future_data_terminate); int action_id_data_terminate = future_data_terminate.GetHashCode(); Console.WriteLine(this.ThisFacetInstance + "/" + this.Rank + ": WORKFLOW 6-1"); Task_binding_data.invoke(ITaskPortData.READ_SOURCE); Console.WriteLine(this.ThisFacetInstance + "/" + this.Rank + ": WORKFLOW 6-2"); Task_binding_split_first.invoke(ITaskPortAdvance.READ_CHUNK); Task_binding_split_first.invoke(ITaskPortAdvance.PERFORM); Console.WriteLine(this.ThisFacetInstance + "/" + this.Rank + ": WORKFLOW 7"); IList <Thread> bag_of_tasks = new List <Thread> (); bool terminate = false; do { Console.WriteLine(this.ThisFacetInstance + "/" + this.Rank + ": WORKFLOW 8 - LOOP"); IActionFuture action = future_iteration.waitAny(); int action_id = action.GetHashCode(); if (action_id == action_id_split_first_chunk_ready) { Thread t1 = new Thread((ThreadStart) delegate() { Console.WriteLine("INVOKE GUSTY0 READ_CHUNK - BEFORE"); Task_gusty0.invoke(ITaskPortAdvance.READ_CHUNK); Console.WriteLine("INVOKE GUSTY0 READ_CHUNK - AFTER"); IActionFuture future_gusty0_perform = null; Thread thread_gusty0_perform = Task_gusty0.invoke(ITaskPortAdvance.PERFORM, gusty0_perform, out future_gusty0_perform); Console.WriteLine("END INVOKE SPLITTER CHUNK_READY"); }); bag_of_tasks.Add(t1); t1.Start(); Thread t2 = new Thread((ThreadStart) delegate() { Console.WriteLine("INVOKE SPLITTER READ_CHUNK"); Task_binding_split_first.invoke(ITaskPortAdvance.READ_CHUNK); Console.WriteLine("INVOKE SPLITTER PERFORM"); Task_binding_split_first.invoke(ITaskPortAdvance.PERFORM); Console.WriteLine("INVOKE SPLITTER READ_CHUNK/PERFORM - AFTER"); IActionFuture future_split_chunk_ready_ = null; Task_binding_split_first.invoke(ITaskPortAdvance.CHUNK_READY, out future_split_chunk_ready_); action_id_split_first_chunk_ready = future_split_chunk_ready_.GetHashCode(); future_iteration.addAction(future_split_chunk_ready_); }); Console.WriteLine("THREAD LAUNCHED 1"); bag_of_tasks.Add(t2); t2.Start(); } else if (action_id == action_id_gusty0_chunk_ready) { Task_gusty0.invoke(ITaskPortAdvance.CHUNK_READY, out future_gusty0_chunk_ready); action_id_gusty0_chunk_ready = future_gusty0_chunk_ready.GetHashCode(); future_iteration.addAction(future_gusty0_chunk_ready); Thread t = new Thread((ThreadStart) delegate() { Console.WriteLine("INVOKE STEP1 READ_CHUNK - BEFORE"); // 110 executados (o 48 completou nos pares, mas não progrediu aqui (????), motivo do erro. Task_binding_step1.invoke(ITaskPortAdvance.READ_CHUNK); // Console.WriteLine("INVOKE STEP1 READ_CHUNK - AFTER"); // 47 completados IActionFuture future_step1_perform = null; Thread thread_step1_perform = Task_binding_step1.invoke(ITaskPortAdvance.PERFORM, step1_perform, out future_step1_perform); Console.WriteLine("END INVOKE GUSTY0 CHUNK_READY"); }); t.Start(); bag_of_tasks.Add(t); Console.WriteLine("THREAD LAUNCHED 2"); } else if (action_id == action_id_step1_chunk_ready) { Task_binding_step1.invoke(ITaskPortAdvance.CHUNK_READY, out future_step1_chunk_ready); action_id_step1_chunk_ready = future_step1_chunk_ready.GetHashCode(); future_iteration.addAction(future_step1_chunk_ready); Thread t = new Thread((ThreadStart) delegate() { Console.WriteLine("INVOKE GUSTY1 READ_CHUNK - BEFORE"); Task_gusty1.invoke(ITaskPortAdvance.READ_CHUNK); // **** Console.WriteLine("INVOKE GUSTY1 READ_CHUNK - AFTER"); IActionFuture future_gusty1_perform = null; Thread thread_gusty1_perform = Task_gusty1.invoke(ITaskPortAdvance.PERFORM, gusty1_perform, out future_gusty1_perform); Console.WriteLine("END INVOKE STEP1 CHUNK_READY"); }); t.Start(); bag_of_tasks.Add(t); Console.WriteLine("THREAD LAUNCHED 3"); } else if (action_id == action_id_gusty1_chunk_ready) { Task_gusty1.invoke(ITaskPortAdvance.CHUNK_READY, out future_gusty1_chunk_ready); action_id_gusty1_chunk_ready = future_gusty1_chunk_ready.GetHashCode(); future_iteration.addAction(future_gusty1_chunk_ready); Thread t = new Thread((ThreadStart) delegate() { Console.WriteLine("INVOKE SPLITTER NEXT READ_CHUNK - BEFORE"); Task_binding_split_next.invoke(ITaskPortAdvance.READ_CHUNK); // **** Console.WriteLine("INVOKE SPLITTER NEXT READ_CHUNK - AFTER"); IActionFuture future_split_perform = null; Thread thread_split_perform = Task_binding_split_next.invoke(ITaskPortAdvance.PERFORM, split_perform, out future_split_perform); Console.WriteLine("END INVOKE GUSTY1 CHUNK_READY"); }); t.Start(); bag_of_tasks.Add(t); Console.WriteLine("THREAD LAUNCHED 4"); } else if (action_id == action_id_split_next_chunk_ready) { Thread t1 = new Thread((ThreadStart) delegate() { Console.WriteLine("INVOKE GUSTY0 READ_CHUNK NEXT - BEFORE"); Task_gusty0.invoke(ITaskPortAdvance.READ_CHUNK); // Console.WriteLine("INVOKE GUSTY0 READ_CHUNK NEXT - AFTER"); IActionFuture future_gusty0_perform = null; Thread thread_gusty0_perform = Task_gusty0.invoke(ITaskPortAdvance.PERFORM, gusty0_perform, out future_gusty0_perform); Console.WriteLine("END INVOKE SPLIT NEXT CHUNK_READY"); }); bag_of_tasks.Add(t1); t1.Start(); Thread t2 = new Thread((ThreadStart) delegate() { Console.WriteLine("INVOKE SPLITTER NEXT READ_CHUNK"); Task_binding_split_next.invoke(ITaskPortAdvance.READ_CHUNK); Console.WriteLine("INVOKE SPLITTER NEXT PERFORM"); Task_binding_split_next.invoke(ITaskPortAdvance.PERFORM); Console.WriteLine("INVOKE SPLITTER NEXT READ_CHUNK/PERFORM - AFTER"); IActionFuture future_split_next_chunk_ready_ = null; Task_binding_split_next.invoke(ITaskPortAdvance.CHUNK_READY, out future_split_next_chunk_ready_); action_id_split_next_chunk_ready = future_split_next_chunk_ready_.GetHashCode(); future_iteration.addAction(future_split_next_chunk_ready_); }); Console.WriteLine("THREAD LAUNCHED 1"); bag_of_tasks.Add(t2); t2.Start(); Console.WriteLine("THREAD LAUNCHED 5"); } else if (action_id == action_id_data_terminate) { Thread t = new Thread((ThreadStart) delegate() { Console.WriteLine("INVOKE DATA WRITE_SINK - BEFORE"); Task_binding_data.invoke(ITaskPortData.WRITE_SINK); Console.WriteLine("INVOKE DATA WRITE_SINK - AFTER"); terminate = true; }); t.Start(); bag_of_tasks.Add(t); Console.WriteLine("THREAD LAUNCHED 6"); } else { Console.WriteLine("UNEXPECTED ERROR: ACTION FUTURE NOT RECOGNIZED ! "); } } while (!terminate); Console.WriteLine("WORKFLOW FINISHED ! "); foreach (Thread t in bag_of_tasks) { t.Join(); } }
public override void main() { Split_channel.TraceFlag = true; Task_binding_data.TraceFlag = true; Console.WriteLine("SPLITER WRITE SINK 0"); Sink.clientConnection(); IPortTypeIterator output_instance = (IPortTypeIterator)Sink.Client; Console.WriteLine("SPLITER WRITE SINK 1"); Task_binding_data.TraceFlag = true; Task_binding_data.invoke(ITaskPortData.READ_SOURCE); // Do nothing ... Console.WriteLine("SPLITER WRITE SINK 3"); // RECEIVE PAIRS FROM THE REDUCERS (next iterations) IDictionary <int, Tuple <int, int> > unit_ref = new Dictionary <int, Tuple <int, int> > (); unit_ref = new Dictionary <int, Tuple <int, int> > (); int senders_size = 0; foreach (int i in this.FacetIndexes[FACET_REDUCE]) { int nr0 = senders_size; senders_size += this.UnitSizeInFacet [i] ["gusty_collector"]; for (int k = 0, j = nr0; j < senders_size; k++, j++) { unit_ref [j] = new Tuple <int, int> (i, k); } } Console.WriteLine("SPLITER WRITE SINK 4"); Thread t_output = new Thread(new ThreadStart(delegate { Task_binding_data.invoke(ITaskPortData.TERMINATE); Task_binding_data.invoke(ITaskPortData.WRITE_SINK); Sink.startWriteSink(); })); t_output.Start(); Thread[] receive_pairs_iteration_thread = new Thread[senders_size]; for (int i = 0; i < senders_size; i++) { receive_pairs_iteration_thread [i] = new Thread(new ParameterizedThreadStart(delegate(object i_obj) { int ii = (int)i_obj; receive_pairs_iteration(output_instance, senders_size, unit_ref[ii]); })); receive_pairs_iteration_thread [i].Start(i); } for (int i = 0; i < senders_size; i++) { receive_pairs_iteration_thread [i].Join(); } output_instance.finish(); t_output.Join(); Console.WriteLine("SPLITER WRITE SINK - WRITTER FINISH"); Split_channel.TraceFlag = false; Task_binding_data.TraceFlag = false; }