public void read_send_graph(int gusty_size, IDictionary <int, Tuple <int, int> > unit_ref)
{
IIteratorInstance <IKVPair <IInteger, GIF> > input_instance_graph = (IIteratorInstance <IKVPair <IInteger, GIF> >)Collect_graph.Client;
object bin_object = null;
IActionFuture sync_perform;
Partition_function_gif.NumberOfPartitions = gusty_size;
bool end_iteration_gif = false; bool set_table = true;
while (!end_iteration_gif)
{
Task_binding_shuffle.invoke(ITaskPortAdvance.READ_CHUNK);
Task_binding_shuffle.invoke(ITaskPortAdvance.PERFORM, out sync_perform);
IList <IKVPairInstance <IInteger, GIF> >[] buffer_gif = new IList <IKVPairInstance <IInteger, GIF> > [gusty_size];
for (int i = 0; i < gusty_size; i++)
{
buffer_gif [i] = new List <IKVPairInstance <IInteger, GIF> > ();
}
IKVPairInstance <IInteger, GIF> item = null;
if (!input_instance_graph.has_next())
{
end_iteration_gif = true;
}
int count = 0;
while (input_instance_graph.fetch_next(out bin_object))
{
item = (IKVPairInstance <IInteger, GIF>)bin_object;
this.Input_key_gif.Instance = item.Value; //item.Key;
Partition_function_gif.go();
int index = ((IIntegerInstance)this.Output_key_gif.Instance).Value;
buffer_gif [index].Add(item);
Console.WriteLine(this.GlobalRank + ": SHUFFLER MAP Collector Graph count=" + (count++));
}
// PERFORM
for (int i = 0; i < gusty_size; i++)
{
Step_channel.Send(buffer_gif [i], unit_ref [i], end_iteration_gif ? TAG_SHUFFLE_OMV_END_CHUNK : TAG_SHUFFLE_OMV_NEW_CHUNK);
}
sync_perform.wait();
Task_binding_shuffle.invoke(ITaskPortAdvance.CHUNK_READY);
}
Partition_function.PartitionTABLE = Partition_function_gif.PartitionTABLE;
}
public void receive_write_gif(int senders_size, IDictionary <int, Tuple <int, int> > unit_ref)
{
IIteratorInstance <IKVPair <IInteger, IIterator <GIF> > > output_instance_gifs = (IIteratorInstance <IKVPair <IInteger, IIterator <GIF> > >)Output_gifs.Instance;
Feed_graph.Server = output_instance_gifs;
bool[] finished_stream = new bool[senders_size];
for (int i = 0; i < senders_size; i++)
{
finished_stream [i] = false;
}
int count_finished_streams = 0;
while (count_finished_streams < senders_size) // take next chunk ...
{
IActionFuture sync_perform;
Task_binding_shuffle.invoke(ITaskPortAdvance.READ_CHUNK);
Task_binding_shuffle.invoke(ITaskPortAdvance.PERFORM, out sync_perform);
IDictionary <object, IIteratorInstance <GIF> > kv_cache = new Dictionary <object, IIteratorInstance <GIF> > ();
// PERFORM
for (int i = 0; i < senders_size; i++)
{
if (!finished_stream [i])
{
IList <IKVPairInstance <IInteger, GIF> > buffer;
CompletedStatus status;
Step_channel.Receive(unit_ref [i], MPI.Communicator.anyTag, out buffer, out status);
foreach (IKVPairInstance <IInteger, GIF> kv in buffer)
{
IIteratorInstance <GIF> iterator = null;
if (!kv_cache.ContainsKey(kv.Key))
{
iterator = Value_factory_gif.newIteratorInstance();
kv_cache.Add(kv.Key, iterator);
IKVPairInstance <IInteger, IIterator <GIF> > item = (IKVPairInstance <IInteger, IIterator <GIF> >)Output_gifs.createItem();
item.Key = kv.Key;
item.Value = iterator;
output_instance_gifs.put(item);
}
else
{
kv_cache.TryGetValue(kv.Key, out iterator);
}
iterator.put(kv.Value);
}
if (status.Tag == TAG_SHUFFLE_OMV_END_CHUNK)
{
count_finished_streams++;
finished_stream [i] = true;
}
}
}
output_instance_gifs.finish();
foreach (IIteratorInstance <GIF> iterator in kv_cache.Values)
{
iterator.finish();
}
sync_perform.wait();
// CHUNK_READY
Task_binding_shuffle.invoke(ITaskPortAdvance.CHUNK_READY); //****
}
output_instance_gifs.finish();
output_instance_gifs.finish();
}
public override void main()
{
Console.WriteLine(this.Rank + ": SHUFFLER REDUCE COLLECTOR START");
// IIteratorInstance<IKVPair<TKey,IIterator<TValue>>> output_instance = (IIteratorInstance<IKVPair<TKey,IIterator<TValue>>>) Output.Instance;
// Feed_pairs.Server = output_instance;
// DETERMINE COMMUNICATION SOURCEs
IDictionary <int, Tuple <int, int> > unit_ref = new Dictionary <int, Tuple <int, int> > ();
int nf = this.FacetMultiplicity [FACET_MAP];
int senders_size = 0;
foreach (int i in this.FacetIndexes[FACET_MAP])
{
int nr0 = senders_size;
senders_size += this.UnitSizeInFacet [i] ["gusty_collector"];
for (int k = 0, j = nr0; j < senders_size; k++, j++)
{
unit_ref [j] = new Tuple <int, int> (i /*,0 INDEX OF gusty_collector*/, k);
}
}
receive_write_gif(senders_size, unit_ref);
IIteratorInstance <IKVPair <TKey, IIterator <TValue> > > output_instance = (IIteratorInstance <IKVPair <TKey, IIterator <TValue> > >)Output.Instance;
Feed_pairs.Server = output_instance;
bool end_computation = false;
while (!end_computation) // next iteration
{
bool[] finished_stream = new bool[senders_size];
for (int i = 0; i < senders_size; i++)
{
finished_stream [i] = false;
}
int count_finished_streams = 0;
end_computation = true;
while (count_finished_streams < senders_size) // take next chunk ...
{
Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...1");
IActionFuture sync_perform;
Task_binding_shuffle.invoke(ITaskPortAdvance.READ_CHUNK);
Task_binding_shuffle.invoke(ITaskPortAdvance.PERFORM, out sync_perform);
Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...2");
IDictionary <object, IIteratorInstance <TValue> > kv_cache = new Dictionary <object, IIteratorInstance <TValue> > ();
// PERFORM
for (int i = 0; i < senders_size; i++)
{
if (!finished_stream[i])
{
Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...3 i=" + i);
IList <IKVPairInstance <TKey, TValue> > buffer;
CompletedStatus status;
Step_channel.Receive(unit_ref[i], MPI.Communicator.anyTag, out buffer, out status);
Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...4");
foreach (IKVPairInstance <TKey, TValue> kv in buffer)
{
IIteratorInstance <TValue> iterator = null;
if (!kv_cache.ContainsKey(kv.Key))
{
iterator = Value_factory.newIteratorInstance();
kv_cache.Add(kv.Key, iterator);
IKVPairInstance <TKey, IIterator <TValue> > item = (IKVPairInstance <TKey, IIterator <TValue> >)Output.createItem();
item.Key = kv.Key;
item.Value = iterator;
output_instance.put(item);
}
else
{
kv_cache.TryGetValue(kv.Key, out iterator);
}
iterator.put(kv.Value);
}
Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...5");
if (status.Tag == TAG_SHUFFLE_OMV_END_CHUNK)
{
count_finished_streams++;
finished_stream [i] = true;
}
else
{
end_computation = false;
}
Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...6");
}
}
output_instance.finish();
foreach (IIteratorInstance <TValue> iterator in kv_cache.Values)
{
iterator.finish();
}
Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...7-1");
sync_perform.wait();
Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...7-2");
// CHUNK_READY
Task_binding_shuffle.invoke(ITaskPortAdvance.CHUNK_READY); //****
Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...8");
}
output_instance.finish();
Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...9 - FINISH");
}
}
public override void main()
{
Console.WriteLine(this.GlobalRank + ": STARTING SHUFFLER MAP ...1");
// IIteratorInstance<IKVPair<TKey,TValue>> input_instance = (IIteratorInstance<IKVPair<TKey,TValue>>) Collect_pairs.Client;
object bin_object = null;
IActionFuture sync_perform;
// DETERMINE COMMMUNICATION TARGETs
IDictionary <int, Tuple <int, int> > unit_ref = new Dictionary <int, Tuple <int, int> > ();
int nf = this.FacetMultiplicity [FACET_REDUCE];
int gusty_size = 0;
Console.WriteLine(this.GlobalRank + ": STARTING SHUFFLER MAP ...2");
foreach (int i in this.FacetIndexes[FACET_REDUCE])
{
int nr0 = gusty_size;
Console.WriteLine(this.GlobalRank + ": STARTING SHUFFLER MAP ...3 - BEGIN 1 - i=" + i);
foreach (KeyValuePair <int, IDictionary <string, int> > ttt in this.UnitSizeInFacet)
{
Console.WriteLine(this.GlobalRank + ": STARTING SHUFFLER MAP ...3 - " + (ttt.Value == null));
foreach (KeyValuePair <string, int> tttt in ttt.Value)
{
Console.WriteLine(this.GlobalRank + ": STARTING SHUFFLER MAP ...3 --- " + ttt.Key + " / " + tttt.Key + " / " + tttt.Value);
}
}
gusty_size += this.UnitSizeInFacet [i] ["gusty_feeder"];
Console.WriteLine(this.GlobalRank + ": STARTING SHUFFLER MAP ...3 - BEGIN 2 - " + i);
for (int k = 0, j = nr0; j < gusty_size; k++, j++)
{
unit_ref [j] = new Tuple <int, int> (i /*,0 INDEX OF gusty_feeder*/, k);
}
Console.WriteLine(this.GlobalRank + ": STARTING SHUFFLER MAP ...3 - END - " + i);
}
read_send_graph(gusty_size, unit_ref);
IIteratorInstance <IKVPair <TKey, TValue> > input_instance = (IIteratorInstance <IKVPair <TKey, TValue> >)Collect_pairs.Client;
Partition_function.NumberOfPartitions = gusty_size;
bool end_computation = false;
while (!end_computation) // next iteration
{
end_computation = true;
bool end_iteration = false;
while (!end_iteration) // take next chunk ...
{
Console.WriteLine(this.GlobalRank + ": SHUFFLER MAP ...4");
Task_binding_shuffle.invoke(ITaskPortAdvance.READ_CHUNK);
Task_binding_shuffle.invoke(ITaskPortAdvance.PERFORM, out sync_perform);
Console.WriteLine(this.GlobalRank + ": SHUFFLER MAP ...5");
IList <IKVPairInstance <TKey, TValue> >[] buffer = new IList <IKVPairInstance <TKey, TValue> > [gusty_size];
for (int i = 0; i < gusty_size; i++)
{
buffer [i] = new List <IKVPairInstance <TKey, TValue> > ();
}
Console.WriteLine(this.GlobalRank + ": SHUFFLER MAP ...6");
IKVPairInstance <TKey, TValue> item = null;
if (!input_instance.has_next())
{
end_iteration = true;
}
else
{
end_computation = false;
}
int count = 0;
while (input_instance.fetch_next(out bin_object))
{
Console.WriteLine(this.GlobalRank + ": SHUFFLER MAP ...6 count=" + count);
item = (IKVPairInstance <TKey, TValue>)bin_object;
this.Input_key.Instance = item.Key;
Partition_function.go();
int index = ((IIntegerInstance)this.Output_key.Instance).Value;
buffer [index].Add(item);
Console.WriteLine(this.GlobalRank + ": SHUFFLER MAP ...7 count=" + count);
count++;
}
Console.WriteLine(this.GlobalRank + ": SHUFFLER MAP ...9");
for (int i = 0; i < buffer.Length; i++)
{
Console.WriteLine("SHUFFLER MAP ...9 - buffer[" + i + "]=" + buffer[i].Count);
}
// PERFORM
for (int i = 0; i < gusty_size; i++)
{
Step_channel.Send(buffer [i], unit_ref [i], end_iteration ? TAG_SHUFFLE_OMV_END_CHUNK : TAG_SHUFFLE_OMV_NEW_CHUNK);
}
Console.WriteLine(this.GlobalRank + ": SHUFFLER MAP ...10");
sync_perform.wait();
Console.WriteLine(this.GlobalRank + ": SHUFFLER MAP ...11");
Task_binding_shuffle.invoke(ITaskPortAdvance.CHUNK_READY); //****
Console.WriteLine(this.GlobalRank + ": SHUFFLER MAP ...12");
}
Console.WriteLine(this.GlobalRank + ": SHUFFLER MAP ...13");
}
Console.WriteLine(this.GlobalRank + ": SHUFFLER MAP ...14");
}
public override void main()
{
Console.WriteLine(this.ThisFacetInstance + "/" + this.Rank + ": WORKFLOW 1");
Task_binding_data.TraceFlag = Task_map.TraceFlag = Task_gusty.TraceFlag = Task_binding_shuffle.TraceFlag = Task_binding_split_first.TraceFlag = Task_binding_split_next.TraceFlag = true;
Task_binding_data.TraceFlag = Task_binding_shuffle.TraceFlag = Task_binding_split_first.TraceFlag = Task_binding_split_next.TraceFlag = true;
IActionFutureSet future_iteration = null;
Console.WriteLine(this.ThisFacetInstance + "/" + this.Rank + ": WORKFLOW 2-1");
IActionFuture future_split_first_chunk_ready = null;
Task_binding_split_first.invoke(ITaskPortAdvance.CHUNK_READY, out future_split_first_chunk_ready);
int action_id_split_first_chunk_ready = future_split_first_chunk_ready.GetHashCode();
future_iteration = future_split_first_chunk_ready.createSet();
Console.WriteLine(this.ThisFacetInstance + "/" + this.Rank + ": WORKFLOW 2-2");
IActionFuture future_split_next_chunk_ready = null;
Task_binding_split_next.invoke(ITaskPortAdvance.CHUNK_READY, out future_split_next_chunk_ready);
int action_id_split_next_chunk_ready = future_split_next_chunk_ready.GetHashCode();
future_iteration.addAction(future_split_next_chunk_ready);
Console.WriteLine(this.ThisFacetInstance + "/" + this.Rank + ": WORKFLOW 3");
IActionFuture future_map_chunk_ready = null;
Task_map.invoke(ITaskPortAdvance.CHUNK_READY, out future_map_chunk_ready);
int action_id_map_chunk_ready = future_map_chunk_ready.GetHashCode();
future_iteration.addAction(future_map_chunk_ready);
Console.WriteLine(this.ThisFacetInstance + "/" + this.Rank + ": WORKFLOW 4");
IActionFuture future_shuffle_chunk_ready = null;
Task_binding_shuffle.invoke(ITaskPortAdvance.CHUNK_READY, out future_shuffle_chunk_ready);
int action_id_shuffle_chunk_ready = future_shuffle_chunk_ready.GetHashCode();
future_iteration.addAction(future_shuffle_chunk_ready);
Console.WriteLine(this.ThisFacetInstance + "/" + this.Rank + ": WORKFLOW 4");
IActionFuture future_gusty_chunk_ready = null;
Task_gusty.invoke(ITaskPortAdvance.CHUNK_READY, out future_gusty_chunk_ready);
int action_id_gusty_chunk_ready = future_gusty_chunk_ready.GetHashCode();
future_iteration.addAction(future_gusty_chunk_ready);
Console.WriteLine(this.ThisFacetInstance + "/" + this.Rank + ": WORKFLOW 5");
IActionFuture future_data_terminate = null;
Task_binding_data.invoke(ITaskPortData.TERMINATE, out future_data_terminate);
future_iteration.addAction(future_data_terminate);
int action_id_data_terminate = future_data_terminate.GetHashCode();
Console.WriteLine(this.ThisFacetInstance + "/" + this.Rank + ": WORKFLOW 6-1");
Task_binding_data.invoke(ITaskPortData.READ_SOURCE);
Console.WriteLine(this.ThisFacetInstance + "/" + this.Rank + ": WORKFLOW 6-2");
Task_binding_split_first.invoke(ITaskPortAdvance.READ_CHUNK);
Task_binding_split_first.invoke(ITaskPortAdvance.PERFORM);
Console.WriteLine(this.ThisFacetInstance + "/" + this.Rank + ": WORKFLOW 7");
IList <Thread> bag_of_tasks = new List <Thread> ();
bool terminate = false;
do
{
Console.WriteLine(this.ThisFacetInstance + "/" + this.Rank + ": WORKFLOW 8 - LOOP");
IActionFuture action = future_iteration.waitAny();
int action_id = action.GetHashCode();
if (action_id == action_id_split_first_chunk_ready)
{
Thread t1 = new Thread((ThreadStart) delegate()
{
Console.WriteLine("INVOKE MAPPER READ_CHUNK - BEFORE");
Task_map.invoke(ITaskPortAdvance.READ_CHUNK);
Console.WriteLine("INVOKE MAPPER READ_CHUNK - AFTER");
IActionFuture future_map_perform = null;
Thread thread_map_perform = Task_map.invoke(ITaskPortAdvance.PERFORM, map_perform, out future_map_perform);
Console.WriteLine("END INVOKE SPLITTER CHUNK_READY");
});
bag_of_tasks.Add(t1);
t1.Start();
Thread t2 = new Thread((ThreadStart) delegate()
{
Console.WriteLine("INVOKE SPLITTER READ_CHUNK");
Task_binding_split_first.invoke(ITaskPortAdvance.READ_CHUNK);
Console.WriteLine("INVOKE SPLITTER PERFORM");
Task_binding_split_first.invoke(ITaskPortAdvance.PERFORM);
Console.WriteLine("INVOKE SPLITTER READ_CHUNK/PERFORM - AFTER");
IActionFuture future_split_chunk_ready_ = null;
Task_binding_split_first.invoke(ITaskPortAdvance.CHUNK_READY, out future_split_chunk_ready_);
action_id_split_first_chunk_ready = future_split_chunk_ready_.GetHashCode();
future_iteration.addAction(future_split_chunk_ready_);
});
Console.WriteLine("THREAD LAUNCHED 1");
bag_of_tasks.Add(t2);
t2.Start();
}
else if (action_id == action_id_map_chunk_ready)
{
Task_map.invoke(ITaskPortAdvance.CHUNK_READY, out future_map_chunk_ready);
action_id_map_chunk_ready = future_map_chunk_ready.GetHashCode();
future_iteration.addAction(future_map_chunk_ready);
Thread t = new Thread((ThreadStart) delegate()
{
Console.WriteLine("INVOKE SHUFFLER READ_CHUNK - BEFORE"); // 110 executados (o 48 completou nos pares, mas não progrediu aqui (????), motivo do erro.
Task_binding_shuffle.invoke(ITaskPortAdvance.READ_CHUNK); //
Console.WriteLine("INVOKE SHUFFLER READ_CHUNK - AFTER"); // 47 completados
IActionFuture future_shuffle_perform = null;
Thread thread_shuffle_perform = Task_binding_shuffle.invoke(ITaskPortAdvance.PERFORM, shuffle_perform, out future_shuffle_perform);
Console.WriteLine("END INVOKE MAPPER CHUNK_READY");
});
t.Start();
bag_of_tasks.Add(t);
Console.WriteLine("THREAD LAUNCHED 2");
}
else if (action_id == action_id_shuffle_chunk_ready)
{
Task_binding_shuffle.invoke(ITaskPortAdvance.CHUNK_READY, out future_shuffle_chunk_ready);
action_id_shuffle_chunk_ready = future_shuffle_chunk_ready.GetHashCode();
future_iteration.addAction(future_shuffle_chunk_ready);
Thread t = new Thread((ThreadStart) delegate()
{
Console.WriteLine("INVOKE REDUCER READ_CHUNK - BEFORE");
Task_gusty.invoke(ITaskPortAdvance.READ_CHUNK); // ****
Console.WriteLine("INVOKE REDUCER READ_CHUNK - AFTER");
IActionFuture future_gusty_perform = null;
Thread thread_gusty_perform = Task_gusty.invoke(ITaskPortAdvance.PERFORM, gusty_perform, out future_gusty_perform);
Console.WriteLine("END INVOKE SHUFFLER CHUNK_READY");
});
t.Start();
bag_of_tasks.Add(t);
Console.WriteLine("THREAD LAUNCHED 3");
}
else if (action_id == action_id_gusty_chunk_ready)
{
Task_gusty.invoke(ITaskPortAdvance.CHUNK_READY, out future_gusty_chunk_ready);
action_id_gusty_chunk_ready = future_gusty_chunk_ready.GetHashCode();
future_iteration.addAction(future_gusty_chunk_ready);
Thread t = new Thread((ThreadStart) delegate()
{
Console.WriteLine("INVOKE SPLITTER NEXT READ_CHUNK - BEFORE");
Task_binding_split_next.invoke(ITaskPortAdvance.READ_CHUNK); // ****
Console.WriteLine("INVOKE SPLITTER NEXT READ_CHUNK - AFTER");
IActionFuture future_split_perform = null;
Thread thread_split_perform = Task_binding_split_next.invoke(ITaskPortAdvance.PERFORM, split_perform, out future_split_perform);
Console.WriteLine("END INVOKE REDUCER CHUNK_READY");
});
t.Start();
bag_of_tasks.Add(t);
Console.WriteLine("THREAD LAUNCHED 4");
}
else if (action_id == action_id_split_next_chunk_ready)
{
Thread t1 = new Thread((ThreadStart) delegate()
{
Console.WriteLine("INVOKE MAP READ_CHUNK NEXT - BEFORE");
Task_map.invoke(ITaskPortAdvance.READ_CHUNK); //
Console.WriteLine("INVOKE MAP READ_CHUNK NEXT - AFTER");
IActionFuture future_map_perform = null;
Thread thread_map_perform = Task_map.invoke(ITaskPortAdvance.PERFORM, map_perform, out future_map_perform);
Console.WriteLine("END INVOKE SPLIT NEXT CHUNK_READY");
});
bag_of_tasks.Add(t1);
t1.Start();
Thread t2 = new Thread((ThreadStart) delegate()
{
Console.WriteLine("INVOKE SPLITTER NEXT READ_CHUNK");
Task_binding_split_next.invoke(ITaskPortAdvance.READ_CHUNK);
Console.WriteLine("INVOKE SPLITTER NEXT PERFORM");
Task_binding_split_next.invoke(ITaskPortAdvance.PERFORM);
Console.WriteLine("INVOKE SPLITTER NEXT READ_CHUNK/PERFORM - AFTER");
IActionFuture future_split_next_chunk_ready_ = null;
Task_binding_split_next.invoke(ITaskPortAdvance.CHUNK_READY, out future_split_next_chunk_ready_);
action_id_split_next_chunk_ready = future_split_next_chunk_ready_.GetHashCode();
future_iteration.addAction(future_split_next_chunk_ready_);
});
Console.WriteLine("THREAD LAUNCHED 1");
bag_of_tasks.Add(t2);
t2.Start();
Console.WriteLine("THREAD LAUNCHED 5");
}
else if (action_id == action_id_data_terminate)
{
Thread t = new Thread((ThreadStart) delegate()
{
Console.WriteLine("INVOKE DATA WRITE_SINK - BEFORE");
Task_binding_data.invoke(ITaskPortData.WRITE_SINK);
Console.WriteLine("INVOKE DATA WRITE_SINK - AFTER");
terminate = true;
});
t.Start();
bag_of_tasks.Add(t);
Console.WriteLine("THREAD LAUNCHED 6");
}
else
{
Console.WriteLine("UNEXPECTED ERROR: ACTION FUTURE NOT RECOGNIZED ! ");
}
} while (!terminate);
Console.WriteLine("WORKFLOW FINISHED ! ");
foreach (Thread t in bag_of_tasks)
{
t.Join();
}
}
