public void read_send_graph(int gusty_size, IDictionary <int, Tuple <int, int> > unit_ref)
{
IIteratorInstance <IKVPair <IInteger, GIF> > input_instance_graph = (IIteratorInstance <IKVPair <IInteger, GIF> >)Collect_graph.Client;
object bin_object = null;
IActionFuture sync_perform;
Partition_function_gif.NumberOfPartitions = gusty_size;
bool end_iteration_gif = false; bool set_table = true;
while (!end_iteration_gif)
{
Task_binding_shuffle.invoke(ITaskPortAdvance.READ_CHUNK);
Task_binding_shuffle.invoke(ITaskPortAdvance.PERFORM, out sync_perform);
IList <IKVPairInstance <IInteger, GIF> >[] buffer_gif = new IList <IKVPairInstance <IInteger, GIF> > [gusty_size];
for (int i = 0; i < gusty_size; i++)
{
buffer_gif [i] = new List <IKVPairInstance <IInteger, GIF> > ();
}
IKVPairInstance <IInteger, GIF> item = null;
if (!input_instance_graph.has_next())
{
end_iteration_gif = true;
}
int count = 0;
while (input_instance_graph.fetch_next(out bin_object))
{
item = (IKVPairInstance <IInteger, GIF>)bin_object;
this.Input_key_gif.Instance = item.Value; //item.Key;
Partition_function_gif.go();
int index = ((IIntegerInstance)this.Output_key_gif.Instance).Value;
buffer_gif [index].Add(item);
Console.WriteLine(this.GlobalRank + ": SHUFFLER MAP Collector Graph count=" + (count++));
}
// PERFORM
for (int i = 0; i < gusty_size; i++)
{
Step_channel.Send(buffer_gif [i], unit_ref [i], end_iteration_gif ? TAG_SHUFFLE_OMV_END_CHUNK : TAG_SHUFFLE_OMV_NEW_CHUNK);
}
sync_perform.wait();
Task_binding_shuffle.invoke(ITaskPortAdvance.CHUNK_READY);
}
Partition_function.PartitionTABLE = Partition_function_gif.PartitionTABLE;
}
private void graph_creator()
{
IIteratorInstance <IKVPair <IInteger, IIterator <GIF> > > input_instance_gif = (IIteratorInstance <IKVPair <IInteger, IIterator <GIF> > >)Collect_graph.Client;
IIteratorInstance <IKVPair <IInteger, GIF> > output_instance_gif = (IIteratorInstance <IKVPair <IInteger, GIF> >)Output_gif.Instance;
Feed_graph.Server = output_instance_gif;
IActionFuture sync_perform;
bool end_iteration = false;
while (!end_iteration) // take next chunk ...
{
Task_gusty.invoke(ITaskPortAdvance.READ_CHUNK);
Task_gusty.invoke(ITaskPortAdvance.PERFORM, out sync_perform);
IKVPairInstance <IInteger, IIterator <GIF> > kvpair = null;
object kvpair_object;
object o;
if (!input_instance_gif.has_next())
{
end_iteration = true;
}
while (input_instance_gif.fetch_next(out kvpair_object))
{
kvpair = (IKVPairInstance <IInteger, IIterator <GIF> >)kvpair_object;
Graph_values.Instance = kvpair;
Gusty_function.graph_creator();
}
sync_perform.wait();
}
//IActionFuture gusty_chunk_ready;
//Task_gusty.invoke (ITaskPortAdvance.CHUNK_READY, out gusty_chunk_ready); //***
output_instance_gif.finish();
//gusty_chunk_ready.wait ();
}
private void send_to_mappers()
{
Console.WriteLine(this.Rank + ": IJoinGustyCollector 1");
IIteratorInstance <IKVPair <OKey, OValue> > output_instance = (IIteratorInstance <IKVPair <OKey, OValue> >)Output_pairs.Instance;
IIteratorInstance <IKVPair <IKey, IValue> > input_instance = (IIteratorInstance <IKVPair <IKey, IValue> >)Input_pairs.Instance;
object bin_object = null;
// DETERMINE COMMUNICATION TARGETs
Tuple <int, int> sink_ref = new Tuple <int, int> (this.FacetIndexes [FACET_SINK] [0], 0);
IDictionary <int, Tuple <int, int> > unit_ref = new Dictionary <int, Tuple <int, int> > ();
int m_size = 0;
foreach (int i in this.FacetIndexes[FACET_MAP])
{
int nr0 = m_size;
m_size += this.UnitSizeInFacet [i] ["gusty_feeder"];
for (int k = 0, j = nr0; j < m_size; k++, j++)
{
unit_ref [j] = new Tuple <int, int> (i /*,0 index of MAP_FEEDER*/, k);
}
}
Console.WriteLine(this.Rank + ": IJoinGustyCollector 2");
IActionFuture sync_perform;
bool end_computation = false;
while (!end_computation) // new iteration
{
Console.WriteLine(this.Rank + ": IJoinGustyCollector LOOP");
end_computation = true;
//Console.WriteLine (this.Rank + ": IJoinGustyCollector ITERATE 1");
//Task_binding_split_next.invoke (ITaskPortAdvance.READ_CHUNK); //****
//Task_binding_split_next.invoke (ITaskPortAdvance.PERFORM, out sync_perform);
//Console.WriteLine (this.Rank + ": IJoinGustyCollector ITERATE 2");
// SEND TO SINK
IList <IKVPairInstance <OKey, OValue> > buffer_sink = new List <IKVPairInstance <OKey, OValue> >();
end_computation = !input_instance.has_next();
Thread thread_send_output = new Thread(new ThreadStart(delegate {
int count1 = 0;
while (output_instance.fetch_next(out bin_object))
{
IKVPairInstance <OKey, OValue> item = (IKVPairInstance <OKey, OValue>)bin_object;
buffer_sink.Add(item);
if (count1 % CHUNK_SIZE == 0)
{
// PERFORM
Console.WriteLine(this.Rank + ": IJoinGustyCollector SINK SEND CHUNK 3-1 count=" + count1);
Split_channel.Send(buffer_sink, sink_ref, TAG_SPLIT_NEW_CHUNK);
Console.WriteLine(this.Rank + ": IJoinGustyCollector SINK SEND CHUNK 3-2 count=" + count1);
buffer_sink.Clear();
}
count1++;
}
if (buffer_sink.Count > 0)
{
Console.WriteLine(this.Rank + ": IJoinGustyCollector SINK SEND CHUNK 3-3 count=" + count1);
Split_channel.Send(buffer_sink, sink_ref, TAG_SPLIT_NEW_CHUNK);
Console.WriteLine(this.Rank + ": IJoinGustyCollector SINK SEND CHUNK 3-4 count=" + count1);
}
}));
thread_send_output.Start();
// SEND BACK TO MAPPER (new iteration)
Bin_function_iterate.NumberOfPartitions = m_size;
IList <IKVPairInstance <OKey, OValue> >[] buffer = new IList <IKVPairInstance <OKey, OValue> > [m_size];
for (int i = 0; i < m_size; i++)
{
buffer [i] = new List <IKVPairInstance <OKey, OValue> > ();
}
Console.WriteLine(this.Rank + ": IJoinGustyCollector ITERATE 3 end_computation=" + end_computation);
int count = 0;
while (input_instance.fetch_next(out bin_object))
{
Console.WriteLine(this.Rank + ": IJoinGustyCollector ITERATE LOOP 4-1 count=" + count);
IKVPairInstance <OKey, OValue> item = (IKVPairInstance <OKey, OValue>)bin_object;
this.Input_key_iterate.Instance = item.Key;
Bin_function_iterate.go();
int index = ((IIntegerInstance)this.Output_key_iterate.Instance).Value;
buffer [index].Add(item);
if (count % CHUNK_SIZE == 0)
{
Task_binding_split_next.invoke(ITaskPortAdvance.READ_CHUNK);
Task_binding_split_next.invoke(ITaskPortAdvance.PERFORM, out sync_perform);
for (int i = 0; i < m_size; i++)
{
Console.WriteLine("SPLITTER REDUCE COLLECTOR - Sending chunk of " + buffer[i].Count + " elements");
Split_channel.Send(buffer [i], unit_ref [i], TAG_SPLIT_NEW_CHUNK);
buffer [i].Clear();
}
sync_perform.wait();
Task_binding_split_next.invoke(ITaskPortAdvance.CHUNK_READY);
Console.WriteLine(this.Rank + ": IJoinGustyCollector ITERATE 3-5");
}
Console.WriteLine(this.Rank + ": IJoinGustyCollector ITERATE LOOP 4-2 count=" + count);
count++;
}
Console.WriteLine(this.Rank + ": IJoinGustyCollector ITERATE 5");
Task_binding_split_next.invoke(ITaskPortAdvance.READ_CHUNK);
Task_binding_split_next.invoke(ITaskPortAdvance.PERFORM, out sync_perform);
// SEND REMAINING PAIRS AND CLOSES THE CHUNK LIST
Console.WriteLine(this.Rank + ": IJoinGustyCollector ITERATE 6-1");
for (int i = 0; i < m_size; i++)
{
Split_channel.Send(buffer [i], unit_ref [i], TAG_SPLIT_END_CHUNK);
}
sync_perform.wait();
Console.WriteLine(this.Rank + ": IJoinGustyCollector ITERATE 7-1");
Task_binding_split_next.invoke(ITaskPortAdvance.CHUNK_READY);
Console.WriteLine(this.Rank + ": IJoinGustyCollector ITERATE 7-2");
thread_send_output.Join();
//sync_perform.wait ();
Console.WriteLine(this.Rank + ": IJoinGustyCollector ITERATE 7-3");
}
Console.WriteLine(this.Rank + ": IJoinGustyCollector END COMPUTATION ");
input_instance.finish();
output_instance.finish();
Split_channel.Send(new List <IKVPairInstance <OKey, OValue> >(), sink_ref, TAG_SPLIT_END_CHUNK);
}
private void readPair_OMK_OMVs()
{
Console.WriteLine(this.Rank + ": REDUCE 1");
IIteratorInstance <IKVPair <TKey, IIterator <TValue> > > input_instance = (IIteratorInstance <IKVPair <TKey, IIterator <TValue> > >)Collect_pairs.Client;
IIteratorInstance <IKVPair <OKey, OValue> > output_instance = (IIteratorInstance <IKVPair <OKey, OValue> >)Output.Instance;
Feed_pairs.Server = output_instance;
IActionFuture sync_perform;
// TODO: Dividir em chunks a saa de pares (OKey,OValue)
Console.WriteLine(this.Rank + ": REDUCER 2");
// object gusty_lock = new object ();
bool end_computation = false;
while (!end_computation) // new iteration
{
IDictionary <object, object> cont_dict = new Dictionary <object, object> ();
Console.WriteLine(this.Rank + ": REDUCER LOOP");
end_computation = true;
bool end_iteration = false;
while (!end_iteration) // take next chunk ...
{
Console.WriteLine(this.Rank + ": REDUCER ITERATE 1");
Task_gusty.invoke(ITaskPortAdvance.READ_CHUNK);
Task_gusty.invoke(ITaskPortAdvance.PERFORM, out sync_perform);
Console.WriteLine(this.Rank + ": REDUCER ITERATE 2");
IKVPairInstance <TKey, IIterator <TValue> > kvpair = null;
object kvpair_object;
if (!input_instance.has_next())
{
end_iteration = true;
}
else
{
end_computation = false;
}
int count = 0;
while (input_instance.fetch_next(out kvpair_object))
{
Console.WriteLine(this.Rank + ": REDUCER ITERATE INNER LOOP 3 count=" + count);
kvpair = (IKVPairInstance <TKey, IIterator <TValue> >)kvpair_object;
object acc_value;
if (!cont_dict.TryGetValue(kvpair.Key, out acc_value))
{
cont_dict[kvpair.Key] = new object();
}
else
{
((IDataInstance)Continue_value.Instance).ObjValue = acc_value;
}
Input_values.Instance = kvpair;
Gusty_function.go();
cont_dict [kvpair.Key] = ((IDataInstance)((IKVPairInstance <OKey, OValue>)Output_value.Instance).Value).ObjValue;
count++;
}
Console.WriteLine(this.Rank + ": REDUCER ITERATE 4 count=" + count);
sync_perform.wait();
Console.WriteLine(this.Rank + ": REDUCER ITERATE 5");
}
Console.WriteLine(this.Rank + ": REDUCER ITERATE END ITERATION");
int chunk_counter = 1;
IActionFuture gusty_chunk_ready;
Task_gusty.invoke(ITaskPortAdvance.CHUNK_READY, out gusty_chunk_ready); //***
foreach (KeyValuePair <object, object> output_pair in cont_dict)
{
IKVPairInstance <OKey, OValue> new_pair = (IKVPairInstance <OKey, OValue>)Output_value.newInstance();
new_pair.Key = output_pair.Key;
new_pair.Value = output_pair.Value;
output_instance.put(new_pair);
}
output_instance.finish();
gusty_chunk_ready.wait();
Console.WriteLine(this.Rank + ": REDUCER ITERATE FINISH");
}
Console.WriteLine(this.Rank + ": REDUCER FINISH ... ");
}
public override void main()
{
Console.WriteLine(this.GlobalRank + ": STARTING SHUFFLER MAP ...1");
// IIteratorInstance<IKVPair<TKey,TValue>> input_instance = (IIteratorInstance<IKVPair<TKey,TValue>>) Collect_pairs.Client;
object bin_object = null;
IActionFuture sync_perform;
// DETERMINE COMMMUNICATION TARGETs
IDictionary <int, Tuple <int, int> > unit_ref = new Dictionary <int, Tuple <int, int> > ();
int nf = this.FacetMultiplicity [FACET_REDUCE];
int gusty_size = 0;
Console.WriteLine(this.GlobalRank + ": STARTING SHUFFLER MAP ...2");
foreach (int i in this.FacetIndexes[FACET_REDUCE])
{
int nr0 = gusty_size;
Console.WriteLine(this.GlobalRank + ": STARTING SHUFFLER MAP ...3 - BEGIN 1 - i=" + i);
foreach (KeyValuePair <int, IDictionary <string, int> > ttt in this.UnitSizeInFacet)
{
Console.WriteLine(this.GlobalRank + ": STARTING SHUFFLER MAP ...3 - " + (ttt.Value == null));
foreach (KeyValuePair <string, int> tttt in ttt.Value)
{
Console.WriteLine(this.GlobalRank + ": STARTING SHUFFLER MAP ...3 --- " + ttt.Key + " / " + tttt.Key + " / " + tttt.Value);
}
}
gusty_size += this.UnitSizeInFacet [i] ["gusty_feeder"];
Console.WriteLine(this.GlobalRank + ": STARTING SHUFFLER MAP ...3 - BEGIN 2 - " + i);
for (int k = 0, j = nr0; j < gusty_size; k++, j++)
{
unit_ref [j] = new Tuple <int, int> (i /*,0 INDEX OF gusty_feeder*/, k);
}
Console.WriteLine(this.GlobalRank + ": STARTING SHUFFLER MAP ...3 - END - " + i);
}
read_send_graph(gusty_size, unit_ref);
IIteratorInstance <IKVPair <TKey, TValue> > input_instance = (IIteratorInstance <IKVPair <TKey, TValue> >)Collect_pairs.Client;
Partition_function.NumberOfPartitions = gusty_size;
bool end_computation = false;
while (!end_computation) // next iteration
{
end_computation = true;
bool end_iteration = false;
while (!end_iteration) // take next chunk ...
{
Console.WriteLine(this.GlobalRank + ": SHUFFLER MAP ...4");
Task_binding_shuffle.invoke(ITaskPortAdvance.READ_CHUNK);
Task_binding_shuffle.invoke(ITaskPortAdvance.PERFORM, out sync_perform);
Console.WriteLine(this.GlobalRank + ": SHUFFLER MAP ...5");
IList <IKVPairInstance <TKey, TValue> >[] buffer = new IList <IKVPairInstance <TKey, TValue> > [gusty_size];
for (int i = 0; i < gusty_size; i++)
{
buffer [i] = new List <IKVPairInstance <TKey, TValue> > ();
}
Console.WriteLine(this.GlobalRank + ": SHUFFLER MAP ...6");
IKVPairInstance <TKey, TValue> item = null;
if (!input_instance.has_next())
{
end_iteration = true;
}
else
{
end_computation = false;
}
int count = 0;
while (input_instance.fetch_next(out bin_object))
{
Console.WriteLine(this.GlobalRank + ": SHUFFLER MAP ...6 count=" + count);
item = (IKVPairInstance <TKey, TValue>)bin_object;
this.Input_key.Instance = item.Key;
Partition_function.go();
int index = ((IIntegerInstance)this.Output_key.Instance).Value;
buffer [index].Add(item);
Console.WriteLine(this.GlobalRank + ": SHUFFLER MAP ...7 count=" + count);
count++;
}
Console.WriteLine(this.GlobalRank + ": SHUFFLER MAP ...9");
for (int i = 0; i < buffer.Length; i++)
{
Console.WriteLine("SHUFFLER MAP ...9 - buffer[" + i + "]=" + buffer[i].Count);
}
// PERFORM
for (int i = 0; i < gusty_size; i++)
{
Step_channel.Send(buffer [i], unit_ref [i], end_iteration ? TAG_SHUFFLE_OMV_END_CHUNK : TAG_SHUFFLE_OMV_NEW_CHUNK);
}
Console.WriteLine(this.GlobalRank + ": SHUFFLER MAP ...10");
sync_perform.wait();
Console.WriteLine(this.GlobalRank + ": SHUFFLER MAP ...11");
Task_binding_shuffle.invoke(ITaskPortAdvance.CHUNK_READY); //****
Console.WriteLine(this.GlobalRank + ": SHUFFLER MAP ...12");
}
Console.WriteLine(this.GlobalRank + ": SHUFFLER MAP ...13");
}
Console.WriteLine(this.GlobalRank + ": SHUFFLER MAP ...14");
}
