private void receive_pairs_iteration(IPortTypeIterator output_instance, int senders_size, Tuple <int, int> unit_ref)
{
CompletedStatus status;
do
{
Console.WriteLine("SPLITER WRITE SINK 7-1 facet=" + unit_ref.Item1 + "/rank=" + unit_ref.Item2);
IList <IKVPairInstance <OKey, OValue> > buffer;
lock (cs_recv) Split_channel.Receive(unit_ref, MPI.Communicator.anyTag, out buffer, out status);
foreach (IKVPairInstance <OKey, OValue> kv in buffer)
{
output_instance.put(kv);
}
Console.WriteLine("SPLITER WRITE SINK 7-2 facet=" + unit_ref.Item1 + "/rank=" + unit_ref.Item2);
} while (status.Tag == TAG_SPLIT_NEW_CHUNK);
Console.WriteLine("SPLITER WRITE SINK - OUTPUT RECEIVED");
}
//CONFIG
public void readSource()
{
Console.WriteLine(this.GlobalRank + ": STARTING SPLITTER READSOURCE...1");
output_instance_gifs = (IIteratorInstance <IKVPair <IInteger, IIterator <GIF> > >)Output_gifs.Instance;
Feed_graph.Server = output_instance_gifs;
Console.WriteLine(this.GlobalRank + ": STARTING SPLITTER READSOURCE...2");
// RECEIVE PAIR FROM THE SOURCE (1st iteration)
Tuple <int, int> unit_ref_source = new Tuple <int, int> (this.FacetIndexes [FACET_SOURCE] [0], 0);
// Thread[] threads_receive = new Thread[senders_size];
// for (int i = 0; i < senders_size; i++)
// {
// threads_receive [i] = new Thread ((ParameterizedThreadStart)delegate(object unit_ref_obj) {
// Tuple<int,int> unit_ref_i = (Tuple<int,int>)unit_ref_obj;
// receive_pairs_iteration (unit_ref_i);
// });
// }
// TODO: READ_SOURCE é necessário ? Não no map feeder. Tirar fatia de Task_binding data ...
Task_binding_data.TraceFlag = true;
Task_binding_data.invoke(ITaskPortData.READ_SOURCE);
// Do nothing ...
IList <IKVPairInstance <IInteger, GIF> > buffer;
object buffer_obj;
CompletedStatus status;
do
{
IActionFuture sync_perform;
Console.WriteLine(this.Rank + ": SPLITTER MAP FEEDER - BEFORE READ_CHUNK");
Task_binding_split_first.invoke(ITaskPortAdvance.READ_CHUNK); //****
Task_binding_split_first.invoke(ITaskPortAdvance.PERFORM, out sync_perform);
IDictionary <object, IIteratorInstance <GIF> > kv_cache = new Dictionary <object, IIteratorInstance <GIF> > ();
Console.WriteLine(this.Rank + ": SPLITTER MAP FEEDER !!! PERFORM OK !");
Split_channel.Receive(unit_ref_source, MPI.Communicator.anyTag, out buffer, out status);
Console.WriteLine(this.Rank + ": CHUNK PAIRS RECEIVED !!! from source buffer.Count=" + buffer.Count);
//foreach (IKVPairInstance<IInteger,GIF> kv in buffer)
// output_instance_gif.put (kv);
foreach (IKVPairInstance <IInteger, GIF> kv in buffer)
{
IIteratorInstance <GIF> iterator = null;
if (!kv_cache.ContainsKey(kv.Key))
{
iterator = Value_factory_gif.newIteratorInstance();
kv_cache.Add(kv.Key, iterator);
IKVPairInstance <IInteger, IIterator <GIF> > item = (IKVPairInstance <IInteger, IIterator <GIF> >)Output_gifs.createItem();
item.Key = kv.Key;
item.Value = iterator;
output_instance_gifs.put(item);
}
else
{
kv_cache.TryGetValue(kv.Key, out iterator);
}
iterator.put(kv.Value);
}
output_instance_gifs.finish();
foreach (IIteratorInstance <IValue> iterator in kv_cache.Values)
{
iterator.finish();
}
sync_perform.wait();
// CHUNK_READY
Console.WriteLine(this.Rank + ": SPLITTER MAP FEEDER 1");
Task_binding_split_first.invoke(ITaskPortAdvance.CHUNK_READY);
Console.WriteLine(this.Rank + ": SPLITTER MAP FEEDER 2");
} while (status.Tag != TAG_SPLIT_END_CHUNK);
output_instance_gifs.finish();
Console.WriteLine(this.Rank + ": FINISH READING CHUNKS OF SOURCE");
}
public void iterate()
{
IList <IKVPairInstance <IKey, IValue> > buffer;
object buffer_obj;
CompletedStatus status;
Console.WriteLine(this.GlobalRank + ": STARTING SPLITTER ITERATE...1");
output_instance = (IIteratorInstance <IKVPair <IKey, IIterator <IValue> > >)Output.Instance;
Feed_pairs.Server = output_instance;
Console.WriteLine(this.GlobalRank + ": STARTING SPLITTER ITERATE...2");
IDictionary <int, Tuple <int, int> > unit_ref = new Dictionary <int, Tuple <int, int> > ();
Console.WriteLine(this.GlobalRank + ": STARTING SPLITTER ...3");
// RECEIVE PAIRS FROM THE REDUCERS (next iterations)
int senders_size = 0;
foreach (int i in this.FacetIndexes[FACET_REDUCE])
{
Console.WriteLine(this.GlobalRank + ": STARTING SPLITTER ...4 -- i=" + i);
int nr0 = senders_size;
senders_size += this.UnitSizeInFacet [i] ["gusty_collector"];
for (int k = 0, j = nr0; j < senders_size; k++, j++)
{
unit_ref [j] = new Tuple <int, int> (i, k);
}
}
bool end_computation = false;
while (!end_computation)
{
end_computation = true;
int count_finished_streams = 0;
while (count_finished_streams < senders_size)
{
IActionFuture sync_perform;
Console.WriteLine(this.Rank + ": SPLITTER MAP FEEDER NEXT 1");
Task_binding_split_next.invoke(ITaskPortAdvance.READ_CHUNK);
Task_binding_split_next.invoke(ITaskPortAdvance.PERFORM, out sync_perform);
IDictionary <object, IIteratorInstance <IValue> > kv_cache = new Dictionary <object, IIteratorInstance <IValue> > ();
Console.WriteLine(this.Rank + ": SPLITTER MAP FEEDER NEXT 2");
for (int i = 0; i < senders_size; i++)
{
Console.WriteLine(this.Rank + ": SPLITTER MAP FEEDER NEXT LOOP - before receive");
Split_channel.Receive(unit_ref [i], MPI.Communicator.anyTag, out buffer_obj, out status);
Console.WriteLine(this.Rank + ": CHUNK PAIRS RECEIVED !!! from gusty_collector of index " + i + " / count_finished_streams=" + count_finished_streams);
if (status.Tag == TAG_SPLIT_END_CHUNK)
{
count_finished_streams++;
}
else
{
end_computation = false;
}
try {
buffer = (IList <IKVPairInstance <IKey, IValue> >)buffer_obj;
//foreach (IKVPairInstance<IKey,IValue> kv in buffer)
// output_instance.put (kv);
foreach (IKVPairInstance <IKey, IValue> kv in buffer)
{
IIteratorInstance <IValue> iterator = null;
if (!kv_cache.ContainsKey(kv.Key))
{
iterator = Value_factory.newIteratorInstance();
kv_cache.Add(kv.Key, iterator);
IKVPairInstance <IKey, IIterator <IValue> > item = (IKVPairInstance <IKey, IIterator <IValue> >)Output.createItem();
item.Key = kv.Key;
item.Value = iterator;
output_instance.put(item);
}
else
{
kv_cache.TryGetValue(kv.Key, out iterator);
}
iterator.put(kv.Value);
}
} catch (InvalidCastException e) {
Console.WriteLine("SPLITTER MAPPER: incompatible input !");
count_finished_streams++;
}
}
output_instance.finish();
foreach (IIteratorInstance <IValue> iterator in kv_cache.Values)
{
iterator.finish();
}
Console.WriteLine(this.Rank + ": SPLITTER MAP FEEDER NEXT 3");
sync_perform.wait();
Console.WriteLine(this.Rank + ": SPLITTER MAP FEEDER NEXT 4");
// CHUNK_READY
Task_binding_split_next.invoke(ITaskPortAdvance.CHUNK_READY);
Console.WriteLine(this.Rank + ": SPLITTER MAP FEEDER NEXT 5");
}
}
Console.WriteLine(this.Rank + ": SPLITTER MAP FEEDER FINISH 1 !");
Task_binding_data.invoke(ITaskPortData.TERMINATE);
Task_binding_data.invoke(ITaskPortData.WRITE_SINK);
Console.WriteLine(this.Rank + ": SPLITTER MAP FEEDER FINISH 2 !");
}
private void send_startup()
{
IIteratorInstance <IKVPair <IKey, IValue> > input_instance = (IIteratorInstance <IKVPair <IKey, IValue> >)Input_pairs.Instance;
object bin_object = null;
// DETERMINE COMMUNICATION TARGETs
IDictionary <int, Tuple <int, int> > unit_ref = new Dictionary <int, Tuple <int, int> > ();
int m_size = 0;
foreach (int i in this.FacetIndexes[FACET_MAP])
{
int nr0 = m_size;
m_size += this.UnitSizeInFacet [i] ["gusty_feeder"];
for (int k = 0, j = nr0; j < m_size; k++, j++)
{
unit_ref [j] = new Tuple <int, int> (i /*,0 index of MAP_FEEDER*/, k);
}
}
IActionFuture sync_perform;
// SEND STARTUP TO MAPPER (new iteration)
Bin_function_iterate.NumberOfPartitions = m_size;
IList <IKVPairInstance <OKey, OValue> >[] buffer = new IList <IKVPairInstance <OKey, OValue> > [m_size];
for (int i = 0; i < m_size; i++)
{
buffer [i] = new List <IKVPairInstance <OKey, OValue> > ();
}
int count = 0;
while (input_instance.fetch_next(out bin_object))
{
IKVPairInstance <OKey, OValue> item = (IKVPairInstance <OKey, OValue>)bin_object;
this.Input_key_iterate.Instance = item.Key;
Bin_function_iterate.go();
int index = ((IIntegerInstance)this.Output_key_iterate.Instance).Value;
buffer [index].Add(item);
if (count % CHUNK_SIZE == 0)
{
Task_binding_split_next.invoke(ITaskPortAdvance.READ_CHUNK);
Task_binding_split_next.invoke(ITaskPortAdvance.PERFORM, out sync_perform);
for (int i = 0; i < m_size; i++)
{
Split_channel.Send(buffer [i], unit_ref [i], TAG_SPLIT_NEW_CHUNK);
buffer [i].Clear();
}
sync_perform.wait();
Task_binding_split_next.invoke(ITaskPortAdvance.CHUNK_READY);
}
count++;
}
Task_binding_split_next.invoke(ITaskPortAdvance.READ_CHUNK);
Task_binding_split_next.invoke(ITaskPortAdvance.PERFORM, out sync_perform);
// SEND REMAINING PAIRS AND CLOSES THE CHUNK LIST
for (int i = 0; i < m_size; i++)
{
Split_channel.Send(buffer [i], unit_ref [i], TAG_SPLIT_END_CHUNK);
}
sync_perform.wait();
Task_binding_split_next.invoke(ITaskPortAdvance.CHUNK_READY);
//sync_perform.wait ();
}
private void send_to_mappers()
{
Console.WriteLine(this.Rank + ": IJoinGustyCollector 1");
IIteratorInstance <IKVPair <OKey, OValue> > output_instance = (IIteratorInstance <IKVPair <OKey, OValue> >)Output_pairs.Instance;
IIteratorInstance <IKVPair <IKey, IValue> > input_instance = (IIteratorInstance <IKVPair <IKey, IValue> >)Input_pairs.Instance;
object bin_object = null;
// DETERMINE COMMUNICATION TARGETs
Tuple <int, int> sink_ref = new Tuple <int, int> (this.FacetIndexes [FACET_SINK] [0], 0);
IDictionary <int, Tuple <int, int> > unit_ref = new Dictionary <int, Tuple <int, int> > ();
int m_size = 0;
foreach (int i in this.FacetIndexes[FACET_MAP])
{
int nr0 = m_size;
m_size += this.UnitSizeInFacet [i] ["gusty_feeder"];
for (int k = 0, j = nr0; j < m_size; k++, j++)
{
unit_ref [j] = new Tuple <int, int> (i /*,0 index of MAP_FEEDER*/, k);
}
}
Console.WriteLine(this.Rank + ": IJoinGustyCollector 2");
IActionFuture sync_perform;
bool end_computation = false;
while (!end_computation) // new iteration
{
Console.WriteLine(this.Rank + ": IJoinGustyCollector LOOP");
end_computation = true;
//Console.WriteLine (this.Rank + ": IJoinGustyCollector ITERATE 1");
//Task_binding_split_next.invoke (ITaskPortAdvance.READ_CHUNK); //****
//Task_binding_split_next.invoke (ITaskPortAdvance.PERFORM, out sync_perform);
//Console.WriteLine (this.Rank + ": IJoinGustyCollector ITERATE 2");
// SEND TO SINK
IList <IKVPairInstance <OKey, OValue> > buffer_sink = new List <IKVPairInstance <OKey, OValue> >();
end_computation = !input_instance.has_next();
Thread thread_send_output = new Thread(new ThreadStart(delegate {
int count1 = 0;
while (output_instance.fetch_next(out bin_object))
{
IKVPairInstance <OKey, OValue> item = (IKVPairInstance <OKey, OValue>)bin_object;
buffer_sink.Add(item);
if (count1 % CHUNK_SIZE == 0)
{
// PERFORM
Console.WriteLine(this.Rank + ": IJoinGustyCollector SINK SEND CHUNK 3-1 count=" + count1);
Split_channel.Send(buffer_sink, sink_ref, TAG_SPLIT_NEW_CHUNK);
Console.WriteLine(this.Rank + ": IJoinGustyCollector SINK SEND CHUNK 3-2 count=" + count1);
buffer_sink.Clear();
}
count1++;
}
if (buffer_sink.Count > 0)
{
Console.WriteLine(this.Rank + ": IJoinGustyCollector SINK SEND CHUNK 3-3 count=" + count1);
Split_channel.Send(buffer_sink, sink_ref, TAG_SPLIT_NEW_CHUNK);
Console.WriteLine(this.Rank + ": IJoinGustyCollector SINK SEND CHUNK 3-4 count=" + count1);
}
}));
thread_send_output.Start();
// SEND BACK TO MAPPER (new iteration)
Bin_function_iterate.NumberOfPartitions = m_size;
IList <IKVPairInstance <OKey, OValue> >[] buffer = new IList <IKVPairInstance <OKey, OValue> > [m_size];
for (int i = 0; i < m_size; i++)
{
buffer [i] = new List <IKVPairInstance <OKey, OValue> > ();
}
Console.WriteLine(this.Rank + ": IJoinGustyCollector ITERATE 3 end_computation=" + end_computation);
int count = 0;
while (input_instance.fetch_next(out bin_object))
{
Console.WriteLine(this.Rank + ": IJoinGustyCollector ITERATE LOOP 4-1 count=" + count);
IKVPairInstance <OKey, OValue> item = (IKVPairInstance <OKey, OValue>)bin_object;
this.Input_key_iterate.Instance = item.Key;
Bin_function_iterate.go();
int index = ((IIntegerInstance)this.Output_key_iterate.Instance).Value;
buffer [index].Add(item);
if (count % CHUNK_SIZE == 0)
{
Task_binding_split_next.invoke(ITaskPortAdvance.READ_CHUNK);
Task_binding_split_next.invoke(ITaskPortAdvance.PERFORM, out sync_perform);
for (int i = 0; i < m_size; i++)
{
Console.WriteLine("SPLITTER REDUCE COLLECTOR - Sending chunk of " + buffer[i].Count + " elements");
Split_channel.Send(buffer [i], unit_ref [i], TAG_SPLIT_NEW_CHUNK);
buffer [i].Clear();
}
sync_perform.wait();
Task_binding_split_next.invoke(ITaskPortAdvance.CHUNK_READY);
Console.WriteLine(this.Rank + ": IJoinGustyCollector ITERATE 3-5");
}
Console.WriteLine(this.Rank + ": IJoinGustyCollector ITERATE LOOP 4-2 count=" + count);
count++;
}
Console.WriteLine(this.Rank + ": IJoinGustyCollector ITERATE 5");
Task_binding_split_next.invoke(ITaskPortAdvance.READ_CHUNK);
Task_binding_split_next.invoke(ITaskPortAdvance.PERFORM, out sync_perform);
// SEND REMAINING PAIRS AND CLOSES THE CHUNK LIST
Console.WriteLine(this.Rank + ": IJoinGustyCollector ITERATE 6-1");
for (int i = 0; i < m_size; i++)
{
Split_channel.Send(buffer [i], unit_ref [i], TAG_SPLIT_END_CHUNK);
}
sync_perform.wait();
Console.WriteLine(this.Rank + ": IJoinGustyCollector ITERATE 7-1");
Task_binding_split_next.invoke(ITaskPortAdvance.CHUNK_READY);
Console.WriteLine(this.Rank + ": IJoinGustyCollector ITERATE 7-2");
thread_send_output.Join();
//sync_perform.wait ();
Console.WriteLine(this.Rank + ": IJoinGustyCollector ITERATE 7-3");
}
Console.WriteLine(this.Rank + ": IJoinGustyCollector END COMPUTATION ");
input_instance.finish();
output_instance.finish();
Split_channel.Send(new List <IKVPairInstance <OKey, OValue> >(), sink_ref, TAG_SPLIT_END_CHUNK);
}
public override void main()
{
Console.WriteLine(this.Rank + ": SPLITTER 1 ");
IPortTypeIterator input_instance = (IPortTypeIterator)Source.Client;
// TODO: será que READ_SOURCE é necessária ???
Task_binding_data.TraceFlag = true;
Task_binding_data.invoke(ITaskPortData.READ_SOURCE);
// CALCULATE SubGraph TARGET
int nfr = this.FacetMultiplicity [FACET_REDUCE];
int r_size = 0;
foreach (int i in this.FacetIndexes[FACET_REDUCE])
{
r_size += this.UnitSizeInFacet [i] ["gusty_collector"];
}
Console.WriteLine(this.Rank + ": SPLITTER 2 ");
Source.startReadSource(r_size); // subgrafos divididos em GIF, com destino final nos r_size redutores
Console.WriteLine(this.Rank + ": SPLITTER 3 ");
object bin_object = null;
bool set_table_partition = true;
// CALCULATE TARGETs
IDictionary <int, Tuple <int, int> > unit_ref = new Dictionary <int, Tuple <int, int> > ();
int nf = this.FacetMultiplicity [FACET_MAP];
int m_size = 0;
foreach (int i in this.FacetIndexes[FACET_MAP])
{
int nr0 = m_size;
m_size += this.UnitSizeInFacet [i] ["gusty_feeder"];
for (int k = 0, j = nr0; j < m_size; k++, j++)
{
unit_ref [j] = new Tuple <int, int> (i /*, 0 INDEX OF gusty_feeder */, k);
}
}
Console.WriteLine(this.Rank + ": SPLITTER 4 ");
Task_binding_split_first.TraceFlag = true;
Split_channel.TraceFlag = true;
Thread t_output = new Thread(new ThreadStart(delegate
{
Task_binding_data.invoke(ITaskPortData.TERMINATE);
Task_binding_data.invoke(ITaskPortData.WRITE_SINK);
}));
t_output.Start();
Console.WriteLine(this.Rank + ": SPLITTER 5 ");
bool end_iteration = false;
while (!end_iteration) // take next chunk
{
IActionFuture sync_perform;
Console.WriteLine(this.Rank + ": SPLITTER READ SOURCE - BEFORE READ_CHUNK " + m_size);
/* All the pairs will be read from the source in a single chunk */
Task_binding_split_first.invoke(ITaskPortAdvance.READ_CHUNK); //****
Task_binding_split_first.invoke(ITaskPortAdvance.PERFORM, out sync_perform);
IList <IKVPairInstance <IInteger, GIF> >[] buffer = new IList <IKVPairInstance <IInteger, GIF> > [m_size];
for (int i = 0; i < m_size; i++)
{
buffer [i] = new List <IKVPairInstance <IInteger, GIF> > ();
}
Console.WriteLine(this.Rank + ": BEGIN READING CHUNKS and distributing to MAPPERS m_size=" + m_size);
if (!input_instance.has_next())
{
end_iteration = true;
}
Bin_function_gif.NumberOfPartitions = m_size;
while (input_instance.fetch_next(out bin_object))
{
IKVPairInstance <IInteger, GIF> item = (IKVPairInstance <IInteger, GIF>)bin_object;
this.Input_key_gif.Instance = item.Value; //item.Key;
Bin_function_gif.go();
int index = ((IIntegerInstance)this.Output_key_gif.Instance).Value;
buffer [index].Add(item);
}
Console.WriteLine(this.Rank + ": END READING CHUNKS and distributing to MAPPERS");
for (int i = 0; i < m_size; i++)
{
Console.WriteLine("buffer[" + i + "].Count = " + buffer [i].Count);
}
Console.WriteLine(this.Rank + ": BEGIN SENDING CHUNKS to the MAPPERS ");
// PERFORM
for (int i = 0; i < m_size; i++)
{
Split_channel.Send(buffer [i], unit_ref [i], end_iteration ? TAG_SPLIT_END_CHUNK : TAG_SPLIT_NEW_CHUNK);
}
sync_perform.wait();
Console.WriteLine(this.Rank + ": END SENDING CHUNKS to the MAPPERS ");
Task_binding_split_first.invoke(ITaskPortAdvance.CHUNK_READY);
}
Console.WriteLine(this.Rank + ": FINISH SPLITTER READ SOURCE ");
t_output.Join();
Console.WriteLine(this.Rank + ": FINISH SPLITTER READ SOURCE - INVOKE TERMINATE");
}
