private void receive_pairs_iteration(IPortTypeIterator output_instance, int senders_size, IDictionary<int,Tuple<int,int>> unit_ref)
{
while (true) // next iteration
{
int count_finished_streams = 0;
while (count_finished_streams == 0) // take next chunk ...
{
// PERFORM
for (int i = 0; i < senders_size; i++)
{
IList<object /*IKVPairInstance<ORK,ORV>*/> buffer;
CompletedStatus status;
Split_channel.Receive (unit_ref [i], MPI.Communicator.anyTag, out buffer, out status);
foreach (object /*IKVPairInstance<ORK,ORV>*/ kv in buffer)
output_instance.put (kv);
if (status.Tag == TAG_SPLIT_PAIR_FINISH)
count_finished_streams++;
}
output_instance.finish ();
Debug.Assert (count_finished_streams == 0 || count_finished_streams == senders_size);
// CHUNK_READY
Task_port_split.invoke (ITaskPortAdvance.CHUNK_READY);
}
// send empty iterator to inform the mappers the end of the iteration ...
output_instance.finish ();
}
}
private void receive_pairs_iteration(IPortTypeIterator output_instance, int senders_size, Tuple <int, int> unit_ref)
{
CompletedStatus status;
do
{
Console.WriteLine("SPLITER WRITE SINK 7-1 facet=" + unit_ref.Item1 + "/rank=" + unit_ref.Item2);
IList <IKVPairInstance <OKey, OValue> > buffer;
lock (cs_recv) Split_channel.Receive(unit_ref, MPI.Communicator.anyTag, out buffer, out status);
foreach (IKVPairInstance <OKey, OValue> kv in buffer)
{
output_instance.put(kv);
}
Console.WriteLine("SPLITER WRITE SINK 7-2 facet=" + unit_ref.Item1 + "/rank=" + unit_ref.Item2);
} while (status.Tag == TAG_SPLIT_NEW_CHUNK);
Console.WriteLine("SPLITER WRITE SINK - OUTPUT RECEIVED");
}
public override void main()
{
Console.WriteLine(this.Rank + ": SPLITTER 1 ");
IPortTypeIterator input_instance = (IPortTypeIterator)Source.Client;
// TODO: será que READ_SOURCE é necessária ???
Task_binding_data.TraceFlag = true;
Task_binding_data.invoke(ITaskPortData.READ_SOURCE);
// CALCULATE SubGraph TARGET
int nfr = this.FacetMultiplicity [FACET_REDUCE];
int r_size = 0;
foreach (int i in this.FacetIndexes[FACET_REDUCE])
{
r_size += this.UnitSizeInFacet [i] ["gusty_collector"];
}
Console.WriteLine(this.Rank + ": SPLITTER 2 ");
Source.startReadSource(r_size); // subgrafos divididos em GIF, com destino final nos r_size redutores
Console.WriteLine(this.Rank + ": SPLITTER 3 ");
object bin_object = null;
bool set_table_partition = true;
// CALCULATE TARGETs
IDictionary <int, Tuple <int, int> > unit_ref = new Dictionary <int, Tuple <int, int> > ();
int nf = this.FacetMultiplicity [FACET_MAP];
int m_size = 0;
foreach (int i in this.FacetIndexes[FACET_MAP])
{
int nr0 = m_size;
m_size += this.UnitSizeInFacet [i] ["gusty_feeder"];
for (int k = 0, j = nr0; j < m_size; k++, j++)
{
unit_ref [j] = new Tuple <int, int> (i /*, 0 INDEX OF gusty_feeder */, k);
}
}
Console.WriteLine(this.Rank + ": SPLITTER 4 ");
Task_binding_split_first.TraceFlag = true;
Split_channel.TraceFlag = true;
Thread t_output = new Thread(new ThreadStart(delegate
{
Task_binding_data.invoke(ITaskPortData.TERMINATE);
Task_binding_data.invoke(ITaskPortData.WRITE_SINK);
}));
t_output.Start();
Console.WriteLine(this.Rank + ": SPLITTER 5 ");
bool end_iteration = false;
while (!end_iteration) // take next chunk
{
IActionFuture sync_perform;
Console.WriteLine(this.Rank + ": SPLITTER READ SOURCE - BEFORE READ_CHUNK " + m_size);
/* All the pairs will be read from the source in a single chunk */
Task_binding_split_first.invoke(ITaskPortAdvance.READ_CHUNK); //****
Task_binding_split_first.invoke(ITaskPortAdvance.PERFORM, out sync_perform);
IList <IKVPairInstance <IInteger, GIF> >[] buffer = new IList <IKVPairInstance <IInteger, GIF> > [m_size];
for (int i = 0; i < m_size; i++)
{
buffer [i] = new List <IKVPairInstance <IInteger, GIF> > ();
}
Console.WriteLine(this.Rank + ": BEGIN READING CHUNKS and distributing to MAPPERS m_size=" + m_size);
if (!input_instance.has_next())
{
end_iteration = true;
}
Bin_function_gif.NumberOfPartitions = m_size;
while (input_instance.fetch_next(out bin_object))
{
IKVPairInstance <IInteger, GIF> item = (IKVPairInstance <IInteger, GIF>)bin_object;
this.Input_key_gif.Instance = item.Value; //item.Key;
Bin_function_gif.go();
int index = ((IIntegerInstance)this.Output_key_gif.Instance).Value;
buffer [index].Add(item);
}
Console.WriteLine(this.Rank + ": END READING CHUNKS and distributing to MAPPERS");
for (int i = 0; i < m_size; i++)
{
Console.WriteLine("buffer[" + i + "].Count = " + buffer [i].Count);
}
Console.WriteLine(this.Rank + ": BEGIN SENDING CHUNKS to the MAPPERS ");
// PERFORM
for (int i = 0; i < m_size; i++)
{
Split_channel.Send(buffer [i], unit_ref [i], end_iteration ? TAG_SPLIT_END_CHUNK : TAG_SPLIT_NEW_CHUNK);
}
sync_perform.wait();
Console.WriteLine(this.Rank + ": END SENDING CHUNKS to the MAPPERS ");
Task_binding_split_first.invoke(ITaskPortAdvance.CHUNK_READY);
}
Console.WriteLine(this.Rank + ": FINISH SPLITTER READ SOURCE ");
t_output.Join();
Console.WriteLine(this.Rank + ": FINISH SPLITTER READ SOURCE - INVOKE TERMINATE");
}
public override void main()
{
Split_channel.TraceFlag = true;
Task_binding_data.TraceFlag = true;
Console.WriteLine("SPLITER WRITE SINK 0");
Sink.clientConnection();
IPortTypeIterator output_instance = (IPortTypeIterator)Sink.Client;
Console.WriteLine("SPLITER WRITE SINK 1");
Task_binding_data.TraceFlag = true;
Task_binding_data.invoke(ITaskPortData.READ_SOURCE);
// Do nothing ...
Console.WriteLine("SPLITER WRITE SINK 3");
// RECEIVE PAIRS FROM THE REDUCERS (next iterations)
IDictionary <int, Tuple <int, int> > unit_ref = new Dictionary <int, Tuple <int, int> > ();
unit_ref = new Dictionary <int, Tuple <int, int> > ();
int senders_size = 0;
foreach (int i in this.FacetIndexes[FACET_REDUCE])
{
int nr0 = senders_size;
senders_size += this.UnitSizeInFacet [i] ["gusty_collector"];
for (int k = 0, j = nr0; j < senders_size; k++, j++)
{
unit_ref [j] = new Tuple <int, int> (i, k);
}
}
Console.WriteLine("SPLITER WRITE SINK 4");
Thread t_output = new Thread(new ThreadStart(delegate
{
Task_binding_data.invoke(ITaskPortData.TERMINATE);
Task_binding_data.invoke(ITaskPortData.WRITE_SINK);
Sink.startWriteSink();
}));
t_output.Start();
Thread[] receive_pairs_iteration_thread = new Thread[senders_size];
for (int i = 0; i < senders_size; i++)
{
receive_pairs_iteration_thread [i] = new Thread(new ParameterizedThreadStart(delegate(object i_obj) {
int ii = (int)i_obj;
receive_pairs_iteration(output_instance, senders_size, unit_ref[ii]);
}));
receive_pairs_iteration_thread [i].Start(i);
}
for (int i = 0; i < senders_size; i++)
{
receive_pairs_iteration_thread [i].Join();
}
output_instance.finish();
t_output.Join();
Console.WriteLine("SPLITER WRITE SINK - WRITTER FINISH");
Split_channel.TraceFlag = false;
Task_binding_data.TraceFlag = false;
}
