private void receive_pairs_iteration(IPortTypeIterator output_instance, int senders_size, IDictionary<int,Tuple<int,int>> unit_ref)
{
while (true) // next iteration
{
int count_finished_streams = 0;
while (count_finished_streams == 0) // take next chunk ...
{
// PERFORM
for (int i = 0; i < senders_size; i++)
{
IList<object /*IKVPairInstance<ORK,ORV>*/> buffer;
CompletedStatus status;
Split_channel.Receive (unit_ref [i], MPI.Communicator.anyTag, out buffer, out status);
foreach (object /*IKVPairInstance<ORK,ORV>*/ kv in buffer)
output_instance.put (kv);
if (status.Tag == TAG_SPLIT_PAIR_FINISH)
count_finished_streams++;
}
output_instance.finish ();
Debug.Assert (count_finished_streams == 0 || count_finished_streams == senders_size);
// CHUNK_READY
Task_port_split.invoke (ITaskPortAdvance.CHUNK_READY);
}
// send empty iterator to inform the mappers the end of the iteration ...
output_instance.finish ();
}
}
private void file_reader()
{
string fileName = System.Environment.GetEnvironmentVariable("PATH_GRAPH_FILE");
IInputFormatInstance extractor_input_format = (IInputFormatInstance)server.fetchFileContentObject();
extractor_input_format.PARTITION_SIZE = partition_size;
IDictionary <int, IInputFormatInstance> sub_formats = extractor_input_format.extractBins(fileName);
this.PartitionTABLE = extractor_input_format.PartitionTABLE;
foreach (IInputFormatInstance shard in sub_formats.Values)
{
IKVPairInstance <IInteger, IInputFormat> item = (IKVPairInstance <IInteger, IInputFormat>)client.createItem();
int first_vertex_id_from_partition = shard.firstVertex(shard.PARTID);
((IIntegerInstance)item.Key).Value = first_vertex_id_from_partition; //counter_write_global;
item.Value = shard;
client.put(item);
counter_write_chunk++; //counter_write_global++;
if (counter_write_chunk >= CHUNK_SIZE)
{
Console.WriteLine("NEW CHUNK size=" + counter_write_chunk);
counter_write_chunk = 0;
client.finish();
}
}
client.finish();
client.finish();
Console.WriteLine("FINISHING READING DATA SOURCE");
}
public override void main()
{
Split_channel.TraceFlag = true;
Task_binding_data.TraceFlag = true;
Console.WriteLine("SPLITER WRITE SINK 0");
Sink.clientConnection();
IPortTypeIterator output_instance = (IPortTypeIterator)Sink.Client;
Console.WriteLine("SPLITER WRITE SINK 1");
Task_binding_data.TraceFlag = true;
Task_binding_data.invoke(ITaskPortData.READ_SOURCE);
// Do nothing ...
Console.WriteLine("SPLITER WRITE SINK 3");
// RECEIVE PAIRS FROM THE REDUCERS (next iterations)
IDictionary <int, Tuple <int, int> > unit_ref = new Dictionary <int, Tuple <int, int> > ();
unit_ref = new Dictionary <int, Tuple <int, int> > ();
int senders_size = 0;
foreach (int i in this.FacetIndexes[FACET_REDUCE])
{
int nr0 = senders_size;
senders_size += this.UnitSizeInFacet [i] ["gusty_collector"];
for (int k = 0, j = nr0; j < senders_size; k++, j++)
{
unit_ref [j] = new Tuple <int, int> (i, k);
}
}
Console.WriteLine("SPLITER WRITE SINK 4");
Thread t_output = new Thread(new ThreadStart(delegate
{
Task_binding_data.invoke(ITaskPortData.TERMINATE);
Task_binding_data.invoke(ITaskPortData.WRITE_SINK);
Sink.startWriteSink();
}));
t_output.Start();
Thread[] receive_pairs_iteration_thread = new Thread[senders_size];
for (int i = 0; i < senders_size; i++)
{
receive_pairs_iteration_thread [i] = new Thread(new ParameterizedThreadStart(delegate(object i_obj) {
int ii = (int)i_obj;
receive_pairs_iteration(output_instance, senders_size, unit_ref[ii]);
}));
receive_pairs_iteration_thread [i].Start(i);
}
for (int i = 0; i < senders_size; i++)
{
receive_pairs_iteration_thread [i].Join();
}
output_instance.finish();
t_output.Join();
Console.WriteLine("SPLITER WRITE SINK - WRITTER FINISH");
Split_channel.TraceFlag = false;
Task_binding_data.TraceFlag = false;
}
