public void emite()
{
string saida = ((IStringInstance)Input_data.Instance).Value;
IIteratorInstance <IKVPair <IInteger, IPageNode> > output = (IIteratorInstance <IKVPair <IInteger, IPageNode> >)Output_data.Instance;
double X = 0.0;
string[] lines = saida.Split(new char[] { System.Environment.NewLine[0] });
string[] pr = lines[0].Split(' ');
if (pr [0].Equals("X"))
{
X = double.Parse(pr [1]) / dic.Count;
}
else
{
System.Console.WriteLine("Error of index to X");
}
lines [0] = "";
foreach (string line in lines)
{
if (!line.Trim().Equals(""))
{
string[] prank = line.Split(' ');
//if (b && !prank [0].Equals ("X")) {
IKVPairInstance <IInteger, IPageNode> kvpair = dic [int.Parse(prank [0])];
IPageNodeInstance no = (IPageNodeInstance)kvpair.Value;
no.PgrankInstance = double.Parse(prank [1]) + X;
output.put(kvpair);
//}
}
}
output.finish();
}
public override void main()
{
Trace.WriteLine(WorldComm.Rank + ": STARTING BREAK IN LINE (SPLIT FUNCTION)");
IStringInstance input_data_instance = (IStringInstance)Input_data.Instance;
IIteratorInstance <IKVPair <IInteger, IString> > output_data_instance = (IIteratorInstance <IKVPair <IInteger, IString> >)Output_data.Instance;
string s = input_data_instance.Value;
string[] lines = s.Split(new char[] { System.Environment.NewLine[0] });
int line_counter = 0;
foreach (string line in lines)
{
if (!line.Trim().Equals(""))
{
Trace.WriteLine(WorldComm.Rank + ": LINE = " + line);
IKVPairInstance <IInteger, IString> line_pair = (IKVPairInstance <IInteger, IString>)Output_data.createItem();
((IIntegerInstance)line_pair.Key).Value = line_counter++;
((IStringInstance)line_pair.Value).Value = line;
output_data_instance.put(line_pair);
}
}
Trace.WriteLine(WorldComm.Rank + ": FINISH BREAK IN LINES #1 !!! " + output_data_instance.GetHashCode());
output_data_instance.finish();
Trace.WriteLine(WorldComm.Rank + ": FINISH BREAK IN LINES #2 !!!");
}
public override void main()
{
IIteratorInstance <IKVPair <IMK, IMV> > input_instance = (IIteratorInstance <IKVPair <IMK, IMV> >)Input.Instance;
IIteratorInstance <IKVPair <OMK, OMV> > output_instance = (IIteratorInstance <IKVPair <OMK, OMV> >)Output.Instance;
object bin_object = null;
int count = 0;
while (input_instance.fetch_next(out bin_object))
{
Trace.WriteLine(Rank + ": LOOP MAPPER !!!" + (count++));
IKVPairInstance <IMK, IMV> bin = (IKVPairInstance <IMK, IMV>)bin_object;
Trace.WriteLine(Rank + ": " + (Map_value.Instance == null) + " $$$$ " + bin.Value.GetType());
Map_key.Instance = bin.Key;
Map_value.Instance = bin.Value;
Map_function.go();
}
Trace.WriteLine(Rank + ": FINISH MAPPER !!!");
output_instance.finish();
}
public override void main()
{
IIteratorInstance <IKVPair <IVertex, IDataTriangle> > output_pairs = (IIteratorInstance <IKVPair <IVertex, IDataTriangle> >)Output_pairs.Instance;
IIteratorInstance <IKVPair <IVertex, IDataTriangle> > input_pairs = (IIteratorInstance <IKVPair <IVertex, IDataTriangle> >)Input_pairs.Instance;
object pair;
// For 3 iterations: startup_push, gust0, gust1
while ((iterate_num++) < 2)
{
while (iterate_pairs.fetch_next(out pair))
{
input_pairs.put(pair);
}
input_pairs.finish();
}
input_pairs.finish();
// Finally, all pairs are sent to the output.
while (iterate_pairs.fetch_next(out pair))
{
output_pairs.put(pair);
}
output_pairs.finish();
}
public override void main()
{
Trace.WriteLine(Rank + ": START COMBINE FUNCTION #1");
IIteratorInstance <ORV> input_data_instance = (IIteratorInstance <ORV>)Input_data.Instance;
IKVPairInstance <IInteger, IIterator <IPathInfo> > output_data_instance = (IKVPairInstance <IInteger, IIterator <IPathInfo> >)Output_data.Instance;
Trace.WriteLine(Rank + ": START COMBINE FUNCTION #2");
IIntegerInstance done_flag = (IIntegerInstance)output_data_instance.Key;
IIteratorInstance <IPathInfo> output_distances = (IIteratorInstance <IPathInfo>)output_data_instance.Value;
Trace.WriteLine(Rank + ": START COMBINE FUNCTION #3");
bool done = true;
object o;
while (input_data_instance.fetch_next(out o))
{
IKVPairInstance <IInteger, IIterator <IPathInfo> > kv = (IKVPairInstance <IInteger, IIterator <IPathInfo> >)o;
IIntegerInstance k = (IIntegerInstance)kv.Key;
Trace.WriteLine(Rank + ": LOOP #1 COMBINE FUNCTION " + k.Value);
output_distances.putAll((IIteratorInstance <IPathInfo>)kv.Value);
done = done && k.Value == 1;
}
output_distances.finish();
done_flag.Value = done ? 1 : 0;
Trace.WriteLine(Rank + ": END COMBINE FUNCTION " + done);
}
public override void main()
{
Trace.WriteLine(Rank + ": START SPLIT PATH INFO 1");
IIteratorInstance <IPathInfo> input_data_instance = (IIteratorInstance <IPathInfo>)Input_data.Instance;
IIteratorInstance <IKVPair <IInteger, IPathInfo> > output_data_instance = (IIteratorInstance <IKVPair <IInteger, IPathInfo> >)Output_data.Instance;
Trace.WriteLine(Rank + ": START SPLIT PATH INFO 2");
object path_info;
while (input_data_instance.fetch_next(out path_info))
{
Trace.WriteLine(Rank + ": LOOP SPLIT PATH INFO " + path_info.GetType());
IPathInfoInstance path_info_instance = (IPathInfoInstance)path_info;
IKVPairInstance <IInteger, IPathInfo> path_info_output = (IKVPairInstance <IInteger, IPathInfo>)Output_data.createItem();
((IIntegerInstance)path_info_output.Key).Value = ((Info)path_info_instance.Value).vertex;
path_info_output.Value = path_info_instance;
Trace.WriteLine(Rank + ": vertex = " + path_info_output.Key);
output_data_instance.put(path_info_output);
}
output_data_instance.finish();
Trace.Write(Rank + ": STOP SPLIT PATH INFO ");
}
/* Recebimento de ORVs das unidades source */
public void receiveCombineORVs()
{
int listenFinishedObject = 0;
Trace.WriteLine(WorldComm.Rank + ": START COMBINER TARGET !!!");
object orv;
MPI.CompletedStatus status;
while (listenFinishedObject < size_reducers)
{
Trace.WriteLine(WorldComm.Rank + ": BEGIN RECEIVE COMBINER TARGET to ");
comm.Receive <object>(MPI.Unsafe.MPI_ANY_SOURCE, MPI.Unsafe.MPI_ANY_TAG, out orv, out status);
Trace.WriteLine(WorldComm.Rank + ": END RECEIVE COMBINER TARGET to " + (status.Source));
if (status.Tag == TAG_COMBINER_ORV_FINISH)
{
listenFinishedObject = listenFinishedObject + 1;
}
else
{
combine_input_data_instance.put(orv);
}
}
combine_input_data_instance.finish();
Trace.WriteLine(WorldComm.Rank + ": FINISH COMBINER TARGET !!!");
}
public override void main()
{
/* 1. Ler pares chave (OMK) e valores (OMV) de Input.
* 2. Para cada par, atribuir a Key e Values e chamar Reduce_function.go();
* 3. Pegar o resultado de Reduction_function.go() de Output_reduce (ORV)
* e colocar no iterator Output.
*/
IIteratorInstance <IKVPair <OMK, IIterator <OMV> > > input_instance = (IIteratorInstance <IKVPair <OMK, IIterator <OMV> > >)Input.Instance;
IIteratorInstance <ORV> output_instance = (IIteratorInstance <ORV>)Output.Instance;
//long t0 = (long)(DateTime.UtcNow - (new DateTime (1970, 1, 1, 0, 0, 0, DateTimeKind.Utc))).TotalMilliseconds;
object kvpair_object;
while (input_instance.fetch_next(out kvpair_object))
{
IKVPairInstance <OMK, IIterator <OMV> > kvpair = (IKVPairInstance <OMK, IIterator <OMV> >)kvpair_object;
Input_reduce.Instance = kvpair;
Reduce_function.go();
active++;
active = active - Reduce_function.Active;
}
string candidates_buffer = sincronize();
while (active > 0)
{
active = 0;
string[] lines = candidates_buffer.Split(new char[] { System.Environment.NewLine[0] });
foreach (string line in lines)
{
if (!line.Trim().Equals(""))
{
string[] values = line.Split(' ');
Reduce_function.step(int.Parse(values[0]), double.Parse(values[1]));
active++;
active = active - Reduce_function.Active;
}
}
candidates_buffer = sincronize();
}
Reduce_function.apply();
output_instance.put(Output_reduce.Instance);
output_instance.finish();
//long t1 = (long)(DateTime.UtcNow - (new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc))).TotalMilliseconds;
//writeFile ("./logTIME-REDUCE", "TIME: " + (t1 - t0) + " ms."+System.Environment.NewLine+"BSP");
}
public override void main()
{
Console.WriteLine("################################*IAplicacaoImpl Start ###############################");
IDoubleInstance tDouble = (IDoubleInstance)T_double.Instance;
IKVPairInstance <IString, IDouble> tKVPair = (IKVPairInstance <IString, IDouble>)T_kvpair.Instance; //.newInstance ();
IIntegerInstance tInteger = (IIntegerInstance)T_integer.Instance;
IIteratorInstance <IInteger> tIterator = (IIteratorInstance <IInteger>)T_iterator.Instance; //.newInstance ();
IStringInstance tString = (IStringInstance)T_string.Instance;
tDouble.Value = 0.8;
tInteger.Value = 8;
tString.Value = "Tipo String";
((IStringInstance)tKVPair.Key).Value = tString.Value;
((IDoubleInstance)tKVPair.Value).Value = tDouble.Value;
tIterator.put(tInteger);
tIterator.finish();
object o; int count = 0;
while (tIterator.fetch_next(out o))
{
IIntegerInstance oI = (IIntegerInstance)o;
Console.WriteLine("Item Iterator " + (count++) + ": " + oI.Value);
}
Console.WriteLine("tDouble.Value=" + tDouble.Value + " : " + "tInteger.Value=" + tInteger.Value + " : " + "tString.Value=" + tString.Value);
Console.WriteLine("tKVPair.Key=" + ((IStringInstance)tKVPair.Key).Value + " : " + "tKVPair.Value=" + ((IDoubleInstance)tKVPair.Value).Value);
IPageNodeInstance instance = (IPageNodeInstance)T_aplicacao_node.Instance;
((IIntegerInstance)instance.IdInstance).Value = 8;
IIntegerInstance item = (IIntegerInstance)T_aplicacao_node.Edge_node.createItem();
item.Value = 7;
instance.NeighborsInstance.put(item);
instance.NeighborsInstance.put(((IIntegerInstance)instance.IdInstance));
instance.NeighborsInstance.finish();
while (instance.NeighborsInstance.fetch_next(out o))
{
IIntegerInstance II = (IIntegerInstance)o;
Console.WriteLine("%%%%%%%%%%%%%%%%%%%%%%%%%%%% Valor no Iterator=" + II.Value);
}
Console.WriteLine("%%%%%%%%%%%%%%%%%%%%%%%%%%%% Resultado Id_node=" + ((IIntegerInstance)instance.IdInstance).Value + " PGRank=" + instance.PgrankInstance.Value);
Console.WriteLine("################################ IAplicacaoImpl End ###############################");
}
public void gust0() //emite saídas
{
IIteratorInstance <IKVPair <IInteger, IDataSSSP> > output_value_instance = (IIteratorInstance <IKVPair <IInteger, IDataSSSP> >)Output.Instance;
bool any_emite = false;
foreach (bool any in emite)
{
any_emite = any_emite || any;
}
if (!any_emite && halt_sum == 0)
{
output_value_instance.finish(); // TerminatedFunctionSSSP é avisado com um finish(), preparando-se para a emissão definitiva de saída
IKVPairInstance <IInteger, IDataSSSP> ITEM = (IKVPairInstance <IInteger, IDataSSSP>)Output.createItem();
((IIntegerInstance)ITEM.Key).Value = this.partid;
((IDataSSSPInstance)ITEM.Value).Path_size = messages [((IIntegerInstance)ITEM.Key).Value];
((IDataSSSPInstance)ITEM.Value).Halt = 0;
output_value_instance.put(ITEM);
}
else
{
for (int i = 0; i < partition_size; i++)
{
IKVPairInstance <IInteger, IDataSSSP> ITEM = (IKVPairInstance <IInteger, IDataSSSP>)Output.createItem();
((IIntegerInstance)ITEM.Key).Value = i;
if (partition_own [i] || !emite [i])
{
((IDataSSSPInstance)ITEM.Value).Path_size = new Dictionary <int, float> ();
}
else
{
((IDataSSSPInstance)ITEM.Value).Path_size = messages [i];
}
((IDataSSSPInstance)ITEM.Value).Halt = any_emite ? 1 : 0;
output_value_instance.put(ITEM);
}
}
emite = new bool[partition_size];
halt_sum = 0;
}
public override void main()
{
IIteratorInstance <ORV> input_data_instance = (IIteratorInstance <ORV>)Input_data.Instance;
IIteratorInstance <ORV> output_data_instance = (IIteratorInstance <ORV>)Output_data.Instance;
Trace.WriteLine(WorldComm.Rank + ": START COMBINE FUNCTION !!! ");
object item_object;
int count = 0;
while (input_data_instance.fetch_next(out item_object))
{
Trace.WriteLine(WorldComm.Rank + ": COMBINE_FUNCTION_LOOP 1 " + (count++) + " " + item_object.GetType());
output_data_instance.put(item_object);
}
Trace.WriteLine(WorldComm.Rank + ": OUT LOOP COMBINE FUNCTION !!!");
output_data_instance.finish();
Trace.WriteLine(WorldComm.Rank + ": FINISH COMBINE FUNCTION !!!");
}
private void graph_creator()
{
IIteratorInstance <IKVPair <IInteger, IIterator <GIF> > > input_instance_gif = (IIteratorInstance <IKVPair <IInteger, IIterator <GIF> > >)Collect_graph.Client;
IIteratorInstance <IKVPair <IInteger, GIF> > output_instance_gif = (IIteratorInstance <IKVPair <IInteger, GIF> >)Output_gif.Instance;
Feed_graph.Server = output_instance_gif;
IActionFuture sync_perform;
bool end_iteration = false;
while (!end_iteration) // take next chunk ...
{
Task_gusty.invoke(ITaskPortAdvance.READ_CHUNK);
Task_gusty.invoke(ITaskPortAdvance.PERFORM, out sync_perform);
IKVPairInstance <IInteger, IIterator <GIF> > kvpair = null;
object kvpair_object;
object o;
if (!input_instance_gif.has_next())
{
end_iteration = true;
}
while (input_instance_gif.fetch_next(out kvpair_object))
{
kvpair = (IKVPairInstance <IInteger, IIterator <GIF> >)kvpair_object;
Graph_values.Instance = kvpair;
Gusty_function.graph_creator();
}
sync_perform.wait();
}
//IActionFuture gusty_chunk_ready;
//Task_gusty.invoke (ITaskPortAdvance.CHUNK_READY, out gusty_chunk_ready); //***
output_instance_gif.finish();
//gusty_chunk_ready.wait ();
}
public override void main()
{
Console.WriteLine(this.Rank + ": TERMINATE FUNCTION 1");
IIteratorInstance <IKVPair <OKey, OValue> > output_pairs = (IIteratorInstance <IKVPair <OKey, OValue> >)Output_pairs.Instance;
IIteratorInstance <IKVPair <IKey, IValue> > input_pairs = (IIteratorInstance <IKVPair <IKey, IValue> >)Input_pairs.Instance;
Console.WriteLine(this.Rank + ": TERMINATE FUNCTION 2");
input_pairs.finish();
// All pairs are sent to the output, i.e. the algorithm is not iterative.
object pair;
while (iterate_pairs.fetch_next(out pair))
{
output_pairs.put(pair);
}
output_pairs.finish();
Console.WriteLine(this.Rank + ": TERMINATE FUNCTION 3");
}
public override void main()
{
int count = 0;
// 1. recebe os bins enviados pelo gerente (unidade source),
// através do MPI, e os insere no Target_data.
Trace.WriteLine(WorldComm.Rank + ": STARTING SCATTER SPLIT DATA TARGET");
MPI.CompletedStatus status;
int source_rank = this.UnitRanks["source"][0];
object bin_key;
object bin_value;
IIteratorInstance <IKVPair <IMK, IMV> > target_data_instance = (IIteratorInstance <IKVPair <IMK, IMV> >)Target_data.Instance;
Trace.WriteLine(WorldComm.Rank + ": BEGIN RECEIVE BIN KEY from " + source_rank + "count=" + (count++));
comm.Receive <object> (source_rank, MPI.Unsafe.MPI_ANY_TAG, out bin_key, out status);
Trace.WriteLine(WorldComm.Rank + ": END RECEIVE BIN KEY from " + source_rank);
while (status.Tag != TAG_SPLITTER_IMK_FINISH)
{
Trace.WriteLine(WorldComm.Rank + ": BEGIN RECEIVE BIN VALUE from " + source_rank);
comm.Receive <object> (source_rank, TAG_SPLITTER_IMV, out bin_value, out status);
Trace.WriteLine(WorldComm.Rank + ": END RECEIVE BIN VALUE from " + source_rank);
IKVPairInstance <IMK, IMV> pair = (IKVPairInstance <IMK, IMV>)Target_data.createItem();
pair.Key = bin_key;
pair.Value = bin_value;
target_data_instance.put(pair);
Trace.WriteLine(WorldComm.Rank + ": BEGIN RECEIVE BIN KEY from " + source_rank + "count=" + (count++));
comm.Receive <object> (source_rank, MPI.Unsafe.MPI_ANY_TAG, out bin_key, out status);
Trace.WriteLine(WorldComm.Rank + ": END RECEIVE BIN KEY from " + source_rank);
}
Trace.WriteLine(WorldComm.Rank + ": FINISH ALL BIN KEYs #1 !!!");
target_data_instance.finish();
Trace.WriteLine(WorldComm.Rank + ": FINISH ALL BIN KEYs #2 !!!");
}
public override void main()
{
int di = int.MaxValue;
int dmin = int.MaxValue;
int done = 1;
Trace.WriteLine(Rank + ": START REDUCE FUNCTION");
IKVPairInstance <IInteger, IIterator <IPathInfo> > input_instance = (IKVPairInstance <IInteger, IIterator <IPathInfo> >)Input_values.Instance;
IIntegerInstance k = (IIntegerInstance)input_instance.Key;
IIteratorInstance <IPathInfo> v = (IIteratorInstance <IPathInfo>)input_instance.Value;
int k_int = k.Value;
if (!neighbours.ContainsKey(k_int))
{
neighbours [k_int] = new Dictionary <int, int> ();
}
object o;
while (v.fetch_next(out o))
{
IPathInfoInstance item = (IPathInfoInstance)o;
Info item_info = (Info)item.Value;
Trace.WriteLine(Rank + ": REDUCE FUNCTION LOOP " + item_info);
switch (item_info.info_type)
{
case Info.PATH_INFO_TYPE_DISTANCE_TRIAL:
DistanceInfo distance_item_info_1 = (DistanceInfo)item_info;
int tmp = distance_item_info_1.distance;
dmin = (int)min(dmin, tmp);
break;
case Info.PATH_INFO_TYPE_DISTANCE_PARTIAL:
DistanceInfo distance_item_info_2 = (DistanceInfo)item_info;
di = distance_item_info_2.distance;
break;
case Info.PATH_INFO_TYPE_EDGE:
EdgeInfo edge_item_info = (EdgeInfo)item_info;
IDictionary <int, int> output_neibours = neighbours[k_int];
int n = edge_item_info.vertex_other;
int d = 0;
if (!output_neibours.TryGetValue(n, out d))
{
output_neibours[n] = edge_item_info.weight;
}
else
if (edge_item_info.weight < d)
{
output_neibours[n] = edge_item_info.weight;
}
break;
}
}
Trace.WriteLine(Rank + ": REDUCE FUNCTION AFTER LOOP #1 ");
IKVPairInstance <IInteger, IIterator <IPathInfo> > orv = (IKVPairInstance <IInteger, IIterator <IPathInfo> >)Output_value.newInstance();
IIteratorInstance <IPathInfo> buffer = (IIteratorInstance <IPathInfo>)orv.Value;
Trace.WriteLine(Rank + ": REDUCE FUNCTION OUT LOOP #2 ");
//IPathInfo path_info = buffer.createItem();
dmin = (int)min(dmin, di);
if (dmin != di)
{
foreach (KeyValuePair <int, int> kv in neighbours[k_int])
{
//IPathInfoInstance path_info_instance = (IPathInfoInstance) path_info.newInstance ();
IPathInfoInstance path_info_instance = (IPathInfoInstance)buffer.createItem();
DistanceInfo distance_info = new DistanceInfo();
distance_info.info_type = Info.PATH_INFO_TYPE_DISTANCE_TRIAL;
distance_info.vertex = kv.Key;
distance_info.distance = kv.Value + dmin;
path_info_instance.Value = distance_info;
buffer.put(path_info_instance);
Trace.WriteLine(Rank + ": REDUCE FUNCTION OTHER LOOP " + distance_info);
}
done = 0;
}
((IIntegerInstance)orv.Key).Value = done;
IPathInfoInstance path_info_instance_2 = (IPathInfoInstance)buffer.createItem();
DistanceInfo distance_info_2 = new DistanceInfo();
distance_info_2.info_type = Info.PATH_INFO_TYPE_DISTANCE_PARTIAL;
distance_info_2.vertex = k.Value;
distance_info_2.distance = dmin;
path_info_instance_2.Value = distance_info_2;
buffer.put(path_info_instance_2);
Trace.WriteLine(Rank + ": REDUCE FUNCTION OTHER LOOP OUT " + distance_info_2);
buffer.finish();
Trace.WriteLine(Rank + ": FINISH REDUCE FUNCTION");
}
private void send_to_mappers()
{
Console.WriteLine(this.Rank + ": IJoinGustyCollector 1");
IIteratorInstance <IKVPair <OKey, OValue> > output_instance = (IIteratorInstance <IKVPair <OKey, OValue> >)Output_pairs.Instance;
IIteratorInstance <IKVPair <IKey, IValue> > input_instance = (IIteratorInstance <IKVPair <IKey, IValue> >)Input_pairs.Instance;
object bin_object = null;
// DETERMINE COMMUNICATION TARGETs
Tuple <int, int> sink_ref = new Tuple <int, int> (this.FacetIndexes [FACET_SINK] [0], 0);
IDictionary <int, Tuple <int, int> > unit_ref = new Dictionary <int, Tuple <int, int> > ();
int m_size = 0;
foreach (int i in this.FacetIndexes[FACET_MAP])
{
int nr0 = m_size;
m_size += this.UnitSizeInFacet [i] ["gusty_feeder"];
for (int k = 0, j = nr0; j < m_size; k++, j++)
{
unit_ref [j] = new Tuple <int, int> (i /*,0 index of MAP_FEEDER*/, k);
}
}
Console.WriteLine(this.Rank + ": IJoinGustyCollector 2");
IActionFuture sync_perform;
bool end_computation = false;
while (!end_computation) // new iteration
{
Console.WriteLine(this.Rank + ": IJoinGustyCollector LOOP");
end_computation = true;
//Console.WriteLine (this.Rank + ": IJoinGustyCollector ITERATE 1");
//Task_binding_split_next.invoke (ITaskPortAdvance.READ_CHUNK); //****
//Task_binding_split_next.invoke (ITaskPortAdvance.PERFORM, out sync_perform);
//Console.WriteLine (this.Rank + ": IJoinGustyCollector ITERATE 2");
// SEND TO SINK
IList <IKVPairInstance <OKey, OValue> > buffer_sink = new List <IKVPairInstance <OKey, OValue> >();
end_computation = !input_instance.has_next();
Thread thread_send_output = new Thread(new ThreadStart(delegate {
int count1 = 0;
while (output_instance.fetch_next(out bin_object))
{
IKVPairInstance <OKey, OValue> item = (IKVPairInstance <OKey, OValue>)bin_object;
buffer_sink.Add(item);
if (count1 % CHUNK_SIZE == 0)
{
// PERFORM
Console.WriteLine(this.Rank + ": IJoinGustyCollector SINK SEND CHUNK 3-1 count=" + count1);
Split_channel.Send(buffer_sink, sink_ref, TAG_SPLIT_NEW_CHUNK);
Console.WriteLine(this.Rank + ": IJoinGustyCollector SINK SEND CHUNK 3-2 count=" + count1);
buffer_sink.Clear();
}
count1++;
}
if (buffer_sink.Count > 0)
{
Console.WriteLine(this.Rank + ": IJoinGustyCollector SINK SEND CHUNK 3-3 count=" + count1);
Split_channel.Send(buffer_sink, sink_ref, TAG_SPLIT_NEW_CHUNK);
Console.WriteLine(this.Rank + ": IJoinGustyCollector SINK SEND CHUNK 3-4 count=" + count1);
}
}));
thread_send_output.Start();
// SEND BACK TO MAPPER (new iteration)
Bin_function_iterate.NumberOfPartitions = m_size;
IList <IKVPairInstance <OKey, OValue> >[] buffer = new IList <IKVPairInstance <OKey, OValue> > [m_size];
for (int i = 0; i < m_size; i++)
{
buffer [i] = new List <IKVPairInstance <OKey, OValue> > ();
}
Console.WriteLine(this.Rank + ": IJoinGustyCollector ITERATE 3 end_computation=" + end_computation);
int count = 0;
while (input_instance.fetch_next(out bin_object))
{
Console.WriteLine(this.Rank + ": IJoinGustyCollector ITERATE LOOP 4-1 count=" + count);
IKVPairInstance <OKey, OValue> item = (IKVPairInstance <OKey, OValue>)bin_object;
this.Input_key_iterate.Instance = item.Key;
Bin_function_iterate.go();
int index = ((IIntegerInstance)this.Output_key_iterate.Instance).Value;
buffer [index].Add(item);
if (count % CHUNK_SIZE == 0)
{
Task_binding_split_next.invoke(ITaskPortAdvance.READ_CHUNK);
Task_binding_split_next.invoke(ITaskPortAdvance.PERFORM, out sync_perform);
for (int i = 0; i < m_size; i++)
{
Console.WriteLine("SPLITTER REDUCE COLLECTOR - Sending chunk of " + buffer[i].Count + " elements");
Split_channel.Send(buffer [i], unit_ref [i], TAG_SPLIT_NEW_CHUNK);
buffer [i].Clear();
}
sync_perform.wait();
Task_binding_split_next.invoke(ITaskPortAdvance.CHUNK_READY);
Console.WriteLine(this.Rank + ": IJoinGustyCollector ITERATE 3-5");
}
Console.WriteLine(this.Rank + ": IJoinGustyCollector ITERATE LOOP 4-2 count=" + count);
count++;
}
Console.WriteLine(this.Rank + ": IJoinGustyCollector ITERATE 5");
Task_binding_split_next.invoke(ITaskPortAdvance.READ_CHUNK);
Task_binding_split_next.invoke(ITaskPortAdvance.PERFORM, out sync_perform);
// SEND REMAINING PAIRS AND CLOSES THE CHUNK LIST
Console.WriteLine(this.Rank + ": IJoinGustyCollector ITERATE 6-1");
for (int i = 0; i < m_size; i++)
{
Split_channel.Send(buffer [i], unit_ref [i], TAG_SPLIT_END_CHUNK);
}
sync_perform.wait();
Console.WriteLine(this.Rank + ": IJoinGustyCollector ITERATE 7-1");
Task_binding_split_next.invoke(ITaskPortAdvance.CHUNK_READY);
Console.WriteLine(this.Rank + ": IJoinGustyCollector ITERATE 7-2");
thread_send_output.Join();
//sync_perform.wait ();
Console.WriteLine(this.Rank + ": IJoinGustyCollector ITERATE 7-3");
}
Console.WriteLine(this.Rank + ": IJoinGustyCollector END COMPUTATION ");
input_instance.finish();
output_instance.finish();
Split_channel.Send(new List <IKVPairInstance <OKey, OValue> >(), sink_ref, TAG_SPLIT_END_CHUNK);
}
private void readPair_OMK_OMVs()
{
Console.WriteLine(this.Rank + ": REDUCE 1");
IIteratorInstance <IKVPair <TKey, IIterator <TValue> > > input_instance = (IIteratorInstance <IKVPair <TKey, IIterator <TValue> > >)Collect_pairs.Client;
IIteratorInstance <IKVPair <OKey, OValue> > output_instance = (IIteratorInstance <IKVPair <OKey, OValue> >)Output.Instance;
Feed_pairs.Server = output_instance;
IActionFuture sync_perform;
// TODO: Dividir em chunks a saa de pares (OKey,OValue)
Console.WriteLine(this.Rank + ": REDUCER 2");
// object gusty_lock = new object ();
bool end_computation = false;
while (!end_computation) // new iteration
{
IDictionary <object, object> cont_dict = new Dictionary <object, object> ();
Console.WriteLine(this.Rank + ": REDUCER LOOP");
end_computation = true;
bool end_iteration = false;
while (!end_iteration) // take next chunk ...
{
Console.WriteLine(this.Rank + ": REDUCER ITERATE 1");
Task_gusty.invoke(ITaskPortAdvance.READ_CHUNK);
Task_gusty.invoke(ITaskPortAdvance.PERFORM, out sync_perform);
Console.WriteLine(this.Rank + ": REDUCER ITERATE 2");
IKVPairInstance <TKey, IIterator <TValue> > kvpair = null;
object kvpair_object;
if (!input_instance.has_next())
{
end_iteration = true;
}
else
{
end_computation = false;
}
int count = 0;
while (input_instance.fetch_next(out kvpair_object))
{
Console.WriteLine(this.Rank + ": REDUCER ITERATE INNER LOOP 3 count=" + count);
kvpair = (IKVPairInstance <TKey, IIterator <TValue> >)kvpair_object;
object acc_value;
if (!cont_dict.TryGetValue(kvpair.Key, out acc_value))
{
cont_dict[kvpair.Key] = new object();
}
else
{
((IDataInstance)Continue_value.Instance).ObjValue = acc_value;
}
Input_values.Instance = kvpair;
Gusty_function.go();
cont_dict [kvpair.Key] = ((IDataInstance)((IKVPairInstance <OKey, OValue>)Output_value.Instance).Value).ObjValue;
count++;
}
Console.WriteLine(this.Rank + ": REDUCER ITERATE 4 count=" + count);
sync_perform.wait();
Console.WriteLine(this.Rank + ": REDUCER ITERATE 5");
}
Console.WriteLine(this.Rank + ": REDUCER ITERATE END ITERATION");
int chunk_counter = 1;
IActionFuture gusty_chunk_ready;
Task_gusty.invoke(ITaskPortAdvance.CHUNK_READY, out gusty_chunk_ready); //***
foreach (KeyValuePair <object, object> output_pair in cont_dict)
{
IKVPairInstance <OKey, OValue> new_pair = (IKVPairInstance <OKey, OValue>)Output_value.newInstance();
new_pair.Key = output_pair.Key;
new_pair.Value = output_pair.Value;
output_instance.put(new_pair);
}
output_instance.finish();
gusty_chunk_ready.wait();
Console.WriteLine(this.Rank + ": REDUCER ITERATE FINISH");
}
Console.WriteLine(this.Rank + ": REDUCER FINISH ... ");
}
public override void main()
{
Trace.WriteLine(Rank + ": PARTITIONER (FETCH VALUES TARGET) - START ");
MPI.CompletedStatus status;
IKVPairInstance <OMK, OMV> kv;
int senders_size = this.UnitSize ["mapper"];
IIteratorInstance <IKVPair <OMK, IIterator <OMV> > > reduce_job_instance = (IIteratorInstance <IKVPair <OMK, IIterator <OMV> > >)Reduce_job.Instance;
IDictionary <object, IIteratorInstance <OMV> > kv_cache = new Dictionary <object, IIteratorInstance <OMV> >();
int finished_senders = 0;
Trace.WriteLine(Rank + ": PARTITIONER (FETCH VALUES TARGET) BEGIN RECEIVE 1 !");
comm.Receive <IKVPairInstance <OMK, OMV> >(MPI.Unsafe.MPI_ANY_SOURCE, MPI.Unsafe.MPI_ANY_TAG, out kv, out status);
Trace.WriteLine(Rank + ": PARTITIONER (FETCH VALUES TARGET) END RECEIVE 1 ! tag=" + status.Tag + ", source=" + status.Source);
int count = 0;
bool last_finished = false;
if (status.Tag == TAG_FETCHVALUES_OMV_FINISH)
{
finished_senders++;
last_finished = true;
}
while (finished_senders < senders_size)
{
if (!last_finished)
{
IIteratorInstance <OMV> iterator = null;
if (!kv_cache.ContainsKey(kv.Key))
{
Trace.WriteLine(Rank + ": PARTITIONER (FETCH VALUES TARGET) LOOP RECV 1 !" + (count++) + " " + kv.Key.GetType());
iterator = Reduce_job_values_factory.newIteratorInstance();
kv_cache.Add(kv.Key, iterator);
IKVPairInstance <OMK, IIterator <OMV> > item = (IKVPairInstance <OMK, IIterator <OMV> >)Reduce_job.createItem();
item.Key = kv.Key;
item.Value = iterator;
reduce_job_instance.put(item);
}
else
{
Trace.WriteLine(Rank + ": PARTITIONER (FETCH VALUES TARGET) LOOP RECV 2 !" + (count++));
kv_cache.TryGetValue(kv.Key, out iterator);
}
iterator.put(kv.Value);
}
else
{
last_finished = false;
Trace.WriteLine(Rank + ": PARTITIONER (FETCH VALUES TARGET) - FINISH DETECTED ");
}
Trace.WriteLine(Rank + ": PARTITIONER (FETCH VALUES TARGET) BEGIN RECEIVE n !");
comm.Receive <IKVPairInstance <OMK, OMV> >(MPI.Unsafe.MPI_ANY_SOURCE, MPI.Unsafe.MPI_ANY_TAG, out kv, out status);
Trace.WriteLine(Rank + ": PARTITIONER (FETCH VALUES TARGET) END RECEIVE n ! tag=" + status.Tag + ", source=" + status.Source);
if (status.Tag == TAG_FETCHVALUES_OMV_FINISH)
{
finished_senders++;
last_finished = true;
}
}
foreach (KeyValuePair <object, IIteratorInstance <OMV> > kv_item in kv_cache)
{
kv_item.Value.finish();
}
reduce_job_instance.finish();
Trace.WriteLine(Rank + ": PARTITIONER (FETCH VALUES TARGET) - FINISH ");
}
private void createPageNodes(string fileContent)
{
IIteratorInstance <IKVPair <IInteger, IPageNode> > output = (IIteratorInstance <IKVPair <IInteger, IPageNode> >)Output_data.Instance;
IDictionary <int, IDictionary <int, IPageNodeInstance> > dictionary = new Dictionary <int, IDictionary <int, IPageNodeInstance> >();
IList <IKVPairInstance <IInteger, IPageNode> > PAGENODES = new List <IKVPairInstance <IInteger, IPageNode> >();
string[] lines = fileContent.Split(new char[] { System.Environment.NewLine[0] });
foreach (string line in lines)
{
if (!line.Trim().Equals(""))
{
IPageNodeInstance V, W, temp = null;
IDictionary <int, IPageNodeInstance> referenceV, referenceW = null;
int[] KEY = new int[2];
string[] vwID = line.Split(' ');
for (int k = 0; k < 2; k++)
{
KEY [k] = int.Parse(vwID [k]);
}
if (!dictionary.TryGetValue(KEY [0], out referenceV))
{
IKVPairInstance <IInteger, IPageNode> kvpair = (IKVPairInstance <IInteger, IPageNode>)Output_data.createItem();
V = (IPageNodeInstance)kvpair.Value;
V.IdInstance = KEY [0];
((IIntegerInstance)kvpair.Key).Value = V.IdInstance;
referenceV = new Dictionary <int, IPageNodeInstance> ();
dictionary [KEY [0]] = referenceV;
referenceV [KEY [0]] = V;
PAGENODES.Add(kvpair);
}
if (!dictionary.TryGetValue(KEY [1], out referenceW))
{
IKVPairInstance <IInteger, IPageNode> kvpair = (IKVPairInstance <IInteger, IPageNode>)Output_data.createItem();
W = (IPageNodeInstance)kvpair.Value;
W.IdInstance = KEY [1];
((IIntegerInstance)kvpair.Key).Value = W.IdInstance;
referenceW = new Dictionary <int, IPageNodeInstance> ();
dictionary [KEY [1]] = referenceW;
referenceW [KEY [1]] = W;
PAGENODES.Add(kvpair);
}
if (!referenceV.TryGetValue(KEY [1], out temp))
{
V = referenceV [KEY [0]];
W = referenceW [KEY [1]];
referenceV [KEY [1]] = W;
V.NeighborsInstance.Add(W.IdInstance);
}
}
}
IEnumerator <IKVPairInstance <IInteger, IPageNode> > iterator = PAGENODES.GetEnumerator();
while (iterator.MoveNext())
{
IKVPairInstance <IInteger, IPageNode> kvpair = iterator.Current;
IIntegerInstance k = (IIntegerInstance)kvpair.Key;
output.put(kvpair);
dic [k.Value] = kvpair;
}
output.finish();
}
public void receive_write_gif(int senders_size, IDictionary <int, Tuple <int, int> > unit_ref)
{
IIteratorInstance <IKVPair <IInteger, IIterator <GIF> > > output_instance_gifs = (IIteratorInstance <IKVPair <IInteger, IIterator <GIF> > >)Output_gifs.Instance;
Feed_graph.Server = output_instance_gifs;
bool[] finished_stream = new bool[senders_size];
for (int i = 0; i < senders_size; i++)
{
finished_stream [i] = false;
}
int count_finished_streams = 0;
while (count_finished_streams < senders_size) // take next chunk ...
{
IActionFuture sync_perform;
Task_binding_shuffle.invoke(ITaskPortAdvance.READ_CHUNK);
Task_binding_shuffle.invoke(ITaskPortAdvance.PERFORM, out sync_perform);
IDictionary <object, IIteratorInstance <GIF> > kv_cache = new Dictionary <object, IIteratorInstance <GIF> > ();
// PERFORM
for (int i = 0; i < senders_size; i++)
{
if (!finished_stream [i])
{
IList <IKVPairInstance <IInteger, GIF> > buffer;
CompletedStatus status;
Step_channel.Receive(unit_ref [i], MPI.Communicator.anyTag, out buffer, out status);
foreach (IKVPairInstance <IInteger, GIF> kv in buffer)
{
IIteratorInstance <GIF> iterator = null;
if (!kv_cache.ContainsKey(kv.Key))
{
iterator = Value_factory_gif.newIteratorInstance();
kv_cache.Add(kv.Key, iterator);
IKVPairInstance <IInteger, IIterator <GIF> > item = (IKVPairInstance <IInteger, IIterator <GIF> >)Output_gifs.createItem();
item.Key = kv.Key;
item.Value = iterator;
output_instance_gifs.put(item);
}
else
{
kv_cache.TryGetValue(kv.Key, out iterator);
}
iterator.put(kv.Value);
}
if (status.Tag == TAG_SHUFFLE_OMV_END_CHUNK)
{
count_finished_streams++;
finished_stream [i] = true;
}
}
}
output_instance_gifs.finish();
foreach (IIteratorInstance <GIF> iterator in kv_cache.Values)
{
iterator.finish();
}
sync_perform.wait();
// CHUNK_READY
Task_binding_shuffle.invoke(ITaskPortAdvance.CHUNK_READY); //****
}
output_instance_gifs.finish();
output_instance_gifs.finish();
}
public override void main()
{
Console.WriteLine(this.Rank + ": SHUFFLER REDUCE COLLECTOR START");
// IIteratorInstance<IKVPair<TKey,IIterator<TValue>>> output_instance = (IIteratorInstance<IKVPair<TKey,IIterator<TValue>>>) Output.Instance;
// Feed_pairs.Server = output_instance;
// DETERMINE COMMUNICATION SOURCEs
IDictionary <int, Tuple <int, int> > unit_ref = new Dictionary <int, Tuple <int, int> > ();
int nf = this.FacetMultiplicity [FACET_MAP];
int senders_size = 0;
foreach (int i in this.FacetIndexes[FACET_MAP])
{
int nr0 = senders_size;
senders_size += this.UnitSizeInFacet [i] ["gusty_collector"];
for (int k = 0, j = nr0; j < senders_size; k++, j++)
{
unit_ref [j] = new Tuple <int, int> (i /*,0 INDEX OF gusty_collector*/, k);
}
}
receive_write_gif(senders_size, unit_ref);
IIteratorInstance <IKVPair <TKey, IIterator <TValue> > > output_instance = (IIteratorInstance <IKVPair <TKey, IIterator <TValue> > >)Output.Instance;
Feed_pairs.Server = output_instance;
bool end_computation = false;
while (!end_computation) // next iteration
{
bool[] finished_stream = new bool[senders_size];
for (int i = 0; i < senders_size; i++)
{
finished_stream [i] = false;
}
int count_finished_streams = 0;
end_computation = true;
while (count_finished_streams < senders_size) // take next chunk ...
{
Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...1");
IActionFuture sync_perform;
Task_binding_shuffle.invoke(ITaskPortAdvance.READ_CHUNK);
Task_binding_shuffle.invoke(ITaskPortAdvance.PERFORM, out sync_perform);
Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...2");
IDictionary <object, IIteratorInstance <TValue> > kv_cache = new Dictionary <object, IIteratorInstance <TValue> > ();
// PERFORM
for (int i = 0; i < senders_size; i++)
{
if (!finished_stream[i])
{
Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...3 i=" + i);
IList <IKVPairInstance <TKey, TValue> > buffer;
CompletedStatus status;
Step_channel.Receive(unit_ref[i], MPI.Communicator.anyTag, out buffer, out status);
Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...4");
foreach (IKVPairInstance <TKey, TValue> kv in buffer)
{
IIteratorInstance <TValue> iterator = null;
if (!kv_cache.ContainsKey(kv.Key))
{
iterator = Value_factory.newIteratorInstance();
kv_cache.Add(kv.Key, iterator);
IKVPairInstance <TKey, IIterator <TValue> > item = (IKVPairInstance <TKey, IIterator <TValue> >)Output.createItem();
item.Key = kv.Key;
item.Value = iterator;
output_instance.put(item);
}
else
{
kv_cache.TryGetValue(kv.Key, out iterator);
}
iterator.put(kv.Value);
}
Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...5");
if (status.Tag == TAG_SHUFFLE_OMV_END_CHUNK)
{
count_finished_streams++;
finished_stream [i] = true;
}
else
{
end_computation = false;
}
Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...6");
}
}
output_instance.finish();
foreach (IIteratorInstance <TValue> iterator in kv_cache.Values)
{
iterator.finish();
}
Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...7-1");
sync_perform.wait();
Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...7-2");
// CHUNK_READY
Task_binding_shuffle.invoke(ITaskPortAdvance.CHUNK_READY); //****
Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...8");
}
output_instance.finish();
Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...9 - FINISH");
}
}
