private void receive_pairs_iteration(IPortTypeIterator output_instance, int senders_size, IDictionary<int,Tuple<int,int>> unit_ref) { while (true) // next iteration { int count_finished_streams = 0; while (count_finished_streams == 0) // take next chunk ... { // PERFORM for (int i = 0; i < senders_size; i++) { IList<object /*IKVPairInstance<ORK,ORV>*/> buffer; CompletedStatus status; Split_channel.Receive (unit_ref [i], MPI.Communicator.anyTag, out buffer, out status); foreach (object /*IKVPairInstance<ORK,ORV>*/ kv in buffer) output_instance.put (kv); if (status.Tag == TAG_SPLIT_PAIR_FINISH) count_finished_streams++; } output_instance.finish (); Debug.Assert (count_finished_streams == 0 || count_finished_streams == senders_size); // CHUNK_READY Task_port_split.invoke (ITaskPortAdvance.CHUNK_READY); } // send empty iterator to inform the mappers the end of the iteration ... output_instance.finish (); } }
private void file_reader() { string fileName = System.Environment.GetEnvironmentVariable("PATH_GRAPH_FILE"); IInputFormatInstance extractor_input_format = (IInputFormatInstance)server.fetchFileContentObject(); extractor_input_format.PARTITION_SIZE = partition_size; IDictionary <int, IInputFormatInstance> sub_formats = extractor_input_format.extractBins(fileName); this.PartitionTABLE = extractor_input_format.PartitionTABLE; foreach (IInputFormatInstance shard in sub_formats.Values) { IKVPairInstance <IInteger, IInputFormat> item = (IKVPairInstance <IInteger, IInputFormat>)client.createItem(); int first_vertex_id_from_partition = shard.firstVertex(shard.PARTID); ((IIntegerInstance)item.Key).Value = first_vertex_id_from_partition; //counter_write_global; item.Value = shard; client.put(item); counter_write_chunk++; //counter_write_global++; if (counter_write_chunk >= CHUNK_SIZE) { Console.WriteLine("NEW CHUNK size=" + counter_write_chunk); counter_write_chunk = 0; client.finish(); } } client.finish(); client.finish(); Console.WriteLine("FINISHING READING DATA SOURCE"); }
public override void main() { Split_channel.TraceFlag = true; Task_binding_data.TraceFlag = true; Console.WriteLine("SPLITER WRITE SINK 0"); Sink.clientConnection(); IPortTypeIterator output_instance = (IPortTypeIterator)Sink.Client; Console.WriteLine("SPLITER WRITE SINK 1"); Task_binding_data.TraceFlag = true; Task_binding_data.invoke(ITaskPortData.READ_SOURCE); // Do nothing ... Console.WriteLine("SPLITER WRITE SINK 3"); // RECEIVE PAIRS FROM THE REDUCERS (next iterations) IDictionary <int, Tuple <int, int> > unit_ref = new Dictionary <int, Tuple <int, int> > (); unit_ref = new Dictionary <int, Tuple <int, int> > (); int senders_size = 0; foreach (int i in this.FacetIndexes[FACET_REDUCE]) { int nr0 = senders_size; senders_size += this.UnitSizeInFacet [i] ["gusty_collector"]; for (int k = 0, j = nr0; j < senders_size; k++, j++) { unit_ref [j] = new Tuple <int, int> (i, k); } } Console.WriteLine("SPLITER WRITE SINK 4"); Thread t_output = new Thread(new ThreadStart(delegate { Task_binding_data.invoke(ITaskPortData.TERMINATE); Task_binding_data.invoke(ITaskPortData.WRITE_SINK); Sink.startWriteSink(); })); t_output.Start(); Thread[] receive_pairs_iteration_thread = new Thread[senders_size]; for (int i = 0; i < senders_size; i++) { receive_pairs_iteration_thread [i] = new Thread(new ParameterizedThreadStart(delegate(object i_obj) { int ii = (int)i_obj; receive_pairs_iteration(output_instance, senders_size, unit_ref[ii]); })); receive_pairs_iteration_thread [i].Start(i); } for (int i = 0; i < senders_size; i++) { receive_pairs_iteration_thread [i].Join(); } output_instance.finish(); t_output.Join(); Console.WriteLine("SPLITER WRITE SINK - WRITTER FINISH"); Split_channel.TraceFlag = false; Task_binding_data.TraceFlag = false; }