public void iterate()
        {
            IList <IKVPairInstance <IKey, IValue> > buffer;
            object          buffer_obj;
            CompletedStatus status;

            Console.WriteLine(this.GlobalRank + ": STARTING SPLITTER ITERATE...1");
            output_instance   = (IIteratorInstance <IKVPair <IKey, IIterator <IValue> > >)Output.Instance;
            Feed_pairs.Server = output_instance;
            Console.WriteLine(this.GlobalRank + ": STARTING SPLITTER ITERATE...2");

            IDictionary <int, Tuple <int, int> > unit_ref = new Dictionary <int, Tuple <int, int> > ();

            Console.WriteLine(this.GlobalRank + ": STARTING SPLITTER ...3");
            // RECEIVE PAIRS FROM THE REDUCERS (next iterations)
            int senders_size = 0;

            foreach (int i in this.FacetIndexes[FACET_REDUCE])
            {
                Console.WriteLine(this.GlobalRank + ": STARTING SPLITTER ...4 -- i=" + i);
                int nr0 = senders_size;
                senders_size += this.UnitSizeInFacet [i] ["gusty_collector"];
                for (int k = 0, j = nr0; j < senders_size; k++, j++)
                {
                    unit_ref [j] = new Tuple <int, int> (i, k);
                }
            }

            bool end_computation = false;

            while (!end_computation)
            {
                end_computation = true;
                int count_finished_streams = 0;

                while (count_finished_streams < senders_size)
                {
                    IActionFuture sync_perform;

                    Console.WriteLine(this.Rank + ": SPLITTER MAP FEEDER NEXT 1");

                    Task_binding_split_next.invoke(ITaskPortAdvance.READ_CHUNK);
                    Task_binding_split_next.invoke(ITaskPortAdvance.PERFORM, out sync_perform);

                    IDictionary <object, IIteratorInstance <IValue> > kv_cache = new Dictionary <object, IIteratorInstance <IValue> > ();

                    Console.WriteLine(this.Rank + ": SPLITTER MAP FEEDER NEXT 2");

                    for (int i = 0; i < senders_size; i++)
                    {
                        Console.WriteLine(this.Rank + ": SPLITTER MAP FEEDER NEXT LOOP - before receive");

                        Split_channel.Receive(unit_ref [i], MPI.Communicator.anyTag, out buffer_obj, out status);

                        Console.WriteLine(this.Rank + ": CHUNK PAIRS RECEIVED !!! from gusty_collector of index " + i + " / count_finished_streams=" + count_finished_streams);

                        if (status.Tag == TAG_SPLIT_END_CHUNK)
                        {
                            count_finished_streams++;
                        }
                        else
                        {
                            end_computation = false;
                        }

                        try {
                            buffer = (IList <IKVPairInstance <IKey, IValue> >)buffer_obj;

                            //foreach (IKVPairInstance<IKey,IValue> kv in buffer)
                            //	output_instance.put (kv);

                            foreach (IKVPairInstance <IKey, IValue> kv in buffer)
                            {
                                IIteratorInstance <IValue> iterator = null;
                                if (!kv_cache.ContainsKey(kv.Key))
                                {
                                    iterator = Value_factory.newIteratorInstance();
                                    kv_cache.Add(kv.Key, iterator);
                                    IKVPairInstance <IKey, IIterator <IValue> > item = (IKVPairInstance <IKey, IIterator <IValue> >)Output.createItem();
                                    item.Key   = kv.Key;
                                    item.Value = iterator;
                                    output_instance.put(item);
                                }
                                else
                                {
                                    kv_cache.TryGetValue(kv.Key, out iterator);
                                }

                                iterator.put(kv.Value);
                            }
                        } catch (InvalidCastException e) {
                            Console.WriteLine("SPLITTER MAPPER: incompatible input !");
                            count_finished_streams++;
                        }
                    }

                    output_instance.finish();
                    foreach (IIteratorInstance <IValue> iterator in kv_cache.Values)
                    {
                        iterator.finish();
                    }

                    Console.WriteLine(this.Rank + ": SPLITTER MAP FEEDER NEXT 3");

                    sync_perform.wait();

                    Console.WriteLine(this.Rank + ": SPLITTER MAP FEEDER NEXT 4");

                    // CHUNK_READY
                    Task_binding_split_next.invoke(ITaskPortAdvance.CHUNK_READY);

                    Console.WriteLine(this.Rank + ": SPLITTER MAP FEEDER NEXT 5");
                }
            }

            Console.WriteLine(this.Rank + ": SPLITTER MAP FEEDER FINISH 1 !");

            Task_binding_data.invoke(ITaskPortData.TERMINATE);
            Task_binding_data.invoke(ITaskPortData.WRITE_SINK);

            Console.WriteLine(this.Rank + ": SPLITTER MAP FEEDER FINISH 2 !");
        }
Esempio n. 2
0
        public override void main()
        {
            Console.WriteLine(this.Rank + ": SHUFFLER REDUCE COLLECTOR START");

//			IIteratorInstance<IKVPair<TKey,IIterator<TValue>>> output_instance = (IIteratorInstance<IKVPair<TKey,IIterator<TValue>>>) Output.Instance;
//			Feed_pairs.Server = output_instance;

            // DETERMINE COMMUNICATION SOURCEs
            IDictionary <int, Tuple <int, int> > unit_ref = new Dictionary <int, Tuple <int, int> > ();
            int nf           = this.FacetMultiplicity [FACET_MAP];
            int senders_size = 0;

            foreach (int i in this.FacetIndexes[FACET_MAP])
            {
                int nr0 = senders_size;
                senders_size += this.UnitSizeInFacet [i] ["gusty_collector"];
                for (int k = 0, j = nr0; j < senders_size; k++, j++)
                {
                    unit_ref [j] = new Tuple <int, int> (i /*,0 INDEX OF gusty_collector*/, k);
                }
            }

            receive_write_gif(senders_size, unit_ref);

            IIteratorInstance <IKVPair <TKey, IIterator <TValue> > > output_instance = (IIteratorInstance <IKVPair <TKey, IIterator <TValue> > >)Output.Instance;

            Feed_pairs.Server = output_instance;

            bool end_computation = false;

            while (!end_computation)               // next iteration
            {
                bool[] finished_stream = new bool[senders_size];
                for (int i = 0; i < senders_size; i++)
                {
                    finished_stream [i] = false;
                }

                int count_finished_streams = 0;

                end_computation = true;

                while (count_finished_streams < senders_size)                     // take next chunk ...
                {
                    Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...1");

                    IActionFuture sync_perform;

                    Task_binding_shuffle.invoke(ITaskPortAdvance.READ_CHUNK);
                    Task_binding_shuffle.invoke(ITaskPortAdvance.PERFORM, out sync_perform);

                    Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...2");

                    IDictionary <object, IIteratorInstance <TValue> > kv_cache = new Dictionary <object, IIteratorInstance <TValue> > ();

                    // PERFORM
                    for (int i = 0; i < senders_size; i++)
                    {
                        if (!finished_stream[i])
                        {
                            Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...3 i=" + i);

                            IList <IKVPairInstance <TKey, TValue> > buffer;
                            CompletedStatus status;
                            Step_channel.Receive(unit_ref[i], MPI.Communicator.anyTag, out buffer, out status);

                            Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...4");

                            foreach (IKVPairInstance <TKey, TValue> kv in buffer)
                            {
                                IIteratorInstance <TValue> iterator = null;
                                if (!kv_cache.ContainsKey(kv.Key))
                                {
                                    iterator = Value_factory.newIteratorInstance();
                                    kv_cache.Add(kv.Key, iterator);
                                    IKVPairInstance <TKey, IIterator <TValue> > item = (IKVPairInstance <TKey, IIterator <TValue> >)Output.createItem();
                                    item.Key   = kv.Key;
                                    item.Value = iterator;
                                    output_instance.put(item);
                                }
                                else
                                {
                                    kv_cache.TryGetValue(kv.Key, out iterator);
                                }

                                iterator.put(kv.Value);
                            }

                            Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...5");

                            if (status.Tag == TAG_SHUFFLE_OMV_END_CHUNK)
                            {
                                count_finished_streams++;
                                finished_stream [i] = true;
                            }
                            else
                            {
                                end_computation = false;
                            }

                            Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...6");
                        }
                    }

                    output_instance.finish();
                    foreach (IIteratorInstance <TValue> iterator in kv_cache.Values)
                    {
                        iterator.finish();
                    }

                    Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...7-1");

                    sync_perform.wait();

                    Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...7-2");

                    // CHUNK_READY
                    Task_binding_shuffle.invoke(ITaskPortAdvance.CHUNK_READY);                        //****

                    Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...8");
                }

                output_instance.finish();

                Console.WriteLine(this.GlobalRank + ": SHUFFLER REDUCER...9 - FINISH");
            }
        }