Example #1
0
        private ClusteredArrayList FinalizeReducers()
        {
            ClusteredArrayList reducersOutput = new ClusteredArrayList();

            lock (mutex)
            {
                IEnumerator it = reducers.Values.GetEnumerator();
                while (it.MoveNext())
                {
                    Alachisoft.NCache.Runtime.MapReduce.KeyValuePair result = ((IReducer)it.Current).FinishReduce();
                    reducersOutput.Add(new TaskOutputPair(result.Key, result.Value));
                }
            }
            return(reducersOutput);
        }
Example #2
0
        public void Run()
        {
            try
            {
                if (parent.Context != null)
                {
                    if (parent.Context.NCacheLog.IsInfoEnabled)
                    {
                        parent.Context.NCacheLog.Info("ReducerTask(" + parent.TaskId + ").Start", "Reducer task is started");
                    }
                }

                bool isCompeletedSuccessfully = true;

                while (isAlive)
                {
                    try
                    {
                        object currObj = null;
                        lock (ReducerInputQueue)
                        {
                            if (ReducerInputQueue.Count == 0 && !isMappercompleted)
                            {
                                Monitor.Wait(ReducerInputQueue);
                            }
                            if (ReducerInputQueue.Count > 0)
                            {
                                currObj = ReducerInputQueue.Dequeue();
                            }
                        }
                        if (currObj != null)
                        {
                            Alachisoft.NCache.Runtime.MapReduce.KeyValuePair entry = (Alachisoft.NCache.Runtime.MapReduce.KeyValuePair)currObj;
                            Object   key = entry.Key;
                            IReducer r   = null;
                            lock (mutex)
                            {
                                if (!reducers.ContainsKey(key))
                                {
                                    r = reducerFactory.Create(key);
                                    r.BeginReduce();
                                    reducers.Add(key, r);
                                }
                                else
                                {
                                    r = (IReducer)reducers[key];
                                }
                            }
                            r.Reduce(entry.Value);
                            ReducedCount = ReducedCount + 1; // increment the reducedCount
                            if (parent.Context.PerfStatsColl != null)
                            {
                                parent.Context.PerfStatsColl.IncrementReducedPerSecRate();
                            }
                        }
                        else
                        {
                            if (isMappercompleted)
                            {
                                parent.PersistOutput(FinalizeReducers());
                                if (parent.Context.NCacheLog.IsInfoEnabled)
                                {
                                    parent.Context.NCacheLog.Info("ReducerTask(" + parent.TaskId + ").Run ", "Reducer Completed, output persisted.");
                                }

                                break;
                            }
                        }
                    }
                    catch (Exception ex)
                    {
                        if (parent.ExceptionCount < parent.MaxExceptions)
                        {
                            if (parent.Context.NCacheLog != null)
                            {
                                parent.Context.NCacheLog.Error("ReducerTask(" + parent.TaskId + ").Run", " Exception:" + ex.Message);
                            }
                            parent.ExceptionCount = parent.ExceptionCount + 1;
                        }
                        else
                        {
                            isCompeletedSuccessfully = false;
                            parent.LocalReducerFailed();

                            lock (mutex)
                            {
                                //Dispose Reducers
                                foreach (IReducer rr in reducers.Values)
                                {
                                    rr.Dispose();
                                }
                                reducers.Clear();
                            }

                            break;
                        }
                    }
                }

                if (isCompeletedSuccessfully && isAlive)
                {
                    if (parent.Context.NCacheLog.IsInfoEnabled)
                    {
                        parent.Context.NCacheLog.Info("ReducerTask (" + parent.TaskId + ").Run ", "Reduced Total Keys : " + this.ReducedCount);
                    }
                    parent.LocalReducerCompleted();
                    lock (mutex)
                    {
                        foreach (IReducer rr in reducers.Values)
                        {
                            rr.Dispose();
                        }
                        reducers.Clear();
                    }
                }
            }
            catch (Exception e)
            {
                try
                {
                    parent.LocalReducerFailed();
                }
                catch (Exception)
                {}
            }
        }