Beispiel #1
0
        private Object ReducerFailed(MapReduceOperation operation)
        {
            string taskID = (string)operation.TaskID;

            if (runningTasks.ContainsKey(taskID))
            {
                MapReduceTask t = (MapReduceTask)runningTasks[taskID];
                t.ReducerFailed(operation);
            }
            return(true);
        }
Beispiel #2
0
 private object ReducerCompleted(MapReduceOperation operation)
 {
     lock (_lock) {
         string taskID = (string)operation.TaskID;
         if (runningTasks.ContainsKey(taskID))
         {
             MapReduceTask t = (MapReduceTask)runningTasks[taskID];
             t.ReducerCompleted(operation);
         }
         return(true);
     }
 }
Beispiel #3
0
        private object RegisterTaskNotification(MapReduceOperation operation)
        {
            string taskID = (string)operation.TaskID;

            if (taskID == null || string.IsNullOrEmpty(taskID))
            {
                throw new OperationFailedException("Task can not be null or empty");
            }

            TaskBase t = null;

            if (runningTasks.ContainsKey(taskID))
            {
                t = (TaskBase)runningTasks[taskID];
                t.AddTaskCallbackInfo((TaskCallbackInfo)operation.CallbackInfo);
            }
            else if (waitingTasks.ContainsKey(taskID))
            {
                t = (TaskBase)waitingTasks[taskID];
                t.AddTaskCallbackInfo((TaskCallbackInfo)operation.CallbackInfo);
            }
            else if (taskOutputStore != null && taskOutputStore.TaskOutputExists(taskID))
            {
                //Put listener info into taskoutput and then call notify client async
                TaskOutput output = taskOutputStore.TaskOutput(taskID);
                if (output != null)
                {
                    output.AddToListeners(operation.CallbackInfo);
                }

                if (_context.CacheImpl != null && _context.CacheInternal != null)
                {
                    EventContext eventContext = new EventContext();
                    eventContext.TaskStatus = TaskStatus.Success;

                    IList listeners = new ArrayList();
                    listeners.Add((TaskCallbackInfo)operation.CallbackInfo);

                    _context.CacheInternal.NotifyTaskCallback(taskID, listeners, true, null, eventContext);
                }
            }
            else if (this.cancelledTask.Contains(taskID))
            {
                throw new OperationFailedException("Task with Specified TaskId was cancelled.");
            }
            else
            {
                throw new OperationFailedException("Task with Specified Task id does not exists");
            }

            return(true);
        }
Beispiel #4
0
        public void SendReducerData()
        {
            TaskOutputPair entry = null;

            while (ReducerDataQueue.Count > 0)
            {
                try {
                    lock (ReducerDataQueue) {
                        object ent = ReducerDataQueue.Dequeue();
                        if (ent is DictionaryEntry)
                        {
                            entry = new TaskOutputPair(((DictionaryEntry)ent).Key, ((DictionaryEntry)ent).Value);
                        }
                    }
                    if (entry != null)
                    {
                        if (!ReducerConfigured)
                        {
                            localOutput.Add(entry);
                        }
                        else
                        {
                            MapReduceOperation op = new MapReduceOperation();
                            op.Data   = entry;
                            op.OpCode = MapReduceOpCodes.ReceiveReducerData;
                            op.TaskID = this.TaskId;
#if !CLIENT
                            if (distributionMgr != null)
                            {
                                Address target = distributionMgr.SelectNode((string)entry.Key, "");

                                if (target.Equals(((ClusterCacheBase)_parent).Cluster.LocalAddress))
                                {
                                    _parent.InternalCache.TaskOperationReceived(op);
                                }
                                else
                                {
                                    _parent.SendMapReduceOperation(target, op);
                                }
                            }
                            else
#endif
                            _parent.InternalCache.TaskOperationReceived(op);
                        }
                    }
                } catch (Exception ex) {
                    _parent.Context.NCacheLog.Error("MapReduceTask(" + this.TaskId + ").SendToReducers", "Exception: " + ex.Message);
                }
                //Get Target Node check if local enqueue into reducer queue otherwise send to target node
            }
        }
Beispiel #5
0
 public object RemoveFromSubmitted(MapReduceOperation operation)
 {
     try
     {
         if (this.submittedTasks.ContainsKey(operation.TaskID))
         {
             this.submittedTasks.Remove(operation.TaskID);
         }
         return(true);
     }
     catch (Exception ex)
     {
         return(false);
     }
 }
Beispiel #6
0
        public object TaskOperationReceived(MapReduceOperation operation)
        {
            try
            {
                if (_taskTracker == null)
                {
                    throw new GeneralFailureException("No instance is available to process the task Requests.");
                }

                return(this._taskTracker.TaskOperationRecieved(operation));
            }
            catch (Exception ex)
            {
                throw new OperationFailedException(ex.Message);
            }
        }
Beispiel #7
0
        public void ReducerFailed(MapReduceOperation operation)
        {
            Address source = operation.Source != null ? operation.Source : new Address(_parent.Context.Render.IPAddress, _parent.Context.Render.Port);

            if (participants != null)
            {
                NodeTaskStatus status = (NodeTaskStatus)participants[source];
                status.ReducerStatus = ReducerStatus.Failed;
            }
            if (_context.NCacheLog.IsInfoEnabled)
            {
                _context.NCacheLog.Info("MapReduceTask(" + this.TaskId + ").ReducerFailed", "Reducer is failed on '" + source.IpAddress.ToString() + "'");
            }
            this.Callback.Invoke(new CallbackResult(this, Notifications.TaskStatus.Failure, "Reducer Failed on" + source.IpAddress.ToString()));
            this.StopTask();
        }
Beispiel #8
0
        private object RecievedReducerData(MapReduceOperation operation)
        {
            string taskID = (string)operation.TaskID;

            if (runningTasks.ContainsKey(taskID))
            {
                MapReduceTask t = (MapReduceTask)runningTasks[taskID];
                t.EnqueueReducerInput((Common.MapReduce.TaskOutputPair)operation.Data);
            }
            else
            {
                if (_context.NCacheLog.IsInfoEnabled)
                {
                    _context.NCacheLog.Info("TaskTracker.recievedReducerData", "task does not exist in running tasks " + taskID);
                }
            }
            return(true);
        }
Beispiel #9
0
 public void MapperCompleted(MapReduceOperation operation)
 {
     lock (mutex){
         Address source = operation.Source != null ? operation.Source : new Address(_parent.Context.Render.IPAddress, _parent.Context.Render.Port);
         if (participants != null)
         {
             NodeTaskStatus status = (NodeTaskStatus)participants[source];
             if (status != null)
             {
                 status.MapperStatus = MapperStatus.Completed;
             }
         }
         if (_context.NCacheLog.IsInfoEnabled)
         {
             _context.NCacheLog.Info("MapReduceTask(" + this.TaskId + ").MapperCompleted", "Mapped is completed on '" + source.IpAddress.ToString() + "'");
         }
         CheckMappersCompleted();
     }
 }
Beispiel #10
0
 public object RemoveFromRunning(MapReduceOperation operation)
 {
     try
     {
         if (this.runningTasks.ContainsKey(operation.TaskID))
         {
             this.runningTasks.Remove(operation.TaskID);
             if (_context.PerfStatsColl != null)
             {
                 _context.PerfStatsColl.DecrementRunningTasks();
             }
         }
         return(true);
     }
     catch (Exception ex)
     {
         return(false);
     }
 }
Beispiel #11
0
        private void SendParticipantReducerFailedMessage()
        {
            MapReduceOperation operation = new MapReduceOperation();

            operation.TaskID = this.TaskId;
            operation.OpCode = MapReduceOpCodes.ReducerFailed;

            try
            {
                if (!isLocal)
                {
                    _parent.SendMapReduceOperation(new System.Collections.ArrayList(participants.Keys), operation);
                }
            }
            catch (Exception ex)
            {
                _parent.Context.NCacheLog.Error("MapReduceTask(" + this.TaskId + ").SendParticipantReducerFailedMessage", "Failed: " + ex.Message);
            }
            ReducerFailed(operation);
        }
Beispiel #12
0
        private object UnregisterTaskNotification(MapReduceOperation operation)
        {
            String taskID = (String)operation.TaskID;

            if (taskID == null || string.IsNullOrEmpty(taskID))
            {
                throw new OperationFailedException("Task can not be null or empty");
            }

            TaskBase t = null;

            if (runningTasks.ContainsKey(taskID))
            {
                t = (TaskBase)runningTasks[taskID];
                t.RemoveTaskCallbackInfo((TaskCallbackInfo)operation.CallbackInfo);
            }
            else if (waitingTasks.ContainsKey(taskID))
            {
                t = (TaskBase)waitingTasks[taskID];
                t.RemoveTaskCallbackInfo((TaskCallbackInfo)operation.CallbackInfo);
            }
            else if (taskOutputStore != null && taskOutputStore.TaskOutputExists(taskID))
            {
                //remove listener info from taskoutput
                TaskOutput output = taskOutputStore.TaskOutput(taskID);
                if (output != null)
                {
                    output.RemoveFromListeners(operation.CallbackInfo);
                }
            }
            else if (this.cancelledTask.Contains(taskID))
            {
                throw new OperationFailedException("Task with Specified TaskId was cancelled.");
            }
            else
            {
                throw new OperationFailedException("Task with Specified Task id does not exists");
            }

            return(true);
        }
Beispiel #13
0
        private object CancelTask(MapReduceOperation operation)
        {
            string taskID = (string)operation.TaskID;

            lock (_lock)
            {
                if (runningTasks.ContainsKey(taskID))
                {
                    TaskBase t = (TaskBase)runningTasks[taskID];
                    t.StopTask();
                    t.Dispose();

                    this.cancelledTask.Add(taskID);

                    if (_context.CacheImpl != null && _context.CacheInternal != null)
                    {
                        EventContext eventContext = new EventContext();
                        eventContext.TaskStatus        = TaskStatus.Cancelled;
                        eventContext.TaskFailureReason = "Task was Cancelled by user.";
                        _context.CacheInternal.NotifyTaskCallback(t.TaskId, t.CallbackListeners, false, null, eventContext);
                    }
                    runningTasks.Remove(taskID);
                    if (_context.PerfStatsColl != null)
                    {
                        _context.PerfStatsColl.DecrementRunningTasks();
                    }
                    if (_context.NCacheLog.IsInfoEnabled)
                    {
                        _context.NCacheLog.Info("TaskTracker.CancelTask", "Task with task ID '" + taskID.ToUpper() + "' has been cancelled.");
                    }
                }
                else
                {
                    _context.NCacheLog.Error("TaskTracker.CancelTask", "Task with task ID '" + taskID.ToUpper() + "' does not exist.");
                }
            }
            return(true);
        }
Beispiel #14
0
        private Object NextRecord(MapReduceOperation operation)
        {
            TaskEnumeratorPointer pointer = (TaskEnumeratorPointer)operation.Data;

            // Task cancellation check
            if (this.cancelledTask.Contains(pointer.TaskId))
            {
                throw new OperationFailedException("Task with specified Task ID was Cancelled.");
            }

            TaskEnumeratorResult result = null;

            if (taskOutputStore != null)
            {
                try {
                    result             = taskOutputStore.NextRecord(pointer);
                    result.NodeAddress = _context.Render.IPAddress.ToString();
                } catch (InvalidTaskEnumeratorException ex) {
                    _context.NCacheLog.Error("TaskTracker.GetTaskEnumerator", ex.Message);
                    throw ex;
                }
            }
            return(result);
        }
Beispiel #15
0
        private object GetTaskEnumerator(MapReduceOperation operation)
        {
            TaskEnumeratorPointer pointer = (TaskEnumeratorPointer)operation.Data;

            if (clientAddress == null)
            {
                try {
                    if (_context.Render != null)
                    {
                        clientAddress = new Address(_context.Render.IPAddress, _context.Render.Port);
                    }
                } catch (Exception ex) {
                    throw new InvalidTaskEnumeratorException(ex.Message, ex);
                }
            }
            pointer.ClientAddress = clientAddress;

            if (clusterAddress == null)
            {
#if !CLIENT
                if (_context.CacheImpl is ClusterCacheBase)
                {
                    clusterAddress = ((ClusterCacheBase)_context.CacheImpl).Cluster.LocalAddress;
                }
                else if (_context.CacheImpl is LocalCacheImpl)
                {
                    clusterAddress = new Address(_context.Render.IPAddress, _context.Render.Port);
                }
#else
                clusterAddress = new Address(_context.Render.IPAddress, _context.Render.Port);
#endif
            }

            pointer.ClusterAddress = clusterAddress;

            // Task Cancellation check.
            if (this.cancelledTask.Contains(pointer.TaskId))
            {
                throw new OperationFailedException("Task with specified Task ID was cancelled");
            }

            TaskEnumeratorResult result = null;
            if (taskOutputStore != null)
            {
                try {
                    result = taskOutputStore.GetTaskEnumerator(pointer);
                    //result.NodeAddress = _context.Render.IPAddress.ToString();
                    result.Pointer = pointer;
                } catch (InvalidTaskEnumeratorException ex) {
                    _context.NCacheLog.Error("TaskTracker.GetTaskEnumerator", ex.Message);
                    throw ex;
                }
            }

            if (_context.NCacheLog.IsInfoEnabled)
            {
                _context.NCacheLog.Info("TaskTracker.GetTaskEnumerator", "Task Enumerator provided to client.");
            }

            return(result);
        }
Beispiel #16
0
        public object TaskOperationRecieved(MapReduceOperation operation)
        {
            MapReduceOpCodes opCode = operation.OpCode;

            switch (opCode)
            {
            case MapReduceOpCodes.GetTaskSequence:
                return(GetTaskSequence());

            case MapReduceOpCodes.SubmitMapReduceTask:
                return(SubmitMapReduceTask(operation));

            case MapReduceOpCodes.StartTask:
                return(StartMapReduceTask(operation));

            case MapReduceOpCodes.CancelTask:
                return(CancelTask(operation));

            case MapReduceOpCodes.CancellAllTasks:
                return(CancelAllTasks(true));

            case MapReduceOpCodes.GetRunningTasks:
                return(GetRunningTasks());

            case MapReduceOpCodes.GetTaskEnumerator:
                return(GetTaskEnumerator(operation));

            case MapReduceOpCodes.GetNextRecord:
                return(NextRecord(operation));

            case MapReduceOpCodes.RegisterTaskNotification:
                return(RegisterTaskNotification(operation));

            case MapReduceOpCodes.UnregisterTaskNotification:
                return(UnregisterTaskNotification(operation));

            case MapReduceOpCodes.GetTaskStatus:
                return(GetTaskProgress(operation));

            case MapReduceOpCodes.RemoveFromRunningList:
                return(RemoveFromRunning(operation));

            case MapReduceOpCodes.RemoveFromSubmittedList:
                return(RemoveFromSubmitted(operation));

            case MapReduceOpCodes.MapperCompleted:
                return(MapperCompleted(operation));

            case MapReduceOpCodes.MapperFailed:
                return(MapperFailed(operation));

            case MapReduceOpCodes.ReducerCompleted:
                return(ReducerCompleted(operation));

            case MapReduceOpCodes.ReducerFailed:
                return(ReducerFailed(operation));

            case MapReduceOpCodes.ReceiveReducerData:
                return(RecievedReducerData(operation));
            }
            return(null);
        }
Beispiel #17
0
 private object StartMapReduceTask(MapReduceOperation operation)
 {
     return(StartTask(operation.TaskID, operation.SequenceID));
 }