private Object ReducerFailed(MapReduceOperation operation) { string taskID = (string)operation.TaskID; if (runningTasks.ContainsKey(taskID)) { MapReduceTask t = (MapReduceTask)runningTasks[taskID]; t.ReducerFailed(operation); } return(true); }
private object ReducerCompleted(MapReduceOperation operation) { lock (_lock) { string taskID = (string)operation.TaskID; if (runningTasks.ContainsKey(taskID)) { MapReduceTask t = (MapReduceTask)runningTasks[taskID]; t.ReducerCompleted(operation); } return(true); } }
private object RegisterTaskNotification(MapReduceOperation operation) { string taskID = (string)operation.TaskID; if (taskID == null || string.IsNullOrEmpty(taskID)) { throw new OperationFailedException("Task can not be null or empty"); } TaskBase t = null; if (runningTasks.ContainsKey(taskID)) { t = (TaskBase)runningTasks[taskID]; t.AddTaskCallbackInfo((TaskCallbackInfo)operation.CallbackInfo); } else if (waitingTasks.ContainsKey(taskID)) { t = (TaskBase)waitingTasks[taskID]; t.AddTaskCallbackInfo((TaskCallbackInfo)operation.CallbackInfo); } else if (taskOutputStore != null && taskOutputStore.TaskOutputExists(taskID)) { //Put listener info into taskoutput and then call notify client async TaskOutput output = taskOutputStore.TaskOutput(taskID); if (output != null) { output.AddToListeners(operation.CallbackInfo); } if (_context.CacheImpl != null && _context.CacheInternal != null) { EventContext eventContext = new EventContext(); eventContext.TaskStatus = TaskStatus.Success; IList listeners = new ArrayList(); listeners.Add((TaskCallbackInfo)operation.CallbackInfo); _context.CacheInternal.NotifyTaskCallback(taskID, listeners, true, null, eventContext); } } else if (this.cancelledTask.Contains(taskID)) { throw new OperationFailedException("Task with Specified TaskId was cancelled."); } else { throw new OperationFailedException("Task with Specified Task id does not exists"); } return(true); }
public void SendReducerData() { TaskOutputPair entry = null; while (ReducerDataQueue.Count > 0) { try { lock (ReducerDataQueue) { object ent = ReducerDataQueue.Dequeue(); if (ent is DictionaryEntry) { entry = new TaskOutputPair(((DictionaryEntry)ent).Key, ((DictionaryEntry)ent).Value); } } if (entry != null) { if (!ReducerConfigured) { localOutput.Add(entry); } else { MapReduceOperation op = new MapReduceOperation(); op.Data = entry; op.OpCode = MapReduceOpCodes.ReceiveReducerData; op.TaskID = this.TaskId; #if !CLIENT if (distributionMgr != null) { Address target = distributionMgr.SelectNode((string)entry.Key, ""); if (target.Equals(((ClusterCacheBase)_parent).Cluster.LocalAddress)) { _parent.InternalCache.TaskOperationReceived(op); } else { _parent.SendMapReduceOperation(target, op); } } else #endif _parent.InternalCache.TaskOperationReceived(op); } } } catch (Exception ex) { _parent.Context.NCacheLog.Error("MapReduceTask(" + this.TaskId + ").SendToReducers", "Exception: " + ex.Message); } //Get Target Node check if local enqueue into reducer queue otherwise send to target node } }
public object RemoveFromSubmitted(MapReduceOperation operation) { try { if (this.submittedTasks.ContainsKey(operation.TaskID)) { this.submittedTasks.Remove(operation.TaskID); } return(true); } catch (Exception ex) { return(false); } }
public object TaskOperationReceived(MapReduceOperation operation) { try { if (_taskTracker == null) { throw new GeneralFailureException("No instance is available to process the task Requests."); } return(this._taskTracker.TaskOperationRecieved(operation)); } catch (Exception ex) { throw new OperationFailedException(ex.Message); } }
public void ReducerFailed(MapReduceOperation operation) { Address source = operation.Source != null ? operation.Source : new Address(_parent.Context.Render.IPAddress, _parent.Context.Render.Port); if (participants != null) { NodeTaskStatus status = (NodeTaskStatus)participants[source]; status.ReducerStatus = ReducerStatus.Failed; } if (_context.NCacheLog.IsInfoEnabled) { _context.NCacheLog.Info("MapReduceTask(" + this.TaskId + ").ReducerFailed", "Reducer is failed on '" + source.IpAddress.ToString() + "'"); } this.Callback.Invoke(new CallbackResult(this, Notifications.TaskStatus.Failure, "Reducer Failed on" + source.IpAddress.ToString())); this.StopTask(); }
private object RecievedReducerData(MapReduceOperation operation) { string taskID = (string)operation.TaskID; if (runningTasks.ContainsKey(taskID)) { MapReduceTask t = (MapReduceTask)runningTasks[taskID]; t.EnqueueReducerInput((Common.MapReduce.TaskOutputPair)operation.Data); } else { if (_context.NCacheLog.IsInfoEnabled) { _context.NCacheLog.Info("TaskTracker.recievedReducerData", "task does not exist in running tasks " + taskID); } } return(true); }
public void MapperCompleted(MapReduceOperation operation) { lock (mutex){ Address source = operation.Source != null ? operation.Source : new Address(_parent.Context.Render.IPAddress, _parent.Context.Render.Port); if (participants != null) { NodeTaskStatus status = (NodeTaskStatus)participants[source]; if (status != null) { status.MapperStatus = MapperStatus.Completed; } } if (_context.NCacheLog.IsInfoEnabled) { _context.NCacheLog.Info("MapReduceTask(" + this.TaskId + ").MapperCompleted", "Mapped is completed on '" + source.IpAddress.ToString() + "'"); } CheckMappersCompleted(); } }
public object RemoveFromRunning(MapReduceOperation operation) { try { if (this.runningTasks.ContainsKey(operation.TaskID)) { this.runningTasks.Remove(operation.TaskID); if (_context.PerfStatsColl != null) { _context.PerfStatsColl.DecrementRunningTasks(); } } return(true); } catch (Exception ex) { return(false); } }
private void SendParticipantReducerFailedMessage() { MapReduceOperation operation = new MapReduceOperation(); operation.TaskID = this.TaskId; operation.OpCode = MapReduceOpCodes.ReducerFailed; try { if (!isLocal) { _parent.SendMapReduceOperation(new System.Collections.ArrayList(participants.Keys), operation); } } catch (Exception ex) { _parent.Context.NCacheLog.Error("MapReduceTask(" + this.TaskId + ").SendParticipantReducerFailedMessage", "Failed: " + ex.Message); } ReducerFailed(operation); }
private object UnregisterTaskNotification(MapReduceOperation operation) { String taskID = (String)operation.TaskID; if (taskID == null || string.IsNullOrEmpty(taskID)) { throw new OperationFailedException("Task can not be null or empty"); } TaskBase t = null; if (runningTasks.ContainsKey(taskID)) { t = (TaskBase)runningTasks[taskID]; t.RemoveTaskCallbackInfo((TaskCallbackInfo)operation.CallbackInfo); } else if (waitingTasks.ContainsKey(taskID)) { t = (TaskBase)waitingTasks[taskID]; t.RemoveTaskCallbackInfo((TaskCallbackInfo)operation.CallbackInfo); } else if (taskOutputStore != null && taskOutputStore.TaskOutputExists(taskID)) { //remove listener info from taskoutput TaskOutput output = taskOutputStore.TaskOutput(taskID); if (output != null) { output.RemoveFromListeners(operation.CallbackInfo); } } else if (this.cancelledTask.Contains(taskID)) { throw new OperationFailedException("Task with Specified TaskId was cancelled."); } else { throw new OperationFailedException("Task with Specified Task id does not exists"); } return(true); }
private object CancelTask(MapReduceOperation operation) { string taskID = (string)operation.TaskID; lock (_lock) { if (runningTasks.ContainsKey(taskID)) { TaskBase t = (TaskBase)runningTasks[taskID]; t.StopTask(); t.Dispose(); this.cancelledTask.Add(taskID); if (_context.CacheImpl != null && _context.CacheInternal != null) { EventContext eventContext = new EventContext(); eventContext.TaskStatus = TaskStatus.Cancelled; eventContext.TaskFailureReason = "Task was Cancelled by user."; _context.CacheInternal.NotifyTaskCallback(t.TaskId, t.CallbackListeners, false, null, eventContext); } runningTasks.Remove(taskID); if (_context.PerfStatsColl != null) { _context.PerfStatsColl.DecrementRunningTasks(); } if (_context.NCacheLog.IsInfoEnabled) { _context.NCacheLog.Info("TaskTracker.CancelTask", "Task with task ID '" + taskID.ToUpper() + "' has been cancelled."); } } else { _context.NCacheLog.Error("TaskTracker.CancelTask", "Task with task ID '" + taskID.ToUpper() + "' does not exist."); } } return(true); }
private Object NextRecord(MapReduceOperation operation) { TaskEnumeratorPointer pointer = (TaskEnumeratorPointer)operation.Data; // Task cancellation check if (this.cancelledTask.Contains(pointer.TaskId)) { throw new OperationFailedException("Task with specified Task ID was Cancelled."); } TaskEnumeratorResult result = null; if (taskOutputStore != null) { try { result = taskOutputStore.NextRecord(pointer); result.NodeAddress = _context.Render.IPAddress.ToString(); } catch (InvalidTaskEnumeratorException ex) { _context.NCacheLog.Error("TaskTracker.GetTaskEnumerator", ex.Message); throw ex; } } return(result); }
private object GetTaskEnumerator(MapReduceOperation operation) { TaskEnumeratorPointer pointer = (TaskEnumeratorPointer)operation.Data; if (clientAddress == null) { try { if (_context.Render != null) { clientAddress = new Address(_context.Render.IPAddress, _context.Render.Port); } } catch (Exception ex) { throw new InvalidTaskEnumeratorException(ex.Message, ex); } } pointer.ClientAddress = clientAddress; if (clusterAddress == null) { #if !CLIENT if (_context.CacheImpl is ClusterCacheBase) { clusterAddress = ((ClusterCacheBase)_context.CacheImpl).Cluster.LocalAddress; } else if (_context.CacheImpl is LocalCacheImpl) { clusterAddress = new Address(_context.Render.IPAddress, _context.Render.Port); } #else clusterAddress = new Address(_context.Render.IPAddress, _context.Render.Port); #endif } pointer.ClusterAddress = clusterAddress; // Task Cancellation check. if (this.cancelledTask.Contains(pointer.TaskId)) { throw new OperationFailedException("Task with specified Task ID was cancelled"); } TaskEnumeratorResult result = null; if (taskOutputStore != null) { try { result = taskOutputStore.GetTaskEnumerator(pointer); //result.NodeAddress = _context.Render.IPAddress.ToString(); result.Pointer = pointer; } catch (InvalidTaskEnumeratorException ex) { _context.NCacheLog.Error("TaskTracker.GetTaskEnumerator", ex.Message); throw ex; } } if (_context.NCacheLog.IsInfoEnabled) { _context.NCacheLog.Info("TaskTracker.GetTaskEnumerator", "Task Enumerator provided to client."); } return(result); }
public object TaskOperationRecieved(MapReduceOperation operation) { MapReduceOpCodes opCode = operation.OpCode; switch (opCode) { case MapReduceOpCodes.GetTaskSequence: return(GetTaskSequence()); case MapReduceOpCodes.SubmitMapReduceTask: return(SubmitMapReduceTask(operation)); case MapReduceOpCodes.StartTask: return(StartMapReduceTask(operation)); case MapReduceOpCodes.CancelTask: return(CancelTask(operation)); case MapReduceOpCodes.CancellAllTasks: return(CancelAllTasks(true)); case MapReduceOpCodes.GetRunningTasks: return(GetRunningTasks()); case MapReduceOpCodes.GetTaskEnumerator: return(GetTaskEnumerator(operation)); case MapReduceOpCodes.GetNextRecord: return(NextRecord(operation)); case MapReduceOpCodes.RegisterTaskNotification: return(RegisterTaskNotification(operation)); case MapReduceOpCodes.UnregisterTaskNotification: return(UnregisterTaskNotification(operation)); case MapReduceOpCodes.GetTaskStatus: return(GetTaskProgress(operation)); case MapReduceOpCodes.RemoveFromRunningList: return(RemoveFromRunning(operation)); case MapReduceOpCodes.RemoveFromSubmittedList: return(RemoveFromSubmitted(operation)); case MapReduceOpCodes.MapperCompleted: return(MapperCompleted(operation)); case MapReduceOpCodes.MapperFailed: return(MapperFailed(operation)); case MapReduceOpCodes.ReducerCompleted: return(ReducerCompleted(operation)); case MapReduceOpCodes.ReducerFailed: return(ReducerFailed(operation)); case MapReduceOpCodes.ReceiveReducerData: return(RecievedReducerData(operation)); } return(null); }
private object StartMapReduceTask(MapReduceOperation operation) { return(StartTask(operation.TaskID, operation.SequenceID)); }