public void OnHttpRequest(ReefHttpRequest requet, ReefHttpResponse response) { LOGGER.Log(Level.Info, string.Format(CultureInfo.CurrentCulture, "HelloHttpHandler OnHttpRequest: URL: {0}, QueryString: {1}, inputStream: {2}.", requet.Url, requet.Querystring, ByteUtilities.ByteArrarysToString(requet.InputStream))); response.Status = HttpStatusCode.OK; response.OutputStream = ByteUtilities.StringToByteArrays("Byte array returned from HelloHttpHandler in CLR!!!"); }
/// <summary> /// Creates a mock completed update task with the taskId specified /// </summary> /// <param name="taskId"></param> /// <returns></returns> private static ICompletedTask CreateMockCompletedUpdateTask(string taskId) { var completedTask = CreateMockCompletedTask(taskId); completedTask.Message.Returns(ByteUtilities.StringToByteArrays(TaskManager.UpdateTaskCompleted)); return(completedTask); }
/// <summary> /// THIS ASSUMES THAT IT IS CALLED ON A THREAD HOLDING THE LOCK ON THE HeartBeatManager /// </summary> /// <param name="e"></param> private void HandleTaskInitializationException(TaskClientCodeException e) { byte[] error; try { error = ByteUtilities.SerializeToBinaryFormat(e); } catch (SerializationException se) { error = ByteUtilities.SerializeToBinaryFormat( TaskClientCodeException.CreateWithNonSerializableInnerException(e, se)); } var avroFailedTask = new AvroFailedTask { identifier = e.TaskId, cause = error, data = ByteUtilities.StringToByteArrays(e.ToString()), message = e.Message }; var taskStatus = new TaskStatusProto { context_id = e.ContextId, task_id = e.TaskId, result = AvroJsonSerializer <AvroFailedTask> .ToBytes(avroFailedTask), state = State.FAILED }; LOGGER.Log(Level.Error, "Sending Heartbeat for a failed task: {0}", taskStatus); _heartBeatManager.OnNext(taskStatus); }
public TaskStatusProto ToProto() { Check(); TaskStatusProto taskStatusProto = new TaskStatusProto() { context_id = _contextId, task_id = _taskId, state = GetProtoState(), }; if (_result.IsPresent()) { taskStatusProto.result = ByteUtilities.CopyBytesFrom(_result.Value); } else if (_lastException.IsPresent()) { //final Encoder<Throwable> codec = new ObjectSerializableCodec<>(); //final byte[] error = codec.encode(_lastException.get()); byte[] error = ByteUtilities.StringToByteArrays(_lastException.Value.ToString()); taskStatusProto.result = ByteUtilities.CopyBytesFrom(error); } else if (_state == TaskState.Running) { foreach (TaskMessage message in GetMessages()) { TaskStatusProto.TaskMessageProto taskMessageProto = new TaskStatusProto.TaskMessageProto() { source_id = message.MessageSourceId, message = ByteUtilities.CopyBytesFrom(message.Message), }; taskStatusProto.task_message.Add(taskMessageProto); } } return(taskStatusProto); }
/// <summary> /// Performs IMRU iterations on update side /// </summary> /// <returns></returns> protected override byte[] TaskBody(byte[] memento) { UpdateResult <TMapInput, TResult> updateResult = null; try { updateResult = _updateTask.Initialize(); } catch (Exception e) { HandleTaskAppException(e); } while (!_cancellationSource.IsCancellationRequested && updateResult.HasMapInput) { using ( var message = new MapInputWithControlMessage <TMapInput>(updateResult.MapInput, MapControlMessage.AnotherRound)) { _dataAndControlMessageSender.Send(message); } if (_invokeGc) { Logger.Log(Level.Verbose, "Calling Garbage Collector"); GC.Collect(); GC.WaitForPendingFinalizers(); } var input = _dataReceiver.Reduce(_cancellationSource); try { updateResult = _updateTask.Update(input); if (updateResult.HasResult) { _resultHandler.HandleResult(updateResult.Result); _done = true; } } catch (Exception e) { HandleTaskAppException(e); } } if (!_cancellationSource.IsCancellationRequested) { MapInputWithControlMessage <TMapInput> stopMessage = new MapInputWithControlMessage <TMapInput>(MapControlMessage.Stop); _dataAndControlMessageSender.Send(stopMessage); } if (_done) { return(ByteUtilities.StringToByteArrays(TaskManager.UpdateTaskCompleted)); } return(null); }
public static void BuildHttpResponse( ReefHttpResponse response, HttpStatusCode httpStatusCode, string strResponse) { response.Status = httpStatusCode; response.OutputStream = ByteUtilities.StringToByteArrays(strResponse); }
private void CloseRunningTasks() { foreach (var task in _runningTasks) { Logger.Log(Level.Info, "Dispose running task from driver:" + task.Key); task.Value.Dispose(ByteUtilities.StringToByteArrays(TaskManager.CloseTaskByDriver)); } _runningTasks.Clear(); }
internal static string CreateTestToken() { var t1 = new SecurityToken( Identifier1, Identifier1, ByteUtilities.StringToByteArrays(TokenKey1), Encoding.ASCII.GetBytes(Password1)); return(JsonConvert.SerializeObject(t1)); }
public void OnError(Exception error) { LOGGER.Log(Level.Error, "job excemption", error); JobStatusProto proto = new JobStatusProto(); proto.identifier = _jobId; proto.state = State.FAILED; proto.exception = ByteUtilities.StringToByteArrays(error.Message); _clock.Dispose(); }
public TaskStatusProto ToProto() { // This is locked because the Task continuation thread which sets the // result is potentially different from the HeartBeat thread. lock (_heartBeatManager) { Check(); TaskStatusProto taskStatusProto = new TaskStatusProto() { context_id = ContextId, task_id = TaskId, state = GetProtoState() }; if (_result.IsPresent()) { taskStatusProto.result = ByteUtilities.CopyBytesFrom(_result.Value); } else if (_lastException.IsPresent()) { byte[] error; try { error = ByteUtilities.SerializeToBinaryFormat(_lastException.Value); } catch (SerializationException se) { error = ByteUtilities.SerializeToBinaryFormat( NonSerializableTaskException.UnableToSerialize(_lastException.Value, se)); } var avroFailedTask = new AvroFailedTask { identifier = _taskId, cause = error, data = ByteUtilities.StringToByteArrays(_lastException.Value.ToString()), message = _lastException.Value.Message }; taskStatusProto.result = AvroJsonSerializer <AvroFailedTask> .ToBytes(avroFailedTask); } else if (_state == TaskState.Running) { foreach (TaskMessage message in GetMessages()) { TaskStatusProto.TaskMessageProto taskMessageProto = new TaskStatusProto.TaskMessageProto() { source_id = message.MessageSourceId, message = ByteUtilities.CopyBytesFrom(message.Message), }; taskStatusProto.task_message.Add(taskMessageProto); } } return(taskStatusProto); } }
/// <summary> /// Called when receiving an http request from Java side /// </summary> /// <param name="httpMessage">The HTTP message.</param> public void OnNext(IHttpMessage httpMessage) { LOGGER.Log(Level.Info, "HttpHandler OnNext is called"); string requestString = httpMessage.GetRequestString(); if (requestString != null && requestString.Equals(SPEC)) { LOGGER.Log(Level.Info, "HttpHandler OnNext, requestString:" + requestString); LOGGER.Log(Level.Info, "HttpHandler OnNext, port number:" + httpServerPort.PortNumber); httpMessage.SetUriSpecification(GetAllSpecifications()); } else { LOGGER.Log(Level.Info, "HttpHandler OnNext, handling http request."); byte[] byteData = httpMessage.GetQueryReuestData(); AvroHttpRequest avroHttpRequest = AvroHttpSerializer.FromBytesWithJson(byteData); LOGGER.Log(Level.Info, string.Format(CultureInfo.CurrentCulture, "HttpHandler OnNext, requestData:", avroHttpRequest)); string spec = GetSpecification(avroHttpRequest.PathInfo); if (spec != null) { LOGGER.Log(Level.Info, "HttpHandler OnNext, target:" + spec); ReefHttpRequest request = ToHttpRequest(avroHttpRequest); foreach (var h in request.Header) { LOGGER.Log(Level.Info, string.Format(CultureInfo.CurrentCulture, "HttpRequest Header-key: {0}, value: {1}.", h.Key, h.Value)); } ReefHttpResponse response = new ReefHttpResponse(); IHttpHandler handler; eventHandlers.TryGetValue(spec.ToLower(CultureInfo.CurrentCulture), out handler); byte[] responseData; if (handler != null) { LOGGER.Log(Level.Info, "HttpHandler OnNext, get eventHandler:" + handler.GetSpecification()); handler.OnHttpRequest(request, response); responseData = response.OutputStream; } else { responseData = ByteUtilities.StringToByteArrays(string.Format(CultureInfo.CurrentCulture, "No event handler found at CLR side for {0}.", spec)); } httpMessage.SetQueryResponseData(responseData); } } }
/// <summary> /// Add the RunningTask to _runningTasks and dispose the last received running task /// </summary> public void OnNext(IRunningTask value) { lock (_lock) { Logger.Log(Level.Info, "Received running task:" + value.Id + " from container [" + value.ActiveContext.EvaluatorId + "]"); _runningTasks.Add(value.Id, value); if (_runningTasks.Count == 4) { Logger.Log(Level.Info, "Dispose running task from driver:" + value.Id); value.Dispose(ByteUtilities.StringToByteArrays(TaskManager.CloseTaskByDriver)); _runningTasks.Remove(value.Id); } } }
public void TestHttpRequestSerializationJasonRoundTrip() { AvroHttpRequest r = CreateAvroHttpRequest(); string str = AvroHttpSerializer.ToJson(r); byte[] bytes = ByteUtilities.StringToByteArrays(str); var r1 = AvroHttpSerializer.FromBytesWithJson(bytes); var ri = ByteUtilities.ByteArraysToString(r.InputStream); var ri1 = ByteUtilities.ByteArraysToString(r1.InputStream); Assert.Equal(ri, ri1); Assert.Equal(r.QueryString, r1.QueryString); }
/// <summary> /// THIS ASSUMES THAT IT IS CALLED ON A THREAD HOLDING THE LOCK ON THE HeartBeatManager /// </summary> /// <param name="e"></param> private void HandleTaskException(TaskClientCodeException e) { LOGGER.Log(Level.Error, "TaskClientCodeException", e); byte[] exception = ByteUtilities.StringToByteArrays(e.ToString()); TaskStatusProto taskStatus = new TaskStatusProto() { context_id = e.ContextId, task_id = e.TaskId, result = exception, state = State.FAILED }; LOGGER.Log(Level.Error, string.Format(CultureInfo.InvariantCulture, "Sending Heartbeatb for a failed task: {0}", taskStatus.ToString())); _heartBeatManager.OnNext(taskStatus); }
public void OnNext(IRunningTask runningTask) { IActiveContext context = runningTask.ActiveContext; string messageStr = string.Format( CultureInfo.InvariantCulture, "HelloRunningTaskHandler: Task [{0}] is running. Evaluator id: [{1}].", runningTask.Id, context.EvaluatorId); Console.WriteLine(messageStr); byte[] message = ByteUtilities.StringToByteArrays(messageStr); runningTask.Send(message); }
public void OnNext(IActiveContext activeContext) { const string taskId = "TaskID"; Logger.Log(Level.Info, SendingMessageToContextLog); activeContext.SendMessage(ByteUtilities.StringToByteArrays(Message)); var taskConfiguration = TaskConfiguration.ConfigurationModule .Set(TaskConfiguration.Identifier, taskId) .Set(TaskConfiguration.Task, GenericType <MessageTask> .Class) .Set(TaskConfiguration.OnMessage, GenericType <MessageTask> .Class) .Set(TaskConfiguration.OnSendMessage, GenericType <MessageTask> .Class) .Build(); activeContext.SubmitTask(taskConfiguration); }
public void OnNext(IRunningTask runningTask) { IActiveContext context = runningTask.ActiveContext; Console.WriteLine(string.Format( CultureInfo.InvariantCulture, "HelloDriverRestartRunningTaskHandler: Task [{0}] is running after driver restart. Evaluator id: [{1}].", runningTask.Id, context.EvaluatorId)); runningTask.Send(ByteUtilities.StringToByteArrays( string.Format( CultureInfo.InvariantCulture, "Hello, task {0}! Glad to know that you are still running in Evaluator {1} after driver restart!", runningTask.Id, context.EvaluatorId))); }
/// <summary> /// THIS ASSUMES THAT IT IS CALLED ON A THREAD HOLDING THE LOCK ON THE HeartBeatManager /// </summary> /// <param name="e"></param> private void HandlContextException(ContextClientCodeException e) { LOGGER.Log(Level.Error, "ContextClientCodeException", e); byte[] exception = ByteUtilities.StringToByteArrays(e.ToString()); ContextStatusProto contextStatusProto = new ContextStatusProto() { context_id = e.ContextId, context_state = ContextStatusProto.State.FAIL, error = exception }; if (e.ParentId.IsPresent()) { contextStatusProto.parent_id = e.ParentId.Value; } LOGGER.Log(Level.Error, string.Format(CultureInfo.InvariantCulture, "Sending Heartbeat for a failed context: {0}", contextStatusProto.ToString())); _heartBeatManager.OnNext(contextStatusProto); }
private void HandleContextException(Exception e, string contextId, string parentContextId) { lock (_heartBeatManager) { LOGGER.Log(Level.Warning, "ContextException", e); byte[] exception = ByteUtilities.StringToByteArrays(e.ToString()); var contextStatusProto = new ContextStatusProto { context_id = contextId, context_state = ContextStatusProto.State.FAIL, parent_id = parentContextId, error = exception }; LOGGER.Log(Level.Error, "Sending Heartbeat for a failed context: {0}", contextStatusProto); _heartBeatManager.OnNext(contextStatusProto); } }
/// <summary> /// Get runtime configuration /// </summary> private static IConfiguration GetRuntimeConfiguration(string[] args) { var token = new SecurityToken( TrustedApplicationTokenIdentifier, TrustedApplicationTokenIdentifier, ByteUtilities.StringToByteArrays(args[0]), Encoding.ASCII.GetBytes(args[1])); var clientConfig = YARNClientConfiguration.ConfigurationModule .Set(YARNClientConfiguration.SecurityTokenStr, JsonConvert.SerializeObject(token)) .Build(); var tcpPortConfig = TcpPortConfigurationModule.ConfigurationModule .Set(TcpPortConfigurationModule.PortRangeStart, args.Length > 2 ? args[2] : DefaultPortRangeStart) .Set(TcpPortConfigurationModule.PortRangeCount, args.Length > 3 ? args[3] : DefaultPortRangeCount) .Build(); return(Configurations.Merge(clientConfig, tcpPortConfig)); }
/// <summary> /// Handles RuntimeStop /// </summary> /// <param name="runtimeStop"></param> public void OnNext(RuntimeStop runtimeStop) { LOGGER.Log(Level.Info, "RuntimeStop: " + runtimeStop); if (runtimeStop.Exception != null) { string exceptionMessage = runtimeStop.Exception.Message; LOGGER.Log(Level.Warning, "Sending runtime error:" + exceptionMessage); RuntimeErrorProto runtimeErrorProto = new RuntimeErrorProto(); runtimeErrorProto.message = exceptionMessage; runtimeErrorProto.exception = ByteUtilities.StringToByteArrays(exceptionMessage); runtimeErrorProto.name = "REEF"; _runtimeErrorHandler.OnNext(runtimeErrorProto); LOGGER.Log(Level.Warning, "DONE Sending runtime error: " + exceptionMessage); } lock (_evaluators) { foreach (EvaluatorManager evaluatorManager in _evaluators.Values) { LOGGER.Log(Level.Warning, "Unclean shutdown of evaluator: " + evaluatorManager.Id); evaluatorManager.Dispose(); } } try { _heartbeatConnectionChannel.Dispose(); _errorChannel.Dispose(); Optional <Exception> e = runtimeStop.Exception != null ? Optional <Exception> .Of(runtimeStop.Exception) : Optional <Exception> .Empty(); _clientJobStatusHandler.Dispose(e); LOGGER.Log(Level.Info, "driver manager closed"); } catch (Exception e) { Exceptions.Caught(e, Level.Error, "Error disposing Driver manager", LOGGER); Exceptions.Throw(new InvalidOperationException("Cannot dispose driver manager"), LOGGER); } }
internal static IList <string> CreateTestTokens() { var t1 = new SecurityToken( Identifier1, Identifier1, ByteUtilities.StringToByteArrays(TokenKey1), Encoding.ASCII.GetBytes(Password1)); var t2 = new SecurityToken( Identifier2, Identifier2, ByteUtilities.StringToByteArrays(TokenKey2), Encoding.ASCII.GetBytes(Password2)); return(new List <string>() { JsonConvert.SerializeObject(t1), JsonConvert.SerializeObject(t2) }); }
/// <summary> /// It gets HTTP request and close the contexts after receiving all the completed tasks /// </summary> /// <param name="requet"></param> /// <param name="resonse"></param> public void OnHttpRequest(ReefHttpRequest requet, ReefHttpResponse resonse) { var msg = string.Format(CultureInfo.CurrentCulture, "HelloTaskCompletedHandler OnHttpRequest: URL: {0}, QueryString: {1}, inputStream: {2}.", requet.Url, requet.Querystring, ByteUtilities.ByteArraysToString(requet.InputStream)); Logger.Log(Level.Info, msg); resonse.Status = HttpStatusCode.OK; while (_completedTasks.Count != 2) { Thread.Sleep(1000); Logger.Log(Level.Info, "_completedTasks received:" + _completedTasks.Count); } resonse.OutputStream = ByteUtilities.StringToByteArrays("Stopped!!!"); foreach (var t in _completedTasks) { t.ActiveContext.Dispose(); } _completedTasks.Clear(); }
public TaskRuntime(IInjector taskInjector, string contextId, string taskId, HeartBeatManager heartBeatManager, string memento = null) { _injector = taskInjector; _heartBeatManager = heartBeatManager; Optional <ISet <ITaskMessageSource> > messageSources = Optional <ISet <ITaskMessageSource> > .Empty(); try { _task = _injector.GetInstance <ITask>(); } catch (Exception e) { Org.Apache.Reef.Utilities.Diagnostics.Exceptions.CaughtAndThrow(new InvalidOperationException("Unable to inject task.", e), Level.Error, "Unable to inject task.", LOGGER); } try { ITaskMessageSource taskMessageSource = _injector.GetInstance <ITaskMessageSource>(); messageSources = Optional <ISet <ITaskMessageSource> > .Of(new HashSet <ITaskMessageSource>() { taskMessageSource }); } catch (Exception e) { Org.Apache.Reef.Utilities.Diagnostics.Exceptions.Caught(e, Level.Warning, "Cannot inject task message source with error: " + e.StackTrace, LOGGER); // do not rethrow since this is benign } try { _nameClient = _injector.GetInstance <INameClient>(); _heartBeatManager.EvaluatorSettings.NameClient = _nameClient; } catch (InjectionException) { LOGGER.Log(Level.Warning, "Cannot inject name client from task configuration."); // do not rethrow since user is not required to provide name client } LOGGER.Log(Level.Info, "task message source injected"); _currentStatus = new TaskStatus(_heartBeatManager, contextId, taskId, messageSources); _memento = memento == null ? Optional <byte[]> .Empty() : Optional <byte[]> .Of(ByteUtilities.StringToByteArrays(memento)); }
private void Handle(Exception e) { lock (_heartBeatManager) { Logger.Log(Level.Error, string.Format(CultureInfo.InvariantCulture, "evaluator {0} failed with exception", _evaluatorId), e); _state = State.FAILED; string errorMessage = string.Format( CultureInfo.InvariantCulture, "failed with error [{0}] with message [{1}] and stack trace [{2}]", e, e.Message, e.StackTrace); EvaluatorStatusProto evaluatorStatusProto = new EvaluatorStatusProto() { evaluator_id = _evaluatorId, error = ByteUtilities.StringToByteArrays(errorMessage), state = _state }; _heartBeatManager.OnNext(evaluatorStatusProto); _contextManager.Dispose(); } }
/// <summary> /// Get runtime configuration /// </summary> private static IConfiguration GetRuntimeConfigurationForYarn(string[] args) { var token = new SecurityToken( TrustedApplicationTokenIdentifier, TrustedApplicationTokenIdentifier, ByteUtilities.StringToByteArrays(args[0]), Encoding.ASCII.GetBytes(args[1])); var clientConfig = YARNClientConfiguration.ConfigurationModule .Set(YARNClientConfiguration.SecurityTokenStr, JsonConvert.SerializeObject(token)) .Build(); var tcpPortConfig = TcpPortConfigurationModule.ConfigurationModule .Set(TcpPortConfigurationModule.PortRangeStart, args.Length > 3 ? args[3] : DefaultPortRangeStart) .Set(TcpPortConfigurationModule.PortRangeCount, args.Length > 4 ? args[4] : DefaultPortRangeCount) .Build(); var c = TangFactory.GetTang().NewConfigurationBuilder() .BindIntNamedParam <NumberOfContainers>(args[2]) .Build(); return(Configurations.Merge(clientConfig, tcpPortConfig, c)); }
/// <summary> /// Creates an Avro HTTP request for unit tests. /// </summary> /// <returns>AvroHttpRequest.</returns> private AvroHttpRequest CreateAvroHttpRequest() { AvroHttpRequest r = new AvroHttpRequest(); r.Header = new List <HeaderEntry>(); HeaderEntry he1 = new HeaderEntry(); he1.key = "a"; he1.value = "xxx"; HeaderEntry he2 = new HeaderEntry(); he2.key = "b"; he2.value = "yyy"; r.Header.Add(he1); r.Header.Add(he2); r.HttpMethod = "POST"; r.InputStream = ByteUtilities.StringToByteArrays("test binary stream data"); r.PathInfo = "/reef/evaluators"; r.QueryString = "id=12&id=34&a=b"; r.RequestUrl = "http://localhost:8080/reef/evaluators?id=12&id=34&a=b"; return(r); }
public TaskStatusProto ToProto() { // This is locked because the Task continuation thread which sets the // result is potentially different from the HeartBeat thread. lock (_stateLock) { Check(); TaskStatusProto taskStatusProto = new TaskStatusProto() { context_id = ContextId, task_id = TaskId, state = GetProtoState() }; if (_result.IsPresent()) { taskStatusProto.result = ByteUtilities.CopyBytesFrom(_result.Value); } else if (_lastException.IsPresent()) { byte[] error = ByteUtilities.StringToByteArrays(_lastException.Value.ToString()); taskStatusProto.result = ByteUtilities.CopyBytesFrom(error); } else if (_state == TaskState.Running) { foreach (TaskMessage message in GetMessages()) { TaskStatusProto.TaskMessageProto taskMessageProto = new TaskStatusProto.TaskMessageProto() { source_id = message.MessageSourceId, message = ByteUtilities.CopyBytesFrom(message.Message), }; taskStatusProto.task_message.Add(taskMessageProto); } } return(taskStatusProto); } }
public byte[] Encode(ProcessedResults results) { return(ByteUtilities.StringToByteArrays(results.Loss + "+" + string.Join("@", results.Means.Select(m => m.ToString())))); }
public void OnNext(IRunningTask runningTask) { LOGGER.Log(Level.Info, string.Format(CultureInfo.InvariantCulture, "TaskMessegingRunningTaskHandler: {0} is to send message {1}.", runningTask.Id, Message)); runningTask.Send(ByteUtilities.StringToByteArrays(Message)); }