public override async Task <DataEventReply> NotifyDataAvailable(DataEventRequest request, ServerCallContext context) { _workQueue.QueueOrchestrationWork(request); return(await Task.FromResult(new DataEventReply { IsSuccess = true })); }
public async Task <DataEventReply> PublishData(DataEventRequest request) { return(await _client.NotifyDataAvailableAsync(request)); }
public async Task <DataEventReply> ProcessDataEvent(DataEventRequest req) { var workflowDefinition = _registry.RetrieveWorkflow(); int eventSourcePosition = _workTracker.GetPositionInWorkflowForRequest(req.RequestId); if (eventSourcePosition == workflowDefinition.Steps.Count - 1) { // todo here I should finish a span that started all the way when the data chunk was first registered. _logger.LogInformation("Workflow finished for 1 data chunk"); await _workTrackingSemaphore.WaitAsync(); try { _workTracker.MarkWorkAsFinished(req.RequestId); } finally { _workTrackingSemaphore.Release(); } return(new DataEventReply { IsSuccess = true }); } var nextStep = workflowDefinition.Steps[eventSourcePosition + 1]; GrpcChannel channel = null; var stepTriggerRequest = new StepTriggerRequest { Metadata = req.Metadata, RequestId = Guid.NewGuid().ToString(), DataSink = nextStep.DataSink, DataSource = nextStep.DataSource }; if (_configuration["UseDataLocality"] == "false") { await _workTrackingSemaphore.WaitAsync(); try { _workTracker.MarkWorkAsFinished(req.RequestId); } finally { _workTrackingSemaphore.Release(); } var url = $"http://{_configuration[$"{nextStep.ComputeImage}_SERVICE_HOST"]}:{_configuration[$"{nextStep.ComputeImage}_SERVICE_PORT"]}"; _logger.LogInformation("Not using data locality, falling back to the service: {url}", url); await _workTrackingSemaphore.WaitAsync(); try { _workTracker.MarkWorkAsStarted(stepTriggerRequest.RequestId, eventSourcePosition + 1, "noName"); } finally { _workTrackingSemaphore.Release(); } channel = GrpcChannel.ForAddress(url); } else { await _workTrackingSemaphore.WaitAsync(); try { _workTracker.MarkWorkAsFinished(req.RequestId); var channelChoice = await this._requestRouter.GetGrpcChannelForRequest(nextStep.ComputeImage, req.Metadata.DataLocalization); // The request router will return null if there is no available pod. if (channelChoice == null) { _logger.LogInformation("Channel choice is null. We will re-queue the request"); // TODO this could be a bit more informative (like canRetry=true) return(new DataEventReply { IsSuccess = false }); } channel = channelChoice.GrpcChannel; _workTracker.MarkWorkAsStarted(stepTriggerRequest.RequestId, eventSourcePosition + 1, channelChoice.PodChoice.Name()); } catch (Exception e) { _logger.LogError(e, "Exception for data locality stuff"); throw; } finally { _workTrackingSemaphore.Release(); } } var client = new SidecarService.SidecarServiceClient(channel); _logger.LogInformation($"Triggering the computation for a step {nextStep.ComputeImage}"); await client.TriggerStepAsync(stepTriggerRequest); return(new DataEventReply { IsSuccess = true }); }