public override async Task <Empty> TriggerStep(StepTriggerRequest request, ServerCallContext context)
{
_logger.LogInformation("Sidecar received a trigger step notification");
return(await this._implementation.TriggerStep(request));
}
public async Task <DataEventReply> ProcessDataEvent(DataEventRequest req)
{
var workflowDefinition = _registry.RetrieveWorkflow();
int eventSourcePosition = _workTracker.GetPositionInWorkflowForRequest(req.RequestId);
if (eventSourcePosition == workflowDefinition.Steps.Count - 1)
{
// todo here I should finish a span that started all the way when the data chunk was first registered.
_logger.LogInformation("Workflow finished for 1 data chunk");
await _workTrackingSemaphore.WaitAsync();
try
{
_workTracker.MarkWorkAsFinished(req.RequestId);
}
finally
{
_workTrackingSemaphore.Release();
}
return(new DataEventReply
{
IsSuccess = true
});
}
var nextStep = workflowDefinition.Steps[eventSourcePosition + 1];
GrpcChannel channel = null;
var stepTriggerRequest = new StepTriggerRequest
{
Metadata = req.Metadata,
RequestId = Guid.NewGuid().ToString(),
DataSink = nextStep.DataSink,
DataSource = nextStep.DataSource
};
if (_configuration["UseDataLocality"] == "false")
{
await _workTrackingSemaphore.WaitAsync();
try
{
_workTracker.MarkWorkAsFinished(req.RequestId);
}
finally
{
_workTrackingSemaphore.Release();
}
var url =
$"http://{_configuration[$"{nextStep.ComputeImage}_SERVICE_HOST"]}:{_configuration[$"{nextStep.ComputeImage}_SERVICE_PORT"]}";
_logger.LogInformation("Not using data locality, falling back to the service: {url}", url);
await _workTrackingSemaphore.WaitAsync();
try
{
_workTracker.MarkWorkAsStarted(stepTriggerRequest.RequestId, eventSourcePosition + 1, "noName");
}
finally
{
_workTrackingSemaphore.Release();
}
channel = GrpcChannel.ForAddress(url);
}
else
{
await _workTrackingSemaphore.WaitAsync();
try
{
_workTracker.MarkWorkAsFinished(req.RequestId);
var channelChoice =
await this._requestRouter.GetGrpcChannelForRequest(nextStep.ComputeImage,
req.Metadata.DataLocalization);
// The request router will return null if there is no available pod.
if (channelChoice == null)
{
_logger.LogInformation("Channel choice is null. We will re-queue the request");
// TODO this could be a bit more informative (like canRetry=true)
return(new DataEventReply
{
IsSuccess = false
});
}
channel = channelChoice.GrpcChannel;
_workTracker.MarkWorkAsStarted(stepTriggerRequest.RequestId, eventSourcePosition + 1,
channelChoice.PodChoice.Name());
}
catch (Exception e)
{
_logger.LogError(e, "Exception for data locality stuff");
throw;
}
finally
{
_workTrackingSemaphore.Release();
}
}
var client = new SidecarService.SidecarServiceClient(channel);
_logger.LogInformation($"Triggering the computation for a step {nextStep.ComputeImage}");
await client.TriggerStepAsync(stepTriggerRequest);
return(new DataEventReply
{
IsSuccess = true
});
}
public async Task <Empty> TriggerStep(StepTriggerRequest request)
{
// Make sure we are not processing more than 3 requests at a time
await _parallelCallSemaphore.WaitAsync();
try
{
var metadata = request.Metadata;
var reqId = request.RequestId;
var fileName = Guid.NewGuid().ToString();
var targetPathForDataDaeomn = $"/store/inputs/{fileName}";
// 1. Ask the data source to download the data.
_logger.LogInformation(
$"Downloading data from the data source. TargetPath = {targetPathForDataDaeomn}");
var pullDataRequest = new PullDataRequest
{
Metadata = metadata,
TargetPath = targetPathForDataDaeomn
};
var src = await _adapterProvider.GetSourceForName(request.DataSource) ?? _dataSource;
await src.DownloadData(pullDataRequest);
_logger.LogInformation("Passing the data to the compute step");
var computeStepRequest = new ComputeStepRequest
{
LocalPath = $"/in/{fileName}"
};
_logger.LogInformation("Triggered the compute step, awaiting responses");
var activity = _source.StartActivity("ComputeStepService/TriggerCompute-Single");
await foreach (var response in _computeStep.TriggerCompute(computeStepRequest))
{
var fName = Path.GetFileName(response.OutputFilePath);
_logger.LogInformation($"Publishing data to the data sink /store/outputs/{fName}");
// stop the activity cause we got the response and set it to null.
activity?.Stop();
activity = null;
var sink = await _adapterProvider.GetSinkForName(request.DataSink) ?? _dataSink;
var reply = await sink.PushData(new PushDataRequest
{
SourceFilePath = $"/store/outputs/{fName}",
// Delete the input file.
DeletePath = targetPathForDataDaeomn
});
_logger.LogInformation("Publishing metadata to the orchestrator service");
await _orchestrator.PublishData(new DataEventRequest
{
Metadata = reply.GeneratedMetadata,
RequestId = reqId
});
}
return(new Empty());
}
finally
{
// indicate 1 thread finished.
_parallelCallSemaphore.Release();
}
}
