public IAsyncEnumerable <ComputeStepReply> TriggerCompute(ComputeStepRequest request)
{
_logger.LogInformation("Triggering GRPC call for compute");
var call = this._client.TriggerCompute(request);
return(call.ResponseStream.ReadAllAsync());
}
public override async Task TriggerCompute(ComputeStepRequest request, IServerStreamWriter <ComputeStepReply> responseStream, ServerCallContext context)
{
_logger.LogInformation($"Received computation request: {request}");
await foreach (var reply in implementation.TriggerCompute(request))
{
_logger.LogInformation($"Writing the reply to the response stream");
await responseStream.WriteAsync(reply);
}
}
public async IAsyncEnumerable <ComputeStepReply> TriggerCompute(ComputeStepRequest request)
{
await using var inputFile = File.OpenRead(request.LocalPath);
using var binaryReader = new BinaryReader(inputFile);
var outputFilePath = $"{this.outputPath}/Step_{Guid.NewGuid().ToString()}";
await using var outputFile = File.OpenWrite(outputFilePath);
await inputFile.CopyToAsync(outputFile);
var reply = new ComputeStepReply
{
OutputFilePath = outputFilePath
};
inputFile.Close();
outputFile.Close();
yield return(reply);
}
public async Task <Empty> TriggerStep(StepTriggerRequest request)
{
// Make sure we are not processing more than 3 requests at a time
await _parallelCallSemaphore.WaitAsync();
try
{
var metadata = request.Metadata;
var reqId = request.RequestId;
var fileName = Guid.NewGuid().ToString();
var targetPathForDataDaeomn = $"/store/inputs/{fileName}";
// 1. Ask the data source to download the data.
_logger.LogInformation(
$"Downloading data from the data source. TargetPath = {targetPathForDataDaeomn}");
var pullDataRequest = new PullDataRequest
{
Metadata = metadata,
TargetPath = targetPathForDataDaeomn
};
var src = await _adapterProvider.GetSourceForName(request.DataSource) ?? _dataSource;
await src.DownloadData(pullDataRequest);
_logger.LogInformation("Passing the data to the compute step");
var computeStepRequest = new ComputeStepRequest
{
LocalPath = $"/in/{fileName}"
};
_logger.LogInformation("Triggered the compute step, awaiting responses");
var activity = _source.StartActivity("ComputeStepService/TriggerCompute-Single");
await foreach (var response in _computeStep.TriggerCompute(computeStepRequest))
{
var fName = Path.GetFileName(response.OutputFilePath);
_logger.LogInformation($"Publishing data to the data sink /store/outputs/{fName}");
// stop the activity cause we got the response and set it to null.
activity?.Stop();
activity = null;
var sink = await _adapterProvider.GetSinkForName(request.DataSink) ?? _dataSink;
var reply = await sink.PushData(new PushDataRequest
{
SourceFilePath = $"/store/outputs/{fName}",
// Delete the input file.
DeletePath = targetPathForDataDaeomn
});
_logger.LogInformation("Publishing metadata to the orchestrator service");
await _orchestrator.PublishData(new DataEventRequest
{
Metadata = reply.GeneratedMetadata,
RequestId = reqId
});
}
return(new Empty());
}
finally
{
// indicate 1 thread finished.
_parallelCallSemaphore.Release();
}
}
