/// <summary>
/// Process the request with a local data source specified by its definition
/// </summary>
/// <param name="request">A request to process</param>
/// <param name="dataSource">A data source definition to process the request with</param>
/// <returns></returns>
private async Task <IFetchResponseWithProvenance> ProcessWithLocalDataSourceAsync(IFetchRequest request, ExtendedDataSourceDefinition dataSource)
{
if (dataSource.IsFederated)
{
throw new InvalidOperationException("Expected local data source definition, while passed definition is federated data source");
}
var instance = await DataSourceHandlerCache.GetInstanceAsync(dataSource.HandlerTypeName, dataSource.Uri);
StandaloneRequestContext ctx = new StandaloneRequestContext(
TranslateRequestIntoDsNamings(request, dataSource),
this,
dataSource.DsToEnvMapping,
instance.Storage,
dataSource.ID);
var values = await instance.Handler.ProcessRequestAsync(ctx);
var uncertainty = ctx.ReportUncertainty();
if (uncertainty == null)
{
uncertainty = ArrayHelper.GetConstantArray <double>(request.Domain.GetDataArrayShape(), Double.MaxValue);
}
return(new FetchResponseWithProvenance(request, values, uncertainty,
ArrayHelper.GetConstantArray <ushort>(request.Domain.GetDataArrayShape(), dataSource.ID)));
}
/// <summary>
/// Returns null if there are no local data source that can handle specified variable
/// </summary>
/// <param name="request"></param>
/// <param name="localDataSources"></param>
/// <returns></returns>
private async Task <IFetchResponseWithProvenance> ProcessWithLocalDataSourcesAsync(IFetchRequest request, ExtendedDataSourceDefinition[] localDataSources)
{
IFetchResponseWithProvenance localResult;
traceSource.TraceEvent(TraceEventType.Start, 6, "Local data sources are starting processing");
// Start individual fetches
if (localDataSources.Length == 0)
{
localResult = null;
}
else if (localDataSources.Length == 1) //merge of local data source won't be performed
{
traceSource.TraceEvent(TraceEventType.Start, 5, "Starting processing the request with the only data source (without further results merging)");
localResult = await ProcessWithLocalDataSourceAsync(request, localDataSources[0]);
traceSource.TraceEvent(TraceEventType.Stop, 5, "Finished processing the request with the only data source (without further results merging)");
}
else
{
var localFetches = await Task.WhenAll(
localDataSources.Select(s => DataSourceHandlerCache.GetInstanceAsync(s.HandlerTypeName, s.Uri).ContinueWith <Tuple <Task <Array>, DependentRequestContext> >((initTask, dataSource) =>
{
var handler = initTask.Result;
var ds = (ExtendedDataSourceDefinition)dataSource;
var ctx = new DependentRequestContext(TranslateRequestIntoDsNamings(request, ds),
this,
ds.DsToEnvMapping,
handler.Storage,
ds.ID);
return(new Tuple <Task <Array>, DependentRequestContext>(handler.Handler.ProcessRequestAsync(ctx), ctx));
}, s)));
// waiting all of the tasks either for returning value or uncertainties. Removing tasks that returned values without uncertainties from fetches list
var finishedTasks = await Task.WhenAll(localFetches.Select(t => Task.WhenAny(t.Item1, t.Item2.EvaluateUncertaintyTask)));
// Check if no individual fetch fails
if (finishedTasks.Any(f => f.Status != TaskStatus.RanToCompletion))
{
Array.ForEach(localFetches, f => f.Item2.SetCanceled());
string errorMess = string.Empty;
var failedTask = finishedTasks.First(f => f.Status != TaskStatus.RanToCompletion);
if (failedTask.Exception != null)
{
errorMess += failedTask.Exception.Flatten().ToString();
}
throw new Exception("One or more dependent tasks failed. " + errorMess);
}
var tasksWithValuesReady = finishedTasks.Where(f => f is Task <Array>).ToList();
var finishedLocalFetches = localFetches.Where(f => tasksWithValuesReady.Contains(f.Item1)).ToList();
var uncertaintyEvaluatedLocalFetches = localFetches.Where(f => !tasksWithValuesReady.Contains(f.Item1)).ToList();
// Complete intelligent request of no dependent request waits for uncertainty
if (!uncertaintyEvaluatedLocalFetches.Any())//one or more data source returned value, but no one returned uncertainties. So returning any (first) data source result, as we have to choose what result to ruturn
{
localResult = new FetchResponseWithProvenance(request,
finishedLocalFetches[0].Item1.Result,
ArrayHelper.GetConstantArray <double>(request.Domain.GetDataArrayShape(), Double.MaxValue),
ArrayHelper.GetConstantArray <ushort>(request.Domain.GetDataArrayShape(), finishedLocalFetches[0].Item2.ID));
}
else if (uncertaintyEvaluatedLocalFetches.Count == 1) // exactly one data source reported uncertainty, returning it's values, ignoring others
{
uncertaintyEvaluatedLocalFetches[0].Item2.SetProvenance(null); //all elements are needed
localResult = new FetchResponseWithProvenance(request,
uncertaintyEvaluatedLocalFetches[0].Item1.Result,
uncertaintyEvaluatedLocalFetches[0].Item2.ReportUncertainty(),
ArrayHelper.GetConstantArray <ushort>(request.Domain.GetDataArrayShape(), uncertaintyEvaluatedLocalFetches[0].Item2.ID));
}
else
{
List <ushort> requiredIDs;
var provenance = Array.CreateInstance(typeof(ushort), request.Domain.GetDataArrayShape());
var uncertainty = Array.CreateInstance(typeof(double), request.Domain.GetDataArrayShape());
var isTimeSeriesRequest = request.Domain.TimeRegion.IsTimeSeries;
MergeUncertainty(ref uncertainty, ref provenance, out requiredIDs,
uncertaintyEvaluatedLocalFetches.Select(f => new Tuple <Array, ushort>(f.Item2.ReportUncertainty(), f.Item2.ID)).ToArray());
// Remove all sources that are not present in merged provenance array
for (int i = 0; i < uncertaintyEvaluatedLocalFetches.Count;)
{
if (requiredIDs.Contains(uncertaintyEvaluatedLocalFetches[i].Item2.ID))
{
i++;
}
else
{
uncertaintyEvaluatedLocalFetches[i].Item2.SetCanceled();
uncertaintyEvaluatedLocalFetches.RemoveAt(i);
}
}
uncertaintyEvaluatedLocalFetches.ForEach(f => f.Item2.SetProvenance(provenance));
Array values = Array.CreateInstance(typeof(double), request.Domain.GetDataArrayShape());
Tuple <Array, ushort>[] localResults = new Tuple <Array, ushort> [uncertaintyEvaluatedLocalFetches.Count];
for (int i = 0; i < localResults.Length; i++)
{
localResults[i] = Tuple.Create(await uncertaintyEvaluatedLocalFetches[i].Item1, uncertaintyEvaluatedLocalFetches[i].Item2.ID);
}
MergeValues(provenance, ref values, localResults);
localResult = new FetchResponseWithProvenance(request, values, uncertainty, provenance);
}
}
traceSource.TraceEvent(TraceEventType.Stop, 6, "Local data sources finished computation");
return(localResult);
}
