/// <summary> /// Process the request with a local data source specified by its definition /// </summary> /// <param name="request">A request to process</param> /// <param name="dataSource">A data source definition to process the request with</param> /// <returns></returns> private async Task <IFetchResponseWithProvenance> ProcessWithLocalDataSourceAsync(IFetchRequest request, ExtendedDataSourceDefinition dataSource) { if (dataSource.IsFederated) { throw new InvalidOperationException("Expected local data source definition, while passed definition is federated data source"); } var instance = await DataSourceHandlerCache.GetInstanceAsync(dataSource.HandlerTypeName, dataSource.Uri); StandaloneRequestContext ctx = new StandaloneRequestContext( TranslateRequestIntoDsNamings(request, dataSource), this, dataSource.DsToEnvMapping, instance.Storage, dataSource.ID); var values = await instance.Handler.ProcessRequestAsync(ctx); var uncertainty = ctx.ReportUncertainty(); if (uncertainty == null) { uncertainty = ArrayHelper.GetConstantArray <double>(request.Domain.GetDataArrayShape(), Double.MaxValue); } return(new FetchResponseWithProvenance(request, values, uncertainty, ArrayHelper.GetConstantArray <ushort>(request.Domain.GetDataArrayShape(), dataSource.ID))); }
/// <summary> /// Returns null if there are no local data source that can handle specified variable /// </summary> /// <param name="request"></param> /// <param name="localDataSources"></param> /// <returns></returns> private async Task <IFetchResponseWithProvenance> ProcessWithLocalDataSourcesAsync(IFetchRequest request, ExtendedDataSourceDefinition[] localDataSources) { IFetchResponseWithProvenance localResult; traceSource.TraceEvent(TraceEventType.Start, 6, "Local data sources are starting processing"); // Start individual fetches if (localDataSources.Length == 0) { localResult = null; } else if (localDataSources.Length == 1) //merge of local data source won't be performed { traceSource.TraceEvent(TraceEventType.Start, 5, "Starting processing the request with the only data source (without further results merging)"); localResult = await ProcessWithLocalDataSourceAsync(request, localDataSources[0]); traceSource.TraceEvent(TraceEventType.Stop, 5, "Finished processing the request with the only data source (without further results merging)"); } else { var localFetches = await Task.WhenAll( localDataSources.Select(s => DataSourceHandlerCache.GetInstanceAsync(s.HandlerTypeName, s.Uri).ContinueWith <Tuple <Task <Array>, DependentRequestContext> >((initTask, dataSource) => { var handler = initTask.Result; var ds = (ExtendedDataSourceDefinition)dataSource; var ctx = new DependentRequestContext(TranslateRequestIntoDsNamings(request, ds), this, ds.DsToEnvMapping, handler.Storage, ds.ID); return(new Tuple <Task <Array>, DependentRequestContext>(handler.Handler.ProcessRequestAsync(ctx), ctx)); }, s))); // waiting all of the tasks either for returning value or uncertainties. Removing tasks that returned values without uncertainties from fetches list var finishedTasks = await Task.WhenAll(localFetches.Select(t => Task.WhenAny(t.Item1, t.Item2.EvaluateUncertaintyTask))); // Check if no individual fetch fails if (finishedTasks.Any(f => f.Status != TaskStatus.RanToCompletion)) { Array.ForEach(localFetches, f => f.Item2.SetCanceled()); string errorMess = string.Empty; var failedTask = finishedTasks.First(f => f.Status != TaskStatus.RanToCompletion); if (failedTask.Exception != null) { errorMess += failedTask.Exception.Flatten().ToString(); } throw new Exception("One or more dependent tasks failed. " + errorMess); } var tasksWithValuesReady = finishedTasks.Where(f => f is Task <Array>).ToList(); var finishedLocalFetches = localFetches.Where(f => tasksWithValuesReady.Contains(f.Item1)).ToList(); var uncertaintyEvaluatedLocalFetches = localFetches.Where(f => !tasksWithValuesReady.Contains(f.Item1)).ToList(); // Complete intelligent request of no dependent request waits for uncertainty if (!uncertaintyEvaluatedLocalFetches.Any())//one or more data source returned value, but no one returned uncertainties. So returning any (first) data source result, as we have to choose what result to ruturn { localResult = new FetchResponseWithProvenance(request, finishedLocalFetches[0].Item1.Result, ArrayHelper.GetConstantArray <double>(request.Domain.GetDataArrayShape(), Double.MaxValue), ArrayHelper.GetConstantArray <ushort>(request.Domain.GetDataArrayShape(), finishedLocalFetches[0].Item2.ID)); } else if (uncertaintyEvaluatedLocalFetches.Count == 1) // exactly one data source reported uncertainty, returning it's values, ignoring others { uncertaintyEvaluatedLocalFetches[0].Item2.SetProvenance(null); //all elements are needed localResult = new FetchResponseWithProvenance(request, uncertaintyEvaluatedLocalFetches[0].Item1.Result, uncertaintyEvaluatedLocalFetches[0].Item2.ReportUncertainty(), ArrayHelper.GetConstantArray <ushort>(request.Domain.GetDataArrayShape(), uncertaintyEvaluatedLocalFetches[0].Item2.ID)); } else { List <ushort> requiredIDs; var provenance = Array.CreateInstance(typeof(ushort), request.Domain.GetDataArrayShape()); var uncertainty = Array.CreateInstance(typeof(double), request.Domain.GetDataArrayShape()); var isTimeSeriesRequest = request.Domain.TimeRegion.IsTimeSeries; MergeUncertainty(ref uncertainty, ref provenance, out requiredIDs, uncertaintyEvaluatedLocalFetches.Select(f => new Tuple <Array, ushort>(f.Item2.ReportUncertainty(), f.Item2.ID)).ToArray()); // Remove all sources that are not present in merged provenance array for (int i = 0; i < uncertaintyEvaluatedLocalFetches.Count;) { if (requiredIDs.Contains(uncertaintyEvaluatedLocalFetches[i].Item2.ID)) { i++; } else { uncertaintyEvaluatedLocalFetches[i].Item2.SetCanceled(); uncertaintyEvaluatedLocalFetches.RemoveAt(i); } } uncertaintyEvaluatedLocalFetches.ForEach(f => f.Item2.SetProvenance(provenance)); Array values = Array.CreateInstance(typeof(double), request.Domain.GetDataArrayShape()); Tuple <Array, ushort>[] localResults = new Tuple <Array, ushort> [uncertaintyEvaluatedLocalFetches.Count]; for (int i = 0; i < localResults.Length; i++) { localResults[i] = Tuple.Create(await uncertaintyEvaluatedLocalFetches[i].Item1, uncertaintyEvaluatedLocalFetches[i].Item2.ID); } MergeValues(provenance, ref values, localResults); localResult = new FetchResponseWithProvenance(request, values, uncertainty, provenance); } } traceSource.TraceEvent(TraceEventType.Stop, 6, "Local data sources finished computation"); return(localResult); }