/// <summary>
/// Returns a computational context which is a set of nodes and a set "weights" to apply to the nodes to form a linear combination to get the mean value of the cells
/// </summary>
public async Task <LinearCombinationContext> CreateAsync(IEnumerable <ICellRequest> cells)
{
Stopwatch sw = Stopwatch.StartNew();
var tsAveragerTask = timeSeriesAveragerFactory.CreateAsync();
var liTask = linearInterpolatorFactory.CreateAsync();
var tsAverager = await tsAveragerTask;
var linearInterpolator = await liTask;
sw.Stop();
traceSource.TraceEvent(TraceEventType.Verbose, 1, "Time series averager and lenear interpolator constructed in {0}", sw.Elapsed);
sw = Stopwatch.StartNew();
var resultTasks = cells.Select <ICellRequest, Task <Tuple <ICellRequest, RealValueNodes, IEnumerable <LinearWeight> > > >(async c =>
{
var tsa = (RealValueNodes)(await tsAverager.GetAsync(c));
var w = (IEnumerable <LinearWeight>)(await linearInterpolator.GetLinearWeigthsAsync(tsa, c));
return(Tuple.Create(c, tsa, w));
}).ToArray();
var result = await Task.WhenAll(resultTasks);
sw.Stop();
traceSource.TraceEvent(TraceEventType.Verbose, 2, "Linear weights for {0} cells prepared in {1}", resultTasks.Length, sw.Elapsed);
LinearCombinationContext lcc = new LinearCombinationContext(result);
return(lcc);
}
/// <summary>
/// Implements the logic of sequential uncertainties evaluation, reporting it, acquiring data mask (a subset of the data which is actually needs to be processed), generating the mean values for points left
/// </summary>
/// <param name="context"></param>
/// <returns></returns>
public override sealed async Task <Array> ProcessRequestAsync(IRequestContext context)
{
Stopwatch sw = Stopwatch.StartNew();
var compContext = await compContextFactory.CreateAsync(context);
sw.Stop();
traceSwitch.TraceEvent(TraceEventType.Information, 1, string.Format("Computational context is ready in {0}", sw.Elapsed));
sw = Stopwatch.StartNew();
var uncertaintes = await uncertaintyEvaluator.EvaluateAsync(context, compContext);
sw.Stop();
traceSwitch.TraceEvent(TraceEventType.Information, 2, string.Format("Uncertatinty was evaluated in {0}", sw.Elapsed));
sw = Stopwatch.StartNew();
//WARNING!!!
//TODO: Implicit dependency here. The user of the ProcessRequestAsync EXPECTS that context.GetMaskAsync is called during the execution and waits for it. Separate the method!
var mask = await context.GetMaskAsync(uncertaintes);
sw.Stop();
traceSwitch.TraceEvent(TraceEventType.Verbose, 3, string.Format("FetchEngine returned a bitmask in {0}", sw.Elapsed));
sw = Stopwatch.StartNew();
var result = await valuesAggregator.AggregateAsync(context, compContext, mask);
sw.Stop();
traceSwitch.TraceEvent(TraceEventType.Information, 3, string.Format("Aggregated values were got in {0}", sw.Elapsed));
return(result);
}
public Task <VariogramModule.IVariogram> GetSpatialVariogramAsync(LinearCombination.RealValueNodes nodes)
{
var task = taskFactory.StartNew(new Func <object, VariogramModule.IVariogram>(obj =>
{
Stopwatch sw = Stopwatch.StartNew();
LinearCombination.RealValueNodes localNodes = (LinearCombination.RealValueNodes)obj;
var variogramFitter = new LMDotNetVariogramFitter.Fitter() as VariogramModule.IVariogramFitter;
traceSource.TraceEvent(TraceEventType.Start, 1, "Starting build of emperical variogram");
var pointSet = new EmpVariogramBuilder.PointSet(localNodes.Lats, localNodes.Lons, localNodes.Values);
var dist = FuncConvert.ToFSharpFunc(new Converter <Tuple <double, double>, FSharpFunc <Tuple <double, double>, double> >(t1 =>
FuncConvert.ToFSharpFunc(new Converter <Tuple <double, double>, double>(t2 => SphereMath.GetDistance(t1.Item1, t1.Item2, t2.Item1, t2.Item2)))));
var empVar = EmpVariogramBuilder.EmpiricalVariogramBuilder.BuildEmpiricalVariogram(pointSet, dist);
sw.Stop();
traceSource.TraceEvent(TraceEventType.Stop, 1, string.Format("Emperical variogram is build in {0}", sw.Elapsed));
sw = Stopwatch.StartNew();
traceSource.TraceEvent(TraceEventType.Start, 2, "Starting variogram fitting");
var variogramRes = variogramFitter.Fit(empVar);
sw.Stop();
traceSource.TraceEvent(TraceEventType.Stop, 2, string.Format("Emperical variogram is build in {0}", sw.Elapsed));
if (FSharpOption <VariogramModule.IDescribedVariogram> .get_IsNone(variogramRes))
{
traceSource.TraceEvent(TraceEventType.Error, 3, "Fariogram fitting failed. Falling back to coarse variogram approximation");
return(variogramFitter.GetFallback(empVar));
}
else
{
return(variogramRes.Value);
}
}), nodes);
return(task);
}
public IEnumerable <double> Aggregate(string variable, Array prefetchedData, DataDomain prefetchDataDomain, IEnumerable <ICellRequest> cells)
{
Stopwatch calcSw = Stopwatch.StartNew();
ts.TraceEvent(TraceEventType.Start, 3, "Integration started");
double[] result = IntegrateBatch(variable, prefetchedData, prefetchDataDomain.Origin, PrepareIPsForCells(variable, cells)).ToArray();
calcSw.Stop();
ts.TraceEvent(TraceEventType.Stop, 3, string.Format("Integration took {0}. {1} values produced", calcSw.Elapsed, result.Length));
return(result);
}
public VirtualDataSource()
: base(null)
{
var dynamicType = GetType();
//looking for available static methods
MethodInfo[] methods = dynamicType.GetMethods(BindingFlags.Static | BindingFlags.Public | BindingFlags.FlattenHierarchy);
foreach (var item in methods)
{
//if ((item.ReturnType != typeof(double)) || item.GetParameters().Any(p => p.ParameterType != typeof(double))) //filter out all except double[] params -> double
// continue;
if ((item.ReturnType == typeof(double)) || item.GetParameters().All(p => p.ParameterType == typeof(double))) //found a variable with constant uncertainty
{
variableDefinitionsFound.Add(item.Name, item.GetParameters().Select(p => p.Name).ToArray()); //saving variable and it's parameters names
variableUncertaintyTypes.Add(item.Name, UncertaintyType.Constant); //saving variable's uncertainty type
virtualVariableFunctions.Add(item.Name, parameters => item.Invoke(null, parameters)); //generating funcs for future invocation
ts.TraceEvent(TraceEventType.Information, 1, string.Format("Found float virtual variable definition \"{0}\" of {1} parameters in \"{2}\" type of \"{3}\" assembly",
item.Name,
variableDefinitionsFound[item.Name].Length,
dynamicType.FullName,
Assembly.GetAssembly(dynamicType).FullName));
}
else if ((item.ReturnType == typeof(GaussValue)) || item.GetParameters().All(p => p.ParameterType == typeof(GaussValue))) //found a variable with custom uncertainty
{
variableDefinitionsFound.Add(item.Name, item.GetParameters().Select(p => p.Name).ToArray()); //saving variable and it's parameters names
variableUncertaintyTypes.Add(item.Name, UncertaintyType.CustomCS); //saving variable's uncertainty type
virtualVariableFunctions.Add(item.Name, parameters => item.Invoke(null, parameters)); //generating funcs for future invocation
ts.TraceEvent(TraceEventType.Information, 3, string.Format("Found Gaussian virtual variable (CSharp) definition \"{0}\" of {1} parameters in \"{2}\" type of \"{3}\" assembly",
item.Name,
variableDefinitionsFound[item.Name].Length,
dynamicType.FullName,
Assembly.GetAssembly(dynamicType).FullName));
}
else if ((item.ReturnType == typeof(Tuple <double, double>)) || item.GetParameters().All(p => p.ParameterType == typeof(Tuple <double, double>))) //found a variable with custom uncertainty
{
variableDefinitionsFound.Add(item.Name, item.GetParameters().Select(p => p.Name).ToArray()); //saving variable and it's parameters names
variableUncertaintyTypes.Add(item.Name, UncertaintyType.CustomFS); //saving variable's uncertainty type
virtualVariableFunctions.Add(item.Name, parameters => item.Invoke(null, parameters)); //generating funcs for future invocation
ts.TraceEvent(TraceEventType.Information, 4, string.Format("Found Gaussian virtual variable (FSharp) definition \"{0}\" of {1} parameters in \"{2}\" type of \"{3}\" assembly",
item.Name,
variableDefinitionsFound[item.Name].Length,
dynamicType.FullName,
Assembly.GetAssembly(dynamicType).FullName));
}
}
}
// GET http://.../JsonProxy/getSchema?uri=dmitrov-uri
public SerializableDataSetSchema Get(string uri)
{
traceSource.TraceEvent(TraceEventType.Information, 1, string.Format("Request for JSON schema of dataset {0}", uri));
SerializableDataSetSchema result = null;
if (!String.IsNullOrEmpty(uri))
{
var ds = new AzureBlobDataSet(uri);
result = ConvUtils.GetSerializableSchema(ds);
}
if (result == null)
{
throw new HttpResponseException(HttpStatusCode.InternalServerError);
}
return(result);
}
public Task <double> GetVarianceForCombinationAsync(IPs temporalWeights, ICellRequest cell, double baseNodeVariance)
{
var capturedValues = Tuple.Create(temporalWeights, cell, baseNodeVariance);
return(Task.Factory.StartNew(obj =>
{
Tuple <IPs, ICellRequest, double> typedObj = (Tuple <IPs, ICellRequest, double>)obj;
var name = typedObj.Item2.VariableName;
var processDescription = dict.GetOrAdd(name, new Lazy <IGaussianProcessDescription>(() =>
{
var v = variogramFactory.Create(name);
if (v == null)
{
ts.TraceEvent(TraceEventType.Warning, 1, string.Format("Could not find temporal variogram for the \"{0}\" variable. Skipping uncertainty propagation analysis", name));
}
else
{
ts.TraceEvent(TraceEventType.Information, 2, string.Format("Loaded temporal variogram for \"{0}\" variable", name));
}
return v;
}, System.Threading.LazyThreadSafetyMode.ExecutionAndPublication)).Value;
if (processDescription == null) //Variogram not found.
{
return double.MaxValue;
}
var variogram = processDescription.Variogram;
var tempPrecomputedNugget = variogram.Nugget;
var tempPrecomputedSill = variogram.Sill;
double tempSpatIndependentVariance = double.IsNaN(typedObj.Item3) ? (tempPrecomputedNugget) : (typedObj.Item3);
var tempVarProps = new TemporalVarianceProperties(tempPrecomputedSill - tempPrecomputedNugget + tempSpatIndependentVariance, new Func <double, double, double>((coord1, coord2) => (tempPrecomputedSill - variogram.GetGamma(processDescription.Dist(coord1, coord2)))));
double temporalVariance = GetTemporalVariance(typedObj.Item1, timeAxis, axisLocator.getAproximationGrid(cell.Time), tempVarProps);
Debug.Assert(temporalVariance >= 0.0);
return temporalVariance;
}, capturedValues));
}
public async Task <double[]> AggregateCellsBatchAsync(LinearCombinationContext computationalContext, IEnumerable <ICellRequest> cells)
{
Stopwatch sw = Stopwatch.StartNew();
var contextEnumerator = computationalContext.Combinations.GetEnumerator();
List <double> result = new List <double>();
var exc = new InvalidOperationException("Cells in computationalContext do not correspond to the cells passed to the method");
ICellRequest contextCell;
Tuple <ICellRequest, RealValueNodes, IEnumerable <LinearWeight> > contextElement;
foreach (var cell in cells)
{
do
{
var nextExists = contextEnumerator.MoveNext();
if (!nextExists)
{
throw exc;
}
contextElement = contextEnumerator.Current;
contextCell = contextElement.Item1;
} while (!contextCell.Equals(cell));
double[] nodeValues = contextElement.Item2.Values;
bool isEmpty = true;
double acc = 0.0;
foreach (var linearWeight in contextElement.Item3)
{
acc += linearWeight.Weight * nodeValues[linearWeight.DataIndex];
isEmpty = false;
}
if (isEmpty)
{
result.Add(Double.NaN);
}
else
{
result.Add(acc);
}
}
sw.Stop();
traceSource.TraceEvent(TraceEventType.Information, 1, string.Format("Averages for all cells are computed in {0}", sw.Elapsed));
return(result.ToArray());
}
public async Task <RealValueNodes> GetAsync(ICellRequest cell)
{
var variableName = cell.VariableName;
var timeSegment = cell.Time;
var bounds = timeIntegrator.GetBoundingBox(timeSegment);
if (bounds.IsSingular)
{
double[] empty = new double[0];
return(new RealValueNodes(empty, empty, empty));
}
int[] origin = new int[2];
int[] shape = new int[2];
int timeDimNum = timeDimNumber[variableName];
int stationDimNum = stationsDimNumber[variableName];
int[] stationsIdxs = stationLocator.GetRelevantStationsIndices(cell);
int stationMinIdx = stationsIdxs.Min(), stationMaxIdx = stationsIdxs.Max();
int stationsCount = stationsIdxs.Length;
//data fetching
origin[timeDimNum] = bounds.first;
origin[stationDimNum] = stationMinIdx;
shape[timeDimNum] = bounds.last - bounds.first + 1;
shape[stationDimNum] = stationMaxIdx - stationMinIdx + 1;
traceSource.TraceEvent(TraceEventType.Verbose, 2, "Requesting raw data for cell");
var dataTask = dataStorage.GetDataAsync(variableName, origin, null, shape);
//integration points calculation
IPs timeIntegrationPoints = timeIntegrator.GetTempIPs(timeSegment);
IPs[][] ips = new IPs[stationsCount][];
for (int i = 0; i < stationsCount; i++)
{
var curr = new IPs[2];
ips[i] = curr;
int idx = stationsIdxs[i];
curr[stationDimNum] = new IPs()
{
Weights = new double[] { 1.0 }, Indices = new int[] { idx }, BoundingIndices = new IndexBoundingBox()
{
first = idx, last = idx
}
};
curr[timeDimNum] = timeIntegrationPoints;
}
var data = await dataTask;
object missingValue = missingValuesDict.GetMissingValue(variableName);
if (missingValue != null && missingValue.GetType() != varTypes[variableName])
{
traceSource.TraceEvent(TraceEventType.Warning, 1, string.Format("A missing value of the variable \"{0}\"has different type from variable type. Ignoring MV definition", variableName));
missingValue = null;
}
IEnumerable <Utils.IntegrationResult> obsResults;
if (missingValue != null)
{
obsResults = Utils.ArrayMean.IntegrateSequenceWithMVs2D(variableName, data, missingValue, origin, ips);
}
else
{
obsResults = Utils.ArrayMean.IntegrateSequence2D(variableName, data, origin, ips);
}
double scaleFacotor = dataRepresentationDictionary.ScaleFactors[variableName];
double addOffset = dataRepresentationDictionary.AddOffsets[variableName];
var scaledResults = obsResults.Select(r => (r.Integral / r.SumOfWeights) * scaleFacotor + addOffset).ToArray();
int len = scaledResults.Length;
List <double> valuesList = new List <double>(len);
List <double> latsList = new List <double>(len);
List <double> lonsList = new List <double>(len);
for (int i = 0; i < len; i++)
{
if (!double.IsNaN(scaledResults[i]))
{
valuesList.Add(scaledResults[i]);
latsList.Add(stationsLats[stationsIdxs[i]]);
lonsList.Add(stationsLons[stationsIdxs[i]]);
}
}
return(new RealValueNodes(latsList.ToArray(), lonsList.ToArray(), valuesList.ToArray()));
}
public Task <double> GetVarianceForCombinationAsync(IPs latWeights, IPs lonWeights, ICellRequest cell, double baseNodeVariance)
{
Tuple <IPs, IPs, ICellRequest, double> capturedValues = Tuple.Create(latWeights, lonWeights, cell, baseNodeVariance);
return(Task.Factory.StartNew(obj =>
{
Tuple <IPs, IPs, ICellRequest, double> arg = (Tuple <IPs, IPs, ICellRequest, double>)obj;
var cell2 = arg.Item3;
var name = cell2.VariableName;
double[] targetLats, targetLons;
if (cell2.LatMax == cell2.LatMin && cell2.LonMax == cell2.LonMin)
{
targetLats = new double[] { cell2.LatMax };
targetLons = new double[] { cell2.LonMax };
}
else
{
double latStep = (cell2.LatMax - cell2.LatMin) / (cellDevisionCount - 1);
double lonStep = (cell2.LonMax - cell2.LonMin) / (cellDevisionCount - 1);
targetLats = Enumerable.Range(0, cellDevisionCount).Select(j => cell2.LatMin + j * latStep).ToArray();
targetLons = Enumerable.Range(0, cellDevisionCount).Select(j => cell2.LonMin + j * lonStep).ToArray();
}
//coerce each target location to the closest data grid node. As closest data grid node is representative for its region. (indirect coercing of distance function by moving target locations)
targetLats = Align(latAxis, targetLats);
targetLons = Align(lonAxis, targetLons);
var fieldDescription = dict.GetOrAdd(name, new Lazy <IGaussianFieldDescription>(() =>
{
var v = variogramsFactory.Create(name);
if (v == null)
{
ts.TraceEvent(TraceEventType.Warning, 1, string.Format("Could not find spatial variogram for the \"{0}\" variable. Skipping uncertainty propagation analysis", name));
}
else
{
ts.TraceEvent(TraceEventType.Information, 2, string.Format("Loaded spatial variogram for \"{0}\" variable", name));
}
return v;
}, System.Threading.LazyThreadSafetyMode.ExecutionAndPublication)).Value;
if (fieldDescription == null)
{
return double.MaxValue;
}
var variogram = fieldDescription.Variogram;
var spatPrecomputedSill = variogram.Sill;
var spatPrecomputedNugget = variogram.Nugget;
var coercedBaseVariance = double.IsNaN(arg.Item4) ? (spatPrecomputedNugget) : arg.Item4;
var spatDistance = new Func <double, double, double, double, double>((lat1, lon1, lat2, lon2) => fieldDescription.Dist(lat1, lon1, lat2, lon2));
var spatCovariogram = new Func <double, double>(distance => (spatPrecomputedSill - variogram.GetGamma(distance)));
SpatialVarianceProperties svp = new SpatialVarianceProperties(spatPrecomputedSill - spatPrecomputedNugget + coercedBaseVariance, spatDistance, spatCovariogram);
return GetSpatialVariance(arg.Item1, arg.Item2, latAxis, lonAxis, targetLats, targetLons, svp); //basic scale_factor/add_offset data transform are assumed to be applied during variograms fitting
}, capturedValues));
}
public async Task <double[]> EvaluateCellsBatchAsync(LinearCombinationContext computationalContext, IEnumerable <ICellRequest> cells)
{
var variogramProvider = await variogramProviderFactory.ConstructAsync();
var combinationsIterator = computationalContext.Combinations.GetEnumerator();
traceSource.TraceEvent(TraceEventType.Start, 1, "Filtering out cells that are not requested");
var filtered = cells.Select(cell =>
{
Tuple <ICellRequest, RealValueNodes, IEnumerable <LinearWeight> > currentElement;
ICellRequest contextCell;
do
{
var nextExists = combinationsIterator.MoveNext();
if (!nextExists)
{
throw new InvalidOperationException("Received linear combination does not containg information about requested cell. Context is not synchronize with requested cells sequenece.");
}
currentElement = combinationsIterator.Current;
contextCell = currentElement.Item1;
} while (!contextCell.Equals(cell));
return(currentElement);
}).ToArray();
traceSource.TraceEvent(TraceEventType.Stop, 1, "Filtered out cells that are not requested");
traceSource.TraceEvent(TraceEventType.Start, 2, "Evaluating uncertatinty for sequence of cells using precomputed linear combinations");
var resultsTasks = filtered.AsParallel().AsOrdered().Select(async tuple =>
{
if (tuple.Item2.Lats.Length == 0)
{
return(Double.NaN); //there are no nodes. Out of data request? can't produce uncertainty
}
var variogram = await variogramProvider.GetSpatialVariogramAsync(tuple.Item2);
var cell = tuple.Item1;
var weights = tuple.Item3.ToArray();
var nodes = tuple.Item2;
Debug.Assert(Math.Abs(weights.Sum(w => w.Weight) - 1.0) < 1e-10);
double sill = variogram.Sill;
double cellLat = (cell.LatMax + cell.LatMin) * 0.5;
double cellLon = (cell.LonMax + cell.LonMin) * 0.5;
//var = cov(0)+ sum sum (w[i]*w[j]*cov(i,j))-2.0*sum(w[i]*cov(x,i))
double cov_at_0 = sill;
double acc = cov_at_0; //cov(0)
for (int i = 0; i < weights.Length; i++)
{
double w = weights[i].Weight;
int idx1 = weights[i].DataIndex;
double lat1 = nodes.Lats[idx1];
double lon1 = nodes.Lons[idx1];
for (int j = 0; j < i; j++)
{
int idx2 = weights[j].DataIndex;
double lat2 = nodes.Lats[idx2];
double lon2 = nodes.Lons[idx2];
double dist = SphereMath.GetDistance(lat1, lon1, lat2, lon2);
double cov = sill - variogram.GetGamma(dist);
acc += 2.0 * w * weights[j].Weight * cov;
}
acc += w * w * cov_at_0; //diagonal elements
double dist2 = SphereMath.GetDistance(lat1, lon1, cellLat, cellLon);
double cov2 = sill - variogram.GetGamma(dist2);
acc -= 2.0 * w * cov2;
}
return(acc * 0.5); //as dealing with full variogram instead of semivariogram
}).ToArray();
var results = await Task.WhenAll(resultsTasks);
traceSource.TraceEvent(TraceEventType.Stop, 2, "Evaluated uncertatinty for sequence of cells using precomputed linear combinations");
return(results);
}
