/// <summary>
/// Indexer coerces the type of MissingValue to dataTypes
/// </summary>
/// <param name="index"></param>
/// <returns></returns>
public object this[string index]
{
get {
return(base[index]);
}
set
{
Type dataT = dataTypes[index];
Type valT = value.GetType();
if (valT == dataT)
{
base[index] = value;
}
else
{
try
{
object coercedVal = Convert.ChangeType(value, dataT);
traceSource.TraceEvent(System.Diagnostics.TraceEventType.Warning, 1, string.Format("The missing value attribute for variable {0} has type {1} hovever data has type {2}. Coercing the value from {3} to {4}", index, valT.ToString(), dataT.ToString(), value, coercedVal));
base[index] = coercedVal;
}
catch (InvalidCastException)
{
traceSource.TraceEvent(System.Diagnostics.TraceEventType.Warning, 2, string.Format("The missing value attribute for variable {0} has type {1} which can not be converted to data has type {2}. Ignoring missing value", index, valT.ToString(), dataT.ToString(), value));
}
}
}
}
public async Task <double[]> AggregateCellsBatchAsync(IEnumerable <ICellRequest> cells)
{
ICellRequest[] cellsArray = cells.ToArray();
if (cellsArray.Length == 0)
{
return(new double[0]);
}
else
{
string variable = cellsArray[0].VariableName;
Type dataType = varDataTypes[variable];
int dataElementSizeBytes = Marshal.SizeOf(dataType);
double[] result = new double[cellsArray.Length];
var bb3Dseq = ConvertToBoundingBox3D(cellsArray);
GreedyClustering <BoundingBox3D> clusterer = new GreedyClustering <BoundingBox3D>(new BoundingBoxVolumeSpaceInfo(), clusterSizeInMegabytes * 1024 * 1024 / dataElementSizeBytes /* 128 Mb*/);
Stopwatch clusteringSw = Stopwatch.StartNew();
ts.TraceEvent(TraceEventType.Start, 1, "Clustering started");
var clusterIndeces = clusterer.GetClusters(bb3Dseq);
clusteringSw.Stop();
ts.TraceEvent(TraceEventType.Stop, 1, string.Format("Got {0} clusters in {1}", clusterIndeces.Length, clusteringSw.Elapsed));
ICellRequest[][] clusters = new ICellRequest[clusterIndeces.Length][];
Task[] clusterTasks = new Task[clusterIndeces.Length];
for (int i = 0; i < clusterIndeces.Length; i++)
{
int localI = i;
clusterTasks[i] = Task.Run(async() =>
{
int[] indecesInsideCluster = clusterIndeces[localI];
int clusterSize = indecesInsideCluster.Length;
clusters[localI] = new ICellRequest[clusterSize];
for (int j = 0; j < clusterSize; j++)
{
clusters[localI][j] = cellsArray[indecesInsideCluster[j]];
}
Stopwatch aggClusterSw = Stopwatch.StartNew();
double[] clusterRes = await component.AggregateCellsBatchAsync(clusters[localI]);
aggClusterSw.Stop();
ts.TraceEvent(TraceEventType.Information, 2, string.Format("Aggregated cluster {0}({1} elements) out of {3} in {2}", localI + 1, clusterSize, aggClusterSw.Elapsed, clusterIndeces.Length));
for (int j = 0; j < clusterSize; j++)
{
result[indecesInsideCluster[j]] = clusterRes[j];
}
clusterIndeces[localI] = null;// freeing memory
clusters[localI] = null;
});
}
;
await Task.WhenAll(clusterTasks);
ts.TraceEvent(TraceEventType.Information, 3, "All clusters have been processed");
return(result);
}
}
public async Task <double[]> EvaluateCellsBatchAsync(IEnumerable <ICellRequest> cells)
{
ICellRequest first = cells.FirstOrDefault();
if (first == null)
{
return(new double[0]);
}
else
{
var cellsArray = cells.ToArray();
ts.TraceEvent(TraceEventType.Start, 3, "Uncertainty evaluation started");
Stopwatch sw = Stopwatch.StartNew();
int N = cellsArray.Length;
Task <double>[] resultTasks = new Task <double> [N];
for (int i = 0; i < N; i++)
{
var cell = cellsArray[i];
var coverage = GetIPsForCell(cell);
IPs tempIps = coverage.Item1;
var capturedValues = Tuple.Create(coverage.Item2, coverage.Item3, cell);
double sd = baseNodeUncertatintyProvider.GetBaseNodeStandardDeviation(cell);
resultTasks[i] = temporalVarianceCalculator.GetVarianceForCombinationAsync(tempIps, cell, sd * sd).ContinueWith((t, obj) => //Using continuations to queue all the cells for calculation instead of awaiting for each of them
{
Tuple <IPs, IPs, ICellRequest> captured = (Tuple <IPs, IPs, ICellRequest>)obj;
IPs latIps = captured.Item1, lonIps = captured.Item2;
var cell2 = captured.Item3;
double temporalVariance = t.Result;
return(spatialVarianceCalculator.GetVarianceForCombinationAsync(latIps, lonIps, cell2, temporalVariance).ContinueWith(t2 =>
{
var res = t2.Result;
return Math.Sqrt(res);
}));
}, capturedValues).Unwrap();
}
double[] result = await Task.WhenAll(resultTasks);
sw.Stop();
ts.TraceEvent(TraceEventType.Stop, 3, string.Format("Calculated uncertainty for {0} cells in {1}", N, sw.Elapsed));
return(result);
}
}
public static IStorageResponse PerformRequest(this DataSet storage, IStorageRequest r)
{
if (r.Shape == null && (r.Stride != null || r.Origin != null))
{
throw new InvalidOperationException("Cannot perform data request with non-zero origin or stride and unknown shape");
}
if (!storage.Variables.Contains(r.VariableName))
{
throw new InvalidOperationException("Variable " + r.VariableName + " is not found in dataset");
}
Array data = null;
for (int i = 0; i < RetryTimeouts.Length; i++)
{
try
{
System.Diagnostics.Stopwatch sw = System.Diagnostics.Stopwatch.StartNew();
traceSource.TraceEvent(System.Diagnostics.TraceEventType.Start, 8, string.Format("requesting GetData. var \"{0}\"", r.VariableName));
if (r.Origin == null && r.Shape == null && r.Stride == null)
{
data = storage[r.VariableName].GetData();
}
else if (r.Stride == null) // r.Shape is not null (see preconditions)
{
data = storage[r.VariableName].GetData(r.Origin, r.Shape);
}
else
{
data = storage[r.VariableName].GetData(r.Origin, r.Stride, r.Shape);
}
sw.Stop();
traceSource.TraceEvent(System.Diagnostics.TraceEventType.Stop, 8, string.Format("GetData done in {1}. var \"{0}\"", r.VariableName, sw.Elapsed));
return(new StorageResponse(r, data));
}
catch (Exception exc)
{
int millisecSleep = (int)(RetryTimeouts[i] * (0.9 + random.NextDouble() * 0.2));
traceSource.TraceEvent(System.Diagnostics.TraceEventType.Error, 9, string.Format("GetData failed with {2}. var {1}. sleeping for {0} sec and retrying", millisecSleep * 0.001, r.VariableName, exc.ToString()));
System.Threading.Thread.Sleep(millisecSleep);
}
}
traceSource.TraceEvent(System.Diagnostics.TraceEventType.Critical, 10, string.Format("Request to data set {0} failed after {1} retries", storage.URI, RetryTimeouts.Length));
throw new InvalidOperationException(String.Format("Data is not available after {0} retries", RetryTimeouts.Length));
}
/// <summary>
/// Opens a predownloaded version of the DataSet or downloads it and opens upen download finished
/// </summary>
/// <param name="uriToDownloadFrom"></param>
/// <returns></returns>
public async static Task <DataSet> OpenOrCloneAsync(string uriToDownloadFrom)
{
traceSource.TraceEvent(TraceEventType.Information, 1, "Request for local replication of \"{0}\" NetCDF dataset", uriToDownloadFrom);
lock ("ncFilesCacheDirectoryCreating") {
if (!Directory.Exists(cachePath))
{
Directory.CreateDirectory(cachePath);
}
}
byte[] bytes = uriToDownloadFrom.SelectMany(c => BitConverter.GetBytes(c)).ToArray();
long hash = await SHA1Hash.HashAsync(bytes);
DataSet ds;
if (dict.TryGetValue(uriToDownloadFrom, out ds))
{
return(ds);
}
traceSource.TraceEvent(TraceEventType.Information, 2, "Opening dataset corresponding to \"{0}\"", uriToDownloadFrom);
string filename = string.Format("{0}.nc", hash);
string fullFileName = Path.Combine(cachePath, filename);
if (File.Exists(fullFileName))
{
traceSource.TraceEvent(TraceEventType.Information, 3, "NetCDF dataset \"{0}\" with hash {1} has already been downloaded previously and found on local FS at \"{2}\". opening it.", uriToDownloadFrom, hash, fullFileName);
DataSet ds1 = DataSet.Open(string.Format("msds:nc?file={0}&openMode=readOnly", fullFileName));
return(dict.GetOrAdd(uriToDownloadFrom, ds1));
}
else
{
Stopwatch sw = Stopwatch.StartNew();
string tempName = Path.GetTempFileName();
traceSource.TraceEvent(TraceEventType.Information, 4, "NetCDF dataset \"{0}\" with hash {1} is not found localy. Initiating download to \"{2}\"", uriToDownloadFrom, hash, tempName);
WebClient wc = new WebClient();
await wc.DownloadFileTaskAsync(uriToDownloadFrom, tempName);
sw.Stop();
traceSource.TraceEvent(TraceEventType.Information, 5, "NetCDF dataset \"{0}\" with hash {1} has been downloaded in {2}.", uriToDownloadFrom, hash, sw.Elapsed);
traceSource.TraceEvent(TraceEventType.Information, 5, "Copying downloaded file \"{0}\" into \"{1}\"", tempName, fullFileName);
File.Copy(tempName, fullFileName, true);
File.Delete(tempName);
traceSource.TraceEvent(TraceEventType.Information, 5, "Opening \"{0}\" for \"{1}\"", fullFileName, uriToDownloadFrom);
DataSet ds2 = DataSet.Open(string.Format("msds:nc?file={0}&openMode=readOnly", fullFileName));
return(dict.GetOrAdd(uriToDownloadFrom, ds2));
}
}
protected DataDomain CalcDataDomain(ICellRequest[] cells, ITimeAxisBoundingBoxCalculator timeBBcalc, ISpatGridBoundingBoxCalculator latBBcalc, ISpatGridBoundingBoxCalculator lonBBcalc)
{
if (cells.Length == 0)
{
throw new ArgumentException("cells array must contain elements");
}
string variable = cells[0].VariableName;
//calculating bounding box
IndexBoundingBox timeBB = new IndexBoundingBox();
IndexBoundingBox latBB = new IndexBoundingBox();
IndexBoundingBox lonBB = new IndexBoundingBox();
Stopwatch bbCalc = Stopwatch.StartNew();
ts.TraceEvent(TraceEventType.Start, 1, "Calculating bounding box");
var cellArray = cells.ToArray();
foreach (var geoCellTuple in cellArray)
{
IndexBoundingBox timeBB2 = timeBBcalc.GetBoundingBox(geoCellTuple.Time);
IndexBoundingBox latBB2 = latBBcalc.GetBoundingBox(geoCellTuple.LatMin, geoCellTuple.LatMax);
IndexBoundingBox lonBB2 = lonBBcalc.GetBoundingBox(geoCellTuple.LonMin, geoCellTuple.LonMax);
timeBB = IndexBoundingBox.Union(timeBB, timeBB2);
latBB = IndexBoundingBox.Union(latBB, latBB2);
lonBB = IndexBoundingBox.Union(lonBB, lonBB2);
}
bbCalc.Stop();
ts.TraceEvent(TraceEventType.Stop, 1, string.Format("Bounding box calculated in {0}", bbCalc.Elapsed));
if (timeBB.IsSingular || latBB.IsSingular || lonBB.IsSingular)
{
ts.TraceEvent(TraceEventType.Information, 4, "Bounding box is singular. (all cells are out of the data) Returning NoData");
return(null);
}
else
{
ts.TraceEvent(TraceEventType.Information, 5, string.Format("{0} elements in bounding box",
(timeBB.last - timeBB.first + 1) * (latBB.last - latBB.first + 1) * (lonBB.last - lonBB.first + 1)));
}
var dataDomain = ConstructDataDomain(variable, metadata, lonBB, latBB, timeBB);
return(dataDomain);
}
/// <summary>Creates data source from handler type name and data uri or return existing one.
/// This method is thread safe</summary>
public static async Task <DataSourceInstance> GetInstanceAsync(string typeName, string dataUri)
{
var key = new TypeUriPair
{
TypeName = typeName,
DataUri = dataUri
};
Task <DataSourceInstance> val;
lock (typeof(DataSourceHandlerCache))
{
if (!instances.TryGetValue(key, out val))
{
val = Task.Factory.StartNew(async obj =>
{
TypeUriPair pair = (TypeUriPair)obj;
ts.TraceEvent(TraceEventType.Information, 1, "loading type " + pair.TypeName + " with data uri " + pair.DataUri);
var handlerType = Type.GetType(pair.TypeName);
if (handlerType == null)
{
throw new Exception(String.Format("Type {0} is not found", pair.TypeName));
}
Task <DataSet> storageTask = OpenDataSetWithRetriesAsync(pair.DataUri);;
DataSourceHandler handler;
bool withoutStorage = false;
//trying Async API
var methods = handlerType.GetMethods(System.Reflection.BindingFlags.Static | System.Reflection.BindingFlags.Public);
Func <Type, bool> checkAsyncReturnType = (t =>
{
if (!t.IsGenericType)
{
return(false);
}
var args = t.GetGenericArguments();
if (args.Length != 1)
{
return(false);
}
var genericArg = args[0];
return(genericArg.IsSubclassOf(typeof(DataSourceHandler)));
});
MethodInfo[] asyncCtors = methods.Where(m => m.Name == "CreateAsync" && checkAsyncReturnType(m.ReturnType)).ToArray();
MethodInfo asyncCtor = null;
//Factory with IDataContext parameter is first priority
foreach (var c in asyncCtors)
{
var parameters = c.GetParameters();
if (parameters.Length == 1 && parameters[0].ParameterType.IsInterface && parameters[0].ParameterType == typeof(IStorageContext))
{
ts.TraceEvent(TraceEventType.Information, 2, pair.TypeName + ": Found Async factory method with IStorageContext parameter");
asyncCtor = c; break;
}
}
//if not found, looking for Factory with no parameters
if (asyncCtor == null)
{
foreach (var c in asyncCtors)
{
var parameters = c.GetParameters();
if (parameters.Length == 0)
{
ts.TraceEvent(TraceEventType.Information, 3, pair.TypeName + ": Found Async factory method with no parameters");
withoutStorage = true;
asyncCtor = c; break;
}
}
}
if (asyncCtor != null)
{
Type genericRetType = asyncCtor.ReturnType;
Type genericArg = asyncCtor.ReturnType.GetGenericArguments()[0];
object o;
if (withoutStorage)
{
o = asyncCtor.Invoke(null, new object[0]);
}
else
{
o = asyncCtor.Invoke(null, new object[] { new LinearizingStorageContext(await storageTask) });
}
await(Task) o;
handler = (DataSourceHandler)o.GetType().InvokeMember("Result", BindingFlags.Public | BindingFlags.Instance | BindingFlags.GetProperty, null, o, new object[0]);
return(new DataSourceInstance(handler, await storageTask));
}
//tring old API
var ctor = handlerType.GetConstructor(new Type[] { typeof(IStorageContext) });
if (ctor == null)
{
ctor = handlerType.GetConstructor(new Type[0]);
withoutStorage = true;
}
else
{
ts.TraceEvent(TraceEventType.Information, 4, pair.TypeName + ": Found synchronous constructor with IStorageContext parameter");
}
if (ctor == null)
{
throw new Exception(String.Format("Type {0} does not have required constructor", pair.TypeName));
}
else
{
ts.TraceEvent(TraceEventType.Information, 5, pair.TypeName + ": Found synchronous constructor without parameters");
}
if (withoutStorage)
{
handler = (DataSourceHandler)ctor.Invoke(new object[0]);
}
else
{
handler = (DataSourceHandler)ctor.Invoke(new object[] { new LinearizingStorageContext(await storageTask) });
}
return(new DataSourceInstance(handler, await storageTask));
}, key, TaskCreationOptions.LongRunning).Unwrap();
instances.Add(key, val);
}
}
return(await val);
}
public static AxisDetectionResult SmartDetectAxis(IStorageContext storage)
{
IDataStorageDefinition storageDefinition = storage.StorageDefinition;
string varName = TimeAxisAutodetection.GetTimeVariableName(storageDefinition);
if (string.IsNullOrEmpty(varName))
{
throw new InvalidOperationException("Can't autodetect time axis variable");
}
string timeUnits = storageDefinition.VariablesMetadata[varName].Where(pair => pair.Key.ToLowerInvariant() == "units").Select(p => p.Value).FirstOrDefault() as string;
if (timeUnits == null)
{
throw new InvalidOperationException(string.Format("Can't find units metadata entry for the time axis \"{0}\"", varName));
}
string trimmed = timeUnits.Trim();
string[] splitted = trimmed.Split(new char[] { ' ', '\t', '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries);
if (splitted.Length < 4 || splitted[1].ToLowerInvariant() != "since")
{
throw new InvalidOperationException("Automatic time axis detection failed to determine time axis semantics. Time axis units must be in format \"days|hours|years since YYYY-MM-DD HH:MM:SS\"");
}
DateTime baseTime = new DateTime();
string dateToParse = string.Format("{0} {1}", splitted[2], splitted[3]);
if (dateToParse.Length > 19)
{
dateToParse = dateToParse.Substring(0, 19);
}
bool baseTimeParsed = false;
foreach (var dateFormat in dateFormats)
{
baseTimeParsed = DateTime.TryParseExact(dateToParse, dateFormat, System.Globalization.CultureInfo.InvariantCulture, System.Globalization.DateTimeStyles.AssumeUniversal, out baseTime);
if (baseTimeParsed)
{
traceSource.TraceEvent(TraceEventType.Information, 4, string.Format("base datetime \"{0}\" for axis variable \"{1}\" was successfuly parsed as {2}", dateToParse, varName, baseTime.ToString("u")));
break;
}
else
{
traceSource.TraceEvent(TraceEventType.Information, 4, string.Format("can not parse base datetime \"{0}\" for axis variable \"{1}\" with format {2}", dateToParse, varName, dateFormat));
}
}
if (baseTimeParsed)
{
switch (splitted[0].ToLowerInvariant())
{
case "years":
traceSource.TraceEvent(TraceEventType.Information, 1, "Detected axis suitable for StepFunctionYearsIntegrator");
return(new AxisFound(varName, AxisKind.Years, baseTime));
case "days":
traceSource.TraceEvent(TraceEventType.Information, 2, "Detected axis suitable for StepFunctionDaysIntegrator");
return(new AxisFound(varName, AxisKind.Days, baseTime));
case "hours":
traceSource.TraceEvent(TraceEventType.Information, 3, "Detected axis suitable for StepFunctionHoursIntegrator");
return(new AxisFound(varName, AxisKind.Hours, baseTime));
default:
traceSource.TraceEvent(TraceEventType.Error, 4, string.Format("the offset units in units metadata entry of \"{0}\" can't be parsed. It must be one of the following: years, days or hours", varName));
return(new AxisNotFound());
}
}
else
{
traceSource.TraceEvent(TraceEventType.Error, 5, string.Format("reference datetime in units metadata entry of \"{0}\" can't be parsed. It must be in format \"{1}\", but it is \"{2}\"", varName, dateFormats[0], dateToParse));
}
return(new AxisNotFound());
}
