/// <summary> /// Indexer coerces the type of MissingValue to dataTypes /// </summary> /// <param name="index"></param> /// <returns></returns> public object this[string index] { get { return(base[index]); } set { Type dataT = dataTypes[index]; Type valT = value.GetType(); if (valT == dataT) { base[index] = value; } else { try { object coercedVal = Convert.ChangeType(value, dataT); traceSource.TraceEvent(System.Diagnostics.TraceEventType.Warning, 1, string.Format("The missing value attribute for variable {0} has type {1} hovever data has type {2}. Coercing the value from {3} to {4}", index, valT.ToString(), dataT.ToString(), value, coercedVal)); base[index] = coercedVal; } catch (InvalidCastException) { traceSource.TraceEvent(System.Diagnostics.TraceEventType.Warning, 2, string.Format("The missing value attribute for variable {0} has type {1} which can not be converted to data has type {2}. Ignoring missing value", index, valT.ToString(), dataT.ToString(), value)); } } } }
public async Task <double[]> AggregateCellsBatchAsync(IEnumerable <ICellRequest> cells) { ICellRequest[] cellsArray = cells.ToArray(); if (cellsArray.Length == 0) { return(new double[0]); } else { string variable = cellsArray[0].VariableName; Type dataType = varDataTypes[variable]; int dataElementSizeBytes = Marshal.SizeOf(dataType); double[] result = new double[cellsArray.Length]; var bb3Dseq = ConvertToBoundingBox3D(cellsArray); GreedyClustering <BoundingBox3D> clusterer = new GreedyClustering <BoundingBox3D>(new BoundingBoxVolumeSpaceInfo(), clusterSizeInMegabytes * 1024 * 1024 / dataElementSizeBytes /* 128 Mb*/); Stopwatch clusteringSw = Stopwatch.StartNew(); ts.TraceEvent(TraceEventType.Start, 1, "Clustering started"); var clusterIndeces = clusterer.GetClusters(bb3Dseq); clusteringSw.Stop(); ts.TraceEvent(TraceEventType.Stop, 1, string.Format("Got {0} clusters in {1}", clusterIndeces.Length, clusteringSw.Elapsed)); ICellRequest[][] clusters = new ICellRequest[clusterIndeces.Length][]; Task[] clusterTasks = new Task[clusterIndeces.Length]; for (int i = 0; i < clusterIndeces.Length; i++) { int localI = i; clusterTasks[i] = Task.Run(async() => { int[] indecesInsideCluster = clusterIndeces[localI]; int clusterSize = indecesInsideCluster.Length; clusters[localI] = new ICellRequest[clusterSize]; for (int j = 0; j < clusterSize; j++) { clusters[localI][j] = cellsArray[indecesInsideCluster[j]]; } Stopwatch aggClusterSw = Stopwatch.StartNew(); double[] clusterRes = await component.AggregateCellsBatchAsync(clusters[localI]); aggClusterSw.Stop(); ts.TraceEvent(TraceEventType.Information, 2, string.Format("Aggregated cluster {0}({1} elements) out of {3} in {2}", localI + 1, clusterSize, aggClusterSw.Elapsed, clusterIndeces.Length)); for (int j = 0; j < clusterSize; j++) { result[indecesInsideCluster[j]] = clusterRes[j]; } clusterIndeces[localI] = null;// freeing memory clusters[localI] = null; }); } ; await Task.WhenAll(clusterTasks); ts.TraceEvent(TraceEventType.Information, 3, "All clusters have been processed"); return(result); } }
public async Task <double[]> EvaluateCellsBatchAsync(IEnumerable <ICellRequest> cells) { ICellRequest first = cells.FirstOrDefault(); if (first == null) { return(new double[0]); } else { var cellsArray = cells.ToArray(); ts.TraceEvent(TraceEventType.Start, 3, "Uncertainty evaluation started"); Stopwatch sw = Stopwatch.StartNew(); int N = cellsArray.Length; Task <double>[] resultTasks = new Task <double> [N]; for (int i = 0; i < N; i++) { var cell = cellsArray[i]; var coverage = GetIPsForCell(cell); IPs tempIps = coverage.Item1; var capturedValues = Tuple.Create(coverage.Item2, coverage.Item3, cell); double sd = baseNodeUncertatintyProvider.GetBaseNodeStandardDeviation(cell); resultTasks[i] = temporalVarianceCalculator.GetVarianceForCombinationAsync(tempIps, cell, sd * sd).ContinueWith((t, obj) => //Using continuations to queue all the cells for calculation instead of awaiting for each of them { Tuple <IPs, IPs, ICellRequest> captured = (Tuple <IPs, IPs, ICellRequest>)obj; IPs latIps = captured.Item1, lonIps = captured.Item2; var cell2 = captured.Item3; double temporalVariance = t.Result; return(spatialVarianceCalculator.GetVarianceForCombinationAsync(latIps, lonIps, cell2, temporalVariance).ContinueWith(t2 => { var res = t2.Result; return Math.Sqrt(res); })); }, capturedValues).Unwrap(); } double[] result = await Task.WhenAll(resultTasks); sw.Stop(); ts.TraceEvent(TraceEventType.Stop, 3, string.Format("Calculated uncertainty for {0} cells in {1}", N, sw.Elapsed)); return(result); } }
public static IStorageResponse PerformRequest(this DataSet storage, IStorageRequest r) { if (r.Shape == null && (r.Stride != null || r.Origin != null)) { throw new InvalidOperationException("Cannot perform data request with non-zero origin or stride and unknown shape"); } if (!storage.Variables.Contains(r.VariableName)) { throw new InvalidOperationException("Variable " + r.VariableName + " is not found in dataset"); } Array data = null; for (int i = 0; i < RetryTimeouts.Length; i++) { try { System.Diagnostics.Stopwatch sw = System.Diagnostics.Stopwatch.StartNew(); traceSource.TraceEvent(System.Diagnostics.TraceEventType.Start, 8, string.Format("requesting GetData. var \"{0}\"", r.VariableName)); if (r.Origin == null && r.Shape == null && r.Stride == null) { data = storage[r.VariableName].GetData(); } else if (r.Stride == null) // r.Shape is not null (see preconditions) { data = storage[r.VariableName].GetData(r.Origin, r.Shape); } else { data = storage[r.VariableName].GetData(r.Origin, r.Stride, r.Shape); } sw.Stop(); traceSource.TraceEvent(System.Diagnostics.TraceEventType.Stop, 8, string.Format("GetData done in {1}. var \"{0}\"", r.VariableName, sw.Elapsed)); return(new StorageResponse(r, data)); } catch (Exception exc) { int millisecSleep = (int)(RetryTimeouts[i] * (0.9 + random.NextDouble() * 0.2)); traceSource.TraceEvent(System.Diagnostics.TraceEventType.Error, 9, string.Format("GetData failed with {2}. var {1}. sleeping for {0} sec and retrying", millisecSleep * 0.001, r.VariableName, exc.ToString())); System.Threading.Thread.Sleep(millisecSleep); } } traceSource.TraceEvent(System.Diagnostics.TraceEventType.Critical, 10, string.Format("Request to data set {0} failed after {1} retries", storage.URI, RetryTimeouts.Length)); throw new InvalidOperationException(String.Format("Data is not available after {0} retries", RetryTimeouts.Length)); }
/// <summary> /// Opens a predownloaded version of the DataSet or downloads it and opens upen download finished /// </summary> /// <param name="uriToDownloadFrom"></param> /// <returns></returns> public async static Task <DataSet> OpenOrCloneAsync(string uriToDownloadFrom) { traceSource.TraceEvent(TraceEventType.Information, 1, "Request for local replication of \"{0}\" NetCDF dataset", uriToDownloadFrom); lock ("ncFilesCacheDirectoryCreating") { if (!Directory.Exists(cachePath)) { Directory.CreateDirectory(cachePath); } } byte[] bytes = uriToDownloadFrom.SelectMany(c => BitConverter.GetBytes(c)).ToArray(); long hash = await SHA1Hash.HashAsync(bytes); DataSet ds; if (dict.TryGetValue(uriToDownloadFrom, out ds)) { return(ds); } traceSource.TraceEvent(TraceEventType.Information, 2, "Opening dataset corresponding to \"{0}\"", uriToDownloadFrom); string filename = string.Format("{0}.nc", hash); string fullFileName = Path.Combine(cachePath, filename); if (File.Exists(fullFileName)) { traceSource.TraceEvent(TraceEventType.Information, 3, "NetCDF dataset \"{0}\" with hash {1} has already been downloaded previously and found on local FS at \"{2}\". opening it.", uriToDownloadFrom, hash, fullFileName); DataSet ds1 = DataSet.Open(string.Format("msds:nc?file={0}&openMode=readOnly", fullFileName)); return(dict.GetOrAdd(uriToDownloadFrom, ds1)); } else { Stopwatch sw = Stopwatch.StartNew(); string tempName = Path.GetTempFileName(); traceSource.TraceEvent(TraceEventType.Information, 4, "NetCDF dataset \"{0}\" with hash {1} is not found localy. Initiating download to \"{2}\"", uriToDownloadFrom, hash, tempName); WebClient wc = new WebClient(); await wc.DownloadFileTaskAsync(uriToDownloadFrom, tempName); sw.Stop(); traceSource.TraceEvent(TraceEventType.Information, 5, "NetCDF dataset \"{0}\" with hash {1} has been downloaded in {2}.", uriToDownloadFrom, hash, sw.Elapsed); traceSource.TraceEvent(TraceEventType.Information, 5, "Copying downloaded file \"{0}\" into \"{1}\"", tempName, fullFileName); File.Copy(tempName, fullFileName, true); File.Delete(tempName); traceSource.TraceEvent(TraceEventType.Information, 5, "Opening \"{0}\" for \"{1}\"", fullFileName, uriToDownloadFrom); DataSet ds2 = DataSet.Open(string.Format("msds:nc?file={0}&openMode=readOnly", fullFileName)); return(dict.GetOrAdd(uriToDownloadFrom, ds2)); } }
protected DataDomain CalcDataDomain(ICellRequest[] cells, ITimeAxisBoundingBoxCalculator timeBBcalc, ISpatGridBoundingBoxCalculator latBBcalc, ISpatGridBoundingBoxCalculator lonBBcalc) { if (cells.Length == 0) { throw new ArgumentException("cells array must contain elements"); } string variable = cells[0].VariableName; //calculating bounding box IndexBoundingBox timeBB = new IndexBoundingBox(); IndexBoundingBox latBB = new IndexBoundingBox(); IndexBoundingBox lonBB = new IndexBoundingBox(); Stopwatch bbCalc = Stopwatch.StartNew(); ts.TraceEvent(TraceEventType.Start, 1, "Calculating bounding box"); var cellArray = cells.ToArray(); foreach (var geoCellTuple in cellArray) { IndexBoundingBox timeBB2 = timeBBcalc.GetBoundingBox(geoCellTuple.Time); IndexBoundingBox latBB2 = latBBcalc.GetBoundingBox(geoCellTuple.LatMin, geoCellTuple.LatMax); IndexBoundingBox lonBB2 = lonBBcalc.GetBoundingBox(geoCellTuple.LonMin, geoCellTuple.LonMax); timeBB = IndexBoundingBox.Union(timeBB, timeBB2); latBB = IndexBoundingBox.Union(latBB, latBB2); lonBB = IndexBoundingBox.Union(lonBB, lonBB2); } bbCalc.Stop(); ts.TraceEvent(TraceEventType.Stop, 1, string.Format("Bounding box calculated in {0}", bbCalc.Elapsed)); if (timeBB.IsSingular || latBB.IsSingular || lonBB.IsSingular) { ts.TraceEvent(TraceEventType.Information, 4, "Bounding box is singular. (all cells are out of the data) Returning NoData"); return(null); } else { ts.TraceEvent(TraceEventType.Information, 5, string.Format("{0} elements in bounding box", (timeBB.last - timeBB.first + 1) * (latBB.last - latBB.first + 1) * (lonBB.last - lonBB.first + 1))); } var dataDomain = ConstructDataDomain(variable, metadata, lonBB, latBB, timeBB); return(dataDomain); }
/// <summary>Creates data source from handler type name and data uri or return existing one. /// This method is thread safe</summary> public static async Task <DataSourceInstance> GetInstanceAsync(string typeName, string dataUri) { var key = new TypeUriPair { TypeName = typeName, DataUri = dataUri }; Task <DataSourceInstance> val; lock (typeof(DataSourceHandlerCache)) { if (!instances.TryGetValue(key, out val)) { val = Task.Factory.StartNew(async obj => { TypeUriPair pair = (TypeUriPair)obj; ts.TraceEvent(TraceEventType.Information, 1, "loading type " + pair.TypeName + " with data uri " + pair.DataUri); var handlerType = Type.GetType(pair.TypeName); if (handlerType == null) { throw new Exception(String.Format("Type {0} is not found", pair.TypeName)); } Task <DataSet> storageTask = OpenDataSetWithRetriesAsync(pair.DataUri);; DataSourceHandler handler; bool withoutStorage = false; //trying Async API var methods = handlerType.GetMethods(System.Reflection.BindingFlags.Static | System.Reflection.BindingFlags.Public); Func <Type, bool> checkAsyncReturnType = (t => { if (!t.IsGenericType) { return(false); } var args = t.GetGenericArguments(); if (args.Length != 1) { return(false); } var genericArg = args[0]; return(genericArg.IsSubclassOf(typeof(DataSourceHandler))); }); MethodInfo[] asyncCtors = methods.Where(m => m.Name == "CreateAsync" && checkAsyncReturnType(m.ReturnType)).ToArray(); MethodInfo asyncCtor = null; //Factory with IDataContext parameter is first priority foreach (var c in asyncCtors) { var parameters = c.GetParameters(); if (parameters.Length == 1 && parameters[0].ParameterType.IsInterface && parameters[0].ParameterType == typeof(IStorageContext)) { ts.TraceEvent(TraceEventType.Information, 2, pair.TypeName + ": Found Async factory method with IStorageContext parameter"); asyncCtor = c; break; } } //if not found, looking for Factory with no parameters if (asyncCtor == null) { foreach (var c in asyncCtors) { var parameters = c.GetParameters(); if (parameters.Length == 0) { ts.TraceEvent(TraceEventType.Information, 3, pair.TypeName + ": Found Async factory method with no parameters"); withoutStorage = true; asyncCtor = c; break; } } } if (asyncCtor != null) { Type genericRetType = asyncCtor.ReturnType; Type genericArg = asyncCtor.ReturnType.GetGenericArguments()[0]; object o; if (withoutStorage) { o = asyncCtor.Invoke(null, new object[0]); } else { o = asyncCtor.Invoke(null, new object[] { new LinearizingStorageContext(await storageTask) }); } await(Task) o; handler = (DataSourceHandler)o.GetType().InvokeMember("Result", BindingFlags.Public | BindingFlags.Instance | BindingFlags.GetProperty, null, o, new object[0]); return(new DataSourceInstance(handler, await storageTask)); } //tring old API var ctor = handlerType.GetConstructor(new Type[] { typeof(IStorageContext) }); if (ctor == null) { ctor = handlerType.GetConstructor(new Type[0]); withoutStorage = true; } else { ts.TraceEvent(TraceEventType.Information, 4, pair.TypeName + ": Found synchronous constructor with IStorageContext parameter"); } if (ctor == null) { throw new Exception(String.Format("Type {0} does not have required constructor", pair.TypeName)); } else { ts.TraceEvent(TraceEventType.Information, 5, pair.TypeName + ": Found synchronous constructor without parameters"); } if (withoutStorage) { handler = (DataSourceHandler)ctor.Invoke(new object[0]); } else { handler = (DataSourceHandler)ctor.Invoke(new object[] { new LinearizingStorageContext(await storageTask) }); } return(new DataSourceInstance(handler, await storageTask)); }, key, TaskCreationOptions.LongRunning).Unwrap(); instances.Add(key, val); } } return(await val); }
public static AxisDetectionResult SmartDetectAxis(IStorageContext storage) { IDataStorageDefinition storageDefinition = storage.StorageDefinition; string varName = TimeAxisAutodetection.GetTimeVariableName(storageDefinition); if (string.IsNullOrEmpty(varName)) { throw new InvalidOperationException("Can't autodetect time axis variable"); } string timeUnits = storageDefinition.VariablesMetadata[varName].Where(pair => pair.Key.ToLowerInvariant() == "units").Select(p => p.Value).FirstOrDefault() as string; if (timeUnits == null) { throw new InvalidOperationException(string.Format("Can't find units metadata entry for the time axis \"{0}\"", varName)); } string trimmed = timeUnits.Trim(); string[] splitted = trimmed.Split(new char[] { ' ', '\t', '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries); if (splitted.Length < 4 || splitted[1].ToLowerInvariant() != "since") { throw new InvalidOperationException("Automatic time axis detection failed to determine time axis semantics. Time axis units must be in format \"days|hours|years since YYYY-MM-DD HH:MM:SS\""); } DateTime baseTime = new DateTime(); string dateToParse = string.Format("{0} {1}", splitted[2], splitted[3]); if (dateToParse.Length > 19) { dateToParse = dateToParse.Substring(0, 19); } bool baseTimeParsed = false; foreach (var dateFormat in dateFormats) { baseTimeParsed = DateTime.TryParseExact(dateToParse, dateFormat, System.Globalization.CultureInfo.InvariantCulture, System.Globalization.DateTimeStyles.AssumeUniversal, out baseTime); if (baseTimeParsed) { traceSource.TraceEvent(TraceEventType.Information, 4, string.Format("base datetime \"{0}\" for axis variable \"{1}\" was successfuly parsed as {2}", dateToParse, varName, baseTime.ToString("u"))); break; } else { traceSource.TraceEvent(TraceEventType.Information, 4, string.Format("can not parse base datetime \"{0}\" for axis variable \"{1}\" with format {2}", dateToParse, varName, dateFormat)); } } if (baseTimeParsed) { switch (splitted[0].ToLowerInvariant()) { case "years": traceSource.TraceEvent(TraceEventType.Information, 1, "Detected axis suitable for StepFunctionYearsIntegrator"); return(new AxisFound(varName, AxisKind.Years, baseTime)); case "days": traceSource.TraceEvent(TraceEventType.Information, 2, "Detected axis suitable for StepFunctionDaysIntegrator"); return(new AxisFound(varName, AxisKind.Days, baseTime)); case "hours": traceSource.TraceEvent(TraceEventType.Information, 3, "Detected axis suitable for StepFunctionHoursIntegrator"); return(new AxisFound(varName, AxisKind.Hours, baseTime)); default: traceSource.TraceEvent(TraceEventType.Error, 4, string.Format("the offset units in units metadata entry of \"{0}\" can't be parsed. It must be one of the following: years, days or hours", varName)); return(new AxisNotFound()); } } else { traceSource.TraceEvent(TraceEventType.Error, 5, string.Format("reference datetime in units metadata entry of \"{0}\" can't be parsed. It must be in format \"{1}\", but it is \"{2}\"", varName, dateFormats[0], dateToParse)); } return(new AxisNotFound()); }