internal TimeIntegratorBasedAverager(IStorageContext storageContext, ITimeAxisAvgProcessing timeIntegrator, IStationLocator stationLocator, double[] lats, double[] lons, string stationsDimName) { this.timeIntegrator = timeIntegrator; this.stationLocator = stationLocator; this.dataStorage = storageContext; this.stationsLats = lats; this.stationsLons = lons; IDataStorageDefinition storageDef = storageContext.StorageDefinition; //determining dimension order var varDimensions = storageContext.StorageDefinition.VariablesDimensions; foreach (string dataVarName in varDimensions.Where(a => a.Value.Length == 2).Select(b => b.Key)) { if (varDimensions[dataVarName][0] == stationsDimName) { stationsDimNumber.Add(dataVarName, 0); timeDimNumber.Add(dataVarName, 1); } else { stationsDimNumber.Add(dataVarName, 1); timeDimNumber.Add(dataVarName, 0); } } dataRepresentationDictionary = new DataRepresentationDictionary(storageContext.StorageDefinition); missingValuesDict = new MissingValuesDictionary(storageContext.StorageDefinition); varTypes = new Dictionary <string, Type>(storageContext.StorageDefinition.VariablesTypes); }
public StorageContextMetadataTimeVarianceExtractor(IDataStorageDefinition storageDefinition, double axisPeriod = double.NaN) { var dict = storageDefinition.VariablesMetadata; var dimDict = storageDefinition.VariablesDimensions; string timeDimName = TimeAxisAutodetection.GetTimeDimension(storageDefinition); foreach (var variable in dict.Keys) { //checking for temporal variogram. Storing it if found if (timeDimName != null) { int idx = Array.IndexOf(dimDict[variable], timeDimName); if (idx >= 0) { var temporalVariogramStorage = new StorageContextVariogramStorage.StorageContextMetadataStorage(storageDefinition, variable, idx.ToString()) as VariogramModule.IVariogramStorage; var materializationResult = temporalVariogramStorage.Materialize(); if (FSharp.Core.FSharpOption <VariogramModule.IVariogram> .get_IsSome(materializationResult)) { processes.Add(variable, new GaussianProcessDescription(materializationResult.Value, axisPeriod)); } } } } }
public GridDefinitionAnalysis(IDataStorageDefinition storageDef, string latArrayName = null, string lonArrayName = null) { if (string.IsNullOrEmpty(latArrayName)) { latArrayName = IntegratorsFactoryHelpers.AutodetectLatName(storageDef); } if (string.IsNullOrEmpty(lonArrayName)) { lonArrayName = IntegratorsFactoryHelpers.AutodetectLonName(storageDef); } missingValuesDictionary = new MissingValuesDictionary(storageDef); dimensionOrderDictioary = new Dictionary <string, DimensionsOrder>(); Dictionary <string, int> ranks = new Dictionary <string, int>(); variableRankDict = ranks; var dataVarsNames = storageDef.VariablesDimensions //looking schema for 2D and 3D data arrays .Where(def => (def.Value.Contains(storageDef.VariablesDimensions[latArrayName][0]) && def.Value.Contains(storageDef.VariablesDimensions[lonArrayName][0]) && (def.Key != latArrayName) && (def.Key != lonArrayName))) .Select(def => def.Key).ToArray(); //analyzing data variables foreach (var dataVarName in dataVarsNames) { //analyzing dimensions dimensionOrderDictioary[dataVarName] = DetermineDimensionsOrder(storageDef, latArrayName, lonArrayName, dataVarName); ranks[dataVarName] = storageDef.VariablesDimensions[dataVarName].Length; } }
public GridClusteringDecorator(IDataStorageDefinition storageDefinition, IBatchValueAggregator component, ITimeAxisBoundingBoxCalculator timeAxisBBcalc, ISpatGridBoundingBoxCalculator latAxisBBcalc, ISpatGridBoundingBoxCalculator lonAxisBBcalc, int clusterSizeInMegabytes = 128) { this.component = component; this.clusterSizeInMegabytes = clusterSizeInMegabytes; this.latAxisBBcalc = latAxisBBcalc; this.lonAxisBBcalc = lonAxisBBcalc; this.timeAxisBBcalc = timeAxisBBcalc; varDataTypes = storageDefinition.VariablesTypes; }
public static string AutodetectLonName(IDataStorageDefinition storageDefinition) { var lonArrayName = DetectArrayName(storageDefinition, possibleLonNames, possibleLonNames); if (string.IsNullOrEmpty(lonArrayName)) { throw new InvalidOperationException("can't auto detect longitude array name"); } return(lonArrayName); }
/// <summary> /// Tries to find an array with one of the specified possible names or with possible substring. Returns null if nothing found /// </summary> /// <param name="storageDefinition">a storage definition to find the axes names in</param> /// <param name="latName">an out parameter to output found latitude array name</param> /// <param name="lonName">an out parameter to output found longitude array name</param> /// <returns></returns> private static string DetectArrayName(IDataStorageDefinition storageDefinition, string[] exactNamesToFind, string[] namePartsToFind) { var oneDimVars = storageDefinition.VariablesDimensions.Where(v => v.Value.Length == 1).Select(v => v.Key); var foundName = oneDimVars.FirstOrDefault(v => exactNamesToFind.Contains(v)); //exact match if (foundName == null) { foundName = oneDimVars.FirstOrDefault(v => namePartsToFind.Any(part => v.Contains(part))); //substring only } return(foundName); }
/// <summary> /// returns null if not found /// </summary> /// <param name="storageDefinition"></param> /// <returns></returns> public static string GetTimeVariableName(IDataStorageDefinition storageDefinition) { var timeDimName = GetTimeDimension(storageDefinition); var foundVars = storageDefinition.VariablesDimensions.Where(p => p.Value.Length == 1 && p.Value[0] == timeDimName).ToArray(); if (foundVars.Length == 0) { return(null); } else { return(foundVars[0].Key); } }
public static Tuple <string, string> AutodetectLatLonNames(IDataStorageDefinition storageDefinition) { for (int i = 0; i < possibleLatNames.Length; i++) { string[] latDim; string[] lonDim; if (storageDefinition.VariablesDimensions.TryGetValue(possibleLatNames[i], out latDim) && storageDefinition.VariablesDimensions.TryGetValue(possibleLonNames[i], out lonDim) && latDim.Length == 1 && lonDim.Length == 1 && latDim[0] == lonDim[0]) { return(new Tuple <string, string>(possibleLatNames[i], possibleLonNames[i])); } } throw new InvalidOperationException("Cannot auto detect pair of latitude and longitude variables"); }
public StorageContextMetadataSpatialVarianceExtractor(IDataStorageDefinition storageDefinition) { var dict = storageDefinition.VariablesMetadata; var dimDict = storageDefinition.VariablesDimensions; foreach (var variable in dict.Keys) { //checking for spatial variogram. Storing it if found var spatialVariogramStorage = new StorageContextVariogramStorage.StorageContextMetadataStorage(storageDefinition, variable, spatialKey) as VariogramModule.IVariogramStorage; var materializationResult = spatialVariogramStorage.Materialize(); if (FSharp.Core.FSharpOption <VariogramModule.IVariogram> .get_IsSome(materializationResult)) { fields.Add(variable, new GaussianFieldDescrption(materializationResult.Value)); } } }
/// <summary> /// Determines the dimensions order for the data array (2D or 3D) /// </summary> /// <param name="storageDefinition">a storage definition to check and to extract the scheme from</param> /// <param name="latName">The name of the array containing latitude values</param> /// <param name="lonName">The name of the array containing longitude values</param> /// <param name="dataName">The name of the data array to determine dimension order for</param> /// <returns></returns> private static DimensionsOrder DetermineDimensionsOrder(IDataStorageDefinition storageDefinition, string latName, string lonName, string dataName) { //checking persistence of lat and lon arrays if (!storageDefinition.VariablesDimensions.ContainsKey(latName)) { throw new ArgumentException("Latitude array (\"" + latName + "\") is not found"); } if (!storageDefinition.VariablesDimensions.ContainsKey(lonName)) { throw new ArgumentException("Longitude array (\"" + lonName + "\") is not found"); } if (storageDefinition.VariablesDimensions[latName].Length > 1) { throw new ArgumentException("Latitude array (\"" + latName + "\") is not an axis. It is multidimensional. One dimensional array is expected"); } if (storageDefinition.VariablesDimensions[lonName].Length > 1) { throw new ArgumentException("Latitude array (\"" + lonName + "\") is not an axis. It is multidimensional. One dimensional array is expected"); } string latDim = storageDefinition.VariablesDimensions[latName][0]; string lonDim = storageDefinition.VariablesDimensions[lonName][0]; DimensionsOrder order = new DimensionsOrder(); order.TimeDimNum = -1; //if there is no time dim string[] dims = storageDefinition.VariablesDimensions[dataName]; for (int i = 0; i < dims.Length; i++) { if (dims[i] == latDim) { order.LatDimNum = i; } if (dims[i] == lonDim) { order.LonDimNum = i; } if (dims[i] != latDim && dims[i] != lonDim) { order.TimeDimNum = i; } } return(order); }
public MissingValuesDictionary(IDataStorageDefinition storageDefinition) { dataTypes = new ReadOnlyDictionary <string, Type>(storageDefinition.VariablesTypes); foreach (var dataVarName in storageDefinition.VariablesMetadata.Keys) { //analyzing metadata var metadata = storageDefinition.VariablesMetadata[dataVarName]; //looking for MV specification foreach (var item in missingValueKeys) { if (metadata.ContainsKey(item)) { this[dataVarName] = metadata[item]; break; } } } }
public DataRepresentationDictionary(IDataStorageDefinition definition) { foreach (var var in definition.VariablesMetadata) { string name = var.Key; if (var.Value.ContainsKey("scale_factor")) { scaleFactors.Add(name, Convert.ToDouble(var.Value["scale_factor"], CultureInfo.InvariantCulture)); } else { scaleFactors.Add(name, 1.0); } if (var.Value.ContainsKey("add_offset")) { addOffsets.Add(name, Convert.ToDouble(var.Value["add_offset"], CultureInfo.InvariantCulture)); } else { addOffsets.Add(name, 0.0); } } }
/// <summary> /// Returns null if not found /// </summary> /// <param name="storageDefinition"></param> /// <returns></returns> public static string GetTimeDimension(IDataStorageDefinition storageDefinition) { var dict = storageDefinition.VariablesMetadata; var dimDict = storageDefinition.VariablesDimensions; string timeDimName = storageDefinition.VariablesDimensions.Where(pair => possibleTimeVarNames.Contains(pair.Key.ToLower()) && pair.Value.Length == 1).Select(pair => pair.Value[0]).FirstOrDefault(); if (timeDimName == null) //checking dimension name itself { timeDimName = storageDefinition.DimensionsLengths.Keys.FirstOrDefault(name => possibleTimeVarNames.Contains(name.ToLower())); } if (timeDimName == null) //checking the var name contains a possible name as a part { timeDimName = storageDefinition.VariablesDimensions.Where(pair => possibleComplexNames.Any(posName => pair.Key.ToLower().Contains(posName)) && pair.Value.Length == 1).Select(pair => pair.Value[0]).FirstOrDefault(); } if (timeDimName == null) //checking the var dimension contains a possible name as a part { timeDimName = storageDefinition.DimensionsLengths.Keys.FirstOrDefault(name => possibleComplexNames.Any(posName => name.ToLower().Contains(posName))); } return(timeDimName); }
public static AxisDetectionResult SmartDetectAxis(IStorageContext storage) { IDataStorageDefinition storageDefinition = storage.StorageDefinition; string varName = TimeAxisAutodetection.GetTimeVariableName(storageDefinition); if (string.IsNullOrEmpty(varName)) { throw new InvalidOperationException("Can't autodetect time axis variable"); } string timeUnits = storageDefinition.VariablesMetadata[varName].Where(pair => pair.Key.ToLowerInvariant() == "units").Select(p => p.Value).FirstOrDefault() as string; if (timeUnits == null) { throw new InvalidOperationException(string.Format("Can't find units metadata entry for the time axis \"{0}\"", varName)); } string trimmed = timeUnits.Trim(); string[] splitted = trimmed.Split(new char[] { ' ', '\t', '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries); if (splitted.Length < 4 || splitted[1].ToLowerInvariant() != "since") { throw new InvalidOperationException("Automatic time axis detection failed to determine time axis semantics. Time axis units must be in format \"days|hours|years since YYYY-MM-DD HH:MM:SS\""); } DateTime baseTime = new DateTime(); string dateToParse = string.Format("{0} {1}", splitted[2], splitted[3]); if (dateToParse.Length > 19) { dateToParse = dateToParse.Substring(0, 19); } bool baseTimeParsed = false; foreach (var dateFormat in dateFormats) { baseTimeParsed = DateTime.TryParseExact(dateToParse, dateFormat, System.Globalization.CultureInfo.InvariantCulture, System.Globalization.DateTimeStyles.AssumeUniversal, out baseTime); if (baseTimeParsed) { traceSource.TraceEvent(TraceEventType.Information, 4, string.Format("base datetime \"{0}\" for axis variable \"{1}\" was successfuly parsed as {2}", dateToParse, varName, baseTime.ToString("u"))); break; } else { traceSource.TraceEvent(TraceEventType.Information, 4, string.Format("can not parse base datetime \"{0}\" for axis variable \"{1}\" with format {2}", dateToParse, varName, dateFormat)); } } if (baseTimeParsed) { switch (splitted[0].ToLowerInvariant()) { case "years": traceSource.TraceEvent(TraceEventType.Information, 1, "Detected axis suitable for StepFunctionYearsIntegrator"); return(new AxisFound(varName, AxisKind.Years, baseTime)); case "days": traceSource.TraceEvent(TraceEventType.Information, 2, "Detected axis suitable for StepFunctionDaysIntegrator"); return(new AxisFound(varName, AxisKind.Days, baseTime)); case "hours": traceSource.TraceEvent(TraceEventType.Information, 3, "Detected axis suitable for StepFunctionHoursIntegrator"); return(new AxisFound(varName, AxisKind.Hours, baseTime)); default: traceSource.TraceEvent(TraceEventType.Error, 4, string.Format("the offset units in units metadata entry of \"{0}\" can't be parsed. It must be one of the following: years, days or hours", varName)); return(new AxisNotFound()); } } else { traceSource.TraceEvent(TraceEventType.Error, 5, string.Format("reference datetime in units metadata entry of \"{0}\" can't be parsed. It must be in format \"{1}\", but it is \"{2}\"", varName, dateFormats[0], dateToParse)); } return(new AxisNotFound()); }
public VariablePresenceCheckDecorator(IDataStorageDefinition storageDef, IBatchUncertaintyEvaluator component) { this.component = component; this.storageDef = storageDef; }