internal TimeIntegratorBasedAverager(IStorageContext storageContext, ITimeAxisAvgProcessing timeIntegrator, IStationLocator stationLocator, double[] lats, double[] lons, string stationsDimName)
{
this.timeIntegrator = timeIntegrator;
this.stationLocator = stationLocator;
this.dataStorage = storageContext;
this.stationsLats = lats;
this.stationsLons = lons;
IDataStorageDefinition storageDef = storageContext.StorageDefinition;
//determining dimension order
var varDimensions = storageContext.StorageDefinition.VariablesDimensions;
foreach (string dataVarName in varDimensions.Where(a => a.Value.Length == 2).Select(b => b.Key))
{
if (varDimensions[dataVarName][0] == stationsDimName)
{
stationsDimNumber.Add(dataVarName, 0);
timeDimNumber.Add(dataVarName, 1);
}
else
{
stationsDimNumber.Add(dataVarName, 1);
timeDimNumber.Add(dataVarName, 0);
}
}
dataRepresentationDictionary = new DataRepresentationDictionary(storageContext.StorageDefinition);
missingValuesDict = new MissingValuesDictionary(storageContext.StorageDefinition);
varTypes = new Dictionary <string, Type>(storageContext.StorageDefinition.VariablesTypes);
}
public StorageContextMetadataTimeVarianceExtractor(IDataStorageDefinition storageDefinition, double axisPeriod = double.NaN)
{
var dict = storageDefinition.VariablesMetadata;
var dimDict = storageDefinition.VariablesDimensions;
string timeDimName = TimeAxisAutodetection.GetTimeDimension(storageDefinition);
foreach (var variable in dict.Keys)
{
//checking for temporal variogram. Storing it if found
if (timeDimName != null)
{
int idx = Array.IndexOf(dimDict[variable], timeDimName);
if (idx >= 0)
{
var temporalVariogramStorage = new StorageContextVariogramStorage.StorageContextMetadataStorage(storageDefinition, variable, idx.ToString()) as VariogramModule.IVariogramStorage;
var materializationResult = temporalVariogramStorage.Materialize();
if (FSharp.Core.FSharpOption <VariogramModule.IVariogram> .get_IsSome(materializationResult))
{
processes.Add(variable, new GaussianProcessDescription(materializationResult.Value, axisPeriod));
}
}
}
}
}
public GridDefinitionAnalysis(IDataStorageDefinition storageDef, string latArrayName = null, string lonArrayName = null)
{
if (string.IsNullOrEmpty(latArrayName))
{
latArrayName = IntegratorsFactoryHelpers.AutodetectLatName(storageDef);
}
if (string.IsNullOrEmpty(lonArrayName))
{
lonArrayName = IntegratorsFactoryHelpers.AutodetectLonName(storageDef);
}
missingValuesDictionary = new MissingValuesDictionary(storageDef);
dimensionOrderDictioary = new Dictionary <string, DimensionsOrder>();
Dictionary <string, int> ranks = new Dictionary <string, int>();
variableRankDict = ranks;
var dataVarsNames = storageDef.VariablesDimensions //looking schema for 2D and 3D data arrays
.Where(def =>
(def.Value.Contains(storageDef.VariablesDimensions[latArrayName][0]) &&
def.Value.Contains(storageDef.VariablesDimensions[lonArrayName][0]) && (def.Key != latArrayName) && (def.Key != lonArrayName)))
.Select(def => def.Key).ToArray();
//analyzing data variables
foreach (var dataVarName in dataVarsNames)
{
//analyzing dimensions
dimensionOrderDictioary[dataVarName] = DetermineDimensionsOrder(storageDef, latArrayName, lonArrayName, dataVarName);
ranks[dataVarName] = storageDef.VariablesDimensions[dataVarName].Length;
}
}
public GridClusteringDecorator(IDataStorageDefinition storageDefinition, IBatchValueAggregator component, ITimeAxisBoundingBoxCalculator timeAxisBBcalc, ISpatGridBoundingBoxCalculator latAxisBBcalc, ISpatGridBoundingBoxCalculator lonAxisBBcalc, int clusterSizeInMegabytes = 128)
{
this.component = component;
this.clusterSizeInMegabytes = clusterSizeInMegabytes;
this.latAxisBBcalc = latAxisBBcalc;
this.lonAxisBBcalc = lonAxisBBcalc;
this.timeAxisBBcalc = timeAxisBBcalc;
varDataTypes = storageDefinition.VariablesTypes;
}
public static string AutodetectLonName(IDataStorageDefinition storageDefinition)
{
var lonArrayName = DetectArrayName(storageDefinition,
possibleLonNames,
possibleLonNames);
if (string.IsNullOrEmpty(lonArrayName))
{
throw new InvalidOperationException("can't auto detect longitude array name");
}
return(lonArrayName);
}
/// <summary>
/// Tries to find an array with one of the specified possible names or with possible substring. Returns null if nothing found
/// </summary>
/// <param name="storageDefinition">a storage definition to find the axes names in</param>
/// <param name="latName">an out parameter to output found latitude array name</param>
/// <param name="lonName">an out parameter to output found longitude array name</param>
/// <returns></returns>
private static string DetectArrayName(IDataStorageDefinition storageDefinition, string[] exactNamesToFind, string[] namePartsToFind)
{
var oneDimVars = storageDefinition.VariablesDimensions.Where(v => v.Value.Length == 1).Select(v => v.Key);
var foundName = oneDimVars.FirstOrDefault(v => exactNamesToFind.Contains(v)); //exact match
if (foundName == null)
{
foundName = oneDimVars.FirstOrDefault(v => namePartsToFind.Any(part => v.Contains(part))); //substring only
}
return(foundName);
}
/// <summary>
/// returns null if not found
/// </summary>
/// <param name="storageDefinition"></param>
/// <returns></returns>
public static string GetTimeVariableName(IDataStorageDefinition storageDefinition)
{
var timeDimName = GetTimeDimension(storageDefinition);
var foundVars = storageDefinition.VariablesDimensions.Where(p => p.Value.Length == 1 && p.Value[0] == timeDimName).ToArray();
if (foundVars.Length == 0)
{
return(null);
}
else
{
return(foundVars[0].Key);
}
}
public static Tuple <string, string> AutodetectLatLonNames(IDataStorageDefinition storageDefinition)
{
for (int i = 0; i < possibleLatNames.Length; i++)
{
string[] latDim;
string[] lonDim;
if (storageDefinition.VariablesDimensions.TryGetValue(possibleLatNames[i], out latDim) &&
storageDefinition.VariablesDimensions.TryGetValue(possibleLonNames[i], out lonDim) &&
latDim.Length == 1 && lonDim.Length == 1 && latDim[0] == lonDim[0])
{
return(new Tuple <string, string>(possibleLatNames[i], possibleLonNames[i]));
}
}
throw new InvalidOperationException("Cannot auto detect pair of latitude and longitude variables");
}
public StorageContextMetadataSpatialVarianceExtractor(IDataStorageDefinition storageDefinition)
{
var dict = storageDefinition.VariablesMetadata;
var dimDict = storageDefinition.VariablesDimensions;
foreach (var variable in dict.Keys)
{
//checking for spatial variogram. Storing it if found
var spatialVariogramStorage = new StorageContextVariogramStorage.StorageContextMetadataStorage(storageDefinition, variable, spatialKey) as VariogramModule.IVariogramStorage;
var materializationResult = spatialVariogramStorage.Materialize();
if (FSharp.Core.FSharpOption <VariogramModule.IVariogram> .get_IsSome(materializationResult))
{
fields.Add(variable, new GaussianFieldDescrption(materializationResult.Value));
}
}
}
/// <summary>
/// Determines the dimensions order for the data array (2D or 3D)
/// </summary>
/// <param name="storageDefinition">a storage definition to check and to extract the scheme from</param>
/// <param name="latName">The name of the array containing latitude values</param>
/// <param name="lonName">The name of the array containing longitude values</param>
/// <param name="dataName">The name of the data array to determine dimension order for</param>
/// <returns></returns>
private static DimensionsOrder DetermineDimensionsOrder(IDataStorageDefinition storageDefinition, string latName, string lonName, string dataName)
{
//checking persistence of lat and lon arrays
if (!storageDefinition.VariablesDimensions.ContainsKey(latName))
{
throw new ArgumentException("Latitude array (\"" + latName + "\") is not found");
}
if (!storageDefinition.VariablesDimensions.ContainsKey(lonName))
{
throw new ArgumentException("Longitude array (\"" + lonName + "\") is not found");
}
if (storageDefinition.VariablesDimensions[latName].Length > 1)
{
throw new ArgumentException("Latitude array (\"" + latName + "\") is not an axis. It is multidimensional. One dimensional array is expected");
}
if (storageDefinition.VariablesDimensions[lonName].Length > 1)
{
throw new ArgumentException("Latitude array (\"" + lonName + "\") is not an axis. It is multidimensional. One dimensional array is expected");
}
string latDim = storageDefinition.VariablesDimensions[latName][0];
string lonDim = storageDefinition.VariablesDimensions[lonName][0];
DimensionsOrder order = new DimensionsOrder();
order.TimeDimNum = -1; //if there is no time dim
string[] dims = storageDefinition.VariablesDimensions[dataName];
for (int i = 0; i < dims.Length; i++)
{
if (dims[i] == latDim)
{
order.LatDimNum = i;
}
if (dims[i] == lonDim)
{
order.LonDimNum = i;
}
if (dims[i] != latDim && dims[i] != lonDim)
{
order.TimeDimNum = i;
}
}
return(order);
}
public MissingValuesDictionary(IDataStorageDefinition storageDefinition)
{
dataTypes = new ReadOnlyDictionary <string, Type>(storageDefinition.VariablesTypes);
foreach (var dataVarName in storageDefinition.VariablesMetadata.Keys)
{
//analyzing metadata
var metadata = storageDefinition.VariablesMetadata[dataVarName];
//looking for MV specification
foreach (var item in missingValueKeys)
{
if (metadata.ContainsKey(item))
{
this[dataVarName] = metadata[item];
break;
}
}
}
}
public DataRepresentationDictionary(IDataStorageDefinition definition)
{
foreach (var var in definition.VariablesMetadata)
{
string name = var.Key;
if (var.Value.ContainsKey("scale_factor"))
{
scaleFactors.Add(name, Convert.ToDouble(var.Value["scale_factor"], CultureInfo.InvariantCulture));
}
else
{
scaleFactors.Add(name, 1.0);
}
if (var.Value.ContainsKey("add_offset"))
{
addOffsets.Add(name, Convert.ToDouble(var.Value["add_offset"], CultureInfo.InvariantCulture));
}
else
{
addOffsets.Add(name, 0.0);
}
}
}
/// <summary>
/// Returns null if not found
/// </summary>
/// <param name="storageDefinition"></param>
/// <returns></returns>
public static string GetTimeDimension(IDataStorageDefinition storageDefinition)
{
var dict = storageDefinition.VariablesMetadata;
var dimDict = storageDefinition.VariablesDimensions;
string timeDimName = storageDefinition.VariablesDimensions.Where(pair => possibleTimeVarNames.Contains(pair.Key.ToLower()) && pair.Value.Length == 1).Select(pair => pair.Value[0]).FirstOrDefault();
if (timeDimName == null) //checking dimension name itself
{
timeDimName = storageDefinition.DimensionsLengths.Keys.FirstOrDefault(name => possibleTimeVarNames.Contains(name.ToLower()));
}
if (timeDimName == null) //checking the var name contains a possible name as a part
{
timeDimName = storageDefinition.VariablesDimensions.Where(pair => possibleComplexNames.Any(posName => pair.Key.ToLower().Contains(posName)) && pair.Value.Length == 1).Select(pair => pair.Value[0]).FirstOrDefault();
}
if (timeDimName == null) //checking the var dimension contains a possible name as a part
{
timeDimName = storageDefinition.DimensionsLengths.Keys.FirstOrDefault(name => possibleComplexNames.Any(posName => name.ToLower().Contains(posName)));
}
return(timeDimName);
}
public static AxisDetectionResult SmartDetectAxis(IStorageContext storage)
{
IDataStorageDefinition storageDefinition = storage.StorageDefinition;
string varName = TimeAxisAutodetection.GetTimeVariableName(storageDefinition);
if (string.IsNullOrEmpty(varName))
{
throw new InvalidOperationException("Can't autodetect time axis variable");
}
string timeUnits = storageDefinition.VariablesMetadata[varName].Where(pair => pair.Key.ToLowerInvariant() == "units").Select(p => p.Value).FirstOrDefault() as string;
if (timeUnits == null)
{
throw new InvalidOperationException(string.Format("Can't find units metadata entry for the time axis \"{0}\"", varName));
}
string trimmed = timeUnits.Trim();
string[] splitted = trimmed.Split(new char[] { ' ', '\t', '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries);
if (splitted.Length < 4 || splitted[1].ToLowerInvariant() != "since")
{
throw new InvalidOperationException("Automatic time axis detection failed to determine time axis semantics. Time axis units must be in format \"days|hours|years since YYYY-MM-DD HH:MM:SS\"");
}
DateTime baseTime = new DateTime();
string dateToParse = string.Format("{0} {1}", splitted[2], splitted[3]);
if (dateToParse.Length > 19)
{
dateToParse = dateToParse.Substring(0, 19);
}
bool baseTimeParsed = false;
foreach (var dateFormat in dateFormats)
{
baseTimeParsed = DateTime.TryParseExact(dateToParse, dateFormat, System.Globalization.CultureInfo.InvariantCulture, System.Globalization.DateTimeStyles.AssumeUniversal, out baseTime);
if (baseTimeParsed)
{
traceSource.TraceEvent(TraceEventType.Information, 4, string.Format("base datetime \"{0}\" for axis variable \"{1}\" was successfuly parsed as {2}", dateToParse, varName, baseTime.ToString("u")));
break;
}
else
{
traceSource.TraceEvent(TraceEventType.Information, 4, string.Format("can not parse base datetime \"{0}\" for axis variable \"{1}\" with format {2}", dateToParse, varName, dateFormat));
}
}
if (baseTimeParsed)
{
switch (splitted[0].ToLowerInvariant())
{
case "years":
traceSource.TraceEvent(TraceEventType.Information, 1, "Detected axis suitable for StepFunctionYearsIntegrator");
return(new AxisFound(varName, AxisKind.Years, baseTime));
case "days":
traceSource.TraceEvent(TraceEventType.Information, 2, "Detected axis suitable for StepFunctionDaysIntegrator");
return(new AxisFound(varName, AxisKind.Days, baseTime));
case "hours":
traceSource.TraceEvent(TraceEventType.Information, 3, "Detected axis suitable for StepFunctionHoursIntegrator");
return(new AxisFound(varName, AxisKind.Hours, baseTime));
default:
traceSource.TraceEvent(TraceEventType.Error, 4, string.Format("the offset units in units metadata entry of \"{0}\" can't be parsed. It must be one of the following: years, days or hours", varName));
return(new AxisNotFound());
}
}
else
{
traceSource.TraceEvent(TraceEventType.Error, 5, string.Format("reference datetime in units metadata entry of \"{0}\" can't be parsed. It must be in format \"{1}\", but it is \"{2}\"", varName, dateFormats[0], dateToParse));
}
return(new AxisNotFound());
}
public VariablePresenceCheckDecorator(IDataStorageDefinition storageDef, IBatchUncertaintyEvaluator component)
{
this.component = component;
this.storageDef = storageDef;
}