public FetchTask(
ISet <ISelection> selections,
IFetchConfiguration configuration,
IImmutableStack <FetchTask> path,
IReadOnlyList <FetchTask> alternativeTasks)
{
Selections = selections;
Configuration = configuration;
Path = path;
AlternativeTasks = alternativeTasks;
Children = new List <FetchTask>();
}
static private void PrettyPrint(IFetchConfiguration config, string heading)
{
ConsoleColor oldColor = Console.ForegroundColor;
Console.ForegroundColor = ConsoleColor.Cyan;
Console.WriteLine(heading);
Console.WriteLine();
Console.WriteLine("Config timestamp: {0}", config.TimeStamp.ToString("yyyy-MM-dd HH:mm:ss.fff"));
Console.WriteLine();
if (config.EnvironmentalVariables.Length == 0)
{
Console.ForegroundColor = oldColor;
Console.WriteLine("There are no environmental variables available");
}
else
{
int maxVarNameLength =
Math.Max(9, Math.Min(15, config.EnvironmentalVariables.Select(v => v.Name.Length).Max()));
int maxUnitsLength =
Math.Max(6, Math.Min(15, config.EnvironmentalVariables.Select(v => v.Units.Length).Max()));
int maxDescrLength =
Math.Max(12, Math.Min(47, config.EnvironmentalVariables.Select(v => v.Description.Length).Max()));
Console.WriteLine("Environmental variables\n");
Console.ForegroundColor = oldColor;
Console.Write("Variable".PadRight(maxVarNameLength + 1));
Console.Write("Units".PadRight(maxUnitsLength + 1));
Console.WriteLine("Description".PadRight(maxDescrLength + 1));
Console.Write("-".PadRight(maxVarNameLength, '-'));
Console.Write(" -".PadRight(maxUnitsLength + 1, '-'));
Console.WriteLine(" -".PadRight(maxDescrLength + 1, '-'));
foreach (var v in config.EnvironmentalVariables)
{
var len = v.Name.Length;
Console.Write(len > maxVarNameLength ?
String.Concat(v.Name.Substring(0, maxVarNameLength - 3), "... ") :
v.Name.PadRight(maxVarNameLength + 1));
len = v.Units.Length;
Console.Write(len > maxUnitsLength ?
String.Concat(v.Units.Substring(0, maxUnitsLength - 3), "... ") :
v.Units.PadRight(maxUnitsLength + 1));
len = v.Description.Length;
Console.WriteLine(len > maxDescrLength ?
String.Concat(v.Description.Substring(0, maxDescrLength - 3), "...") :
v.Description);
}
Console.ForegroundColor = ConsoleColor.Cyan;
Console.Write("\nData sources\n");
Console.ForegroundColor = oldColor;
foreach (var d in config.DataSources)
{
Console.WriteLine("Name:\t\t{0}", d.Name);
Console.WriteLine("ID:\t\t{0}", d.ID);
Console.WriteLine("Description:\t{0}", d.Description);
Console.WriteLine("Copyright:\t{0}", d.Copyright);
Console.WriteLine("Variables:\t{0}", d.ProvidedVariables.Aggregate <string, StringBuilder>(new StringBuilder(), (sb, v) => sb.Append(string.Format("{0} ", v))).ToString());
Console.WriteLine("Location:\t{0}", d.Location);
Console.WriteLine();
}
}
}
public static void MergeTable(IFetchConfiguration config, DataSet dst, Tuple <DataSet, string[]>[] requests)
{
// For faster lookup of data source name from id
var id2name = new Dictionary <int, string>();
foreach (var dsd in config.DataSources)
{
id2name.Add(dsd.ID, dsd.Name);
}
var var2var = new Dictionary <string, Variables>(); // Environment variable short name => data set variables
var cell2row = new Dictionary <Cell, int>(); // Space-time cell => row number
var rowCount = 0;
var optionalRowCount = 0;
var regionRowCount = 0;
var regions = dst.AddVariable <string>("region", "i");
var lat = dst.AddVariable <double>("lat", "i");
var lon = dst.AddVariable <double>("lon", "i");
Variable <double> latmin = null, latmax = null, lonmin = null, lonmax = null;
var start = dst.AddVariable <DateTime>("start", "i");
var end = dst.AddVariable <DateTime>("end", "i");
for (var i = 0; i < requests.Length; i++)
{
using (var src = requests[i].Item1.Clone("msds:memory"))
{
var name = src.Metadata[RequestDataSetFormat.EnvironmentVariableNameKey].ToString();
// Define data source name to use when not provenance is supplied
var noProvDataSource = "";
if (src.Metadata.ContainsKey(RequestDataSetFormat.DataSourceNameKey))
{
var requestedDataSources = (string[])src.Metadata[RequestDataSetFormat.DataSourceNameKey];
if (requestedDataSources.Length == 1)
{
noProvDataSource = requestedDataSources[0];
}
}
var envVar = config.EnvironmentalVariables.Where(ev => ev.Name == name).First();
Variables variables;
if (!var2var.TryGetValue(name, out variables))
{
variables = new Variables();
variables.Values = dst.AddVariable <double>(String.Concat(name, " (", envVar.Units, ")"), "i");
variables.Values.MissingValue = Double.NaN;
variables.Uncertainty = dst.AddVariable <double>(String.Concat(name, "_uncertainty"), "i");
variables.Uncertainty.MissingValue = Double.NaN;
variables.Provenance = dst.AddVariable <string>(String.Concat(name, "_provenance"), "i");
variables.Provenance.MissingValue = null;
var2var.Add(name, variables);
}
foreach (var t in Linearize(src, requests[i].Item2))
{
int row;
if (!cell2row.TryGetValue(t.Item1, out row))
{
row = rowCount++;
cell2row.Add(t.Item1, row);
}
if (t.Item5 != "")
{
regions[row] = t.Item5;
regionRowCount = Math.Max(regionRowCount, row);
}
lat[row] = t.Item1.Lat;
lon[row] = t.Item1.Lon;
start[row] = t.Item1.Start;
end[row] = t.Item1.End;
if (t.Item1.LatMin != t.Item1.Lat)
{
if (latmin == null)
{
latmin = dst.AddVariable <double>("latmin", "i");
latmax = dst.AddVariable <double>("latmax", "i");
lonmin = dst.AddVariable <double>("lonmin", "i");
lonmax = dst.AddVariable <double>("lonmax", "i");
}
latmin[row] = t.Item1.LatMin;
latmax[row] = t.Item1.LatMax;
lonmin[row] = t.Item1.LonMin;
lonmax[row] = t.Item1.LonMax;
optionalRowCount = Math.Max(optionalRowCount, row);
}
variables.Values[row] = t.Item2;
variables.Uncertainty[row] = t.Item3 < Double.MaxValue ? t.Item3 : Double.NaN;
variables.Provenance[row] = (t.Item4 == 65535) ? noProvDataSource : id2name[t.Item4];
variables.RowCount = row;
}
}
}
foreach (var v in var2var)
{
for (var j = v.Value.RowCount + 1; j < rowCount; j++)
{
v.Value.Values[j] = Double.NaN;
v.Value.Uncertainty[j] = Double.NaN;
v.Value.Provenance[j] = null;
}
}
if (latmin != null)
{
for (var j = optionalRowCount + 1; j < rowCount; j++)
{
latmin[j] = Double.NaN;
latmax[j] = Double.NaN;
lonmin[j] = Double.NaN;
lonmax[j] = Double.NaN;
}
}
for (var j = regionRowCount + 1; j < rowCount; j++)
{
regions[j] = "";
}
dst.Commit();
}
