public void DataSourceAdd(string name, string description, string handler = null, string remotename = null, string copyright = "",
string uri = "", List <DataSourceMapping> addMapping = null, List <String> removeMapping = null)
{
ushort?remoteID = null;
string resolvedHandlerAssemblyQualifiedName = null;
//if (string.IsNullOrEmpty(remotename) && !String.IsNullOrEmpty(handler))
// ExtractHandlerAssemblyAndTypeName(handler, out toLoad, out handlerType);
if (!string.IsNullOrEmpty(remotename) && String.IsNullOrEmpty(handler)) //federated
{
IFetchConfiguration remoteConfig;
try
{
RemoteFetchClient client = new RemoteFetchClient(new Uri(uri));
remoteConfig = client.GetConfiguration(DateTime.MaxValue);
}
catch (Exception ex)
{
throw new ArgumentException("Failed to retrieve configuration of the remote service.\n Exception message: " + ex.Message);
}
if (!remoteConfig.DataSources.Any(ds => ds.Name == remotename))
{
throw new ArgumentException("Data source with given name does not exist on the remote service.");
}
remoteID = remoteConfig.DataSources.Where(ds => ds.Name == remotename).FirstOrDefault().ID;
}
else if (string.IsNullOrEmpty(remotename) && !String.IsNullOrEmpty(handler))//local
{
AssemblyStore gac = null;
if (isConnectedToCloud)
{
gac = astore;
}
resolvedHandlerAssemblyQualifiedName = ExtractHandlerAssemblyAndTypeName(handler, gac);
}
else if (!string.IsNullOrEmpty(remotename) && !String.IsNullOrEmpty(handler))
{
throw new ArgumentException("Handler and remote name can not be specified simultaneously.");
}
var localDs = db.GetDataSources(DateTime.MaxValue).ToArray();
if (localDs.All(x => x.Name != name))
{
db.AddDataSource(name, description, copyright, resolvedHandlerAssemblyQualifiedName, ParserHelper.AppendDataSetUriWithDimensions(uri), remoteID, remotename);
}
else
{
throw new ArgumentException("Data source with given name already exists.");
}
SetMappings(name, addMapping, null);
}
/// <summary>Returns the ExtendedConfiguration for the time specifed or null if time is before first timestamp in configuration database</summary>
/// <param name="utcTime">UTC time to fetch the configuration for or DateTime.MaxValue for latest timestamp </param>
/// <returns>ExtendedConfiguration object or null</returns>
public ExtendedConfiguration GetConfiguration(DateTime utcTime)
{
using (FetchConfigurationDataClassesDataContext db = new FetchConfigurationDataClassesDataContext(connectionString))
{
if (utcTime == DateTime.MaxValue)
{
utcTime = (DateTime)db.GetLatestTimeStamp().First().TimeStamp;
}
else if (utcTime < db.GetFirstTimeStamp().First().TimeStamp)
{
return(null);
}
List <ExtendedDataSourceDefinition> dataSourcesList = new List <ExtendedDataSourceDefinition>();
Dictionary <string, VariableDefinition> supportedVars = new Dictionary <string, VariableDefinition>();
var dataSources = db.GetDataSources(utcTime);
var dsVariables = db.GetEnvVariables().ToArray();
foreach (var ds in dataSources)
{
var mapping = db.GetMapping(utcTime, ds.Name).ToArray();
var providedVars = mapping.Where(mp => mp.IsOutbound != null && (bool)mp.IsOutbound).Select(mp => mp.FetchVariableName).ToArray();
ExtendedDataSourceDefinition dsd = new ExtendedDataSourceDefinition((ushort)ds.ID, ds.Name, ds.Description, ds.Copyright, ds.Uri, ds.FullClrTypeName, providedVars,
(ds.RemoteName != null) ? ds.Uri : string.Empty,
mapping.ToDictionary(map => map.FetchVariableName, map => map.DataVariableName),
ds.RemoteName,
(ushort)(ds.RemoteID == null ? -1 : ds.RemoteID));
dataSourcesList.Add(dsd);
foreach (var envVar in providedVars)
{
if (!supportedVars.ContainsKey(envVar))
{
var v = dsVariables.Where(dsv => dsv.DisplayName == envVar).First();
supportedVars[envVar] = new VariableDefinition(v.DisplayName, v.Units, v.Description);
}
}
}
return(new ExtendedConfiguration(utcTime, dataSourcesList.ToArray(), supportedVars.Values.ToArray())
{
FetchEngineTypeName = db.GetFetchEngine(utcTime).First().FullClrTypeName
});
}
}
/// <summary>
/// Produces the configuration for the time specified
/// </summary>
/// <param name="utcTime">A time to produce the configuration for</param>
/// <returns></returns>
public IFetchConfiguration GetConfiguration(DateTime utcTime)
{
using (FetchConfigurationDataClassesDataContext db = new FetchConfigurationDataClassesDataContext(connectionString))
{
if (utcTime == DateTime.MaxValue)
{
utcTime = (DateTime)db.GetLatestTimeStamp().First().TimeStamp;
}
else if (utcTime < db.GetFirstTimeStamp().First().TimeStamp)
{
throw new ArgumentException("No configuration exists for given timestamp");
}
List <IDataSourceDefinition> dataSourcesList = new List <IDataSourceDefinition>();
Dictionary <string, IVariableDefinition> supportedVars = new Dictionary <string, IVariableDefinition>();
var dataSources = db.GetDataSources(utcTime);
var dsVariables = db.GetEnvVariables().ToArray();
foreach (var ds in dataSources)
{
var mappings = db.GetMapping(utcTime, ds.Name);
var providedVars = mappings.Where(mp => mp.IsOutbound != null && (bool)mp.IsOutbound).Select(mp => mp.FetchVariableName).ToArray();
if (providedVars.Length > 0) //otherwise there are no mappings for the data source
{
IDataSourceDefinition dsd = new DataSourceDefinition((ushort)ds.ID, ds.Name, ds.Description, ds.Copyright,
(ds.RemoteName != null) ? ds.Uri : string.Empty,
providedVars);
dataSourcesList.Add(dsd);
foreach (var evn in providedVars)
{
if (!supportedVars.ContainsKey(evn))
{
var v = dsVariables.Where(dsv => dsv.DisplayName == evn).First();
supportedVars[v.DisplayName] = new VariableDefinition(v.DisplayName, v.Units, v.Description);
}
}
}
}
return(new FetchConfiguration(utcTime, dataSourcesList.ToArray(), supportedVars.Values.ToArray()));
}
}
public void ConfigurationScenario()
{
//using (FetchConfigurationDataClassesDataContext db = new FetchConfigurationDataClassesDataContext(connectionString))
//{
//Mapping table uses DataSourceId & Timestamp pair like primary key.
//SQL server makes more than one insertion per 1 millisecond
//Sleep required between SetMapping operation
DateTime TIME_BEFORE_EVERYTHING = DateTime.UtcNow;
DateTime TIME_BEFORE_WC_DISABLED;
DateTime TIME_AFTER_WC_DISABLED;
Thread.Sleep(time);
db.TruncateTables();
db.AddVariable("airt", "Air temperature near surface", "Degrees C");
db.AddVariable("airt_land", "Air temperature near surface (land only area)", "Degrees C");
db.AddVariable("prate", "Precipitation rate", "mm/month");
db.AddVariable("relhum_land", "Relative humidity (land only area)", "percentage");
db.AddDataSource(
"NCEP/NCAR Reanalysis 1 (regular grid)",
"The NCEP/NCAR Reanalysis 1 project is using a state-of-the-art analysis/forecast system to perform data assimilation using past data from 1948 to the present",
"NCEP Reanalysis data provided by the NOAA/OAR/ESRL PSD, Boulder, Colorado, USA, from their Web site at http://www.esrl.noaa.gov/psd/",
"Microsoft.Research.Science.FetchClimate2.DataSources.NCEPReanalysisRegularGridDataSource, NCEPReanalysisDataSource",
"msds:az?name=ReanalysisRegular&DefaultEndpointsProtocol=http&AccountName=fetch&AccountKey=1Y0EOrnCX6ULY8c3iMHg9rrul2BWbPHKsHUceZ7SSh+ShM/q9K0ml49gQm+PE7G7i7zCvrpuT",
null, null);
db.SetMapping("NCEP/NCAR Reanalysis 1 (regular grid)", "airt", "air", true, true);
db.AddDataSource(
"NCEP/NCAR Reanalysis 1 Gauss T62 grid)",
"The NCEP/NCAR Reanalysis 1 project is using a state-of-the-art analysis/forecast system to perform data assimilation using past data from 1948 to the present",
"NCEP Reanalysis data provided by the NOAA/OAR/ESRL PSD, Boulder, Colorado, USA, from their Web site at http://www.esrl.noaa.gov/psd/", "Microsoft.Research.Science.FetchClimate2.DataSources.NCEPReanalysisGaussGridDataSource, NCEPReanalysisDataSource",
"msds:az?name=ReanalysisGaussT62&DefaultEndpointsProtocol=http&AccountName=fetch&AccountKey=1Y0EOrnCX6ULY8c3iMHg9rrul2BWbPHKsHUceZ7SSh+ShM/q9K0ml49gQm+PE7G7i7zCvrpuT",
null, null);
db.SetMapping("NCEP/NCAR Reanalysis 1 Gauss T62 grid)", "prate", "prate", true, true);
db.AddDataSource(
"WorldClim 1.4",
"A set of global climate layers (climate grids) with a spatial resolution of a square kilometer",
"The database is documented in this article: Hijmans, R.J., S.E. Cameron, J.L. Parra, P.G. Jones and A. Jarvis, 2005. Very high resolution interpolated climate surfaces for global land areas. International Journal of Climatology 25: 1965-1978.", "Microsoft.Research.Science.FetchClimate2.DataSources.CRUProcessor, NCEPReanalysisDataSource",
"msds:az?name=CRU_CL_2_0&DefaultEndpointsProtocol=http&AccountName=fetch&AccountKey=1Y0EOrnCX6ULY8c3iMHg9rrul2BWbPHKsHUceZ7SSh+ShM/q9K0ml49gQm+PE7G7i7zCvrpuT",
null, null);
db.SetMapping("WorldClim 1.4", "airt", "tmean", true, true);
db.SetMapping("WorldClim 1.4", "airt_land", "tmean_land", true, true);
db.SetMapping("WorldClim 1.4", "prate", "prec", true, true);
db.AddDataSource(
"CRU CL 2.0",
"High-resolution grid of the average climate in the recent past.",
"Produced by Climatic Research Unit (University of East Anglia). http://www.cru.uea.ac.uk",
"Microsoft.Research.Science.FetchClimate2.DataSources.CRUProcessor, NCEPReanalysisDataSource", "msds:az?name=CRU_CL_2_0&DefaultEndpointsProtocol=http&AccountName=fetch&AccountKey=1Y0EOrnCX6ULY8c3iMHg9rrul2BWbPHKsHUceZ7SSh+ShM/q9K0ml49gQm+PE7G7i7zCvrpuT",
null, null);
db.SetMapping("CRU CL 2.0", "airt", "tmp", true, true);
db.SetMapping("CRU CL 2.0", "airt_land", "tmp", true, true);
db.SetMapping("CRU CL 2.0", "prate", "pre", true, true);
db.SetMapping("CRU CL 2.0", "relhum_land", "reh_land", true, true);
Thread.Sleep(time);
TIME_BEFORE_WC_DISABLED = (DateTime)db.GetLatestTimeStamp().First().TimeStamp;
db.SetMapping("WorldClim 1.4", "airt", "tmean", true, false);
db.SetMapping("WorldClim 1.4", "airt_land", "tmean_land", true, false);
db.SetMapping("WorldClim 1.4", "prate", "prec", true, false);
Thread.Sleep(time);
TIME_AFTER_WC_DISABLED = (DateTime)db.GetLatestTimeStamp().First().TimeStamp;
//must contain nothing
var Sources = db.GetDataSources(TIME_BEFORE_EVERYTHING).ToArray();
Assert.AreEqual(0, Sources.Length);
String[] Expected = new String[]
{
"NCEP/NCAR Reanalysis 1 (regular grid)",
"NCEP/NCAR Reanalysis 1 Gauss T62 grid)",
"WorldClim 1.4",
"CRU CL 2.0"
};
Thread.Sleep(time);
//must contain datasources with mappings and without mappings
Sources = db.GetDataSources(TIME_BEFORE_WC_DISABLED).ToArray();
Assert.AreEqual(Expected.Length, Sources.Length);
foreach (var item in Sources)
{
Assert.IsTrue(Expected.Contains(item.Name));
}
Thread.Sleep(time);
//must contain datasources with mappings and without mappings
//the same as previous, as mappings don't affect the procedure
Sources = db.GetDataSources(TIME_AFTER_WC_DISABLED).ToArray();
Assert.AreEqual(Expected.Length, Sources.Length);
foreach (var item in Sources)
{
Assert.IsTrue(Expected.Contains(item.Name));
}
Expected = new String[]
{
"NCEP/NCAR Reanalysis 1 (regular grid)",
"WorldClim 1.4",
"CRU CL 2.0"
};
Thread.Sleep(time);
//mapping affects the following call
var SourcesForVariable = db.GetDataSourcesForVariable(TIME_BEFORE_WC_DISABLED, "airt").ToArray();
Assert.AreEqual(Expected.Length, SourcesForVariable.Length);
foreach (var item in SourcesForVariable)
{
Assert.IsTrue(Expected.Contains(item.Name));
}
Expected = new String[]
{
"NCEP/NCAR Reanalysis 1 (regular grid)",
"CRU CL 2.0"
};
Thread.Sleep(time);
//mapping affects the following call
SourcesForVariable = db.GetDataSourcesForVariable(TIME_AFTER_WC_DISABLED, "airt").ToArray();
Assert.AreEqual(Expected.Length, SourcesForVariable.Length);
foreach (var item in SourcesForVariable)
{
Assert.IsTrue(Expected.Contains(item.Name));
}
Thread.Sleep(time);
//must contain nothing
SourcesForVariable = db.GetDataSourcesForVariable(TIME_BEFORE_EVERYTHING, "relhum_land").ToArray();
Assert.AreEqual(0, SourcesForVariable.Length);
Expected = new String[]
{
"CRU CL 2.0"
};
Thread.Sleep(time);
SourcesForVariable = db.GetDataSourcesForVariable(TIME_AFTER_WC_DISABLED, "relhum_land").ToArray();
Assert.AreEqual(Expected.Length, SourcesForVariable.Length);
foreach (var item in SourcesForVariable)
{
Assert.IsTrue(Expected.Contains(item.Name));
}
Expected = new String[]
{
"CRU CL 2.0"
};
Thread.Sleep(time);
SourcesForVariable = db.GetDataSourcesForVariable(TIME_BEFORE_WC_DISABLED, "relhum_land").ToArray();
Assert.AreEqual(Expected.Length, SourcesForVariable.Length);
foreach (var item in SourcesForVariable)
{
Assert.IsTrue(Expected.Contains(item.Name));
}
Expected = new String[]
{
"airt",
"prate",
"airt_land",
"relhum_land"
};
Thread.Sleep(time);
var EnvVariables = db.GetEnvVariables(/*TIME_BEFORE_WC_DISABLED*/).ToArray();
Assert.AreEqual(Expected.Length, EnvVariables.Length);
foreach (var item in EnvVariables)
{
Assert.IsTrue(Expected.Contains(item.DisplayName));
}
//EnvVariables = db.GetEnvVariables(TIME_AFTER_WC_DISABLED).ToArray();
//Assert.AreEqual(Expected.Length, EnvVariables.Length);
//foreach (var item in EnvVariables)
//{
// //Debug.WriteLine(String.Format("{0} {1}", item.ID, item.Name));
// Assert.IsTrue(Expected.Contains(item.DisplayName));
//}
db.TruncateTables();
//}
}
