public void DataSourceAdd(string name, string description, string handler = null, string remotename = null, string copyright = "", string uri = "", List <DataSourceMapping> addMapping = null, List <String> removeMapping = null) { ushort?remoteID = null; string resolvedHandlerAssemblyQualifiedName = null; //if (string.IsNullOrEmpty(remotename) && !String.IsNullOrEmpty(handler)) // ExtractHandlerAssemblyAndTypeName(handler, out toLoad, out handlerType); if (!string.IsNullOrEmpty(remotename) && String.IsNullOrEmpty(handler)) //federated { IFetchConfiguration remoteConfig; try { RemoteFetchClient client = new RemoteFetchClient(new Uri(uri)); remoteConfig = client.GetConfiguration(DateTime.MaxValue); } catch (Exception ex) { throw new ArgumentException("Failed to retrieve configuration of the remote service.\n Exception message: " + ex.Message); } if (!remoteConfig.DataSources.Any(ds => ds.Name == remotename)) { throw new ArgumentException("Data source with given name does not exist on the remote service."); } remoteID = remoteConfig.DataSources.Where(ds => ds.Name == remotename).FirstOrDefault().ID; } else if (string.IsNullOrEmpty(remotename) && !String.IsNullOrEmpty(handler))//local { AssemblyStore gac = null; if (isConnectedToCloud) { gac = astore; } resolvedHandlerAssemblyQualifiedName = ExtractHandlerAssemblyAndTypeName(handler, gac); } else if (!string.IsNullOrEmpty(remotename) && !String.IsNullOrEmpty(handler)) { throw new ArgumentException("Handler and remote name can not be specified simultaneously."); } var localDs = db.GetDataSources(DateTime.MaxValue).ToArray(); if (localDs.All(x => x.Name != name)) { db.AddDataSource(name, description, copyright, resolvedHandlerAssemblyQualifiedName, ParserHelper.AppendDataSetUriWithDimensions(uri), remoteID, remotename); } else { throw new ArgumentException("Data source with given name already exists."); } SetMappings(name, addMapping, null); }
/// <summary>Returns the ExtendedConfiguration for the time specifed or null if time is before first timestamp in configuration database</summary> /// <param name="utcTime">UTC time to fetch the configuration for or DateTime.MaxValue for latest timestamp </param> /// <returns>ExtendedConfiguration object or null</returns> public ExtendedConfiguration GetConfiguration(DateTime utcTime) { using (FetchConfigurationDataClassesDataContext db = new FetchConfigurationDataClassesDataContext(connectionString)) { if (utcTime == DateTime.MaxValue) { utcTime = (DateTime)db.GetLatestTimeStamp().First().TimeStamp; } else if (utcTime < db.GetFirstTimeStamp().First().TimeStamp) { return(null); } List <ExtendedDataSourceDefinition> dataSourcesList = new List <ExtendedDataSourceDefinition>(); Dictionary <string, VariableDefinition> supportedVars = new Dictionary <string, VariableDefinition>(); var dataSources = db.GetDataSources(utcTime); var dsVariables = db.GetEnvVariables().ToArray(); foreach (var ds in dataSources) { var mapping = db.GetMapping(utcTime, ds.Name).ToArray(); var providedVars = mapping.Where(mp => mp.IsOutbound != null && (bool)mp.IsOutbound).Select(mp => mp.FetchVariableName).ToArray(); ExtendedDataSourceDefinition dsd = new ExtendedDataSourceDefinition((ushort)ds.ID, ds.Name, ds.Description, ds.Copyright, ds.Uri, ds.FullClrTypeName, providedVars, (ds.RemoteName != null) ? ds.Uri : string.Empty, mapping.ToDictionary(map => map.FetchVariableName, map => map.DataVariableName), ds.RemoteName, (ushort)(ds.RemoteID == null ? -1 : ds.RemoteID)); dataSourcesList.Add(dsd); foreach (var envVar in providedVars) { if (!supportedVars.ContainsKey(envVar)) { var v = dsVariables.Where(dsv => dsv.DisplayName == envVar).First(); supportedVars[envVar] = new VariableDefinition(v.DisplayName, v.Units, v.Description); } } } return(new ExtendedConfiguration(utcTime, dataSourcesList.ToArray(), supportedVars.Values.ToArray()) { FetchEngineTypeName = db.GetFetchEngine(utcTime).First().FullClrTypeName }); } }
/// <summary> /// Produces the configuration for the time specified /// </summary> /// <param name="utcTime">A time to produce the configuration for</param> /// <returns></returns> public IFetchConfiguration GetConfiguration(DateTime utcTime) { using (FetchConfigurationDataClassesDataContext db = new FetchConfigurationDataClassesDataContext(connectionString)) { if (utcTime == DateTime.MaxValue) { utcTime = (DateTime)db.GetLatestTimeStamp().First().TimeStamp; } else if (utcTime < db.GetFirstTimeStamp().First().TimeStamp) { throw new ArgumentException("No configuration exists for given timestamp"); } List <IDataSourceDefinition> dataSourcesList = new List <IDataSourceDefinition>(); Dictionary <string, IVariableDefinition> supportedVars = new Dictionary <string, IVariableDefinition>(); var dataSources = db.GetDataSources(utcTime); var dsVariables = db.GetEnvVariables().ToArray(); foreach (var ds in dataSources) { var mappings = db.GetMapping(utcTime, ds.Name); var providedVars = mappings.Where(mp => mp.IsOutbound != null && (bool)mp.IsOutbound).Select(mp => mp.FetchVariableName).ToArray(); if (providedVars.Length > 0) //otherwise there are no mappings for the data source { IDataSourceDefinition dsd = new DataSourceDefinition((ushort)ds.ID, ds.Name, ds.Description, ds.Copyright, (ds.RemoteName != null) ? ds.Uri : string.Empty, providedVars); dataSourcesList.Add(dsd); foreach (var evn in providedVars) { if (!supportedVars.ContainsKey(evn)) { var v = dsVariables.Where(dsv => dsv.DisplayName == evn).First(); supportedVars[v.DisplayName] = new VariableDefinition(v.DisplayName, v.Units, v.Description); } } } } return(new FetchConfiguration(utcTime, dataSourcesList.ToArray(), supportedVars.Values.ToArray())); } }
public void ConfigurationScenario() { //using (FetchConfigurationDataClassesDataContext db = new FetchConfigurationDataClassesDataContext(connectionString)) //{ //Mapping table uses DataSourceId & Timestamp pair like primary key. //SQL server makes more than one insertion per 1 millisecond //Sleep required between SetMapping operation DateTime TIME_BEFORE_EVERYTHING = DateTime.UtcNow; DateTime TIME_BEFORE_WC_DISABLED; DateTime TIME_AFTER_WC_DISABLED; Thread.Sleep(time); db.TruncateTables(); db.AddVariable("airt", "Air temperature near surface", "Degrees C"); db.AddVariable("airt_land", "Air temperature near surface (land only area)", "Degrees C"); db.AddVariable("prate", "Precipitation rate", "mm/month"); db.AddVariable("relhum_land", "Relative humidity (land only area)", "percentage"); db.AddDataSource( "NCEP/NCAR Reanalysis 1 (regular grid)", "The NCEP/NCAR Reanalysis 1 project is using a state-of-the-art analysis/forecast system to perform data assimilation using past data from 1948 to the present", "NCEP Reanalysis data provided by the NOAA/OAR/ESRL PSD, Boulder, Colorado, USA, from their Web site at http://www.esrl.noaa.gov/psd/", "Microsoft.Research.Science.FetchClimate2.DataSources.NCEPReanalysisRegularGridDataSource, NCEPReanalysisDataSource", "msds:az?name=ReanalysisRegular&DefaultEndpointsProtocol=http&AccountName=fetch&AccountKey=1Y0EOrnCX6ULY8c3iMHg9rrul2BWbPHKsHUceZ7SSh+ShM/q9K0ml49gQm+PE7G7i7zCvrpuT", null, null); db.SetMapping("NCEP/NCAR Reanalysis 1 (regular grid)", "airt", "air", true, true); db.AddDataSource( "NCEP/NCAR Reanalysis 1 Gauss T62 grid)", "The NCEP/NCAR Reanalysis 1 project is using a state-of-the-art analysis/forecast system to perform data assimilation using past data from 1948 to the present", "NCEP Reanalysis data provided by the NOAA/OAR/ESRL PSD, Boulder, Colorado, USA, from their Web site at http://www.esrl.noaa.gov/psd/", "Microsoft.Research.Science.FetchClimate2.DataSources.NCEPReanalysisGaussGridDataSource, NCEPReanalysisDataSource", "msds:az?name=ReanalysisGaussT62&DefaultEndpointsProtocol=http&AccountName=fetch&AccountKey=1Y0EOrnCX6ULY8c3iMHg9rrul2BWbPHKsHUceZ7SSh+ShM/q9K0ml49gQm+PE7G7i7zCvrpuT", null, null); db.SetMapping("NCEP/NCAR Reanalysis 1 Gauss T62 grid)", "prate", "prate", true, true); db.AddDataSource( "WorldClim 1.4", "A set of global climate layers (climate grids) with a spatial resolution of a square kilometer", "The database is documented in this article: Hijmans, R.J., S.E. Cameron, J.L. Parra, P.G. Jones and A. Jarvis, 2005. Very high resolution interpolated climate surfaces for global land areas. International Journal of Climatology 25: 1965-1978.", "Microsoft.Research.Science.FetchClimate2.DataSources.CRUProcessor, NCEPReanalysisDataSource", "msds:az?name=CRU_CL_2_0&DefaultEndpointsProtocol=http&AccountName=fetch&AccountKey=1Y0EOrnCX6ULY8c3iMHg9rrul2BWbPHKsHUceZ7SSh+ShM/q9K0ml49gQm+PE7G7i7zCvrpuT", null, null); db.SetMapping("WorldClim 1.4", "airt", "tmean", true, true); db.SetMapping("WorldClim 1.4", "airt_land", "tmean_land", true, true); db.SetMapping("WorldClim 1.4", "prate", "prec", true, true); db.AddDataSource( "CRU CL 2.0", "High-resolution grid of the average climate in the recent past.", "Produced by Climatic Research Unit (University of East Anglia). http://www.cru.uea.ac.uk", "Microsoft.Research.Science.FetchClimate2.DataSources.CRUProcessor, NCEPReanalysisDataSource", "msds:az?name=CRU_CL_2_0&DefaultEndpointsProtocol=http&AccountName=fetch&AccountKey=1Y0EOrnCX6ULY8c3iMHg9rrul2BWbPHKsHUceZ7SSh+ShM/q9K0ml49gQm+PE7G7i7zCvrpuT", null, null); db.SetMapping("CRU CL 2.0", "airt", "tmp", true, true); db.SetMapping("CRU CL 2.0", "airt_land", "tmp", true, true); db.SetMapping("CRU CL 2.0", "prate", "pre", true, true); db.SetMapping("CRU CL 2.0", "relhum_land", "reh_land", true, true); Thread.Sleep(time); TIME_BEFORE_WC_DISABLED = (DateTime)db.GetLatestTimeStamp().First().TimeStamp; db.SetMapping("WorldClim 1.4", "airt", "tmean", true, false); db.SetMapping("WorldClim 1.4", "airt_land", "tmean_land", true, false); db.SetMapping("WorldClim 1.4", "prate", "prec", true, false); Thread.Sleep(time); TIME_AFTER_WC_DISABLED = (DateTime)db.GetLatestTimeStamp().First().TimeStamp; //must contain nothing var Sources = db.GetDataSources(TIME_BEFORE_EVERYTHING).ToArray(); Assert.AreEqual(0, Sources.Length); String[] Expected = new String[] { "NCEP/NCAR Reanalysis 1 (regular grid)", "NCEP/NCAR Reanalysis 1 Gauss T62 grid)", "WorldClim 1.4", "CRU CL 2.0" }; Thread.Sleep(time); //must contain datasources with mappings and without mappings Sources = db.GetDataSources(TIME_BEFORE_WC_DISABLED).ToArray(); Assert.AreEqual(Expected.Length, Sources.Length); foreach (var item in Sources) { Assert.IsTrue(Expected.Contains(item.Name)); } Thread.Sleep(time); //must contain datasources with mappings and without mappings //the same as previous, as mappings don't affect the procedure Sources = db.GetDataSources(TIME_AFTER_WC_DISABLED).ToArray(); Assert.AreEqual(Expected.Length, Sources.Length); foreach (var item in Sources) { Assert.IsTrue(Expected.Contains(item.Name)); } Expected = new String[] { "NCEP/NCAR Reanalysis 1 (regular grid)", "WorldClim 1.4", "CRU CL 2.0" }; Thread.Sleep(time); //mapping affects the following call var SourcesForVariable = db.GetDataSourcesForVariable(TIME_BEFORE_WC_DISABLED, "airt").ToArray(); Assert.AreEqual(Expected.Length, SourcesForVariable.Length); foreach (var item in SourcesForVariable) { Assert.IsTrue(Expected.Contains(item.Name)); } Expected = new String[] { "NCEP/NCAR Reanalysis 1 (regular grid)", "CRU CL 2.0" }; Thread.Sleep(time); //mapping affects the following call SourcesForVariable = db.GetDataSourcesForVariable(TIME_AFTER_WC_DISABLED, "airt").ToArray(); Assert.AreEqual(Expected.Length, SourcesForVariable.Length); foreach (var item in SourcesForVariable) { Assert.IsTrue(Expected.Contains(item.Name)); } Thread.Sleep(time); //must contain nothing SourcesForVariable = db.GetDataSourcesForVariable(TIME_BEFORE_EVERYTHING, "relhum_land").ToArray(); Assert.AreEqual(0, SourcesForVariable.Length); Expected = new String[] { "CRU CL 2.0" }; Thread.Sleep(time); SourcesForVariable = db.GetDataSourcesForVariable(TIME_AFTER_WC_DISABLED, "relhum_land").ToArray(); Assert.AreEqual(Expected.Length, SourcesForVariable.Length); foreach (var item in SourcesForVariable) { Assert.IsTrue(Expected.Contains(item.Name)); } Expected = new String[] { "CRU CL 2.0" }; Thread.Sleep(time); SourcesForVariable = db.GetDataSourcesForVariable(TIME_BEFORE_WC_DISABLED, "relhum_land").ToArray(); Assert.AreEqual(Expected.Length, SourcesForVariable.Length); foreach (var item in SourcesForVariable) { Assert.IsTrue(Expected.Contains(item.Name)); } Expected = new String[] { "airt", "prate", "airt_land", "relhum_land" }; Thread.Sleep(time); var EnvVariables = db.GetEnvVariables(/*TIME_BEFORE_WC_DISABLED*/).ToArray(); Assert.AreEqual(Expected.Length, EnvVariables.Length); foreach (var item in EnvVariables) { Assert.IsTrue(Expected.Contains(item.DisplayName)); } //EnvVariables = db.GetEnvVariables(TIME_AFTER_WC_DISABLED).ToArray(); //Assert.AreEqual(Expected.Length, EnvVariables.Length); //foreach (var item in EnvVariables) //{ // //Debug.WriteLine(String.Format("{0} {1}", item.ID, item.Name)); // Assert.IsTrue(Expected.Contains(item.DisplayName)); //} db.TruncateTables(); //} }