public static void AssemblyInit(TestContext context)
{
//overriding bug-containing default azure provider with the fixed one
Type t = typeof(DataSetFactory);
var dict = (IDictionary)t.InvokeMember("providersByName", BindingFlags.Static | BindingFlags.NonPublic | BindingFlags.GetField, null, null, null);
dict.Remove("az");
DataSetFactory.Register(typeof(Microsoft.Research.Science.Data.Azure.AzureDataSet));
Trace.WriteLine("Registered Dmitrov Providers");
Trace.WriteLine(DataSetFactory.RegisteredToString());
}
public void RegisterFactory()
{
Type provider = null;
try
{
provider = System.Type.GetType(Type);
}
catch (Exception exc)
{
Trace.WriteLineIf(DataSet.TraceDataSet.TraceError, "DataSet factory: unable to load factory type " + Type + ": " + exc.Message);
return;
}
if (provider == null)
{
Trace.WriteLineIf(DataSet.TraceDataSet.TraceError, "DataSet factory: type " + Type + " is not found.");
return;
}
try
{
if (!String.IsNullOrEmpty(Name))
{
DataSetFactory.Register(Name, provider);
}
else if (!String.IsNullOrEmpty(Extension))
{
DataSetFactory.RegisterExtension(Extension, provider);
}
else
{
DataSetFactory.Register(provider);
}
}
catch (Exception exc)
{
Trace.WriteLineIf(DataSet.TraceDataSet.TraceError, "DataSet factory " + this.ToString() + ": unable register factory: " + exc.Message);
}
}
public static void RtofsForeCastInsertProxy()
{
DataSetFactory.Register(typeof(NetCDFDataSet));
using (DataSet diag = DataSet.Open(_decodeList[0][6]))
{
using (DataSet prog = DataSet.Open(_decodeList[1][6]))
{
var UTCtemp = prog.Variables.ElementAt(10).GetData().GetValue(0);
var SSStemp = prog.Variables.ElementAt(1).GetData();
var SSTtemp = prog.Variables.ElementAt(2).GetData();
var CURRENT_UVtemp = prog.Variables.ElementAt(4).GetData();
var CURRENT_VVtemp = prog.Variables.ElementAt(3).GetData();
var ICETHICKNESStemp = diag.Variables.ElementAt(5).GetData();
var lonLen = 4500;
var latLen = 3298;
var utcTime = new DateTime(Convert.ToInt16(_decodeList[0][0]), Convert.ToInt16(_decodeList[0][1]), Convert.ToInt16(_decodeList[0][2]), Convert.ToInt16(_decodeList[0][3]), 0, 0);
int offset = 0;
var KEY = new int[14841000];
var SSS = new float[14841000];
var SST = new float[14841000];
var CURRENT_UV = new float[14841000];
var CURRENT_VV = new float[14841000];
var ICE_THICKNESS = new float[14841000];
for (int i = 0; i < latLen; i++)
{
for (int j = 0; j < lonLen; j++)
{
offset = i * lonLen + j;
SSS[offset] = (float)SSStemp.GetValue(0, i, j) > 1000 ? 9999 : (float)SSStemp.GetValue(0, i, j);
SST[offset] = (float)SSTtemp.GetValue(0, i, j) > 1000 ? 9999 : (float)SSTtemp.GetValue(0, i, j);
CURRENT_UV[offset] = (float)CURRENT_UVtemp.GetValue(0, 0, i, j) > 1000 ? 9999 : (float)CURRENT_UVtemp.GetValue(0, 0, i, j);
CURRENT_VV[offset] = (float)CURRENT_VVtemp.GetValue(0, 0, i, j) > 1000 ? 9999 : (float)CURRENT_VVtemp.GetValue(0, 0, i, j);
ICE_THICKNESS[offset] = (float)ICETHICKNESStemp.GetValue(0, i, j) > 1000 ? 9999 : (float)ICETHICKNESStemp.GetValue(0, i, j);
}
}
var rtofsproxy = new List <RTOFS_FORECAST>();
var rtofs = new List <RTOFS_FORECAST_PROXY>();
foreach (var item in rtofsPostion)
{
rtofs.Add(new RTOFS_FORECAST_PROXY
{
UTC = utcTime,
LAT = item.LAT,
LON = item.LON,
SSS = SSS[item.KEY],
SST = SST[item.KEY],
ICE_THICKNESS = ICE_THICKNESS[item.KEY],
CURRENT_UV = CURRENT_UV[item.KEY],
CURRENT_VV = CURRENT_VV[item.KEY]
});
rtofsproxy.Add(new RTOFS_FORECAST
{
UTC = utcTime,
LAT = item.LAT,
LON = item.LON,
SSS = SSS[item.KEY],
SST = SST[item.KEY],
ICE_THICKNESS = ICE_THICKNESS[item.KEY],
CURRENT_UV = CURRENT_UV[item.KEY],
CURRENT_VV = CURRENT_VV[item.KEY]
});
}
RtofsMakeJson(rtofsproxy);
_oceanModel.Database.CommandTimeout = 99999;
var rtofsSplit = Split(rtofs, 10000);
foreach (var item in rtofsSplit.Select((value, index) => new { value, index }))
{
EFBatchOperation.For(_oceanModel, _oceanModel.RTOFS_FORECAST_PROXY).InsertAll(item.value);
Thread.Sleep(10);
var progress = Math.Round((double)item.index / 0.259, 2);
_log.Info($"Rtofs Nowcast Db insert :{_decodeList[0][0]}-{_decodeList[0][1]}-{_decodeList[0][2]} {_decodeList[0][3]}:00 / {progress} %");
}
rtofsSplit = null;
rtofs = null;
_log.Info($"Rtofs Forecast Db insert Finish");
var conditionWeatherDb = _logModel.CONDITION_OCEAN_WEATHER.First();
if (conditionWeatherDb.DATE_LAST_OF_RTOFS_JSON < utcTime)
{
conditionWeatherDb.DATE_LAST_FILE_OF_RTOFS_FORECAST = _dateNextRtofsForecastFileToGet;
conditionWeatherDb.FILE_NAME_LAST_OF_RTOFS_FORECAST = _decodeList[0][4];
if (_decodeList[1][4] == "rtofs_glo_2ds_f192_3hrly_prog.nc" || _decodeList[1][4] == "rtofs_glo_2ds_f192_3hrly_diag.nc")
{
conditionWeatherDb.DB_OF_RTOFS_FORECAST = "BASIC";
conditionWeatherDb.DATE_LAST_OF_RTOFS_JSON = utcTime;
conditionWeatherDb.DATE_LAST_OF_RTOFS = utcTime;
}
}
_logModel.SaveChanges();
}
}
}
public void Initialize()
{
if (!DataSetFactory.ContainsProvider("memory"))
{
DataSetFactory.Register(typeof(MemoryDataSet));
}
if (!DataSetFactory.ContainsProvider("ab"))
{
DataSetFactory.Register(typeof(AzureBlobDataSet));
}
string customTempLocalResourcePath =
RoleEnvironment.GetLocalResource("localStorage1").RootPath;
Environment.SetEnvironmentVariable("TMP", customTempLocalResourcePath);
Environment.SetEnvironmentVariable("TEMP", customTempLocalResourcePath);
string jobsDatabaseConnectionString = RoleEnvironment.GetConfigurationSettingValue("FetchClimate.JobsDatabaseConnectionString");
string jobsStorageConnectionString = RoleEnvironment.GetConfigurationSettingValue("FetchClimate.JobsStorageConnectionString");
foreach (TraceListener item in Trace.Listeners)
{
if (!(item is DefaultTraceListener)) // The default trace listener is always in any TraceSource.Listeners collection.
{
AutoRegistratingTraceSource.RegisterTraceListener(item);
WorkerTrace.TraceEvent(TraceEventType.Information, 19, string.Format("TraceListener \"{0}\" registered for accepting data from all AutoRegistratingTraceSources", item.ToString()));
}
}
JobManager.InitializeJobTable(jobsDatabaseConnectionString, false);
configProvider = new SqlExtendedConfigurationProvider(
RoleEnvironment.GetConfigurationSettingValue("ConfigurationDatabaseConnectionString"));
WorkerTrace.TraceEvent(TraceEventType.Information, 6, string.Format("Connected to configuration database. Latest timestamp {0}",
configProvider.GetConfiguration(DateTime.MaxValue).TimeStamp));
azureGAC = new AssemblyStore(RoleEnvironment.GetConfigurationSettingValue("FetchWorker.AssemblyStoreConnectionString"));
//overriding bug-containing default azure provider with the fixed one
Type t = typeof(DataSetFactory);
var dict = (System.Collections.IDictionary)t.InvokeMember("providersByName", BindingFlags.Static | BindingFlags.NonPublic | BindingFlags.GetField, null, null, null);
dict.Remove("az");
DataSetFactory.Register(typeof(Microsoft.Research.Science.Data.Azure.AzureDataSet));
WorkerTrace.TraceEvent(TraceEventType.Verbose, 9, "Available Scientific DataSet providers:\n" + DataSetFactory.RegisteredToString());
if (int.TryParse(RoleEnvironment.GetConfigurationSettingValue("JobTouchPeriod"), out touchPeriodInSeconds))
{
WorkerTrace.TraceEvent(TraceEventType.Verbose, 10, string.Format("Touch period for processing job is set to {0}", touchPeriodInSeconds));
}
else
{
WorkerTrace.TraceEvent(TraceEventType.Warning, 11, string.Format("Failed to read touch period from config. Parsing of value failed. touch period is set to {0}", touchPeriodInSeconds));
}
manager = new JobManager(jobsDatabaseConnectionString, jobsStorageConnectionString);
//Scheduling cleanup
string instanceId = RoleEnvironment.CurrentRoleInstance.Id;
int.TryParse(instanceId.Substring(instanceId.LastIndexOf(".") + 1), out instanceIndex);
if (instanceIndex == 0)
{
double cleanPeriod = 0;
if (!double.TryParse(RoleEnvironment.GetConfigurationSettingValue("HoursBetweenCleanup"), out cleanPeriod))
{
cleanPeriod = 23;
WorkerTrace.TraceEvent(TraceEventType.Warning, 12, "Failed to parse period between clean-ups from configuration. Setting it to default 23 hours.");
}
cleanupTimeSpan = TimeSpan.FromHours(cleanPeriod);
lastCleanUpTime = manager.LastCleanUpTime;
DateTime now = DateTime.UtcNow;
if (now - lastCleanUpTime >= cleanupTimeSpan)
{
manager.SubmitCleanUp();
lastCleanUpTime = now;
}
}
WorkerTrace.TraceEvent(TraceEventType.Verbose, 13, string.Format("starting Allocated memory: {0}Mb", GC.GetTotalMemory(false) / 1024 / 1024));
}
// The Application_Start and Application_End methods are special methods that do not represent HttpApplication events.
// ASP.NET calls them once for the lifetime of the application domain, not for each HttpApplication instance.
// see "ASP.NET Application Life Cycle Overview for IIS 5.0 and 6.0" https://msdn.microsoft.com/en-us/library/ms178473.aspx
// The Global.asax file is used in Integrated mode in IIS 7.0 much as it is used in ASP.NET in IIS 6.0.
protected void Application_Start()
{
Trace.TraceInformation("Global.asax: starting the http application domain (Application_Start).");
// The System.Diagnostics.Trace object gets listeners from Web.config/configuration/system.diagnostics/trace/listeners section.
// The below code copies Trace listeners to all AutoRegistratingTraceSource trace sources.
foreach (TraceListener item in Trace.Listeners)
{
if (!(item is DefaultTraceListener)) // The default trace listener is always in any TraceSource.Listeners collection.
{
Microsoft.Research.Science.FetchClimate2.AutoRegistratingTraceSource.RegisterTraceListener(item);
Trace.TraceInformation("TraceListener \"{0}\" registered for accepting data from all AutoRegistratingTraceSources", item);
}
}
// Initializing Scientific DataSet registration
if (!DataSetFactory.ContainsProvider("memory"))
{
DataSetFactory.Register(typeof(MemoryDataSet));
}
if (!DataSetFactory.ContainsProvider("ab"))
{
DataSetFactory.Register(typeof(AzureBlobDataSet));
}
Trace.TraceInformation("Scientific DataSet providers: " + string.Join(" ", DataSetFactory.GetRegisteredProviders()));
// Initialize JobsManager if necessary
bool firstInstance = RoleEnvironment.CurrentRoleInstance.Id == RoleEnvironment.CurrentRoleInstance.Role.Instances[0].Id;
// The first Frontend role instance initializes the database if necessary.
Microsoft.Research.Science.FetchClimate2.JobManager.InitializeJobTable(FrontendSettings.Current.JobsDatabaseConnectionString, firstInstance);
// Initialize ASP.NET WebAPI
// enabled by Microsoft.AspNet.WebApi.Tracing package
if (FrontendSettings.Current.EnableAspnetDiagnosticTrace)
{
var tw = new System.Web.Http.Tracing.SystemDiagnosticsTraceWriter();
tw.MinimumLevel = System.Web.Http.Tracing.TraceLevel.Debug;
tw.IsVerbose = true;
GlobalConfiguration.Configuration.Services.Replace(typeof(System.Web.Http.Tracing.ITraceWriter), tw);
}
AreaRegistration.RegisterAllAreas();
WebApiConfig.Register(GlobalConfiguration.Configuration);
FilterConfig.RegisterGlobalFilters(GlobalFilters.Filters);
RouteConfig.RegisterRoutes(RouteTable.Routes);
//BundleConfig.RegisterBundles(BundleTable.Bundles);
// enable CORS
GlobalConfiguration.Configuration.MessageHandlers.Add(new CorsMessageHandler());
// configure formatters
var formatters = GlobalConfiguration.Configuration.Formatters;
//formatters.Remove(formatters.XmlFormatter);
formatters.Add(new PlainTextFormatter());
var jsonFormatter = GlobalConfiguration.Configuration.Formatters.OfType <JsonMediaTypeFormatter>().FirstOrDefault();
if (jsonFormatter == null)
{
string msg = "No json media formatter is available due to some internal error";
Trace.TraceError(msg);
throw new Exception(msg);
}
jsonFormatter.SerializerSettings.Converters.Add(new JsonCompliantNumberConverter());
}
public static void ClassInitialize(TestContext dummy)
{
DataSetFactory.Register(typeof(Microsoft.Research.Science.Data.Azure.AzureDataSet));
}
public static void RtofsNowCastInsert()
{
DataSetFactory.Register(typeof(NetCDFDataSet));
using (DataSet diag = DataSet.Open(_decodeList[0][6]))
{
using (DataSet prog = DataSet.Open(_decodeList[1][6]))
{
var UTCtemp = prog.Variables.ElementAt(10).GetData().GetValue(0);
var SSStemp = prog.Variables.ElementAt(1).GetData();
var SSTtemp = prog.Variables.ElementAt(2).GetData();
var CURRENT_UVtemp = prog.Variables.ElementAt(4).GetData();
var CURRENT_VVtemp = prog.Variables.ElementAt(3).GetData();
var ICETHICKNESStemp = diag.Variables.ElementAt(5).GetData();
var lonLen = 4500;
var latLen = 3298;
var utcTime = new DateTime(Convert.ToInt16(_decodeList[0][0]), Convert.ToInt16(_decodeList[0][1]), Convert.ToInt16(_decodeList[0][2]), Convert.ToInt16(_decodeList[0][3]), 0, 0);
int offset = 0;
var KEY = new int[14841000];
var SSS = new float[14841000];
var SST = new float[14841000];
var CURRENT_UV = new float[14841000];
var CURRENT_VV = new float[14841000];
var ICE_THICKNESS = new float[14841000];
for (int i = 0; i < latLen; i++)
{
for (int j = 0; j < lonLen; j++)
{
offset = i * lonLen + j;
SSS[offset] = (float)SSStemp.GetValue(0, i, j) > 1000 ? 9999 : (float)SSStemp.GetValue(0, i, j);
SST[offset] = (float)SSTtemp.GetValue(0, i, j) > 1000 ? 9999 : (float)SSTtemp.GetValue(0, i, j);
CURRENT_UV[offset] = (float)CURRENT_UVtemp.GetValue(0, 0, i, j) > 1000 ? 9999 : (float)CURRENT_UVtemp.GetValue(0, 0, i, j);
CURRENT_VV[offset] = (float)CURRENT_VVtemp.GetValue(0, 0, i, j) > 1000 ? 9999 : (float)CURRENT_VVtemp.GetValue(0, 0, i, j);
ICE_THICKNESS[offset] = (float)ICETHICKNESStemp.GetValue(0, i, j) > 1000 ? 9999 : (float)ICETHICKNESStemp.GetValue(0, i, j);
}
}
var rtofs = new List <RTOFS>();
foreach (var item in rtofsPostion)
{
rtofs.Add(new RTOFS
{
UTC = utcTime,
LAT = item.LAT,
LON = item.LON,
SSS = SSS[item.KEY],
SST = SST[item.KEY],
ICE_THICKNESS = ICE_THICKNESS[item.KEY],
CURRENT_UV = CURRENT_UV[item.KEY],
CURRENT_VV = CURRENT_VV[item.KEY]
});
}
//Nww3MakeJson(wavedataout);
var rtofsDbInsertCheck = new List <RTOFS_DB_INSERT_CHECK>();
int resumeCount = 0;
int insertDataCount = 259200;
var checkDate = utcTime;
_oceanModel.Database.CommandTimeout = 99999;
var checkDB = _oceanModel.RTOFS.FirstOrDefault(s => s.UTC == checkDate);
if (checkDB != null)
{
resumeCount = _oceanModel.RTOFS.Count(s => s.UTC == checkDate);
if (resumeCount == insertDataCount)
{
rtofsDbInsertCheck.Add(new RTOFS_DB_INSERT_CHECK
{
FILE_NAME = _decodeList[1][4].ToLower(),
DATE_OF_FILE = checkDate,
DATE_INSERTED = DateTime.UtcNow
});
rtofsDbInsertCheck.ForEach(s => _oceanModel.RTOFS_DB_INSERT_CHECK.Add(s));
_oceanModel.SaveChanges();
return;
}
}
var rtofsSplit = Split(rtofs, 10000);
foreach (var item in rtofsSplit.Select((value, index) => new { value, index }))
{
if (item.index * 10000 >= resumeCount)
{
EFBatchOperation.For(_oceanModel, _oceanModel.RTOFS).InsertAll(item.value);
Thread.Sleep(500);
}
var progress = Math.Round((double)item.index / 0.259, 2);
_log.Info($"Rtofs Nowcast Db insert :{_decodeList[0][0]}-{_decodeList[0][1]}-{_decodeList[0][2]} {_decodeList[0][3]}:00 / {progress} %");
}
rtofsSplit = null;
rtofs = null;
rtofsDbInsertCheck.Add(new RTOFS_DB_INSERT_CHECK
{
FILE_NAME = _decodeList[1][4].ToLower(),
DATE_OF_FILE = checkDate,
DATE_INSERTED = DateTime.UtcNow
});
rtofsDbInsertCheck.ForEach(s => _oceanModel.RTOFS_DB_INSERT_CHECK.Add(s));
_oceanModel.SaveChanges();
_log.Info($"Wave Db insert Finish");
}
}
}
