public ActionResult TestMerge2()
{
try
{
//Let try to re-create the file...
string appDataFolder = HttpContext.Server.MapPath("~/App_Data");
string timestamp = JassWeatherAPI.fileTimeStamp();
//tas_WRFG_example_2014_2_3_11_10_31_322.nc
string inputFile1 = appDataFolder + "/tas_WRFG_example_2014_2_3_11_10_31_322.nc";
string inputFile3 = appDataFolder + "/new_WRFG__" + timestamp + ".nc";
var dataset1 = DataSet.Open(inputFile1 + "?openMode=open");
var dataset3 = DataSet.Open(inputFile3 + "?openMode=create");
var schema1 = dataset1.GetSchema();
var schema3 = dataset3.GetSchema();
double[] yc = dataset1.GetData <double[]>("yc");
double[] xc = dataset1.GetData <double[]>("xc");
double[] time = dataset1.GetData <double[]>("time");
Single[, ,] temperature = dataset1.GetData <Single[, , ]>("tas");
dataset3.Add <double[]>("time", time, "time");
dataset3.Add <double[]>("xc", xc, "xc");
dataset3.Add <double[]>("yc", yc, "yc");
dataset3.Add <Single[, , ]>("temperature", temperature, "time", "yc", "xc");
//temperature tas (time,50)(yc,109)(xc,134)
var dataset_new = DataSet.Open(inputFile3 + "?openMode=open");
var schema_new = dataset_new.GetSchema();
double[] yc_new = dataset_new.GetData <double[]>("yc");
double[] xc_new = dataset_new.GetData <double[]>("xc");
double[] time_new = dataset_new.GetData <double[]>("time");
Single[, ,] temperature_new = dataset_new.GetData <Single[, , ]>("temperature");
ViewBag.sschema_new = JassWeatherAPI.schema2string(schema_new);
ViewBag.sschema1 = JassWeatherAPI.schema2string(schema1);
ViewBag.sschema3 = JassWeatherAPI.schema2string(schema3);
ViewBag.Message = "Test executed correctly:";
}
catch (Exception e)
{
ViewBag.Message = e.Message;;
}
return(View());
}
public ActionResult TestWriteDataSet2Azure()
{
string rootFolder = HttpContext.Server.MapPath("~/App_Data");
string timestamp = JassWeatherAPI.fileTimeStamp();
try
{
//This is a simple example that reads and writes netCDF files using SDS
//We know how to do this from a normal disk and we want to learn how to do this from azure
//1. Read an original netCDF file from Disk
string input_netCDF_File = rootFolder + "/ftp___ftp.cdc.noaa.gov_Datasets_NARR_pressure_air.201201.nc";
var inputDataSet = DataSet.Open(input_netCDF_File + "?openMode=open");
//read the dimensions that we know they have
Single[] y = inputDataSet.GetData <Single[]>("y");
Single[] x = inputDataSet.GetData <Single[]>("x");
Single[] level = inputDataSet.GetData <Single[]>("level");
//Read a portion of data to make this smaller, collapsing the time dimension to the first row
Int16[, ,] temperature = inputDataSet.GetData <Int16[, , ]>("air",
DataSet.ReduceDim(0), DataSet.FromToEnd(0), DataSet.FromToEnd(0), DataSet.FromToEnd(0));
//CREATING AND READING netCDF on Azure Blob Storage
//I First do it on Disk to warm up and who what I want to achieve.
//Create a new netCDF file on disk for now
string output_netCDF_File = rootFolder + "/testing_azureStorage" + timestamp + ".nc";
var outputDataSet = DataSet.Open(output_netCDF_File + "?openMode=create");
//add the data
outputDataSet.Add <Single[]>("level", level, "level");
outputDataSet.Add <Single[]>("y", y, "y");
outputDataSet.Add <Single[]>("x", x, "x");
outputDataSet.Add <Int16[, , ]>("temperature", temperature, "level", "y", "x");
// Read the created file to make sure if ok.
var outputDataSetNew = DataSet.Open(output_netCDF_File + "?openMode=open");
var schema_new = outputDataSetNew.GetSchema();
Single[] yc_new = outputDataSetNew.GetData <Single[]>("y");
Single[] xc_new = outputDataSetNew.GetData <Single[]>("x");
Single[] level_new = outputDataSetNew.GetData <Single[]>("level");
Int16[, ,] temperature_new = outputDataSetNew.GetData <Int16[, , ]>("temperature");
//Now, let try to do a similar things on azure blob
//Create a new netCDF file on disk for now
//
//Credentials for azure blob storage account
//<add key="StorageConnectionString"
//value="DefaultEndpointsProtocol=https;
//AccountName=envirolytic;
//AccountKey=WN55CUrwdaYDaMoG332M6x9ekXQu4P1uEoxEGsBBXGVr9mRsoKt4FXanp1M0QEgrIqke81Bp78FX7q2EMqqHSg==" />
//
//Database credentials
//"Data Source=tcp:wbio6ii80z.database.windows.net,1433;
//Initial Catalog=envirolytic;
//User ID=envirolytic;
//Password=iswhatyou2!" providerName="System.Data.SqlClient" />
//First we need to define URI
var azureDataSet_URI = new AzureDataSetUri();
azureDataSet_URI.OpenMode = ResourceOpenMode.Create;
azureDataSet_URI.AccountKey = "WN55CUrwdaYDaMoG332M6x9ekXQu4P1uEoxEGsBBXGVr9mRsoKt4FXanp1M0QEgrIqke81Bp78FX7q2EMqqHSg==";
azureDataSet_URI.AccountName = "envirolytic";
azureDataSet_URI.DataSetID = "123456789";
azureDataSet_URI.DataSetName = "testing" + timestamp;
azureDataSet_URI.DefaultEndpointsProtocol = EndpointProtocol.http;
// The next statement will fail with the following exception
//{"Error while getting dataSets Schemas:Failed to get configuration container from blob storage:The specified container does not exist."}
var dataset_from_Azure = AzureDataSet.Open(azureDataSet_URI);
ViewBag.sschema_new = JassWeatherAPI.schema2string(schema_new);
ViewBag.Message = "Test executed correctly:";
}
catch (Exception e)
{
ViewBag.Message = e.Message;;
}
return(View());
}
public ActionResult TestMerge5()
{
long StartingMemory;
DateTime StartingTime = DateTime.Now;
long AfterOpenMemory;
long AfterLoadMemory;
DateTime EndingTime = DateTime.Now;
TimeSpan TotalDelay;
try
{
//Let try to re-create the file...
GC.Collect();
StartingMemory = GC.GetTotalMemory(true);
string appDataFolder = HttpContext.Server.MapPath("~/App_Data");
string timestamp = JassWeatherAPI.fileTimeStamp();
//tas_WRFG_example_2014_2_3_11_10_31_322.nc
string inputFile1 = appDataFolder + "/ftp___ftp.cdc.noaa.gov_Datasets_NARR_pressure_air.201201.nc";
var dataset1 = DataSet.Open(inputFile1 + "?openMode=open");
var schema1 = dataset1.GetSchema();
Single[] y = dataset1.GetData <Single[]>("y");
Single[] x = dataset1.GetData <Single[]>("x");
double[] time = dataset1.GetData <double[]>("time");
Single[] level = dataset1.GetData <Single[]>("level");
string dayString;
int in_year = 2012;
int in_month = 1;
int in_day;
for (var df = 0; df < time.Length; df += 8)
{
in_day = df / 8;
dayString = "" + in_year + "_" + in_month + "_" + in_day;
string inputFile3 = appDataFolder + "/envirolitic_air_2012_1_" + in_day + ".nc";
var dataset3 = DataSet.Open(inputFile3 + "?openMode=create");
AfterOpenMemory = GC.GetTotalMemory(true);
var schema3 = dataset3.GetSchema();
// Int16[,,,] temperature = dataset1.GetData<Int16[,,,]>("air");
Int16[, , ,] temperature = dataset1.GetData <Int16[, , , ]>("air",
DataSet.Range(0, 1, 7), /* removing first dimension from data*/
DataSet.FromToEnd(0), /* removing first dimension from data*/
DataSet.FromToEnd(0),
DataSet.FromToEnd(0));
short tempSample = (short)temperature[0, 0, 0, 0];
AfterLoadMemory = GC.GetTotalMemory(true);
double[] timeday = new double[8];
for (var t = 0; t < 8; t++)
{
timeday[t] = time[t];
}
dataset3.Add <double[]>("time", timeday, "time");
dataset3.Add <Single[]>("level", level, "level");
dataset3.Add <Single[]>("y", y, "y");
dataset3.Add <Single[]>("x", x, "x");
dataset3.Add <Int16[, , , ]>("temperature", temperature, "time", "level", "y", "x");
//temperature tas (time,50)(yc,109)(xc,134)
var dataset_new = DataSet.Open(inputFile3 + "?openMode=open");
var schema_new = dataset_new.GetSchema();
Single[] yc_new = dataset_new.GetData <Single[]>("y");
Single[] xc_new = dataset_new.GetData <Single[]>("x");
double[] time_new = dataset_new.GetData <double[]>("time");
Int16[, , ,] temperature_new = dataset_new.GetData <Int16[, , , ]>("temperature");
ViewBag.sschema_new = JassWeatherAPI.schema2string(schema_new);
ViewBag.sschema1 = JassWeatherAPI.schema2string(schema1);
ViewBag.sschema3 = JassWeatherAPI.schema2string(schema3);
ViewBag.AfterOpenMemory = AfterOpenMemory / 1000000;
ViewBag.AfterLoadMemory = AfterLoadMemory / 1000000;
ViewBag.AfterLoadDiffMemory = AfterLoadMemory - AfterOpenMemory;
ViewBag.temperature_new = temperature_new;
}
EndingTime = DateTime.Now;
TotalDelay = EndingTime - StartingTime;
ViewBag.StartingMemory = StartingMemory / 1000000;
ViewBag.TotalDelay = TotalDelay;
ViewBag.Message = "Test executed correctly:";
}
catch (Exception e)
{
ViewBag.Message = e.Message;;
}
return(View());
}
public ActionResult TestMerge3()
{
long StartingMemory;
DateTime StartingTime = DateTime.Now;
long AfterOpenMemory;
long AfterLoadMemory;
DateTime EndingTime = DateTime.Now;
TimeSpan TotalDelay;
try
{
//Let try to re-create the file...
GC.Collect();
StartingMemory = GC.GetTotalMemory(true);
string appDataFolder = HttpContext.Server.MapPath("~/App_Data");
string timestamp = JassWeatherAPI.fileTimeStamp();
//tas_WRFG_example_2014_2_3_11_10_31_322.nc
string inputFile1 = appDataFolder + "/ftp___ftp.cdc.noaa.gov_Datasets_NARR_pressure_air.201201.nc";
string inputFile3 = appDataFolder + "/new_NARR_air__" + timestamp + ".nc";
var dataset1 = DataSet.Open(inputFile1 + "?openMode=open");
var dataset3 = DataSet.Open(inputFile3 + "?openMode=create");
AfterOpenMemory = GC.GetTotalMemory(true);
var schema1 = dataset1.GetSchema();
var schema3 = dataset3.GetSchema();
Single[] y = dataset1.GetData <Single[]>("y");
Single[] x = dataset1.GetData <Single[]>("x");
double[] time = dataset1.GetData <double[]>("time");
Single[] level = dataset1.GetData <Single[]>("level");
// Int16[,,,] temperature = dataset1.GetData<Int16[,,,]>("air");
Int16[, ,] temperature = dataset1.GetData <Int16[, , ]>("air",
DataSet.FromToEnd(0), /* removing first dimension from data*/
DataSet.ReduceDim(0), /* removing first dimension from data*/
DataSet.FromToEnd(0),
DataSet.FromToEnd(0));
AfterLoadMemory = GC.GetTotalMemory(true);
dataset3.Add <double[]>("time", time, "time");
dataset3.Add <Single[]>("level", level, "level");
dataset3.Add <Single[]>("y", y, "y");
dataset3.Add <Single[]>("x", x, "x");
dataset3.Add <Int16[, , ]>("temperature", temperature, "time", "y", "x");
//temperature tas (time,50)(yc,109)(xc,134)
var dataset_new = DataSet.Open(inputFile3 + "?openMode=open");
var schema_new = dataset_new.GetSchema();
Single[] yc_new = dataset_new.GetData <Single[]>("y");
Single[] xc_new = dataset_new.GetData <Single[]>("x");
double[] time_new = dataset_new.GetData <double[]>("time");
Int16[, ,] temperature_new = dataset_new.GetData <Int16[, , ]>("temperature");
var size = sizeof(Int16);
var c = size;
EndingTime = DateTime.Now;
TotalDelay = EndingTime - StartingTime;
ViewBag.sschema_new = JassWeatherAPI.schema2string(schema_new);
ViewBag.sschema1 = JassWeatherAPI.schema2string(schema1);
ViewBag.sschema3 = JassWeatherAPI.schema2string(schema3);
ViewBag.StartingMemory = StartingMemory / 1000000;
ViewBag.AfterOpenMemory = AfterOpenMemory / 1000000;
ViewBag.AfterLoadMemory = AfterLoadMemory / 1000000;
ViewBag.AfterLoadDiffMemory = AfterLoadMemory - AfterOpenMemory;
EndingTime = DateTime.Now;
TotalDelay = EndingTime - StartingTime;
ViewBag.TotalDelay = TotalDelay;
ViewBag.Message = "Test executed correctly:";
}
catch (Exception e)
{
ViewBag.Message = e.Message;;
}
return(View());
}
public ActionResult TestMerge1()
{
try
{
string appDataFolder = HttpContext.Server.MapPath("~/App_Data");
string timestamp = JassWeatherAPI.fileTimeStamp();
string inputFile1 = appDataFolder + "/one_dimensional_measure_sample_1.csv";
string inputFile3 = appDataFolder + "/merged_file_3_" + timestamp + ".csv";
System.IO.File.Copy(inputFile1, inputFile3);
var dataset1 = DataSet.Open(inputFile1 + "?appendMetadata=true");
var dataset3 = DataSet.Open(inputFile3 + "?appendMetadata=true");
string inputFile2 = appDataFolder + "/one_dimensional_measure_sample_2.csv";
var dataset2 = DataSet.Open(inputFile2 + "?appendMetadata=true");
var schema2 = dataset2.GetSchema();
var schema1 = dataset1.GetSchema();
var x = dataset3.GetData <double[]>("X");
var temp = dataset3.GetData <double[]>("Temp");
var humidity = dataset2.GetData <double[]>("Humidity");
var humidVarID = dataset3.Add <double[]>("Humid", dataset3.Dimensions[0].Name).ID;
dataset3.PutData <double[]>(humidVarID, humidity);
var dim = dataset3.Dimensions;
var x3ID = dataset3["X"].ID;
var temp3ID = dataset3["Temp"].ID;
var humid3ID = dataset3["Humid"].ID;
var x2ID = dataset2["X"].ID;
var humid2ID = dataset2["Humidity"].ID;
var x1ID = dataset1["X"].ID;
var temp1ID = dataset1["Temp"].ID;
dataset3.View();
var schema3 = dataset3.GetSchema();
ViewBag.sschema1 = JassWeatherAPI.schema2string(schema1);
ViewBag.sschema2 = JassWeatherAPI.schema2string(schema2);
ViewBag.sschema3 = JassWeatherAPI.schema2string(schema3);
ViewBag.x3 = dataset3.GetData <double[]>("X");
ViewBag.temp3 = dataset3.GetData <double[]>("Temp");
ViewBag.humid3 = dataset3.GetData <double[]>("Humid");
ViewBag.Message = "Test executed correctly:";
}
catch (Exception e)
{
ViewBag.Message = e.Message;;
}
return(View());
}
public ActionResult GetMeasureValue(int id)
{
JassMeasure jassmeasure = db.JassMeasures.Find(id);
int in_x = jassmeasure.x;
int in_y = jassmeasure.y;
int in_year = jassmeasure.year;
int in_month = jassmeasure.month;
int in_day = jassmeasure.day;
int in_hour3 = jassmeasure.hour3;
int in_level = jassmeasure.level;
string dayString = "" + in_year + "_" + in_month + "_" + in_day;
long StartingMemory;
DateTime StartingTime = DateTime.Now;
long AfterOpenMemory;
long AfterLoadMemory;
DateTime EndingTime = DateTime.Now;
TimeSpan TotalDelay;
try
{
//Let try to re-create the file...
GC.Collect();
StartingMemory = GC.GetTotalMemory(true);
string appDataFolder = HttpContext.Server.MapPath("~/App_Data");
string timestamp = JassWeatherAPI.fileTimeStamp();
//tas_WRFG_example_2014_2_3_11_10_31_322.nc
List <string> files = apiCaller.listFiles_in_AppData();
string inputFile3 = appDataFolder + "/envirolitic_air_" + dayString + ".nc";
var dataset3 = Microsoft.Research.Science.Data.DataSet.Open(inputFile3 + "?openMode=open");
AfterOpenMemory = GC.GetTotalMemory(true);
short[] temperature = dataset3.GetData <short[]>("temperature",
Microsoft.Research.Science.Data.DataSet.FromToEnd(0), //time
Microsoft.Research.Science.Data.DataSet.ReduceDim(in_level), //level
Microsoft.Research.Science.Data.DataSet.ReduceDim(in_y), //y
Microsoft.Research.Science.Data.DataSet.ReduceDim(in_x)); //x
ViewBag.temperature = temperature;
AfterLoadMemory = GC.GetTotalMemory(true);
//To compare
string inputFile1 = appDataFolder + "/ftp___ftp.cdc.noaa.gov_Datasets_NARR_pressure_air.201201.nc";
var dataset1 = Microsoft.Research.Science.Data.DataSet.Open(inputFile1 + "?openMode=open");
var schema1 = dataset1.GetSchema();
Single[] y = dataset1.GetData <Single[]>("y");
Single[] x = dataset1.GetData <Single[]>("x");
double[] time = dataset1.GetData <double[]>("time");
Single[] level = dataset1.GetData <Single[]>("level");
short[] air = dataset1.GetData <short[]>("air",
Microsoft.Research.Science.Data.DataSet.Range(0, 1, 7),
Microsoft.Research.Science.Data.DataSet.ReduceDim(in_level),
Microsoft.Research.Science.Data.DataSet.ReduceDim(in_y),
Microsoft.Research.Science.Data.DataSet.ReduceDim(in_x));
Int16[,,,] airSample = dataset1.GetData <Int16[, , , ]>("air",
Microsoft.Research.Science.Data.DataSet.Range(0, 1, 7),
Microsoft.Research.Science.Data.DataSet.Range(0, 1, 7),
Microsoft.Research.Science.Data.DataSet.Range(0, 1, 7),
Microsoft.Research.Science.Data.DataSet.Range(0, 1, 7));
ViewBag.air = air;
ViewBag.airSample = airSample;
EndingTime = DateTime.Now;
TotalDelay = EndingTime - StartingTime;
ViewBag.StartingMemory = StartingMemory / 1000000;
ViewBag.TotalDelay = TotalDelay;
ViewBag.AfterOpenMemory = AfterOpenMemory / 1000000;
ViewBag.AfterLoadMemory = AfterLoadMemory / 1000000;
ViewBag.AfterLoadDiffMemory = (AfterLoadMemory - AfterOpenMemory) / 1000000;
ViewBag.Message = "Test executed correctly:";
}
catch (Exception e)
{
ViewBag.Message = e.Message;;
}
return(View());
}