public GribImporter(string inputFile) { _inputFile = inputFile; CorrectFilePath(ref _inputFile); bool ok = false; while (!ok) { try { GribEnvironment.Init(); ok = true; } catch (Exception ex) { Console.WriteLine(ex.ToString()); } } using (var gf = new Grib.Api.GribFile(_inputFile)) _messages = gf.ToArray(); }
public static void GfsNowCastInsert() { _log.Info($"Start Insert"); List <GFS_DB_INSERT_CHECK> gfsDbIndertCheck = new List <GFS_DB_INSERT_CHECK>(); DateTime UTC; double[] LAT; double[] LON; double[] temp_surfaceValue; double[] temp_above_groundValue; double[] press_surfaceValue; double[] press_mslValue; GribEnvironment.Init(); List <GribMessage> gfs = new List <GribMessage>(); _log.Info($"Start Insert : {_decodeList[6]}"); using (GribFile file = new GribFile(_decodeList[6])) { file.Context.EnableMultipleFieldMessages = true; var temp_surface = file.Where(d => d.ParameterName == "Temperature" && d.TypeOfLevel == "surface" && d.Level == 0).Single(); if (_oceanModel.GFS_DB_INSERT_CHECK.Where(s => s.DATE_OF_FILE == temp_surface.Time).Count() > 0) { _log.Info("TRIGGER/*/" + "Gfs DB에 " + temp_surface.Time.ToString() + "데이터가 이미 있습니다."); return; } var temp_above_ground = file.Where(d => d.ParameterName == "Temperature" && d.TypeOfLevel == "heightAboveGround" && d.Level == 2).Single(); var press_surface = file.Where(d => d.ParameterName == "Pressure" && d.TypeOfLevel == "surface" && d.Level == 0).Single(); var press_msl = file.Where(d => d.ParameterName == "Pressure reduced to MSL" && d.TypeOfLevel == "meanSea" && d.Level == 0).Single(); //////////////////////////////////////////////////////////////////////// //////////////////DB INSERT //////////////////////////////////////////////////////////////////////// UTC = temp_surface.Time; LAT = temp_surface.GridCoordinateValues.Select(d => d.Latitude).ToArray(); LON = temp_surface.GridCoordinateValues.Select(d => d.Longitude).ToArray(); temp_surface.Values(out temp_surfaceValue); temp_above_ground.Values(out temp_above_groundValue); press_surface.Values(out press_surfaceValue); press_msl.Values(out press_mslValue); } List <GFS> gfsList = new List <GFS>(); for (int j = 0; j < LAT.Length; j++) { //////////////////////////////////////////////////////////////////////// //////////////////예보 데이터 //////////////////////////////////////////////////////////////////////// gfsList.Add(new GFS { UTC = UTC, LAT = Convert.ToDecimal(LAT[j]), LON = Convert.ToDecimal(LON[j]), TEMP_SURFACE = Convert.ToSingle(temp_surfaceValue[j] - 273.15f), TEMP_ABOVE_GROUND = Convert.ToSingle(temp_above_groundValue[j] - 273.15f), PRESSURE_SURFACE = Convert.ToSingle(press_surfaceValue[j]), PRESSURE_MSL = Convert.ToSingle(press_mslValue[j]), }); } //////////////////////////////////////////////////////////////////////// //////////////////DB INSERT //////////////////////////////////////////////////////////////////////// int resumeCount = 0; int insertDataCount = 259920; var checkDate = UTC; _oceanModel.Database.CommandTimeout = 99999; GFS checkDB = _oceanModel.GFS.FirstOrDefault(s => s.UTC == checkDate); if (checkDB != null) { var resumeCount1 = _oceanModel.GFS.Count(s => s.UTC == checkDate); resumeCount = resumeCount1; if (resumeCount == insertDataCount) { gfsDbIndertCheck.Add(new GFS_DB_INSERT_CHECK { FILE_NAME = _decodeList[4].ToLower(), DATE_OF_FILE = UTC, DATE_INSERTED = DateTime.UtcNow }); gfsDbIndertCheck.ForEach(s => _oceanModel.GFS_DB_INSERT_CHECK.Add(s)); _oceanModel.SaveChanges(); return; } _log.Info("TRIGGER/*/" + "Gfs 데이터를 " + resumeCount.ToString() + "개부터 이어 받습니다."); } var gfsSplit = Split(gfsList, 10000); int index = 0; foreach (var item in gfsSplit) { if (index * 10000 >= resumeCount) { EFBatchOperation.For(_oceanModel, _oceanModel.GFS).InsertAll(item); Thread.Sleep(500); } var progress = Math.Round((double)index / 0.26, 2); _log.Info($"Gfs Nowcast Db insert :{_decodeList[0]}-{_decodeList[1]}-{_decodeList[2]} {_decodeList[3]}:00 / {progress} %"); index++; } gfsSplit = null; gfsList = null; gfsDbIndertCheck.Add(new GFS_DB_INSERT_CHECK { FILE_NAME = _decodeList[4].ToLower(), DATE_OF_FILE = UTC, DATE_INSERTED = DateTime.UtcNow }); gfsDbIndertCheck.ForEach(s => _oceanModel.GFS_DB_INSERT_CHECK.Add(s)); _oceanModel.SaveChanges(); _log.Info($"Gfs Db insert Finish"); }
public static void Nww3ForcastInsertProxy() { _log.Info($"forcast Strars"); int elementCount = 19; // 한 셋트 데이터 종류 (변경하지 말것) int needDecordDateSetCount = 81; // 3시간 데이터 2개를 저장. int elementTotalCount = elementCount * needDecordDateSetCount; int dataSetGap = 1; //1는 3시간, 2는 6시간마다 forecast 데이터를 밀어넣음. _oceanModel.Database.ExecuteSqlCommand("TRUNCATE TABLE [" + "NWW3_FORECAST_PROXY" + "]"); _oceanModel.SaveChanges(); var UTC = new DateTime(); var LAT = new double[231120]; var LON = new double[231120]; var HTSGW = new double[231120]; var MWSDIR = new double[231120]; var MWSPER = new double[231120]; var SWELL = new double[231120]; var SWDIR = new double[231120]; var SWPER = new double[231120]; var WVHGT = new double[231120]; var WVDIR = new double[231120]; var WVPER = new double[231120]; var UGRD = new double[231120]; var VGRD = new double[231120]; GribEnvironment.Init(); List <GribMessage> Nww3 = new List <GribMessage>(); using (GribFile gribFile = new GribFile(_decodeList[6])) { gribFile.Context.EnableMultipleFieldMessages = true; elementTotalCount = elementCount * 81; foreach (GribMessage item in gribFile) { Nww3.Add(item); } } List <NWW3_FORECAST_PROXY> waveForecastDataout = new List <NWW3_FORECAST_PROXY> { }; int index = 0; for (int i = 0; i < elementTotalCount; i += elementCount * dataSetGap) { UTC = Nww3.ElementAt(0 + i).Time; LAT = Nww3.ElementAt(0 + i).GridCoordinateValues.Select(d => d.Latitude).ToArray(); LON = Nww3.ElementAt(0 + i).GridCoordinateValues.Select(d => d.Longitude).ToArray(); Nww3.ElementAt(5 + i).Values(out HTSGW); Nww3.ElementAt(8 + i).Values(out MWSDIR); Nww3.ElementAt(6 + i).Values(out MWSPER); Nww3.ElementAt(11 + i).Values(out SWELL); Nww3.ElementAt(17 + i).Values(out SWDIR); Nww3.ElementAt(14 + i).Values(out SWPER); Nww3.ElementAt(10 + i).Values(out WVHGT); Nww3.ElementAt(16 + i).Values(out WVDIR); Nww3.ElementAt(13 + i).Values(out WVPER); Nww3.ElementAt(2 + i).Values(out UGRD); Nww3.ElementAt(3 + i).Values(out VGRD); var nww3Json = new List <NWW3>(); for (int j = 0; j < LAT.Length; j++) { waveForecastDataout.Add(new NWW3_FORECAST_PROXY { UTC = UTC, LAT = Convert.ToDecimal(LAT[j]), LON = Convert.ToDecimal(LON[j]), HTSGW = Convert.ToSingle(HTSGW[j]), MWSDIR = Convert.ToSingle(MWSDIR[j]), MWSPER = Convert.ToSingle(MWSPER[j]), SWELL = Convert.ToSingle(SWELL[j]), SWDIR = Convert.ToSingle(SWDIR[j]), SWPER = Convert.ToSingle(SWPER[j]), WVHGT = Convert.ToSingle(WVHGT[j]), WVDIR = Convert.ToSingle(WVDIR[j]), WVPER = Convert.ToSingle(WVPER[j]), UGRD = Convert.ToSingle(UGRD[j]), VGRD = Convert.ToSingle(VGRD[j]) }); nww3Json.Add(new NWW3 { UTC = UTC, LAT = Convert.ToDecimal(LAT[j]), LON = Convert.ToDecimal(LON[j]), HTSGW = Convert.ToSingle(HTSGW[j]), MWSDIR = Convert.ToSingle(MWSDIR[j]), MWSPER = Convert.ToSingle(MWSPER[j]), SWELL = Convert.ToSingle(SWELL[j]), SWDIR = Convert.ToSingle(SWDIR[j]), SWPER = Convert.ToSingle(SWPER[j]), WVHGT = Convert.ToSingle(WVHGT[j]), WVDIR = Convert.ToSingle(WVDIR[j]), WVPER = Convert.ToSingle(WVPER[j]), UGRD = Convert.ToSingle(UGRD[j]), VGRD = Convert.ToSingle(VGRD[j]) }); } //nowcast json은 이미 만들어서 건너뜀. if (i > 0) { Nww3MakeJson(nww3Json); } var waveForcastSplit = Split(waveForecastDataout, 10000); var length = waveForcastSplit.Count; foreach (var item in waveForcastSplit) { EFBatchOperation.For(_oceanModel, _oceanModel.NWW3_FORECAST_PROXY).InsertAll(item); Thread.Sleep(500); var progress = Math.Round((double)index / 0.23 / (81 / dataSetGap), 2); _log.Info($"Wave Forcast Db insert :{item[0].UTC.ToString("yyyy-MM-dd HH:mm:00")} / {progress} %"); index++; } waveForcastSplit.Clear(); waveForecastDataout.Clear(); } var conditionWeatherDb = _logModel.CONDITION_OCEAN_WEATHER.First(); conditionWeatherDb.DATE_LAST_OF_NWW3 = UTC; conditionWeatherDb.DATE_LAST_OF_NWW3_JSON = UTC; conditionWeatherDb.DB_OF_NWW3_FORECAST = "PROXY"; _logModel.SaveChanges(); }
public static void Nww3NowCastInsert() { _log.Info($"Start Insert"); int elementCount = 19; // 한 셋트 데이터 종류 (변경하지 말것) int needDecordDateSetCount = 2; // 3시간 데이터 2개를 저장. int elementTotalCount = elementCount * needDecordDateSetCount; var UTC = new DateTime[2]; var LAT = new double[2][]; var LON = new double[2][]; var HTSGW = new double[2][]; var MWSDIR = new double[2][]; var MWSPER = new double[2][]; var SWELL = new double[2][]; var SWDIR = new double[2][]; var SWPER = new double[2][]; var WVHGT = new double[2][]; var WVDIR = new double[2][]; var WVPER = new double[2][]; var UGRD = new double[2][]; var VGRD = new double[2][]; GribEnvironment.Init(); List <GribMessage> Nww3 = new List <GribMessage>(); _log.Info($"Start Insert : {_decodeList[6]}"); using (GribFile gribFile = new GribFile(_decodeList[6])) { gribFile.Context.EnableMultipleFieldMessages = true; foreach (GribMessage item in gribFile) { if (item.StepRange == "6") { break; } Nww3.Add(item); } } List <NWW3> wavedataout = new List <NWW3> { }; List <NWW3_DB_INSERT_CHECK> nww3DbInsertCheck = new List <NWW3_DB_INSERT_CHECK>(); int num = 0; for (int i = 0; i < elementTotalCount; i += elementCount) { UTC[num] = Nww3.ElementAt(0 + i).Time; LAT[num] = Nww3.ElementAt(0 + i).GridCoordinateValues.Select(d => d.Latitude).ToArray(); LON[num] = Nww3.ElementAt(0 + i).GridCoordinateValues.Select(d => d.Longitude).ToArray(); Nww3.ElementAt(5 + i).Values(out HTSGW[num]); Nww3.ElementAt(8 + i).Values(out MWSDIR[num]); Nww3.ElementAt(6 + i).Values(out MWSPER[num]); Nww3.ElementAt(11 + i).Values(out SWELL[num]); Nww3.ElementAt(17 + i).Values(out SWDIR[num]); Nww3.ElementAt(14 + i).Values(out SWPER[num]); Nww3.ElementAt(10 + i).Values(out WVHGT[num]); Nww3.ElementAt(16 + i).Values(out WVDIR[num]); Nww3.ElementAt(13 + i).Values(out WVPER[num]); Nww3.ElementAt(2 + i).Values(out UGRD[num]); Nww3.ElementAt(3 + i).Values(out VGRD[num]); num++; } for (int i = 0; i < num; i++) { List <NWW3> nww3Json = new List <NWW3>(); for (int j = 0; j < LAT[i].Length; j++) { wavedataout.Add(new NWW3 { UTC = UTC[i], LAT = Convert.ToDecimal(LAT[i][j]), LON = Convert.ToDecimal(LON[i][j]), HTSGW = Convert.ToSingle(HTSGW[i][j]), MWSDIR = Convert.ToSingle(MWSDIR[i][j]), MWSPER = Convert.ToSingle(MWSPER[i][j]), SWELL = Convert.ToSingle(SWELL[i][j]), SWDIR = Convert.ToSingle(SWDIR[i][j]), SWPER = Convert.ToSingle(SWPER[i][j]), WVHGT = Convert.ToSingle(WVHGT[i][j]), WVDIR = Convert.ToSingle(WVDIR[i][j]), WVPER = Convert.ToSingle(WVPER[i][j]), UGRD = Convert.ToSingle(UGRD[i][j]), VGRD = Convert.ToSingle(VGRD[i][j]), }); nww3Json.Add(new NWW3 { UTC = UTC[i], LAT = Convert.ToDecimal(LAT[i][j]), LON = Convert.ToDecimal(LON[i][j]), HTSGW = Convert.ToSingle(HTSGW[i][j]), MWSDIR = Convert.ToSingle(MWSDIR[i][j]), MWSPER = Convert.ToSingle(MWSPER[i][j]), SWELL = Convert.ToSingle(SWELL[i][j]), SWDIR = Convert.ToSingle(SWDIR[i][j]), SWPER = Convert.ToSingle(SWPER[i][j]), WVHGT = Convert.ToSingle(WVHGT[i][j]), WVDIR = Convert.ToSingle(WVDIR[i][j]), WVPER = Convert.ToSingle(WVPER[i][j]), UGRD = Convert.ToSingle(UGRD[i][j]), VGRD = Convert.ToSingle(VGRD[i][j]), }); } Nww3MakeJson(nww3Json); } int resumeCount = 0; int insertDataCount = 231120; var checkDate = UTC[0]; var checkDate3 = UTC[1]; _oceanModel.Database.CommandTimeout = 99999; var checkDB = _oceanModel.NWW3.FirstOrDefault(s => s.UTC == checkDate); if (checkDB != null) { var resumeCount1 = _oceanModel.NWW3.Count(s => s.UTC == checkDate); var resumeCount2 = _oceanModel.NWW3.Count(s => s.UTC == checkDate3); resumeCount = resumeCount1 + resumeCount2; if (resumeCount == insertDataCount * 2) { nww3DbInsertCheck.Add(new NWW3_DB_INSERT_CHECK { FILE_NAME = _decodeList[4].ToLower(), DATE_OF_FILE = UTC[0], DATE_INSERTED = DateTime.UtcNow }); nww3DbInsertCheck.ForEach(s => _oceanModel.NWW3_DB_INSERT_CHECK.Add(s)); _oceanModel.SaveChanges(); return; } } var waveSplit = Split(wavedataout, 10000); int index = 0; foreach (var item in waveSplit) { if (index * 10000 >= resumeCount) { EFBatchOperation.For(_oceanModel, _oceanModel.NWW3).InsertAll(item); Thread.Sleep(500); } var progress = Math.Round((double)index / 0.46, 2); _log.Info($"Wave Nowcast Db insert :{_decodeList[0]}-{_decodeList[1]}-{_decodeList[2]} {_decodeList[3]}:00 / {progress} %"); index++; } waveSplit = null; wavedataout = null; nww3DbInsertCheck.Add(new NWW3_DB_INSERT_CHECK { FILE_NAME = _decodeList[4].ToLower(), DATE_OF_FILE = UTC[0], DATE_INSERTED = DateTime.UtcNow }); nww3DbInsertCheck.ForEach(s => _oceanModel.NWW3_DB_INSERT_CHECK.Add(s)); _oceanModel.SaveChanges(); _log.Info($"Wave Db insert Finish"); }
public static FileProcessedTracker Run([BlobTrigger("nam-grib-westus-v1/{name}", Connection = "AzureWebJobsStorage")] Stream myBlob, string name, TraceWriter log) { log.Info($"C# Blob trigger function Processed blob\n Name:{name} \n Size: {myBlob.Length} Bytes"); log.Info($"Double Checking if {name} already exists."); var exists = AzureUtilities.CheckIfFileProcessedRowExistsInTableStorage(Constants.NamTrackerTable, Constants.NamTrackerPartitionKey, name, log); if (exists) { log.Info($"{name} Already exists in double check, skipping"); return(null); } log.Info($"Have env: {Environment.GetEnvironmentVariable("GRIB_API_DIR_ROOT")}"); log.Info($"In dir: {Assembly.GetExecutingAssembly().Location}"); string attemptPath = ""; GribUtilities.TryFindBootstrapLibrary(out attemptPath); log.Info($"Attemping to find lib: {attemptPath}"); GribEnvironment.Init(); #if DEBUG == false GribEnvironment.DefinitionsPath = @"D:\home\site\wwwroot\bin\Grib.Api\definitions"; #endif //1. Download stream to temp //TODO: there is supposedly now an ability to read a stream direction in GRIBAPI.Net; investigate to see if its better than storing a temp file string localFileName = AzureUtilities.DownloadBlobToTemp(myBlob, name, log); var rowList = new List <NamTableRow>(); //2. Get values from file using (GribFile file = new GribFile(localFileName)) { log.Info($"Parsing file {name}"); rowList = GribUtilities.ParseNamGribFile(file); } //3. Format in correct table format log.Info($"Attempting to sign in to ad for datalake upload"); var adlsAccountName = CloudConfigurationManager.GetSetting("ADLSAccountName"); //auth secrets var domain = CloudConfigurationManager.GetSetting("Domain"); var webApp_clientId = CloudConfigurationManager.GetSetting("WebAppClientId"); var clientSecret = CloudConfigurationManager.GetSetting("ClientSecret"); var clientCredential = new ClientCredential(webApp_clientId, clientSecret); var creds = ApplicationTokenProvider.LoginSilentAsync(domain, clientCredential).Result; // Create client objects and set the subscription ID var adlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(creds); try { adlsFileSystemClient.FileSystem.UploadFile(adlsAccountName, localFileName, "/nam-grib-westus-v1/" + name, uploadAsBinary: true, overwrite: true); log.Info($"Uploaded file: {localFileName}"); } catch (Exception e) { log.Error($"Upload failed: {e.Message}"); } MemoryStream s = new MemoryStream(); StreamWriter csvWriter = new StreamWriter(s, Encoding.UTF8); csvWriter.WriteLine(NamTableRow.Columns); MemoryStream sLocations = new MemoryStream(); StreamWriter csvLocationsWriter = new StreamWriter(sLocations, Encoding.UTF8); csvLocationsWriter.WriteLine("Lat, Lon"); string fileName = null; foreach (var row in rowList) { if (fileName == null) { fileName = row.PartitionKey + ".csv"; } csvLocationsWriter.WriteLine(row.Lat + "," + row.Lon); csvWriter.WriteLine(row.ToString()); } csvWriter.Flush(); csvLocationsWriter.Flush(); s.Position = 0; sLocations.Position = 0; AzureUtilities.UploadLocationsFile(sLocations, log); sLocations.Flush(); sLocations.Close(); log.Info($"Completed csv creation--attempting to upload to ADLS"); try { adlsFileSystemClient.FileSystem.Create(adlsAccountName, "/nam-csv-westus-v1/" + fileName, s, overwrite: true); log.Info($"Uploaded csv stream: {localFileName}"); } catch (Exception e) { log.Info($"Upload failed: {e.Message}"); } s.Flush(); s.Close(); //delete local temp file File.Delete(localFileName); DateTime date = DateTime.ParseExact(name.Split('.')[0], "yyyyMMdd", null); return(new FileProcessedTracker { ForecastDate = date, PartitionKey = "nam-grib-westus-v1", RowKey = name, Url = "unknown" }); }
public static void GfsForeCastInsertProxy() { DateTime UTC; double[] LAT; double[] LON; double[] temp_surfaceValue; double[] temp_above_groundValue; double[] press_surfaceValue; double[] press_mslValue; GribEnvironment.Init(); List <GribMessage> gfs = new List <GribMessage>(); _log.Info($"Start Insert : {_decodeList[6]}"); using (GribFile file = new GribFile(_decodeList[6])) { file.Context.EnableMultipleFieldMessages = true; var temp_surface = file.Where(d => d.ParameterName == "Temperature" && d.TypeOfLevel == "surface" && d.Level == 0).SingleOrDefault(); var temp_above_ground = file.Where(d => d.ParameterName == "Temperature" && d.TypeOfLevel == "heightAboveGround" && d.Level == 2).SingleOrDefault(); var press_surface = file.Where(d => d.ParameterName == "Pressure" && d.TypeOfLevel == "surface" && d.Level == 0).SingleOrDefault(); var press_msl = file.Where(d => d.ParameterName == "Pressure reduced to MSL" && d.TypeOfLevel == "meanSea" && d.Level == 0).SingleOrDefault(); UTC = temp_surface.Time; LAT = temp_surface.GridCoordinateValues.Select(d => d.Latitude).ToArray(); LON = temp_surface.GridCoordinateValues.Select(d => d.Longitude).ToArray(); temp_surface.Values(out temp_surfaceValue); temp_above_ground.Values(out temp_above_groundValue); press_surface.Values(out press_surfaceValue); press_msl.Values(out press_mslValue); } //ff List <GFS_FORECAST_PROXY> gfsList = new List <GFS_FORECAST_PROXY>(); for (int j = 0; j < LAT.Length; j++) { gfsList.Add(new GFS_FORECAST_PROXY { UTC = UTC, LAT = Convert.ToDecimal(LAT[j]), LON = Convert.ToDecimal(LON[j]), TEMP_SURFACE = Convert.ToSingle(temp_surfaceValue[j] - 273.15f), TEMP_ABOVE_GROUND = Convert.ToSingle(temp_above_groundValue[j] - 273.15f), PRESSURE_SURFACE = Convert.ToSingle(press_surfaceValue[j]), PRESSURE_MSL = Convert.ToSingle(press_mslValue[j]), }); } _oceanModel.Database.CommandTimeout = 99999; var gfsSplit = Split(gfsList, 10000); int index = 0; foreach (var item in gfsSplit) { EFBatchOperation.For(_oceanModel, _oceanModel.GFS_FORECAST_PROXY).InsertAll(item); Thread.Sleep(500); var progress = Math.Round((double)index / 0.26, 2); _log.Info($"Gfs Forecast Db insert :{_decodeList[0]}-{_decodeList[1]}-{_decodeList[2]} {_decodeList[3]}:00 / {progress} %"); index++; } gfsSplit = null; gfsList = null; _log.Info($"Gfs Forecast Db insert :{_decodeList[0]}-{_decodeList[1]}-{_decodeList[2]} {_decodeList[3]}:00 / 100 %"); var conditionWeatherDb = _logModel.CONDITION_OCEAN_WEATHER.First(); conditionWeatherDb.DATE_LAST_FILE_OF_GFS_FORECAST = _dateNextGfsForecastFileToGet; conditionWeatherDb.FILE_NAME_LAST_OF_GFS_FORECAST = _decodeList[4]; if (_decodeList[4] == "gfs.t00z.pgrb2.0p50.f210") { conditionWeatherDb.DB_OF_GFS_FORECAST = "PROXY"; conditionWeatherDb.DATE_LAST_OF_GFS = UTC; //conditionWeatherDb.DATE_LAST_OF_GFS_JSON = UTC; } _logModel.SaveChanges(); }