public static void Run([TimerTrigger("0 */1 * * * *")] TimerInfo myTimer, TraceWriter log)
{
_log = log;
_decodeList = new List <string>();
_log.Info($"Nww3 Decord function executed at: {DateTime.Now}");
if (_functionIsRunningOrNot == true)
{
_log.Info($"Other Instance is Running at: {DateTime.Now}");
return;
}
RemoveDirectoryAndFile();
_functionIsRunningOrNot = true;
try
{
_jobList = _logModel.JOB_LIST_OCEAN_WEATHER.Where(d => d.IS_COMPLETE_JOB == false && d.JOB_TYPE == "GFS_DECODE").ToList();
foreach (var item in _jobList)
{
var blobPath = item.FILE_PATH;
var year = item.FILE_PATH.Split('\\')[0];
var month = item.FILE_PATH.Split('\\')[1];
var day = item.FILE_PATH.Split('\\')[2];
var fileName = item.FILE_PATH.Split('\\')[3];
var hour = Convert.ToString(Convert.ToInt32(item.FILE_PATH.Split('.')[1].Substring(1, 2)) + Convert.ToInt32(item.FILE_PATH.Split('.')[4].Substring(2, 2)));
var localFilePath = "";
_decodeList = new List <string> {
year, month, day, hour, fileName, blobPath, localFilePath
};
GfsRawFileDownLoad();
GfsNowCastInsert();
var whichDb = _logModel.CONDITION_OCEAN_WEATHER.First();
item.IS_COMPLETE_JOB = true;
item.DATE_OF_COMPLETE_JOB_TIME = DateTime.UtcNow;
item.REASON_OF_NOT_COMPLETE = "";
_logModel.SaveChanges();
RemoveDirectoryAndFile();
}
}
catch (Exception e)
{
_log.Error($"Error : : {e.ToString()}");
}
finally
{
RemoveDirectoryAndFile();
_functionIsRunningOrNot = false;
}
}
private static void FtpDownLoad()
{
// Retrieve the storage account
CloudBlobContainer container = _blobClient.GetContainerReference("gfs");
// Ftp Check and Download
using (FtpClient conn = new FtpClient())
{
conn.SocketPollInterval = 1000;
conn.ConnectTimeout = 30000;
conn.ReadTimeout = 300000;
conn.DataConnectionConnectTimeout = 300000;
conn.DataConnectionReadTimeout = 300000;
conn.Host = "ftp.ncep.noaa.gov";
conn.Credentials = new NetworkCredential(_ftpAccountId, _ftpAccountPw);
conn.Connect();
foreach (var item in _fileListToNeedDownload)
{
var checkPoint = 0;
if (conn.FileExists(item[4]))
{
var ftpDownloadCheck = new FTP_RAWDATA_DOWNLOAD_CHECK();
var oceanJobList = new JOB_LIST_OCEAN_WEATHER();
var blobFullPath = Path.Combine(item[0], item[1], item[2], item[5]);
Progress <double> progress = new Progress <double>(x =>
{
// When progress in unknown, -1 will be sent
if (x < 0)
{
}
if (x > checkPoint)
{
checkPoint = checkPoint + 10;
_log.Info($"{item[0]}-{item[1]}-{item[2]} {item[3]}:00 / {item[5]} Progres : {Math.Round(x, 0)} % ");
}
});
conn.RetryAttempts = 3;
ftpDownloadCheck.DATE_OF_FILE = new DateTime(Convert.ToInt32(item[0]), Convert.ToInt32(item[1]), Convert.ToInt32(item[2]), 0, 0, 0);
ftpDownloadCheck.FILE_TYPE = "GFS";
ftpDownloadCheck.FILE_PATH = item[5];
ftpDownloadCheck.TIME_DOWNLOADED = DateTime.UtcNow;
MemoryStream streamFmFtp = new MemoryStream();
conn.Download(streamFmFtp, item[4], progress);
_log.Info($"{item[0]}-{item[1]}-{item[2]} {item[3]}:00 / {item[5]} Progres : Complete");
_log.Info($"========================================================================================================");
CloudBlockBlob blockBlob = container.GetBlockBlobReference(blobFullPath);
streamFmFtp.Position = 0;
using (var fileStream = streamFmFtp)
{
blockBlob.UploadFromStream(streamFmFtp);
}
_oceanModel.FTP_RAWDATA_DOWNLOAD_CHECK.Add(ftpDownloadCheck);
_oceanModel.SaveChanges();
oceanJobList.DATE_OF_JOB_TRIGGERED = DateTime.UtcNow; //job을 등록한 시각
oceanJobList.JOB_TYPE = "GFS_DECODE"; //job 종류
oceanJobList.DATE_OF_COMPLETE_JOB_TIME = new DateTime(2000, 01, 01, 0, 0, 0); //job 완료시간
oceanJobList.IS_COMPLETE_JOB = false; // job 수행여부
oceanJobList.REASON_OF_NOT_COMPLETE = "not started"; // job이 수행되지 않은 이유
oceanJobList.FILE_PATH = blobFullPath; // blobpath
_logModel.JOB_LIST_OCEAN_WEATHER.Add(oceanJobList);
_logModel.SaveChanges();
}
}
conn.Disconnect();
}
}
public static void Run([TimerTrigger("0 */1 * * * *")] TimerInfo myTimer, TraceWriter log)
{
log.Info($"C# Timer trigger function executed at: {DateTime.Now}");
_log = log;
_fileListToNeedDownload = new List <string[]>();
// function 다른 instance가 동작하고 있으면 이중으로 실행하지 않도록 작동 중지.
if (_functionIsRunningOrNot == true)
{
_log.Info($"Other Instance is Running at: {DateTime.Now}");
return;
}
if (rtofsPostion == null)
{
rtofsPostion = _oceanModel.RTOFS_POSITION_CONVERT.ToList();
}
RemoveDirectoryAndFile();
try
{
_functionIsRunningOrNot = true;
if (_logModel.CONDITION_OCEAN_WEATHER.Single().FILE_NAME_LAST_OF_RTOFS_FORECAST == null || _logModel.CONDITION_OCEAN_WEATHER.Single().DATE_LAST_FILE_OF_RTOFS_FORECAST == new DateTime(1900, 01, 01, 0, 0, 0))
{
var conditionWeatherDb = _logModel.CONDITION_OCEAN_WEATHER.First();
conditionWeatherDb.FILE_NAME_LAST_OF_RTOFS_FORECAST = "rtofs_glo_2ds_f024_3hrly_prog.nc";
conditionWeatherDb.DATE_LAST_FILE_OF_RTOFS_FORECAST = new DateTime(DateTime.UtcNow.Year, DateTime.UtcNow.Month, DateTime.UtcNow.AddDays(-1).Day, 0, 0, 0);
_logModel.SaveChanges();
}
var fileNextRtofsForecastFileToGet = _logModel.CONDITION_OCEAN_WEATHER.Single().FILE_NAME_LAST_OF_RTOFS_FORECAST;
var tempHour = fileNextRtofsForecastFileToGet.Split('_')[3].Substring(1);
var hour = Convert.ToInt32(tempHour);
_dateNextRtofsForecastFileToGet = _logModel.CONDITION_OCEAN_WEATHER.Single().DATE_LAST_FILE_OF_RTOFS_FORECAST;
var whichDb = _logModel.CONDITION_OCEAN_WEATHER.First();
if (hour == 192)
{
hour = 24;
_dateNextRtofsForecastFileToGet = _dateNextRtofsForecastFileToGet.AddDays(1);
if (whichDb.DB_OF_RTOFS_FORECAST.ToUpper() == "PROXY")
{
_oceanModel.Database.ExecuteSqlCommand("TRUNCATE TABLE [" + "RTOFS_FORECAST" + "]");
_oceanModel.SaveChanges();
}
else if (whichDb.DB_OF_RTOFS_FORECAST.ToUpper() == "BASIC")
{
_oceanModel.Database.ExecuteSqlCommand("TRUNCATE TABLE [" + "RTOFS_FORECAST_PROXY" + "]");
_oceanModel.SaveChanges();
}
}
else
{
hour = hour + 3;
}
var dateRtofs = _dateNextRtofsForecastFileToGet.ToString("yyyyMMdd");
tempHour = hour.ToString("000");
var fileNameRtofsDiag = "rtofs_glo_2ds_f" + tempHour + "_3hrly_diag.nc";
var fileNameRtofsProg = "rtofs_glo_2ds_f" + tempHour + "_3hrly_prog.nc";
var remoteFilePathDiag = "/pub/data/nccf/com/rtofs/prod/rtofs." + dateRtofs + "/" + fileNameRtofsDiag;
var remoteFilePathProg = "/pub/data/nccf/com/rtofs/prod/rtofs." + dateRtofs + "/" + fileNameRtofsProg;
var localPathDiag = FtpDownLoad(remoteFilePathDiag, fileNameRtofsDiag, _dateNextRtofsForecastFileToGet);
if (localPathDiag == "false")
{
return;
}
_isSuccessDownloadDiag = true;
var localPathProg = FtpDownLoad(remoteFilePathProg, fileNameRtofsProg, _dateNextRtofsForecastFileToGet);
if (localPathProg == "false")
{
return;
}
_isSuccessDownloadProg = true;
if (_isSuccessDownloadDiag && _isSuccessDownloadProg)
{
var year = _dateNextRtofsForecastFileToGet.AddHours(hour).Year.ToString();
var month = _dateNextRtofsForecastFileToGet.AddHours(hour).Month.ToString();
var day = _dateNextRtofsForecastFileToGet.AddHours(hour).Day.ToString();
tempHour = _dateNextRtofsForecastFileToGet.AddHours(hour).Hour.ToString();
_decodeList = new List <string[]>();
_decodeList.Add(new string[] { year, month, day, tempHour, fileNameRtofsDiag, remoteFilePathDiag, localPathDiag });
_decodeList.Add(new string[] { year, month, day, tempHour, fileNameRtofsProg, remoteFilePathProg, localPathProg });
if (whichDb.DB_OF_RTOFS_FORECAST.ToUpper() == "PROXY")
{
RtofsForeCastInsertBasic();
}
else if (whichDb.DB_OF_RTOFS_FORECAST.ToUpper() == "BASIC")
{
RtofsForeCastInsertProxy();
}
RemoveDirectoryAndFile();
}
}
catch (Exception e)
{
log.Info($"error: {e.ToString()}");
}
finally
{
_functionIsRunningOrNot = false;
_isSuccessDownloadDiag = false;
_isSuccessDownloadProg = false;
}
}
public static void Run([TimerTrigger("0 */1 * * * *")] TimerInfo myTimer, TraceWriter log)
{
log.Info($"C# Timer trigger function executed at: {DateTime.Now}");
_log = log;
_fileListToNeedDownload = new List <string[]>();
// function 다른 instance가 동작하고 있으면 이중으로 실행하지 않도록 작동 중지.
if (_functionIsRunningOrNot == true)
{
_log.Info($"Other Instance is Running at: {DateTime.Now}");
return;
}
RemoveDirectoryAndFile();
try
{
_functionIsRunningOrNot = true;
if (_logModel.CONDITION_OCEAN_WEATHER.Single().FILE_NAME_LAST_OF_GFS_FORECAST == null || _logModel.CONDITION_OCEAN_WEATHER.Single().DATE_LAST_FILE_OF_GFS_FORECAST == new DateTime(1900, 01, 01, 0, 0, 0))
{
var conditionWeatherDb = _logModel.CONDITION_OCEAN_WEATHER.First();
conditionWeatherDb.FILE_NAME_LAST_OF_GFS_FORECAST = "gfs.t00z.pgrb2b.0p50.f003";
conditionWeatherDb.DATE_LAST_FILE_OF_GFS_FORECAST = new DateTime(DateTime.UtcNow.Year, DateTime.UtcNow.Month, DateTime.UtcNow.Day, 0, 0, 0);
_logModel.SaveChanges();
}
var fileNextGfsForecastFileToGet = _logModel.CONDITION_OCEAN_WEATHER.Single().FILE_NAME_LAST_OF_GFS_FORECAST;
var tempHour = fileNextGfsForecastFileToGet.Split('.')[4].Substring(1, 3);
var hour = Convert.ToInt32(tempHour);
_dateNextGfsForecastFileToGet = _logModel.CONDITION_OCEAN_WEATHER.Single().DATE_LAST_FILE_OF_GFS_FORECAST;
var whichDb = _logModel.CONDITION_OCEAN_WEATHER.First();
if (hour == 210)
{
hour = 06;
_dateNextGfsForecastFileToGet = _dateNextGfsForecastFileToGet.AddDays(1);
if (whichDb.DB_OF_GFS_FORECAST.ToUpper() == "PROXY")
{
_oceanModel.Database.ExecuteSqlCommand("TRUNCATE TABLE [" + "GFS_FORECAST" + "]");
_oceanModel.SaveChanges();
}
else if (whichDb.DB_OF_GFS_FORECAST.ToUpper() == "BASIC")
{
_oceanModel.Database.ExecuteSqlCommand("TRUNCATE TABLE [" + "GFS_FORECAST_PROXY" + "]");
_oceanModel.SaveChanges();
}
}
else
{
hour = hour + 3;
}
tempHour = hour.ToString("000");
var fileNameGfs = "gfs.t00z.pgrb2.0p50.f" + tempHour;
var tempfolderName = "gfs." + _dateNextGfsForecastFileToGet.ToString("yyyyMMdd") + "00";
var remoteFilePath = "/pub/data/nccf/com/gfs/prod/" + tempfolderName + "/" + fileNameGfs;
var localPath = FtpDownLoad(remoteFilePath, fileNameGfs, _dateNextGfsForecastFileToGet);
if (localPath != "false")
{
var year = _dateNextGfsForecastFileToGet.AddHours(hour).Year.ToString();
var month = _dateNextGfsForecastFileToGet.AddHours(hour).Month.ToString();
var day = _dateNextGfsForecastFileToGet.AddHours(hour).Day.ToString();
tempHour = _dateNextGfsForecastFileToGet.AddHours(hour).Hour.ToString();
_decodeList = new string[] { year, month, day, tempHour, fileNameGfs, remoteFilePath, localPath };
if (whichDb.DB_OF_GFS_FORECAST.ToUpper() == "PROXY")
{
GfsForeCastInsertBasic();
}
else if (whichDb.DB_OF_GFS_FORECAST.ToUpper() == "BASIC")
{
GfsForeCastInsertProxy();
}
RemoveDirectoryAndFile();
}
}
catch (Exception e)
{
log.Info($"error: {e.ToString()}");
}
finally
{
_functionIsRunningOrNot = false;
}
}
private static void FtpDownLoad()
{
CloudBlobContainer container = _blobClient.GetContainerReference("rtofs");
// Ftp Check and Download
using (FtpClient conn = new FtpClient())
{
//ms
conn.SocketPollInterval = 1000;
conn.ConnectTimeout = 30000;
conn.ReadTimeout = 300000;
conn.DataConnectionConnectTimeout = 300000;
conn.DataConnectionReadTimeout = 300000;
conn.Host = "ftp.ncep.noaa.gov";
conn.Credentials = new NetworkCredential(_ftpAccountId, _ftpAccountPw);
conn.Connect();
foreach (var item in _fileListToNeedDownload)
{
var checkPoint = 0;
if (conn.FileExists(item[4]))
{
var ftpNww3DownloadCheck = new FTP_RAWDATA_DOWNLOAD_CHECK();
var oceanJobList = new JOB_LIST_OCEAN_WEATHER();
var blobFullPath = Path.Combine(item[0], item[1], item[2], item[5]);
Progress <double> progress = new Progress <double>(x =>
{
if (x < 0)
{
}
if (x > checkPoint)
{
checkPoint = checkPoint + 5;
_log.Info($"{item[0]}-{item[1]}-{item[2]} {item[3]}:00 / {item[5]} Progres : {Math.Round(x, 0)} % ");
}
});
conn.RetryAttempts = 3;
var dateOfFile = new DateTime(Convert.ToInt16(item[0]), Convert.ToInt16(item[1]), Convert.ToInt16(item[2]), 0, 0, 0);
ftpNww3DownloadCheck.DATE_OF_FILE = dateOfFile;
ftpNww3DownloadCheck.FILE_TYPE = "RTOFS";
ftpNww3DownloadCheck.FILE_PATH = item[5];
ftpNww3DownloadCheck.TIME_DOWNLOADED = DateTime.UtcNow;
MemoryStream streamFmFtp = new MemoryStream();
conn.Download(streamFmFtp, item[4], progress);
_log.Info($"{item[0]}-{item[1]}-{item[2]} {item[3]}:00 / {item[5]} Progres : Complete");
_log.Info($"===============================================================================================================");
CloudBlockBlob blockBlob = container.GetBlockBlobReference(blobFullPath);
streamFmFtp.Position = 0;
using (var fileStream = streamFmFtp)
{
blockBlob.UploadFromStream(streamFmFtp);
}
_oceanModel.FTP_RAWDATA_DOWNLOAD_CHECK.Add(ftpNww3DownloadCheck);
_oceanModel.SaveChanges();
var fileNameOfDiagOrProg = "";
if (item[5].Contains("diag"))
{
fileNameOfDiagOrProg = item[5].Replace("diag", "prog");
}
if (item[5].Contains("prog"))
{
fileNameOfDiagOrProg = item[5].Replace("prog", "diag");
}
var checkDiagData = _oceanModel.FTP_RAWDATA_DOWNLOAD_CHECK.Where(d => d.FILE_TYPE == "RTOFS" && d.DATE_OF_FILE == dateOfFile && d.FILE_PATH == fileNameOfDiagOrProg).Count();
if (checkDiagData > 0)
{
oceanJobList.DATE_OF_JOB_TRIGGERED = DateTime.UtcNow;
oceanJobList.JOB_TYPE = "RTOFS_DECODE";
oceanJobList.DATE_OF_COMPLETE_JOB_TIME = new DateTime(2000, 01, 01, 0, 0, 0);
oceanJobList.IS_COMPLETE_JOB = false;
oceanJobList.REASON_OF_NOT_COMPLETE = "not started";
oceanJobList.FILE_PATH = blobFullPath;
_logModel.JOB_LIST_OCEAN_WEATHER.Add(oceanJobList);
_logModel.SaveChanges();
}
}
}
conn.Disconnect();
}
}
public static void Run([TimerTrigger("0 */1 * * * *")] TimerInfo myTimer, TraceWriter log)
{
_log = log;
_decodeList = new List <string[]>();
_log.Info($"Rtofs Decord function executed at: {DateTime.Now}");
if (_functionIsRunningOrNot == true)
{
_log.Info($"Other Instance is Running at: {DateTime.Now}");
return;
}
RemoveDirectoryAndFile();
_functionIsRunningOrNot = true;
if (rtofsPostion == null)
{
rtofsPostion = _oceanModel.RTOFS_POSITION_CONVERT.ToList();
}
try
{
_jobList = _logModel.JOB_LIST_OCEAN_WEATHER.Where(d => d.IS_COMPLETE_JOB == false && d.JOB_TYPE == "RTOFS_DECODE").ToList();
foreach (var item in _jobList)
{
var blobPath = item.FILE_PATH;
var year = item.FILE_PATH.Split('\\')[0];
var month = item.FILE_PATH.Split('\\')[1];
var day = item.FILE_PATH.Split('\\')[2];
var fileName = item.FILE_PATH.Split('\\')[3];
var hour = item.FILE_PATH.Split('_')[3].Substring(1, 3);
int hourTemp = Convert.ToInt16(hour);
hour = hourTemp.ToString("000");
var fileNameDiag = "rtofs_glo_2ds_n" + hour + "_3hrly_diag.nc";
var fileNameProg = "rtofs_glo_2ds_n" + hour + "_3hrly_prog.nc";
//var commonFileName = "rtofs_glo_2ds_n" + hour + "_3hrly.nc";
var blobPathDiag = year + "\\" + month + "\\" + day + "\\" + fileNameDiag;
var blobPathProg = year + "\\" + month + "\\" + day + "\\" + fileNameProg;
_decodeList.Add(new string[] { year, month, day, hour, fileNameDiag, blobPathDiag, "" });
_decodeList.Add(new string[] { year, month, day, hour, fileNameProg, blobPathProg, "" });
RtofsRawFileDownLoad();
RtofsNowCastInsert();
item.IS_COMPLETE_JOB = true;
item.DATE_OF_COMPLETE_JOB_TIME = DateTime.UtcNow;
item.REASON_OF_NOT_COMPLETE = "";
_logModel.SaveChanges();
}
}
catch (Exception e)
{
_log.Error($"Error : : {e.ToString()}");
}
finally
{
RemoveDirectoryAndFile();
_functionIsRunningOrNot = false;
}
}
