//method为省时候调用,只切割指定的省份 public static bool CreateTileByWATAByCSharpFromProvince(string keyString, string curDatFullname, ref string start, ref string end, ref string datnums, ref string yearmmddForID) { Stopwatch stopwatch = new Stopwatch(); stopwatch.Start(); string[] gridlist = HookHelper.gridsize.Split(','); int gridrow = 1001; int gridcol = 1001; float rainSRCFBL = 0.01f; if (gridlist.Count() == 2) { gridrow = int.Parse(gridlist[0]); gridcol = int.Parse(gridlist[1]); } if (gridlist.Count() == 3) { gridrow = int.Parse(gridlist[0]); gridcol = int.Parse(gridlist[1]); rainSRCFBL = float.Parse(gridlist[2]); } string datPureName = System.IO.Path.GetFileNameWithoutExtension(curDatFullname); //!解析当前dat文件 //!创建数据存储结构 DatFileStruct datStruct = new DatFileStruct(); datStruct.col = gridcol; datStruct.row = gridrow; datStruct.fbl = rainSRCFBL; // 读取文件 BinaryReader br; try { br = new BinaryReader(new FileStream(curDatFullname, FileMode.Open, FileAccess.Read, FileShare.Read)); } catch (IOException e) { Console.WriteLine(string.Format("{0}台风场文件解析场次信息失败,继续下一个", curDatFullname) + DateTime.Now); return(false); } try { //! 第一部分数据 年(year)、月日时(mdh)、该台风总时次(times) 均为整型 3 * 4 个字节 //inFile.read((char*)&datStruct.headerone[0], 3 * sizeof(int)); int year = br.ReadInt32(); int mdh = br.ReadInt32(); int times = br.ReadInt32(); ////// string yearStr = year.ToString(); year = int.Parse(yearStr); /////// datStruct.headerone[0] = year; datStruct.headerone[1] = mdh; datStruct.headerone[2] = times; string mdhSt = mdh.ToString(); if (mdhSt.Length == 5) { mdhSt = String.Format("0{0}", mdhSt); } //System.String.Substring(Int32 startIndex, Int32 length) string ymdhstr = String.Format("{0}{1}", yearStr, mdhSt); //1,2,3位的year前边必须补0,不然会识别错误, 5位的不识别.传入程序的时间以4位为准,统一从2000年开始,追加,模型输出时候纠正时间 string yearStrForCalc = "2020"; //DateTime dt = Convert.ToDateTime(yearStrForCalc + "-" + mdhSt.Substring(0, 2) + "-" + mdhSt.Substring(2, 2) + " " + mdhSt.Substring(4, 2) + ":00:00"); DateTime dt = Convert.ToDateTime(yearStrForCalc + "-" + "01" + "-" + "01" + " " + "00" + ":00:00"); //! 传入到模型中的时间值,用来计算该时间段的水文结果 start = dt.ToString("yyyy-MM-ddTHH:mm"); end = (dt.AddHours(times - 1)).ToString("yyyy-MM-ddTHH:mm"); datnums = times.ToString(); //该变量是根据时间值组合是数字串,后续作为降雨及计算结果的输出文件名称前缀,year前自动补0,与模型计算中更新rainfile中规则一致 yearmmddForID = yearStr + mdhSt.Substring(0, 2) + mdhSt.Substring(2, 2) + mdhSt.Substring(4, 2); //!2、第二部分,是各个场次经纬度列表 datStruct.Lats = new double[times]; datStruct.Lons = new double[times]; for (int tindex = 0; tindex < datStruct.headerone[2]; ++tindex) { double lat = br.ReadDouble(); double lon = br.ReadDouble(); datStruct.Lats[tindex] = lat; datStruct.Lons[tindex] = lon; } //!3、第三部分,是所有场次的网格数据存储,三维数组存放每个时间的网格数据 datStruct.rain = new float[times, datStruct.row, datStruct.col]; for (int tindex = 0; tindex < datStruct.headerone[2]; ++tindex) { datStruct.curRainIndex = tindex; byte[] datbytes = br.ReadBytes(datStruct.row * datStruct.col * 4); for (int r = 0; r < datStruct.row; ++r) { for (int c = 0; c < datStruct.col; ++c) { //float val = br.ReadSingle(); //datStruct.rain[tindex, r, c] = val; datStruct.rain[tindex, r, c] = BitConverter.ToSingle(datbytes, (r * datStruct.col + c) * 4); } } } } catch (IOException e) { Console.WriteLine(string.Format("{0}台风场文件解析场次信息失败,继续下一个", curDatFullname) + DateTime.Now); br.Close(); return(false); } br.Close(); stopwatch.Stop(); Console.WriteLine(string.Format("读取{0}台风场文件耗时{1}", curDatFullname, stopwatch.ElapsedMilliseconds)); stopwatch.Restart(); //!22 数据读取完成,则需要插值到各个计算单元,然后写出 //! 遍历所有的计算单元信息表,写出数据 //! 遍历每个计算单元,然后在其中遍历每个场次的数据 int unitNUM = dbTableConfigs[keyString]["GRID_HSFX_UNIT"].Rows.Count; //unit 表 DataTable grid_unit_tables = dbTableConfigs[keyString]["GRID_HSFX_UNIT"]; int countOfHaveRain = 0; for (int i = 0; i < unitNUM; ++i) { //!单元的信息 string provinceName = keyString; string groovyName = grid_unit_tables.Rows[i]["GroovyName"].ToString(); //!当前场次下某个单元的所有时间文件写出 bool status = WriteAscFileByParams(datPureName, provinceName, groovyName, yearmmddForID, datStruct, grid_unit_tables.Rows[i]); if (status) { countOfHaveRain++; //Console.WriteLine(string.Format("{0}台风场文件在{1}省下{2}单元目录切片成功", curDatFullname, provinceName, groovyName) + DateTime.Now); } else { //Console.WriteLine(string.Format("{0}台风场文件在{1}省下{2}单元目录切片失败", curDatFullname, provinceName, groovyName) + DateTime.Now); } //stopwatch.Stop(); //Console.WriteLine(string.Format("写入{0}台风场第{1}个计算单元耗时{2}", curDatFullname, i, stopwatch.ElapsedMilliseconds)); //stopwatch.Restart(); } stopwatch.Stop(); Console.WriteLine(string.Format("写入{0}台风场文件耗时{1}", curDatFullname, stopwatch.ElapsedMilliseconds)); Console.WriteLine(string.Format("{0}台风场文件在{1}节点下共有{2}个计算单元,其中{3}个计算单元中有有效降雨", curDatFullname, HookHelper.computerNode, unitNUM, countOfHaveRain) + DateTime.Now); //CSVLog if (HookHelper.useCSVLOG.Equals("true")) { CSVData.addData(CSVData.GetRowNumber(), "HostName", System.Net.Dns.GetHostName()); var serverIP = Program.GetLocalIP(HookHelper.serachIP); CSVData.addData(CSVData.GetRowNumber(), "服务器IP", serverIP); CSVData.addData(CSVData.GetRowNumber(), "计算节点", HookHelper.computerNode); CSVData.addData(CSVData.GetRowNumber(), "eventId", Path.GetFileNameWithoutExtension(curDatFullname)); CSVData.addData(CSVData.GetRowNumber(), "计算单元个数", unitNUM); CSVData.addData(CSVData.GetRowNumber(), "有效降雨单元个数", countOfHaveRain); } datStruct.headerone = null; datStruct.rain = null; datStruct.Lons = null; datStruct.Lats = null; return(true); }
public static bool runBySingleCC() { //!! 场次信息回传 // 更新execpath的值 string start = "2000-01-01T00:00"; string end = "2000-01-01T08:00"; string datnums = "95"; string yearmmddForID = "2000010100"; //! 遍历指定目录下的降雨数据 //! 设置计时器,当前场次时间 Stopwatch totalFolderDat = new Stopwatch(); totalFolderDat.Start(); if (!Directory.Exists(HookHelper.rainSRCDirectory)) { Console.WriteLine(string.Format("{0}台风场dat降雨目录不存在 ", HookHelper.rainSRCDirectory) + DateTime.Now); return(false); } FileInfo[] fInfo = GenRainTileByCSharp.GetRaindatList(); int datnum = fInfo.Length; for (int d = 0; d < datnum; ++d) { //每次计算当前台风前先删除当前节点下,所有单元的公共传递文件inputq.csv\rainfile.txt //! 当前dat文件全路径 //! 设置计时器,当前场次时间 Stopwatch oneDat = new Stopwatch(); oneDat.Start(); string curDatFullname = fInfo[d].FullName; Console.WriteLine(string.Format("****************************降雨目录下第{0}场*********A************", d + 1) + DateTime.Now); Console.WriteLine(string.Format("*****************************************************AAA***********") + DateTime.Now); Console.WriteLine(string.Format("*****************************{0}场次Start***********AAAAA**********", curDatFullname) + DateTime.Now); Console.WriteLine(string.Format("***************************************************AAAAAAA*********") + DateTime.Now); Console.WriteLine(string.Format("**************************************************AAAAAAAAA********") + DateTime.Now); //! 遍历每个流域的网格模型 //!执行python切片 //!1、执行切片,调用python执行 if (HookHelper.isgenraintile) { //CSVLog //CSVData.addRow(); //! 设置计时器,当前场次时间 Stopwatch perChangci = new Stopwatch(); perChangci.Start(); bool isGenTilesucess = GenRainTileByCSharp.CreateTileByWATAByCSharp(curDatFullname, ref start, ref end, ref datnums, ref yearmmddForID); perChangci.Stop(); TimeSpan perChangciTime = perChangci.Elapsed; if (!isGenTilesucess) { //Console.WriteLine(string.Format("{0}区域降雨切片执行失败 ", HookHelper.computerNode) + DateTime.Now); HookHelper.Log += string.Format("{0}区域降雨切片执行失败 ", HookHelper.computerNode) + DateTime.Now + ";\r\n"; continue; } else { Console.WriteLine(string.Format("网格{0}场次降雨切片执行耗时:{1}秒", curDatFullname, perChangciTime.TotalMilliseconds / 1000)); HookHelper.Log += string.Format("网格{0}场次降雨切片执行耗时:{1}秒", curDatFullname, perChangciTime.TotalMilliseconds / 1000) + DateTime.Now + ";\r\n"; Console.WriteLine(string.Format("{0}区域降雨切片执行成功 ", HookHelper.computerNode) + DateTime.Now); //CSVLog if (HookHelper.useCSVLOG.Equals("true")) { CSVData.addData(CSVData.GetRowNumber(), "切片时长", perChangciTime.TotalMilliseconds / 1000); } } } //! 启动bat前每场的时间不同,要更新写出execsingle.bat if (dbTableConfigs["china"].Count > 0) { int appnum = dbTableConfigs["china"]["HSFX_ComputeUnit"].Rows.Count; int appValidCount = 0; for (int a = 0; a < appnum; ++a) { //!当前路径 string apppath = dbTableConfigs["china"]["HSFX_ComputeUnit"].Rows[a]["AppPath"].ToString(); string ComputeUnit = dbTableConfigs["china"]["HSFX_ComputeUnit"].Rows[a]["ComputeUnit"].ToString(); string provinceName = dbTableConfigs["china"]["HSFX_ComputeUnit"].Rows[a]["province"].ToString(); //execbat路径 string execpath = apppath + "execsingle.bat"; if (apppath.EndsWith("/")) { execpath = apppath + "execsingle.bat"; } else { execpath = apppath + "\\" + "execsingle.bat"; } if (!dbValues.ContainsKey(provinceName)) { continue; } string outrainTilepath = dbValues[provinceName]["rainTileFolder"]; // 更新execpath的值 bool isUpExec = false; //!覆盖更新通过指定参数到execsingle.bat文件 string datPureName = System.IO.Path.GetFileNameWithoutExtension(curDatFullname); isUpExec = WriteExecBatFile.UpdateExecBatFileByTemplateExecsingle(execpath, ComputeUnit, start, end, datnums, datPureName, outrainTilepath, yearmmddForID); if (isUpExec) { appValidCount++; //Console.WriteLine(string.Format("{0}区域{1}文件execsingle.bat更新成功 ", "china", apppath) + DateTime.Now); } else { //Console.WriteLine(string.Format("{0}区域{1}文件execsingle.bat更新失败 ", "china", apppath) + DateTime.Now); } } Console.WriteLine(string.Format("{0}区域{1}个有效单元的execsingle.bat更新成功 ", "china", appValidCount) + DateTime.Now); //!判断appnum 是否超过了processnum,是则部分启动,等待 //!先根据个数分组,分组后,则循环,则可以等待了 int processGroup = (int)Math.Ceiling((float)appnum / (float)HookHelper.processnum); for (int g = 0; g < processGroup; ++g) { //!循环每个组,pid存在,则执行等待 int perGroupCount = 0; while (pids.Count > 0) { //! 执行等待,然后查询更新pids列表.等待1分钟 Console.WriteLine(string.Format("共{0}个分组", processGroup) + DateTime.Now); Console.WriteLine(string.Format("等待第{0}场{1}文件的第{2}进程组计算完成并关闭,pid进程查询更新等待中,等待时长15秒...", d + 1, curDatFullname, g) + DateTime.Now); System.Threading.Thread.Sleep(1000 * 15 * 1); //kill perGroupCount++; Console.WriteLine(string.Format("已经等待次数{0}次", perGroupCount) + DateTime.Now); if (perGroupCount >= HookHelper.waitcount) { //遍历强制关闭当前场次的所有pid程序 foreach (var item in pids.ToList()) { int curPID = item.Key; Process curProcss = null; try { curProcss = Process.GetProcessById(curPID); } catch (Exception ex) { curProcss = null; } bool isInProcess = curProcss == null ? false : true; if (isInProcess) { //curProcss.Kill(); HookHelper.KillProcessAndChildren(curPID); } } } //! 遍历pids,查询windows process中是否存在这个pid,不存在,则移除 int pidnum = pids.Count; foreach (var item in pids.ToList()) { int curPID = item.Key; Process curProcss = null; try { curProcss = Process.GetProcessById(curPID); } catch (Exception ex) { curProcss = null; } bool isInProcess = curProcss == null ? false : true; if (!isInProcess) { pids.Remove(item.Key); } else { Console.WriteLine(string.Format("单元{0}所在分组{1}计算进行中......需继续等待......", item.Value, g) + DateTime.Now); } } if (pids.Count == 0) { break; } } //当前分组的起始值,和end值 int startPROCESS = g * HookHelper.processnum; int endPROCESS = (g + 1) * HookHelper.processnum; if (g == processGroup - 1) { endPROCESS = appnum; } int validStartUnitModel = 0; for (int a = startPROCESS; a < endPROCESS; ++a) { //!当前路径 string apppath = dbTableConfigs["china"]["HSFX_ComputeUnit"].Rows[a]["AppPath"].ToString(); string ComputeUnit = dbTableConfigs["china"]["HSFX_ComputeUnit"].Rows[a]["ComputeUnit"].ToString(); string ComputeNode = dbTableConfigs["china"]["HSFX_ComputeUnit"].Rows[a]["ComputeNode"].ToString(); //execbat路径 string execpath = apppath + "execsingle.bat"; if (apppath.EndsWith("/")) { execpath = apppath + "execsingle.bat"; } else { execpath = apppath + "\\" + "execsingle.bat"; } //! 启动该exec.bat //! 单元信息 string appunitInfo = ComputeNode + "_" + ComputeUnit + "_" + apppath; bool isOneStart = StartOneByOneExecsingle(execpath, appunitInfo); if (isOneStart) { validStartUnitModel++; HookHelper.Log += string.Format("{0}节点{1}单元{2}路径执行成功 ", HookHelper.computerNode, ComputeUnit, execpath) + DateTime.Now + ";\r\n"; //Console.WriteLine(string.Format("{0}节点{1}单元{2}路径执行成功 ", HookHelper.computerNode, ComputeUnit, execpath) + DateTime.Now); } else { HookHelper.Log += string.Format("{0}节点{1}单元{2}路径执行失败 ", HookHelper.computerNode, ComputeUnit, execpath) + DateTime.Now + ";\r\n"; //Console.WriteLine(string.Format("{0}节点{1}单元{2}路径执行失败 ", HookHelper.computerNode, ComputeUnit, execpath) + DateTime.Now); } } Console.WriteLine(string.Format("{0}节点{1}个有效单元启动命令执行成功 ", HookHelper.computerNode, validStartUnitModel) + DateTime.Now); } } //!循环每个组,pid存在,则执行等待,直至继续运行到下一步,代表一个场次计算结束 int perWaitCount = 0; //如果等待超过1个小时,仍然无法计算,则跳过这个场次,并写出到log中 while (pids.Count > 0) { //! 执行等待,然后查询更新pids列表.等待1分钟 Console.WriteLine(string.Format("等待第{0}场{1}文件计算完成并关闭,pid进程查询更新等待中,等待时长15秒...", d + 1, curDatFullname) + DateTime.Now); System.Threading.Thread.Sleep(1000 * 15 * 1); perWaitCount++; Console.WriteLine(string.Format("已经等待次数{0}次", perWaitCount) + DateTime.Now); if (perWaitCount >= HookHelper.waitcount) { //遍历强制关闭当前场次的所有pid程序 //将该场次值写出到log文件中 string ignoreCCName = curDatFullname; WriteLog.AppendLogMethod(ignoreCCName, "datIgnore"); foreach (var item in pids.ToList()) { int curPID = item.Key; Process curProcss = null; try { curProcss = Process.GetProcessById(curPID); } catch (Exception ex) { curProcss = null; } bool isInProcess = curProcss == null ? false : true; if (isInProcess) { //curProcss.Kill(); HookHelper.KillProcessAndChildren(curPID); } else { Console.WriteLine(string.Format("最后一组单元{0}计算进行中......需继续等待......", item.Value) + DateTime.Now); } } } //! 遍历pids,查询windows process中是否存在这个pid,不存在,则移除 int pidnum = pids.Count; foreach (var item in pids.ToList()) { int curPID = item.Key; Process curProcss = null; try { curProcss = Process.GetProcessById(curPID); } catch (Exception ex) { curProcss = null; } bool isInProcess = curProcss == null ? false : true; if (!isInProcess) { pids.Remove(item.Key); } else { Console.WriteLine(string.Format("最后一组单元{0}计算进行中......需继续等待......", item.Value) + DateTime.Now); } } if (pids.Count == 0) { break; } } Console.WriteLine(string.Format("{0}台风场所有流域计算完成 ", curDatFullname) + DateTime.Now); HookHelper.Log += string.Format("{0}台风场所有流域计算完成 ", curDatFullname) + DateTime.Now + ";\r\n"; Console.WriteLine(string.Format("******************************************************A************") + DateTime.Now); Console.WriteLine(string.Format("*****************************************************AAA***********") + DateTime.Now); Console.WriteLine(string.Format("*****************************{0}场次END*************AAAAA**********", curDatFullname) + DateTime.Now); Console.WriteLine(string.Format("***************************************************AAAAAAA*********") + DateTime.Now); Console.WriteLine(string.Format("**************************************************AAAAAAAAA********") + DateTime.Now); Console.WriteLine(string.Format("****************************降雨目录下第{0}场结束******************", d + 1) + DateTime.Now); oneDat.Stop(); TimeSpan oneDatTime = oneDat.Elapsed; Console.WriteLine(string.Format("网格{0}场次降雨切片->bat信息更新->等待网格流域计算,单场次全流程耗时:{1}秒", curDatFullname, oneDatTime.TotalMilliseconds / 1000)); HookHelper.Log += string.Format("网格{0}场次降雨切片->bat信息更新->等待网格流域计算,单场次全流程耗时:{1}秒", curDatFullname, oneDatTime.TotalMilliseconds / 1000) + DateTime.Now + ";\r\n"; Console.WriteLine(string.Format("####################################################################") + DateTime.Now); Console.WriteLine(string.Format(" ") + DateTime.Now); Console.WriteLine(string.Format(" ") + DateTime.Now); Console.WriteLine(string.Format("####################################################################") + DateTime.Now); //CSVLog if (HookHelper.useCSVLOG.Equals("true")) { CSVData.addData(CSVData.GetRowNumber(), "单场台风时长", oneDatTime.TotalMilliseconds / 1000); CSVData.addData(CSVData.GetRowNumber(), "单场计算时长", oneDatTime.TotalMilliseconds / 1000 - (double)CSVData.LogDataTable.Rows[CSVData.GetRowNumber()]["切片时长"]); } } Console.WriteLine(string.Format("{0}场台风场次逐场次流域计算完成 ", datnum) + DateTime.Now); HookHelper.Log += string.Format("{0}场台风场次逐场次流域计算完成 ", datnum) + DateTime.Now + ";\r\n"; Console.WriteLine(string.Format("*********(-_-) **************{0}场台风场次逐场次流域计算完成*******..~^_^~..***********", datnum) + DateTime.Now); Console.WriteLine(string.Format("*********(-_-) ***************{0}场台风场次逐场次流域计算完成*******..~^_^~..**********", datnum) + DateTime.Now); Console.WriteLine(string.Format("*********(-_-) ***************{0}场台风场次逐场次流域计算完成*******..~^_^~..**********", datnum) + DateTime.Now); Console.WriteLine(string.Format("********(-_-) **************{0}场台风场次逐场次流域计算完成*******..~^_^~..**********", datnum) + DateTime.Now); Console.WriteLine(string.Format("********(-_-) ***************{0}场台风场次逐场次流域计算完成*******..~^_^~..*********", datnum) + DateTime.Now); Console.WriteLine(string.Format("********(-_-) ****************{0}场台风场次逐场次流域计算完成*******..~^_^~..**********", datnum) + DateTime.Now); totalFolderDat.Stop(); TimeSpan totalFolderDatTime = totalFolderDat.Elapsed; Console.WriteLine(string.Format("{0}降雨目录下{1}个降雨文件从降雨切片->bat信息更新->等待网格流域计算,总耗时:{2}秒", HookHelper.rainSRCDirectory, datnum, totalFolderDatTime.TotalMilliseconds / 1000)); HookHelper.Log += string.Format("{0}降雨目录下{1}个降雨文件从降雨切片->bat信息更新->等待网格流域计算,总耗时:{2}秒", HookHelper.rainSRCDirectory, datnum, totalFolderDatTime.TotalMilliseconds / 1000) + DateTime.Now + ";\r\n"; return(true); }