private static DateTime GetFileDateTime(string fname, string mode)
{
switch (mode.ToUpper())
{
case "MOD06":
string dt;
return(DataProcesser.GetOribitTime(fname, out dt));
case "MOD03":
string dt3;
return(DataProcesser.GetOribitTime(fname, out dt3));
case "AIRS":
string outstr;
return(AIRSDataProcesser.GetAIRSDateTimeFromStr(fname, out outstr));
case "ISCCP":
string isccpregex = @".(?<year>\d{4}).(?<month>\d{2}).(?<day>\d{2}).(?<utc>\d{4})";
Match match = Regex.Match(fname, isccpregex);
if (match.Success)
{
int year = int.Parse(match.Groups["year"].Value);
int month = int.Parse(match.Groups["month"].Value);
return(new DateTime(year, month, 1));
}
return(DateTime.MinValue);
case "CLOUDSAT":
string[] parts = fname.Split(new char[] { '_' }, StringSplitOptions.RemoveEmptyEntries);
return(OriginData2Database.GetInfoFromCloudSATName(parts[0]));
default:
return(DateTime.MinValue);
}
}
public int ComputeValidPercent(DataProcesser datapro, string fname, double invalidValue, out float resl)
{
resl = 0;
using (IRasterDataProvider dataprd = GeoDataDriver.Open(fname) as IRasterDataProvider)
{
resl = dataprd.ResolutionX;
return(datapro.ComputeVaildPct(dataprd, invalidValue));
}
}
public AppExecutor()
{
_excelRepo = new ExcelFileParser(@"C:\Users\k.blazevicius\Desktop\test.xlsx");
var sqlConnection = new SqlConnection(ConfigurationManager.ConnectionStrings["GerveSqlDbConnection"].ToString());
var sqlContext = new GerveContext(sqlConnection);
_sqlRepo = new GerveSqlRepository(sqlContext);
_dataProcesser = new DataProcesser <ExcelDto, SqlDto>();
}
private void MODRawDataF2Database(string tableName, List <string> filelist, Action <string> ProBack)
{
//MOD06_L2.A2011001.0350.hdf
//MOD03.A2011001.0350.hdf
ProBack("--------------------------------------------------------------------------------------");
ProBack("MOD06数据入库开始：");
string fName, localfName;
DateTime fDateTime = DateTime.MinValue;
string sensor = "MODIS";
string dt;
foreach (string file in filelist)
{
fName = Path.GetFileName(file);
ProBack("\t" + fName + "入库开始：");
fDateTime = DataProcesser.GetOribitTime(file, out dt);
if (fDateTime == DateTime.MinValue || string.IsNullOrEmpty(dt))
{
ProBack("\t\t解析文件名时间信息失败，入库失败！");
continue;
}
if (Directory.Exists(new DirectoryInfo(_inputDir).Parent.FullName))
{
localfName = file.Replace(new DirectoryInfo(_inputDir).Parent.FullName, "");//Path.Combine(localDir, fName);
}
else
{
localfName = file.Replace(_inputDir, "");
}
try
{
//if (_dbConnect.IshasRecord(tableName, "ImageName", fName) == false)
if (_dbConnect.IshasRecord(tableName, "ImageData", localfName) == false)
{
_dbConnect.InsertRawMOD06Table(tableName, fDateTime, fName, localfName, sensor); //插入,insert
ProBack("\t\t\t文件入库成功！");
}
else
{
//if (_dbConnect.UpdateRawCLDParatable(tableName, fName, localfName))
// ProBack("\t\t\t文件记录更新成功！");
//else
ProBack("\t\t\t文件记录已存在，跳过！");
}
}
catch (System.Exception ex)
{
ProBack("入库错误" + ex.Message);
}
}
ProBack("MOD06数据入库结束！");
ProBack("--------------------------------------------------------------------------------------");
}
private void DoProcess()
{
try
{
if (_frmMode == "MOD06")
{
DataProcesser processer = new DataProcesser(txtDirName.Text, txtOutDir.Text, GetSelectedDatasets(), txtResl.ValueF, _envList.ToArray(), cbxOverlapPrj.Checked, cbxOverlapMosaic.Checked, cbxDirectMosaic.Checked, cbxOnlyPrj.Checked, cbxLostAdded.Checked);
processer.IsOriginResl = cbxOriginResl.Checked;
processer._historyPrjDir = (cbxDirectMosaic.Checked?txtHistoryPrj.Text:null);
if (processer.DoProcess(_state))
{
}
}
else if (_frmMode == "MYD06")
{
MYDDataProcesser processer = new MYDDataProcesser(txtDirName.Text, txtOutDir.Text, GetSelectedDatasets(), txtResl.ValueF, _envList.ToArray(), cbxOverlapPrj.Checked, cbxOverlapMosaic.Checked, cbxDirectMosaic.Checked, cbxOnlyPrj.Checked, cbxLostAdded.Checked);
processer.IsOriginResl = cbxOriginResl.Checked;
processer._historyPrjDir = (cbxDirectMosaic.Checked ? txtHistoryPrj.Text : null);
if (processer.DoProcess(_state))
{
}
}
else
{
AIRSDataProcesser processer = new AIRSDataProcesser(txtDirName.Text, txtOutDir.Text, GetSelectedDatasets(), txtResl.ValueF, _envList.ToArray(), cbxOverlapPrj.Checked, cbxOverlapMosaic.Checked, cbxDirectMosaic.Checked, cbxOnlyPrj.Checked);
processer.IsOriginResl = cbxOriginResl.Checked;
processer._historyPrjDir = (cbxDirectMosaic.Checked ? txtHistoryPrj.Text : null);
if (processer.Process(_state))
{
}
}
runTaskThread = new Thread(new ThreadStart(this.ClearRecycleFiles));
runTaskThread.IsBackground = true;
runTaskThread.Start();
//_state(-1, "开始后台临时文件清理...");
}
catch (System.Exception ex)
{
LogFactory.WriteLine(_frmMode + "预处理ERROR", ex.Message);
_state(-100, ex.Message);
//throw ex;
}
}
private void DoProject(string[] files, Action <int, string> progressCallback)
{
List <string> sets = new List <string>();
foreach (string set in _dataSets)
{
if (!string.IsNullOrEmpty(set))
{
sets.Add(set);
}
}
if (sets.Count == 0)
{
progressCallback(-5, "请选择有效的数据集！");
return;
}
string[] SelectedSets = sets.ToArray();//获取所选数据集中的可处理数据集
if (progressCallback != null)
{
progressCallback(-1, "链接数据库，查询数据集ID开始...");
}
QueryIDs(SelectedSets);
if (progressCallback != null)
{
progressCallback(-1, "查询数据集ID结束！");
}
if (_prjEnvelopes != null && _prjEnvelopes.Count() > 0)
{
StringBuilder outDirBuilder = new StringBuilder(200);
StringBuilder outFilenameBuilder = new StringBuilder(400);
string outDir = string.Empty;
string dataStr = string.Empty;
string dnLabel, setName, outFilename;
//string subDir = "5分钟段产品\\{0}\\AQUA\\AIRS\\{1}\\{2}\\{3}\\{4}\\{5}";
float resl = _isOriginResl ? 1.0f : _resl;
foreach (PrjEnvelopeItem prjItem in _prjEnvelopes)
{
if (progressCallback != null)
{
progressCallback(-1, prjItem.Name + "投影区域投影开始...");
}
int count = files.Length;
for (int i = 0; i < files.Length; i++)
{
if (progressCallback != null)
{
int pct = 6 + (int)(i / (count * 1.0f) * (75 - 6));
progressCallback(pct, "共" + files.Length + "个待投影文件，开始投影第" + (i + 1) + "个...");
}
DateTime dt = GetOribitTime(files[i], out dataStr);
if (dt == DateTime.MinValue || dataStr == "")
{
if (progressCallback != null)
{
progressCallback(-1, "\t获取文件名中的时间信息失败！" + files[i]);
}
continue;
}
//根据文件名中的时间信息确定其归属于白天或夜间文件夹
DateTime timeparts = dt.Date.AddHours(9);
dnLabel = (dt.CompareTo(timeparts) <= 0) ? "day" : "night";
foreach (string item in SelectedSets)
{
//if (progressCallback != null)
// progressCallback(-1, "数据集" + item + "投影开始...");
setName = item.Replace("_", "");
//outDir = Path.Combine(_outputDir, string.Format(subDir, setName, dt.Year, dt.Month, dt.Day, dnLabel, resl));//GetDatasetOutDir(outDirs, item);
//outFilename = Path.Combine(outDir, string.Format("{0}_{1}_{2}_granule_{3}_{4}{5}", setName, "AIRS", prjItem.Name, dataStr, resl.ToString("f2"), ".LDF"));
outDirBuilder.Clear();
outDir = outDirBuilder.Append(Path.Combine(_outputDir, "5分钟段产品\\")).Append(setName).Append("\\AQUA\\AIRS\\").Append(dt.Year + "\\").Append(dt.Month + "\\").Append(dt.Day + "\\").Append(dnLabel + "\\").Append(resl).ToString();
outFilenameBuilder.Clear();
outFilename = outFilenameBuilder.Append(outDir + "\\").Append(setName).Append("_AIRS_").Append(prjItem.Name).Append("_granule_").Append(dataStr + "_").Append(resl.ToString("f2") + ".LDF").ToString();
if (progressCallback != null)
{
progressCallback(-1, "\t\t正在投影" + item + "数据集,文件：" + outFilename);
}
if (File.Exists(outFilename))
{
if (_overlapPrj == false)
{
if (progressCallback != null)
{
progressCallback(-1, "\t\t\t投影文件已存在,跳过！");
}
if (!_allPaths.Contains(outDir))
{
_allPaths.Add(outDir);
}
continue;
}
else
{
File.Delete(outFilename);
if (progressCallback != null)
{
progressCallback(-1, "\t\t\t删除已存在的投影文件成功！");
}
}
}
if (!Directory.Exists(outDir))
{
DataProcesser.TryCreateDstDir(outDir, progressCallback);
}
if (!_allPaths.Contains(outDir))
{
_allPaths.Add(outDir);
}
if (_pro.Project(prjItem, item, files[i], files[i], outFilename, resl, dataStr, _projectLog, _overlapPrj, "AIRS"))
{
if (progressCallback != null)
{
progressCallback(-1, "\t\t\t\t成功！投影处理成功！");
}
}
else
if (progressCallback != null)
{
progressCallback(-1, "\t\t\t\t失败!投影处理出错！");
}
}
if (progressCallback != null)
{
progressCallback(-1, files[i] + "投影结束！");
}
if (progressCallback != null)
{
progressCallback(-1, "-----------------------------------------------");
}
}
if (progressCallback != null)
{
progressCallback(-1, prjItem.Name + "区域投影结束！");
}
}
return;
}
if (progressCallback != null)
{
progressCallback(-1, "没有可用的投影区域！");
}
}
private void DoProcess()
{
if (_originFiles2Base.Count != 0)
{
_originFiles2Base.Clear();
}
if (cbxPrdsLevl.SelectedIndex == 0)//原始数据
{
try
{
string subdataDocDir = Path.Combine(_dataDocDir, "原始数据");
#region 原始数据入库
//_logfName = "RawData2Database";
if (radiMODIS.Checked)
{
GetMODOriginDataFiles(_state);
if (!_originFiles2Base.Keys.Contains("MOD06") || _originFiles2Base["MOD06"].Count < 1)
{
txtErrorLog.Text += "当前路径不存在可入库的MODIS数据！\r\n";
}
}
else if (radiAIRS.Checked)
{
GetAIRSOriginDataFiles(_state);
if (!_originFiles2Base.Keys.Contains("AIRS") || _originFiles2Base["AIRS"].Count < 1)
{
txtErrorLog.Text += "当前路径不存在可入库的AIRS数据！\r\n";
}
}
else if (radiISCCP.Checked)
{
GetISCCPOriginDataFiles(_state);
if (_originFiles2Base.Keys.Contains("ISCCP"))
{
if (_state != null)
{
_state("共计" + _originFiles2Base["ISCCP"].Count + "个待入库ISCCP文件！");
}
else
{
txtErrorLog.Text += "共计" + _originFiles2Base["ISCCP"].Count + "个待入库ISCCP文件！\r\n";
}
}
}
else if (radiCloudSAT.Checked)
{
GetCloudSATOriginDataFiles(_state);
if (!_originFiles2Base.Keys.Contains("CloudSAT") || _originFiles2Base["CloudSAT"].Count < 1)
{
txtErrorLog.Text += "当前路径不存在可入库的CloudSAT数据！\r\n";
}
}
if (_originFiles2Base.Count == 0)
{
MessageBox.Show("当前目录不存在可入库的文件！请重新选择");
return;
}
#region 数据归档
//Dictionary<string, List<string>> uniformOriginFiles = new Dictionary<string, List<string>>();
//foreach (string mode in _originFiles2Base.Keys)
//{
// if (_state != null)
// _state(string.Format("正在归档{0}数据，请稍候...", mode));
// List<string> unifiles = FileToDatabase.Files2UniformDir(_originFiles2Base[mode], _dataDocDir, mode, _state);
// uniformOriginFiles.Add(mode, unifiles);
//}
#endregion
try
{
OriginData2Database OriDbase = new OriginData2Database(subdataDocDir, _originFiles2Base, cbxOverrideRecord.Checked, _state);
OriDbase.IsFiles2UniformDir = cbxData2DocDir.Checked;
OriDbase._DocDir = _dataDocDir;
runTaskThread1 = new Thread(new ThreadStart(OriDbase.CheckFile2Table));
runTaskThread1.IsBackground = true;
if (_state != null)
{
_state(string.Format("开始将数据归档并入库，请稍候..."));
}
runTaskThread1.Start();
}
catch (System.Exception ex)
{
_state(ex.Message);
return;
}
#endregion
}
catch (System.Exception ex)
{
_state(ex.Message);
return;
}
}
else if (cbxPrdsLevl.SelectedIndex == 1)//历史日产品数据
{
#region 日产品数据入库
string fileFilter, sensor;
if (radiAIRS.Checked)
{
fileFilter = "*AIRS*day*.ldf";//文件名的格式有待确认
sensor = "AIRS";
}
else
{
fileFilter = "mod06_*_*.dat";//文件名的格式有待确认
sensor = "MODIS";
}
List <string> dayprdsfiles = Directory.GetFiles(_inputDir, fileFilter, SearchOption.AllDirectories).ToList();
//归档；
//从文件名中解析数据的时间信息、数据集信息、区域信息、分辨率信息、日夜标识信息
//拼接生成数据的归档路径，对数据按照现有的规则进行重命名；//日拼接产品\CloudTopTemperature\TERRA\MODIS\2011\1\day\0.05
List <string> unidayprdsfiles = new List <string>();
//string newfnameformat = "{0}_MOD06_china_day_{1}_0.01.dat";
string newfname = "";
foreach (string oldfile in dayprdsfiles)
{
//mod06_20131201_CMLF.dat
//CloudOpticalThickness_MOD06_china_day_20110101_0.01.LDF
System.IO.File.Move(oldfile, newfname);
}
//入库---注意数据中的无效值问题
//需要查询数据集对应的产品ID及数据集ID，regionID，
//double selfFillValue = _selfFillValue[setName];
//double dayFillValue = _dayFillValue[setName];
//double dayInvalidValue = _dayInvalidValue[setName];
//重新计算有效百分比，granulesCount/granulesTimes、日夜标识DataSource
#region 入库
//归档路径
string tableName = "CP_DayMergeProducts_TB";
ConnectMySqlCloud dbcon = new ConnectMySqlCloud(_dataBaseXml);
string setName, imagedata, regionName, fname;
float resl;
double invalidValue = 32767;
int validPercent, regionID = 1;;
long prdID, datasetID = 0;
string datasource = "";
string[] parts;
DateTime fdate;
DataProcesser datapro = new DataProcesser();
foreach (string newFileName in unidayprdsfiles)
{
imagedata = newFileName.Replace(_dataDocDir, "");
fname = Path.GetFileNameWithoutExtension(newFileName);
parts = fname.Split('_');
if (parts.Length != 6)
{
continue;
}
setName = parts[0];
if (!dbcon.QueryDatasetsID(sensor, setName, out prdID, out datasetID))
{
continue;
}
regionName = parts[2];
if (!dbcon.QueryRegionID(regionName, out regionID))
{
}
if (parts[3].ToLower() == "day")
{
datasource = "D";
}
else
{
datasource = "N";
}
//fdate =DataProcesser.getDayTime(newFileName);
fdate = DateTime.TryParseExact(Path.GetFileNameWithoutExtension(newFileName).Split('_')[4], "yyyyMMdd", System.Globalization.CultureInfo.CurrentCulture, System.Globalization.DateTimeStyles.None, out fdate) ? fdate : DateTime.MinValue;
validPercent = ComputeValidPercent(datapro, newFileName, invalidValue, out resl);
if (!dbcon.IshasRecord(tableName, "ImageName", Path.GetFileName(newFileName)))
{
dbcon.InsertNDayMergeProductsTable(fdate, prdID, datasetID, imagedata, regionID, regionName, sensor, resl, validPercent, 0, "", datasource);
_state(newFileName + "入库完成！");
}
else
{
dbcon.DeleteCLDParatableRecord(tableName, "ImageName", Path.GetFileName(newFileName));
dbcon.InsertNDayMergeProductsTable(fdate, prdID, datasetID, imagedata, regionID, regionName, sensor, resl, validPercent, 0, "", datasource);
_state(newFileName + "入库更新完成！");
}
}
#endregion
#endregion
}
else if (cbxPrdsLevl.SelectedIndex == 2)//周期合成产品
{
//PeriodicSynPrds2Base psp = new PeriodicSynPrds2Base(_inputDir, _logfName, _dataBaseXml);
//psp.StartComp();
}
}
