private void StatisticTen(string dirXun, StringBuilder outnamePrefix, string[] filesSelect, RasterStatics sta, string resl, string label = "day")
{
try
{
string flabel = string.Format("_{0}_{1}.LDF", label, resl);
foreach (string stype in _statics)
{
string outname = "";
if (_statics.Contains("MAX") || _statics.Contains("max"))
{
StringBuilder outmaxname = new StringBuilder();
outmaxname.Append(dirXun).Append("\\Max\\").Append(outnamePrefix.ToString()).Append("_MAX").Append(flabel);
outname = outmaxname.ToString();
}
else if (_statics.Contains("MIN") || _statics.Contains("min"))
{
StringBuilder outminname = new StringBuilder();
outminname.Append(dirXun).Append("\\Min\\").Append(outnamePrefix.ToString()).Append("_MIN").Append(flabel);
outname = outminname.ToString();
}
else if (_statics.Contains("AVG") || _statics.Contains("avg"))
{
StringBuilder outAvgname = new StringBuilder();
outAvgname.Append(dirXun).Append("\\Avg\\").Append(outnamePrefix.ToString()).Append("_AVG").Append(flabel);
outname = outAvgname.ToString();
}
if (File.Exists(outname))
{
if (_overwritefiles == false && _con.IshasRecord(_tableName, "ImageName", Path.GetFileName(outname)))
{
continue;
}
else
{
File.Delete(outname);
}
}
if (!Directory.Exists(Path.GetDirectoryName(outname)))
{
Directory.CreateDirectory(Path.GetDirectoryName(outname));
}
if (sta.PeriodicAvgStat(filesSelect, outname))
{
if (File.Exists(outname))// && !_con.IshasRecord(_tableName, "ImageName", Path.GetFileName(outname)))
{
_periodicsyn.TryCreatePeriodRecord(outname);
}
}
}
}
catch (Exception e)
{
throw e;
//LogFactory.WriteLine(_dataOrigin, e.Message + "\n");
}
}
private void DeleteFiles(string [] files2clear, bool isUpdateDB = false, string tablename = null)
{
foreach (string file in files2clear)
{
try
{
System.IO.File.Delete(file);
if (_state != null)
{
_state(file + "清理完成！");
}
if (isUpdateDB && tablename != null && _DBcon.IshasRecord(tablename, "ImageName", Path.GetFileName(file)))
{
_DBcon.DeleteCLDParatableRecord(tablename, "ImageName", Path.GetFileName(file));
if (_state != null)
{
_state(file + "数据库记录清理完成！");
}
}
}
catch (System.Exception ex)
{
continue;
}
}
if (isUpdateDB && tablename.ToLower() == "cp_cloudsat_tb")
{
_DBcon.UpdateLink();
}
}
private void DoProcess()
{
if (_originFiles2Base.Count != 0)
{
_originFiles2Base.Clear();
}
if (cbxPrdsLevl.SelectedIndex == 0)//原始数据
{
try
{
string subdataDocDir = Path.Combine(_dataDocDir, "原始数据");
#region 原始数据入库
//_logfName = "RawData2Database";
if (radiMODIS.Checked)
{
GetMODOriginDataFiles(_state);
if (!_originFiles2Base.Keys.Contains("MOD06") || _originFiles2Base["MOD06"].Count < 1)
{
txtErrorLog.Text += "当前路径不存在可入库的MODIS数据！\r\n";
}
}
else if (radiAIRS.Checked)
{
GetAIRSOriginDataFiles(_state);
if (!_originFiles2Base.Keys.Contains("AIRS") || _originFiles2Base["AIRS"].Count < 1)
{
txtErrorLog.Text += "当前路径不存在可入库的AIRS数据！\r\n";
}
}
else if (radiISCCP.Checked)
{
GetISCCPOriginDataFiles(_state);
if (_originFiles2Base.Keys.Contains("ISCCP"))
{
if (_state != null)
{
_state("共计" + _originFiles2Base["ISCCP"].Count + "个待入库ISCCP文件！");
}
else
{
txtErrorLog.Text += "共计" + _originFiles2Base["ISCCP"].Count + "个待入库ISCCP文件！\r\n";
}
}
}
else if (radiCloudSAT.Checked)
{
GetCloudSATOriginDataFiles(_state);
if (!_originFiles2Base.Keys.Contains("CloudSAT") || _originFiles2Base["CloudSAT"].Count < 1)
{
txtErrorLog.Text += "当前路径不存在可入库的CloudSAT数据！\r\n";
}
}
if (_originFiles2Base.Count == 0)
{
MessageBox.Show("当前目录不存在可入库的文件！请重新选择");
return;
}
#region 数据归档
//Dictionary<string, List<string>> uniformOriginFiles = new Dictionary<string, List<string>>();
//foreach (string mode in _originFiles2Base.Keys)
//{
// if (_state != null)
// _state(string.Format("正在归档{0}数据，请稍候...", mode));
// List<string> unifiles = FileToDatabase.Files2UniformDir(_originFiles2Base[mode], _dataDocDir, mode, _state);
// uniformOriginFiles.Add(mode, unifiles);
//}
#endregion
try
{
OriginData2Database OriDbase = new OriginData2Database(subdataDocDir, _originFiles2Base, cbxOverrideRecord.Checked, _state);
OriDbase.IsFiles2UniformDir = cbxData2DocDir.Checked;
OriDbase._DocDir = _dataDocDir;
runTaskThread1 = new Thread(new ThreadStart(OriDbase.CheckFile2Table));
runTaskThread1.IsBackground = true;
if (_state != null)
{
_state(string.Format("开始将数据归档并入库，请稍候..."));
}
runTaskThread1.Start();
}
catch (System.Exception ex)
{
_state(ex.Message);
return;
}
#endregion
}
catch (System.Exception ex)
{
_state(ex.Message);
return;
}
}
else if (cbxPrdsLevl.SelectedIndex == 1)//历史日产品数据
{
#region 日产品数据入库
string fileFilter, sensor;
if (radiAIRS.Checked)
{
fileFilter = "*AIRS*day*.ldf";//文件名的格式有待确认
sensor = "AIRS";
}
else
{
fileFilter = "mod06_*_*.dat";//文件名的格式有待确认
sensor = "MODIS";
}
List <string> dayprdsfiles = Directory.GetFiles(_inputDir, fileFilter, SearchOption.AllDirectories).ToList();
//归档；
//从文件名中解析数据的时间信息、数据集信息、区域信息、分辨率信息、日夜标识信息
//拼接生成数据的归档路径，对数据按照现有的规则进行重命名；//日拼接产品\CloudTopTemperature\TERRA\MODIS\2011\1\day\0.05
List <string> unidayprdsfiles = new List <string>();
//string newfnameformat = "{0}_MOD06_china_day_{1}_0.01.dat";
string newfname = "";
foreach (string oldfile in dayprdsfiles)
{
//mod06_20131201_CMLF.dat
//CloudOpticalThickness_MOD06_china_day_20110101_0.01.LDF
System.IO.File.Move(oldfile, newfname);
}
//入库---注意数据中的无效值问题
//需要查询数据集对应的产品ID及数据集ID，regionID，
//double selfFillValue = _selfFillValue[setName];
//double dayFillValue = _dayFillValue[setName];
//double dayInvalidValue = _dayInvalidValue[setName];
//重新计算有效百分比，granulesCount/granulesTimes、日夜标识DataSource
#region 入库
//归档路径
string tableName = "CP_DayMergeProducts_TB";
ConnectMySqlCloud dbcon = new ConnectMySqlCloud(_dataBaseXml);
string setName, imagedata, regionName, fname;
float resl;
double invalidValue = 32767;
int validPercent, regionID = 1;;
long prdID, datasetID = 0;
string datasource = "";
string[] parts;
DateTime fdate;
DataProcesser datapro = new DataProcesser();
foreach (string newFileName in unidayprdsfiles)
{
imagedata = newFileName.Replace(_dataDocDir, "");
fname = Path.GetFileNameWithoutExtension(newFileName);
parts = fname.Split('_');
if (parts.Length != 6)
{
continue;
}
setName = parts[0];
if (!dbcon.QueryDatasetsID(sensor, setName, out prdID, out datasetID))
{
continue;
}
regionName = parts[2];
if (!dbcon.QueryRegionID(regionName, out regionID))
{
}
if (parts[3].ToLower() == "day")
{
datasource = "D";
}
else
{
datasource = "N";
}
//fdate =DataProcesser.getDayTime(newFileName);
fdate = DateTime.TryParseExact(Path.GetFileNameWithoutExtension(newFileName).Split('_')[4], "yyyyMMdd", System.Globalization.CultureInfo.CurrentCulture, System.Globalization.DateTimeStyles.None, out fdate) ? fdate : DateTime.MinValue;
validPercent = ComputeValidPercent(datapro, newFileName, invalidValue, out resl);
if (!dbcon.IshasRecord(tableName, "ImageName", Path.GetFileName(newFileName)))
{
dbcon.InsertNDayMergeProductsTable(fdate, prdID, datasetID, imagedata, regionID, regionName, sensor, resl, validPercent, 0, "", datasource);
_state(newFileName + "入库完成！");
}
else
{
dbcon.DeleteCLDParatableRecord(tableName, "ImageName", Path.GetFileName(newFileName));
dbcon.InsertNDayMergeProductsTable(fdate, prdID, datasetID, imagedata, regionID, regionName, sensor, resl, validPercent, 0, "", datasource);
_state(newFileName + "入库更新完成！");
}
}
#endregion
#endregion
}
else if (cbxPrdsLevl.SelectedIndex == 2)//周期合成产品
{
//PeriodicSynPrds2Base psp = new PeriodicSynPrds2Base(_inputDir, _logfName, _dataBaseXml);
//psp.StartComp();
}
}
private void ISCCPRawDataF2Database(string tableName, List <string> filelist, Action <string> ProBack)
{
ProBack("-------------------------------------------------");
ProBack("ISCCP原始数据入库开始：");
//ISCCP.D2.0.GLOBAL.2008.07.99.0000.GPC
string fullDir, fName, localfName;
int year, month, utcTime;
DateTime fDateTime = DateTime.MinValue;
string pattern = @".(?<year>\d{4}).(?<month>\d{2}).(?<day>\d{2}).(?<utc>\d{4})";
foreach (string file in filelist)
{
fName = Path.GetFileName(file);
ProBack("\t" + fName + "入库开始：");
fullDir = Path.GetDirectoryName(file);
if (Directory.Exists(new DirectoryInfo(_inputDir).Parent.FullName))
{
localfName = file.Replace(new DirectoryInfo(_inputDir).Parent.FullName, "");
}
else
{
localfName = file.Replace(_inputDir, "");
}
Match match = Regex.Match(fName, pattern);
if (match.Success)
{
//LogFactory.WriteLine(_log, "\t\t解析文件名时间信息成功！");
//ProBack("\t\t解析文件名时间信息成功！");
year = int.Parse(match.Groups["year"].Value);
month = int.Parse(match.Groups["month"].Value);
fDateTime = new DateTime(year, month, 1);
utcTime = int.Parse(match.Groups["utc"].Value);
try
{
//if (_dbConnect.IshasRecord(tableName, "ImageName", fName) == false)
if (_dbConnect.IshasRecord(tableName, "ImageData", localfName) == false)
{
_dbConnect.InsertISCCP2Table(tableName, fDateTime, utcTime, fName, localfName); //插入,insert
ProBack("\t\t\t文件入库成功！");
continue;
}
else
{
//if(_dbConnect.UpdateRawCLDParatable(tableName, fName, localfName))
// ProBack("\t\t\t文件记录更新成功！");
//else
ProBack("\t\t\t文件记录已存在，跳过！");
}
}
catch (System.Exception ex)
{
ProBack("入库错误" + ex.Message);
}
}
else
{
ProBack("\t\t解析文件名时间失败，文件入库失败！");
}
}
ProBack("ISCCP原始数据入库结束！");
ProBack("--------------------------------------------------------------------------------------");
}
public void TryCreatePeriodRecord(string file, Action <int, string> progressCallback = null)
{
string fname, localfname, region, sensor = "", statName, datasetname, period;
string[] parts;
long prdID = 0, peridID, staticsID, datasetID;
int year = 0, month = 0, xun = 0, regionID;
//double validPercent;
//CloudTopTemperature_MOD06_china_Ten_2011-11-2_AVG_day_0.05.LDF
//CloudTopTemperature_MOD06_china_Month_2011-11_AVG_day_0.05.LDF
//CloudTopTemperature_MOD06_china_Year_2011_AVG_day_0.05.LDF
if (progressCallback != null)
{
progressCallback(-1, "\t\t\t\t入库开始...," + file);
}
fname = Path.GetFileNameWithoutExtension(file);
localfname = file.Replace(_inputDIR, "");
parts = fname.Split('_');
period = parts[3].ToLower();
if (period != "ten" && period != "month" && period != "year")
{
return;
}
if (!_periodTypeID.ContainsKey(period))
{
return;
}
peridID = _periodTypeID[period];//周期类型
string sta = parts[5].ToUpper();
if (sta != "MIN" && sta != "MAX" && sta != "AVG")
{
return;
}
statName = sta;
staticsID = _statName2ID[sta];//统计类型
string label = parts[1].ToUpper();
if (label == "MOD06" || label == "MYD06" || label == "AIRS")
{
sensor = label;
}
else
{
return;
}
datasetname = parts[0];
if (_dbConnect.QueryDatasetsID(sensor, datasetname, out prdID, out datasetID))
{
region = parts[2];
if (!_regionID.Keys.Contains(region))
{
if (!_dbConnect.QueryRegionID(region.ToLower(), out regionID))
{
return;
}
_regionID.Add(region, regionID);
}
else
{
regionID = _regionID[region];
}
if (regionID == 0)
{
regionID = 1;
}
if (period == "ten")
{
Regex XunDateReg = new Regex(@"(?<year>\d{4})-(?<month>\d{2})-(?<Xun>\d{1})", RegexOptions.Compiled);
Match match = XunDateReg.Match(parts[4]);
if (!match.Success)
{
return;
}
year = Int16.Parse(match.Groups["year"].Value);
month = Int16.Parse(match.Groups["month"].Value);
xun = Int16.Parse(match.Groups["Xun"].Value);
}
else if (period == "month")
{
Regex MonthDateReg = new Regex(@"(?<year>\d{4})-(?<month>\d{2})", RegexOptions.Compiled);
Match match = MonthDateReg.Match(parts[4]);
if (!match.Success)
{
return;
}
year = Int16.Parse(match.Groups["year"].Value);
month = Int16.Parse(match.Groups["month"].Value);
}
else//period == "year"
{
year = Int16.Parse(parts[4]);
}
//GetRsEnvolop(file, out resol, out validPercent);
float resol = float.Parse(parts[parts.Length - 1]);
string datasource = "";
if (parts[parts.Length - 2].ToLower() == "day")
{
datasource = "D";
}
else
{
datasource = "N";
}
if (!_dbConnect.IshasRecord(_tableName, "ImageName", Path.GetFileName(file)))
{
_dbConnect.InsertPeriodicSynProductsTable(prdID, datasetID, localfname, regionID, region, sensor, resol, peridID, year, month, xun, _statName2ID[statName], datasource);
}
else
{
_dbConnect.DeleteCLDParatableRecord(_tableName, "ImageName", Path.GetFileName(file));
_dbConnect.InsertPeriodicSynProductsTable(prdID, datasetID, localfname, regionID, region, sensor, resol, peridID, year, month, xun, _statName2ID[statName], datasource);
}
}
}
private void StatisticTen(string dirXun, StringBuilder outnamePrefix, string[] filesSelect, RasterStatics sta, string resl, string label = "day", Action <int, string> progressCallback = null)
{
try
{
string flabel = string.Format("_{0}_{1}.LDF", label, resl);
int statistic = -1;
StringBuilder outnamebuilder = new StringBuilder();
string outname = "";
foreach (string stype in _statics)
{
outnamebuilder.Clear();
if (_statics.Contains("MAX") || _statics.Contains("max"))
{
statistic = 0;
outnamebuilder.Append(dirXun).Append("\\Max\\").Append(outnamePrefix.ToString()).Append("_MAX").Append(flabel);
outname = outnamebuilder.ToString();
}
else if (_statics.Contains("MIN") || _statics.Contains("min"))
{
statistic = 1;
outnamebuilder.Append(dirXun).Append("\\Min\\").Append(outnamePrefix.ToString()).Append("_MIN").Append(flabel);
outname = outnamebuilder.ToString();
}
else if (_statics.Contains("AVG") || _statics.Contains("avg"))
{
statistic = 2;
outnamebuilder.Append(dirXun).Append("\\Avg\\").Append(outnamePrefix.ToString()).Append("_AVG").Append(flabel);
outname = outnamebuilder.ToString();
}
if (progressCallback != null)
{
progressCallback(-1, "\t\t\t\t开始合成" + stype + "旬产品,输出文件：" + outname);
}
if (File.Exists(outname))
{
if (_overwritefiles == false && _con.IshasRecord(_tableName, "ImageName", Path.GetFileName(outname)))
{
if (!File.Exists(Path.Combine(Path.GetDirectoryName(outname), Path.ChangeExtension(outname, ".overview.png"))))
{
OverViewHelper.OverView(outname, 800);
}
if (progressCallback != null)
{
progressCallback(-1, "\t\t\t\t文件已存在，跳过合成处理！");//+ Path.GetFileName(outname)
}
continue;
}
else
{
File.Delete(outname);
if (progressCallback != null)
{
progressCallback(-1, "\t\t\t\t重新合成处理开始..."); //+ Path.GetFileName(outname)
}
}
}
if (!Directory.Exists(Path.GetDirectoryName(outname)))
{
Directory.CreateDirectory(Path.GetDirectoryName(outname));
}
if ((statistic == 0 && !sta.PeriodicMaxStat(filesSelect, outname, progressCallback)) || (statistic == 1 && !sta.PeriodicMinStat(filesSelect, outname, progressCallback)) ||
(statistic == 2 && !sta.PeriodicAvgStat(filesSelect, outname, null, progressCallback)))
{
if (progressCallback != null)
{
progressCallback(-1, "\t\t\t\t合成处理失败！");// + Path.GetFileName(outname));
}
if (File.Exists(outname))
{
File.Delete(outname);
if (progressCallback != null)
{
progressCallback(-1, "\t\t\t\t删除合成失败文件！");// + Path.GetFileName(outname));
}
}
continue;
}
if (File.Exists(outname))
{
//生成快视图
string filename = OverViewHelper.OverView(outname, 800);
if (progressCallback != null && File.Exists(filename))
{
progressCallback(-1, "\t\t\t\t生成快视图成功！");// 文件:+ Path.GetFileName(outname));
}
if (!_con.IshasRecord(_tableName, "ImageName", Path.GetFileName(outname)))
{
TryCreatePeriodRecord(outname, progressCallback);
if (progressCallback != null && File.Exists(filename))
{
progressCallback(-1, "\t\t\t\t入库成功！");//文件:+ Path.GetFileName(outname));
}
}
else
{
if (progressCallback != null && File.Exists(filename))
{
progressCallback(-1, "\t\t\t\t数据库记录已存在，入库跳过！");//文件:+ Path.GetFileName(outname));
}
}
}
}
}
catch (Exception e)
{
if (progressCallback != null)
{
progressCallback(-5, "合成" + dirXun + "的旬产品失败！" + e.Message);
}
throw e;
}
}
public void Compute(Action <string> ProBack)
{
float[] lats = null, lons = null;
double minX, maxX, minY, maxY;
foreach (long regionid in _allRegions.Keys)
{
PrjEnvelopeItem item = _allRegions[regionid];
minX = item.PrjEnvelope.MinX; //10
maxX = item.PrjEnvelope.MaxX; //60
minY = item.PrjEnvelope.MinY; //65
maxY = item.PrjEnvelope.MaxY; //145
ProBack("\t开始进行CloudSAT数据与" + item.Name + "区域的关联计算...");
Dictionary <Int32, double> granuleNOPct = new Dictionary <Int32, double>();
Dictionary <long, Double> filePct = new Dictionary <long, Double>();
Dictionary <Int32, Int32> granuleNOAllCount = new Dictionary <Int32, Int32>();
string[] fNameParts;
//string fName;
int granuleNO;
float uplat, uplon, downlat, downlon;
foreach (string file in _file2ID.Keys)
{
try
{
//2007101052404_05066_CS_MODIS-AUX_GRANULE_P_R04_E02.hdf
string fname = Path.GetFileName(file);
ProBack("\t\t开始进行" + file + "与" + item.Name + "区域的关联计算...");
fNameParts = Path.GetFileNameWithoutExtension(file).Split(new char[] { '_' }, StringSplitOptions.RemoveEmptyEntries);
if (fNameParts.Length != 8)
{
continue;
}
if (fNameParts[2] != "CS")
{
continue;
}
granuleNO = int.Parse(fNameParts[1]);
if (!granuleNOPct.ContainsKey(granuleNO) && fNameParts[3].ToUpper() != "MODIS-AUX")//每个轨道号数据具有同一个覆盖度
{
int inCount = 0;
CloudsatDataProvider locRst = null;
ProBack("\t\t\t读取" + fname + "的经纬度数据集...");
using (locRst = GeoDataDriver.Open(file) as CloudsatDataProvider)
{
lats = locRst.ReadVdata("Latitude", null, 0, locRst.Height) as float[];
lons = locRst.ReadVdata("Longitude", null, 0, locRst.Height) as float[];
}
ProBack("\t\t\t开始计算" + fname + "与区域的相交...");
int firstpt = -1, lastpt = -1;
for (int i = 0; i < lats.Length; i++)
{
if (lats[i] >= minY && lats[i] <= maxY && lons[i] >= minX && lons[i] <= maxX)
{
if (firstpt == -1)
{
firstpt = i;
}
lastpt = i;
inCount++;
}
}
if (firstpt != -1)//存在覆盖该区域的点
{
#region 计算南点和北点的经纬度
if (firstpt != lastpt)//多于一个点
{
if (lats[firstpt] > lats[lastpt])
{
uplat = lats[firstpt];
uplon = lons[firstpt];
downlat = lats[lastpt];
downlon = lons[lastpt];
}
else
{
uplat = lats[lastpt];
uplon = lons[lastpt];
downlat = lats[firstpt];
downlon = lons[firstpt];
}
}
else
{
uplat = lats[firstpt];
uplon = lons[firstpt];
downlat = lats[firstpt];
downlon = lons[firstpt];
}
#endregion
#region 入库
//granuleNO,regionid,uplat,uplon,downlat,downlon,firstpt,lastpt
if (!_con.IshasRecord("cp_cloudsatinregionlatlon_tb", "GranuleNumber", granuleNO))
{
_con.InsertCloudsatGranule2Region(granuleNO, regionid, uplat, uplon, downlat, downlon, firstpt, lastpt);
}
ProBack("\t\t\t将" + granuleNO + "起始范围计算结果入库成功！");
#endregion
}
granuleNOPct.Add(granuleNO, inCount / (lats.Length * 1.0) * 100.0);
granuleNOAllCount.Add(granuleNO, lats.Length);
}
if (granuleNOAllCount.ContainsKey(granuleNO))
{
if (!_con.IshasRecord("cp_cloudsat2region_tb", "CloudSatID", _file2ID[file]))
{
_con.InsertCloudsat2RegionTable(_file2ID[file], granuleNOAllCount[granuleNO], regionid, granuleNOPct[granuleNO]);
}
ProBack("\t\t将" + fname + "计算结果入库成功！");
}
}
catch (System.Exception ex)
{
ProBack("\t" + file + "处理出错！" + ex.Message);
}
}
}
}
