예제 #1
0
 private void DeleteFiles(string [] files2clear, bool isUpdateDB = false, string tablename = null)
 {
     foreach (string file in files2clear)
     {
         try
         {
             System.IO.File.Delete(file);
             if (_state != null)
             {
                 _state(file + "清理完成!");
             }
             if (isUpdateDB && tablename != null && _DBcon.IshasRecord(tablename, "ImageName", Path.GetFileName(file)))
             {
                 _DBcon.DeleteCLDParatableRecord(tablename, "ImageName", Path.GetFileName(file));
                 if (_state != null)
                 {
                     _state(file + "数据库记录清理完成!");
                 }
             }
         }
         catch (System.Exception ex)
         {
             continue;
         }
     }
     if (isUpdateDB && tablename.ToLower() == "cp_cloudsat_tb")
     {
         _DBcon.UpdateLink();
     }
 }
예제 #2
0
 private void DoProcess()
 {
     if (_originFiles2Base.Count != 0)
     {
         _originFiles2Base.Clear();
     }
     if (cbxPrdsLevl.SelectedIndex == 0)//原始数据
     {
         try
         {
             string subdataDocDir = Path.Combine(_dataDocDir, "原始数据");
             #region 原始数据入库
             //_logfName = "RawData2Database";
             if (radiMODIS.Checked)
             {
                 GetMODOriginDataFiles(_state);
                 if (!_originFiles2Base.Keys.Contains("MOD06") || _originFiles2Base["MOD06"].Count < 1)
                 {
                     txtErrorLog.Text += "当前路径不存在可入库的MODIS数据!\r\n";
                 }
             }
             else if (radiAIRS.Checked)
             {
                 GetAIRSOriginDataFiles(_state);
                 if (!_originFiles2Base.Keys.Contains("AIRS") || _originFiles2Base["AIRS"].Count < 1)
                 {
                     txtErrorLog.Text += "当前路径不存在可入库的AIRS数据!\r\n";
                 }
             }
             else if (radiISCCP.Checked)
             {
                 GetISCCPOriginDataFiles(_state);
                 if (_originFiles2Base.Keys.Contains("ISCCP"))
                 {
                     if (_state != null)
                     {
                         _state("共计" + _originFiles2Base["ISCCP"].Count + "个待入库ISCCP文件!");
                     }
                     else
                     {
                         txtErrorLog.Text += "共计" + _originFiles2Base["ISCCP"].Count + "个待入库ISCCP文件!\r\n";
                     }
                 }
             }
             else if (radiCloudSAT.Checked)
             {
                 GetCloudSATOriginDataFiles(_state);
                 if (!_originFiles2Base.Keys.Contains("CloudSAT") || _originFiles2Base["CloudSAT"].Count < 1)
                 {
                     txtErrorLog.Text += "当前路径不存在可入库的CloudSAT数据!\r\n";
                 }
             }
             if (_originFiles2Base.Count == 0)
             {
                 MessageBox.Show("当前目录不存在可入库的文件!请重新选择");
                 return;
             }
             #region 数据归档
             //Dictionary<string, List<string>> uniformOriginFiles = new Dictionary<string, List<string>>();
             //foreach (string mode in _originFiles2Base.Keys)
             //{
             //    if (_state != null)
             //        _state(string.Format("正在归档{0}数据,请稍候...", mode));
             //    List<string> unifiles = FileToDatabase.Files2UniformDir(_originFiles2Base[mode], _dataDocDir, mode, _state);
             //    uniformOriginFiles.Add(mode, unifiles);
             //}
             #endregion
             try
             {
                 OriginData2Database OriDbase = new OriginData2Database(subdataDocDir, _originFiles2Base, cbxOverrideRecord.Checked, _state);
                 OriDbase.IsFiles2UniformDir = cbxData2DocDir.Checked;
                 OriDbase._DocDir            = _dataDocDir;
                 runTaskThread1 = new Thread(new ThreadStart(OriDbase.CheckFile2Table));
                 runTaskThread1.IsBackground = true;
                 if (_state != null)
                 {
                     _state(string.Format("开始将数据归档并入库,请稍候..."));
                 }
                 runTaskThread1.Start();
             }
             catch (System.Exception ex)
             {
                 _state(ex.Message);
                 return;
             }
             #endregion
         }
         catch (System.Exception ex)
         {
             _state(ex.Message);
             return;
         }
     }
     else if (cbxPrdsLevl.SelectedIndex == 1)//历史日产品数据
     {
         #region 日产品数据入库
         string fileFilter, sensor;
         if (radiAIRS.Checked)
         {
             fileFilter = "*AIRS*day*.ldf";//文件名的格式有待确认
             sensor     = "AIRS";
         }
         else
         {
             fileFilter = "mod06_*_*.dat";//文件名的格式有待确认
             sensor     = "MODIS";
         }
         List <string> dayprdsfiles = Directory.GetFiles(_inputDir, fileFilter, SearchOption.AllDirectories).ToList();
         //归档;
         //从文件名中解析数据的时间信息、数据集信息、区域信息、分辨率信息、日夜标识信息
         //拼接生成数据的归档路径,对数据按照现有的规则进行重命名;//日拼接产品\CloudTopTemperature\TERRA\MODIS\2011\1\day\0.05
         List <string> unidayprdsfiles = new List <string>();
         //string newfnameformat = "{0}_MOD06_china_day_{1}_0.01.dat";
         string newfname = "";
         foreach (string oldfile in dayprdsfiles)
         {
             //mod06_20131201_CMLF.dat
             //CloudOpticalThickness_MOD06_china_day_20110101_0.01.LDF
             System.IO.File.Move(oldfile, newfname);
         }
         //入库---注意数据中的无效值问题
         //需要查询数据集对应的产品ID及数据集ID,regionID,
         //double selfFillValue = _selfFillValue[setName];
         //double dayFillValue = _dayFillValue[setName];
         //double dayInvalidValue = _dayInvalidValue[setName];
         //重新计算有效百分比,granulesCount/granulesTimes、日夜标识DataSource
         #region 入库
         //归档路径
         string            tableName = "CP_DayMergeProducts_TB";
         ConnectMySqlCloud dbcon = new ConnectMySqlCloud(_dataBaseXml);
         string            setName, imagedata, regionName, fname;
         float             resl;
         double            invalidValue = 32767;
         int           validPercent, regionID = 1;;
         long          prdID, datasetID = 0;
         string        datasource = "";
         string[]      parts;
         DateTime      fdate;
         DataProcesser datapro = new DataProcesser();
         foreach (string newFileName in unidayprdsfiles)
         {
             imagedata = newFileName.Replace(_dataDocDir, "");
             fname     = Path.GetFileNameWithoutExtension(newFileName);
             parts     = fname.Split('_');
             if (parts.Length != 6)
             {
                 continue;
             }
             setName = parts[0];
             if (!dbcon.QueryDatasetsID(sensor, setName, out prdID, out datasetID))
             {
                 continue;
             }
             regionName = parts[2];
             if (!dbcon.QueryRegionID(regionName, out regionID))
             {
             }
             if (parts[3].ToLower() == "day")
             {
                 datasource = "D";
             }
             else
             {
                 datasource = "N";
             }
             //fdate =DataProcesser.getDayTime(newFileName);
             fdate        = DateTime.TryParseExact(Path.GetFileNameWithoutExtension(newFileName).Split('_')[4], "yyyyMMdd", System.Globalization.CultureInfo.CurrentCulture, System.Globalization.DateTimeStyles.None, out fdate) ? fdate : DateTime.MinValue;
             validPercent = ComputeValidPercent(datapro, newFileName, invalidValue, out resl);
             if (!dbcon.IshasRecord(tableName, "ImageName", Path.GetFileName(newFileName)))
             {
                 dbcon.InsertNDayMergeProductsTable(fdate, prdID, datasetID, imagedata, regionID, regionName, sensor, resl, validPercent, 0, "", datasource);
                 _state(newFileName + "入库完成!");
             }
             else
             {
                 dbcon.DeleteCLDParatableRecord(tableName, "ImageName", Path.GetFileName(newFileName));
                 dbcon.InsertNDayMergeProductsTable(fdate, prdID, datasetID, imagedata, regionID, regionName, sensor, resl, validPercent, 0, "", datasource);
                 _state(newFileName + "入库更新完成!");
             }
         }
         #endregion
         #endregion
     }
     else if (cbxPrdsLevl.SelectedIndex == 2)//周期合成产品
     {
         //PeriodicSynPrds2Base psp = new PeriodicSynPrds2Base(_inputDir, _logfName, _dataBaseXml);
         //psp.StartComp();
     }
 }
예제 #3
0
        public void TryCreatePeriodRecord(string file, Action <int, string> progressCallback = null)
        {
            string fname, localfname, region, sensor = "", statName, datasetname, period;

            string[] parts;
            long     prdID = 0, peridID, staticsID, datasetID;
            int      year = 0, month = 0, xun = 0, regionID;

            //double validPercent;
            //CloudTopTemperature_MOD06_china_Ten_2011-11-2_AVG_day_0.05.LDF
            //CloudTopTemperature_MOD06_china_Month_2011-11_AVG_day_0.05.LDF
            //CloudTopTemperature_MOD06_china_Year_2011_AVG_day_0.05.LDF
            if (progressCallback != null)
            {
                progressCallback(-1, "\t\t\t\t入库开始...," + file);
            }
            fname      = Path.GetFileNameWithoutExtension(file);
            localfname = file.Replace(_inputDIR, "");
            parts      = fname.Split('_');
            period     = parts[3].ToLower();
            if (period != "ten" && period != "month" && period != "year")
            {
                return;
            }
            if (!_periodTypeID.ContainsKey(period))
            {
                return;
            }
            peridID = _periodTypeID[period];//周期类型
            string sta = parts[5].ToUpper();

            if (sta != "MIN" && sta != "MAX" && sta != "AVG")
            {
                return;
            }
            statName  = sta;
            staticsID = _statName2ID[sta];//统计类型
            string label = parts[1].ToUpper();

            if (label == "MOD06" || label == "MYD06" || label == "AIRS")
            {
                sensor = label;
            }
            else
            {
                return;
            }
            datasetname = parts[0];
            if (_dbConnect.QueryDatasetsID(sensor, datasetname, out prdID, out datasetID))
            {
                region = parts[2];
                if (!_regionID.Keys.Contains(region))
                {
                    if (!_dbConnect.QueryRegionID(region.ToLower(), out regionID))
                    {
                        return;
                    }
                    _regionID.Add(region, regionID);
                }
                else
                {
                    regionID = _regionID[region];
                }
                if (regionID == 0)
                {
                    regionID = 1;
                }
                if (period == "ten")
                {
                    Regex XunDateReg = new Regex(@"(?<year>\d{4})-(?<month>\d{2})-(?<Xun>\d{1})", RegexOptions.Compiled);
                    Match match      = XunDateReg.Match(parts[4]);
                    if (!match.Success)
                    {
                        return;
                    }
                    year  = Int16.Parse(match.Groups["year"].Value);
                    month = Int16.Parse(match.Groups["month"].Value);
                    xun   = Int16.Parse(match.Groups["Xun"].Value);
                }
                else if (period == "month")
                {
                    Regex MonthDateReg = new Regex(@"(?<year>\d{4})-(?<month>\d{2})", RegexOptions.Compiled);
                    Match match        = MonthDateReg.Match(parts[4]);
                    if (!match.Success)
                    {
                        return;
                    }
                    year  = Int16.Parse(match.Groups["year"].Value);
                    month = Int16.Parse(match.Groups["month"].Value);
                }
                else//period == "year"
                {
                    year = Int16.Parse(parts[4]);
                }
                //GetRsEnvolop(file, out resol, out validPercent);
                float  resol      = float.Parse(parts[parts.Length - 1]);
                string datasource = "";
                if (parts[parts.Length - 2].ToLower() == "day")
                {
                    datasource = "D";
                }
                else
                {
                    datasource = "N";
                }
                if (!_dbConnect.IshasRecord(_tableName, "ImageName", Path.GetFileName(file)))
                {
                    _dbConnect.InsertPeriodicSynProductsTable(prdID, datasetID, localfname, regionID, region, sensor, resol, peridID, year, month, xun, _statName2ID[statName], datasource);
                }
                else
                {
                    _dbConnect.DeleteCLDParatableRecord(_tableName, "ImageName", Path.GetFileName(file));
                    _dbConnect.InsertPeriodicSynProductsTable(prdID, datasetID, localfname, regionID, region, sensor, resol, peridID, year, month, xun, _statName2ID[statName], datasource);
                }
            }
        }