コード例 #1
0
        public void ImportDecodesAndProcessWithFlagLimits()
        {
            Logger.EnableLogger();
            FileUtility.CleanTempPath();
            var fn1 = FileUtility.GetTempFileName(".pdb");
            Console.WriteLine(fn1);
            var svr = new SQLiteServer(fn1);
            var db = new TimeSeriesDatabase(svr, Reclamation.TimeSeries.Parser.LookupOption.TableName,false);

            var tmpDir = CopyTestDecodesFileToTempDirectory("decodes_lapo.txt");

               var rtlapo = CreateTempRatingTable("lapo.csv", new double[] {3.50,3.54,3.55,5.54 },
                                             new double[] {1,2,3,10 });
            // set limits  gh: low=3.53, high 3.6,  rate of change/hour 1
               Quality q = new Quality(db);
               q.SaveLimits("instant_lapo_gh", 3.6, 3.53, 1.0);
               q.SaveLimits("instant_lapo_q", 5, 1.1, 0);

               var site = db.GetSiteCatalog();
               site.AddsitecatalogRow("lapo", "", "OR");
               db.Server.SaveTable(site);
            var c = new CalculationSeries("instant_lapo_q");
            c.SiteID = "lapo";
            c.Expression = "FileRatingTable(%site%_gh,\""+rtlapo+"\")";
            db.AddSeries(c);

            //SeriesExpressionParser.Debug = true;
            FileImporter import = new FileImporter(db);
            import.Import(tmpDir,RouteOptions.None,computeDependencies:true,searchPattern:"*.txt");
            db.Inventory();

            var s = db.GetSeriesFromTableName("instant_lapo_gh");
            var expectedFlags = new string[] { "", "", "", "+", "", "", "", "-" };
            for (int i = 0; i < s.Count; i++)
            {
                  Assert.AreEqual(expectedFlags[i], s[i].Flag, " flag not expected ");
            }

            s = db.GetSeriesFromTableName("instant_lapo_q");
            s.Read();
            Assert.IsTrue(s.Count > 0, "No flow data computed lapo");
            s.WriteToConsole(true);
             // computed flows should be: 2 2 2 10 2 2 1
            expectedFlags = new string[]{"","","","+","","","","-"}; //q>=1 and q<= 5
            for (int i = 0; i < s.Count; i++)
            {
                 Assert.AreEqual(expectedFlags[i], s[i].Flag.Trim()," Flag check on Flow (Q) ");
            }

            SeriesExpressionParser.Debug = false;
        }
コード例 #2
0
ファイル: TestImportDMS3.cs プロジェクト: usbr/Pisces
        public void ImportParameterWithUnderscore()
        {
            Logger.OnLogEvent += Logger_OnLogEvent;
            var fn1 = FileUtility.GetTempFileName(".pdb");
            Console.WriteLine(fn1);
            var svr = new SQLiteServer(fn1);
            var db = new TimeSeriesDatabase(svr, Reclamation.TimeSeries.Parser.LookupOption.TableName,false);

            var tmpDir = TestRatingTableDependency.CopyTestDecodesFileToTempDirectory("instant_20150708152901.txt");

            FileImporter import = new FileImporter(db);
            import.Import(tmpDir, RouteOptions.None, computeDependencies: true, searchPattern: "*.txt");
            db.Inventory();
            //Assert.IsFalse(anyErrors);
        }
コード例 #3
0
        public void wrdo()
        {
            Logger.EnableLogger();
            var dir = FileUtility.GetTempPath();
            var testDir = Path.Combine(Globals.TestDataPath, "wrdo");
            FileUtility.CopyFiles(testDir, dir,"*.dat",true,true);

            var c = new Series("daily_wrdo_mm"); //needed to
            //c.Expression = "DailyAverage(instant_wrdo_obm,92)";
            c.TimeInterval = TimeInterval.Daily;
            c.Add("10-1-2014",43.27);
            c.Add("10-2-2014",48.53);
            c.Add("10-3-2014",52.33);
            c.Add("10-4-2014",54.63);
            c.Add("10-5-2014",53.97);
            c.Add("10-6-2014",55.22);
            c.Add("10-7-2014", 55.12);

            db.AddSeries(c);

            c = new CalculationSeries("daily_wrdo_mn");
            c.Expression = "DailyMin(instant_wrdo_obn)";
            c.TimeInterval = TimeInterval.Daily;
            db.AddSeries(c);

            c = new CalculationSeries("daily_wrdo_pc");
            c.Expression = "DailyMidnight(instant_wrdo_pc)";
            c.TimeInterval = TimeInterval.Daily;
            c.Add("10-5-2014", 8);
            db.AddSeries(c);

            c = new CalculationSeries("daily_wrdo_pp");
            c.Expression = "daily_wrdo_pc-daily_wrdo_pc[t-1]";
            c.TimeInterval = TimeInterval.Daily;
            c.Add("10-5-2014", 0);
            db.AddSeries(c);

            c = new CalculationSeries("daily_wrdo_pu");
            c.TimeSeriesDatabase = db;
            c.Expression = "DailyWaterYearRunningTotal(daily_wrdo_pp,daily_wrdo_pu)";
            c.TimeInterval = TimeInterval.Daily;

            c.Add("10-1-2014", 0);
            c.Add("10-2-2014", 0);
            c.Add("10-3-2014", 0);
            c.Add("10-4-2014", 0);
            c.Add("10-5-2014", 0);
            c.Add("10-6-2014", 0);
            db.AddSeries(c);
            c = db.GetSeriesFromTableName("daily_wrdo_pu");

            c.Properties.Set("DaysBack", "4");
            c.Properties.Save();
            c = new CalculationSeries("daily_wrdo_mx");
            c.Expression = "DailyMax(instant_wrdo_obx)";
            c.TimeInterval = TimeInterval.Daily;
            db.AddSeries(c);

            c = new CalculationSeries("daily_wrdo_ym");
            c.Expression = "DailyAverage(instant_wrdo_tp,92)";
            c.TimeInterval = TimeInterval.Daily;
            db.AddSeries(c);

            c = new CalculationSeries("daily_wrdo_wr");
            c.Expression = "DailySum(instant_wrdo_ws,92)/4.0";
            c.TimeInterval = TimeInterval.Daily;
            db.AddSeries(c);

            c = new CalculationSeries("daily_wrdo_sr");
            c.Expression = "DailySum(instant_wrdo_si,92)/4.0";
            c.TimeInterval = TimeInterval.Daily;
            db.AddSeries(c);

            c = new CalculationSeries("daily_wrdo_et");
            c.SiteID = "wrdo"; // needed for expression pre-processor with  %site%
            c.Expression = "DailyEtKimberlyPenman(daily_%site%_mm,daily_%site%_mn,daily_%site%_mx,daily_%site%_ym,daily_%site%_wr,daily_%site%_sr,42.0125,1243.58)";
            c.TimeInterval = TimeInterval.Daily;
            db.AddSeries(c);

            FileImporter fi = new FileImporter(db);
            fi.Import(dir,RouteOptions.None,true,true,"*.dat");

            var s = db.GetSeriesFromTableName("daily_wrdo_ym");
            s.Read();
            Assert.IsTrue(s.Count > 0, " DailyAverage(instant_wrdo_tp,92) failed!");

            s = db.GetSeriesFromTableName("daily_wrdo_et");
            s.Read();
            Assert.AreEqual(0.16,s["10-7-2014"].Value,0.01,"Error with ET");

            s = db.GetSeriesFromTableName("daily_wrdo_pu");
            s.Read();
            Assert.AreEqual(0,s["10-7-2014"].Value, 0.02, "Error with pu");
        }
コード例 #4
0
        public void ImportDecodesWithMissingGageHeight()
        {
            FileUtility.CleanTempPath();
            var fn1 = FileUtility.GetTempFileName(".pdb");
            Console.WriteLine(fn1);
            var svr = new SQLiteServer(fn1);
            var db = new TimeSeriesDatabase(svr, Reclamation.TimeSeries.Parser.LookupOption.TableName,false);
            Logger.EnableLogger();

            var tmpDir = CopyTestDecodesFileToTempDirectory("decodes_mabo_missing_gh.txt");
            var ratingTableFileName =CreateTempRatingTable("mabo.csv", 2.37, 2.8, x => (x*10));
            var c = new CalculationSeries("instant_mabo_q");
            c.Expression = "FileRatingTable(mabo_gh,\""+ratingTableFileName+"\")";
            db.AddSeries(c);

            FileImporter import = new FileImporter(db);
            import.Import(tmpDir, RouteOptions.Outgoing, computeDependencies: true,searchPattern:"*.txt");
            db.Inventory();

            var s = db.GetSeriesFromTableName("instant_mabo_q");

            s.Read();
            Assert.IsTrue(s.CountMissing() == 0);
            Assert.IsTrue(s.Count > 0, "No flow data computed");
        }
コード例 #5
0
ファイル: Program.cs プロジェクト: usbr/Pisces
        static void Main(string[] argList)
        {
            Console.Write("HydrometServer " + Application.ProductVersion +"\n compiled: " + AssemblyUtility.CreationDate()+"\n");
            Console.WriteLine("System Time ="+DateTime.Now);

            Arguments args = new Arguments(argList);
            var p = new OptionSet();

            var cli = "";
            p.Add("cli=", "interface --cli=instant|daily|monthly", x => cli = x);

            try
            {
                p.Parse(argList);
            }
            catch (OptionException e)
            {
                Console.WriteLine(e.Message);
            }
            if (argList.Length == 0)
            {
                ShowHelp(p);
                return;
            }

            string errorFileName = "errors.txt";
            Performance perf = new Performance();
             //   try
            {
            if (args.Contains("debug"))
            {
                Logger.EnableLogger();
                Reclamation.TimeSeries.Parser.SeriesExpressionParser.Debug = true;
            }

            if (args.Contains("import-rating-tables"))
            {// --import-rating-tables=site_list.csv  [--generateNewTables]     : updates usgs,idahopower, and owrd rating tables
                ApplicationTrustPolicy.TrustAll();
                var cfg = args["import-rating-tables"];
                if (File.Exists(cfg))
                {
                    RatingTableDownload.UpdateRatingTables(cfg, args.Contains("generateNewTables"));
                }
                else
                {
                    Console.WriteLine("Error: File not found: " + cfg);
                }
                return;
            }

                if( args.Contains("run-crop-charts"))
                {
                    var str_yr = args["run-crop-charts"].Trim();
                    int year = DateTime.Now.Year;
                    if (str_yr != "")
                        year = Convert.ToInt32(str_yr);

                    string dir = CropDatesDataSet.GetCropOutputDirectory(year);
                    Logger.WriteLine("output dir = " + dir);
                    CropChartGenerator.CreateCropReports(year, dir, HydrometHost.PNLinux);
                    return;
                }

            var db = TimeSeriesDatabase.InitDatabase(args);

            if (cli != "")
            {
              TimeInterval interval = TimeInterval.Irregular;
                    if (cli == "daily")
                        interval = TimeInterval.Daily;

                    Console.WriteLine();
                    HydrometServer.CommandLine.PiscesCommandLine cmd = new CommandLine.PiscesCommandLine(db,interval);
                    cmd.PiscesPrompt();

                return;
            }

            if (args.Contains("error-log"))
            {
                errorFileName = args["error-log"];
                File.AppendAllText(errorFileName,"HydrometServer.exe:  Started " + DateTime.Now.ToString()+"\n");
            }

                string propertyFilter = "";
                if (args.Contains("property-filter"))
                {
                    propertyFilter = args["property-filter"];
                }

                string filter = "";
                if (args.Contains("filter"))
                {
                    filter = args["filter"];
                }

                if (args.Contains("inventory"))
                {
                    db.Inventory();
                }

                if (args.Contains("import")) // import and process data from files
                {
                    bool computeDependencies = args.Contains("computeDependencies");
                    bool computeDailyOnMidnight = args.Contains("computeDailyOnMidnight");

                    string searchPattern = args["import"];

                    if (searchPattern == "")
                        searchPattern = "*";

                    string incomingPath = ConfigurationManager.AppSettings["incoming"];
                    FileImporter importer = new FileImporter(db, DatabaseSaveOptions.Upsert);
                    importer.Import(incomingPath, RouteOptions.Outgoing,computeDependencies,computeDailyOnMidnight,searchPattern);
                }

                DateTime t1;
                DateTime t2;

                SetupDates(args, out t1, out t2);

                if (args.Contains("import-hydromet-instant"))
                {
                    File.AppendAllText(errorFileName, "begin: import-hydromet-instant " + DateTime.Now.ToString() + "\n");
                    ImportHydrometInstant(db, t1.AddDays(-2), t2.AddDays(1), filter, propertyFilter);
                }

                if (args.Contains("import-hydromet-daily"))
                {
                    File.AppendAllText(errorFileName, "begin: import-hydromet-daily " + DateTime.Now.ToString() + "\n");
                    ImportHydrometDaily(db, t1.AddDays(-500), t2, filter, propertyFilter);
                }

                if (args.Contains("import-hydromet-monthly"))
                {
                    File.AppendAllText(errorFileName, "begin: import-hydromet-monthly " + DateTime.Now.ToString() + "\n");
                    ImportHydrometMonthly(db, t1.AddYears(-5), t2.AddDays(1), filter, propertyFilter);
                }

                if (args.Contains("calculate-daily"))
                {
                    DailyTimeSeriesCalculator calc = new DailyTimeSeriesCalculator(db, TimeInterval.Daily,
                        filter,propertyFilter);
                    File.AppendAllText(errorFileName, "begin: calculate-daily " + DateTime.Now.ToString() + "\n");
                    calc.ComputeDailyValues(t1, t2, errorFileName);
                }

                if (args.Contains("update-daily"))
                {
                   string sql = "provider = '" + db.Server.SafeSqlLiteral(args["update-daily"]) + "'";
                   var updateList = db.GetSeriesCatalog(sql);
                   Console.WriteLine("Updating  "+updateList.Count+" Series ");

                   foreach (var item in updateList)
                   {
                       try
                       {
                           Console.Write(item.Name + " ");
                           var s = db.GetSeries(item.id);
                           s.Update(t1, t2);
                       }
                       catch (Exception e)
                       {
                           Console.WriteLine(e.Message);
                       }
                   }
                }

                if (args.Contains("update-period-of-record"))
                {
                    var sc = db.GetSeriesCatalog("isfolder=0");

                    var prop = db.GetSeriesProperties(true);
                    for (int i = 0; i < sc.Count; i++)
                    {
                        var s = db.GetSeries(sc[i].id);
                        var por = s.GetPeriodOfRecord();

                        s.Properties.Set("count", por.Count.ToString());

                        if (por.Count == 0)
                        {
                            s.Properties.Set("t1", "");
                            s.Properties.Set("t2", "");
                        }
                        else
                        {
                            s.Properties.Set("t1", por.T1.ToString("yyyy-MM-dd"));
                            s.Properties.Set("t2", por.T2.ToString("yyyy-MM-dd"));
                        }
                        Console.WriteLine(s.Name);
                    }
                    db.Server.SaveTable(prop);
                }

                db.Server.Cleanup();

                File.AppendAllText(errorFileName, "HydrometServer.exe:  Completed " + DateTime.Now.ToString() + "\n");
            }
            //catch (Exception e )
            //{
            //    Logger.WriteLine(e.Message);
            //    File.AppendAllText(errorFileName, "Error: HydrometServer.exe: \n"+e.Message);
            //    // Console.ReadLine();
            //    throw e;
            //}

            var mem = GC.GetTotalMemory(true);
            long mb = mem / 1024 / 1024;
            Console.WriteLine("Memory Usage: " + mb.ToString() + " Mb");
            perf.Report("HydrometServer: finished ");
        }