public void ImportDecodesAndProcessWithFlagLimits()
{
Logger.EnableLogger();
FileUtility.CleanTempPath();
var fn1 = FileUtility.GetTempFileName(".pdb");
Console.WriteLine(fn1);
var svr = new SQLiteServer(fn1);
var db = new TimeSeriesDatabase(svr, Reclamation.TimeSeries.Parser.LookupOption.TableName,false);
var tmpDir = CopyTestDecodesFileToTempDirectory("decodes_lapo.txt");
var rtlapo = CreateTempRatingTable("lapo.csv", new double[] {3.50,3.54,3.55,5.54 },
new double[] {1,2,3,10 });
// set limits gh: low=3.53, high 3.6, rate of change/hour 1
Quality q = new Quality(db);
q.SaveLimits("instant_lapo_gh", 3.6, 3.53, 1.0);
q.SaveLimits("instant_lapo_q", 5, 1.1, 0);
var site = db.GetSiteCatalog();
site.AddsitecatalogRow("lapo", "", "OR");
db.Server.SaveTable(site);
var c = new CalculationSeries("instant_lapo_q");
c.SiteID = "lapo";
c.Expression = "FileRatingTable(%site%_gh,\""+rtlapo+"\")";
db.AddSeries(c);
//SeriesExpressionParser.Debug = true;
FileImporter import = new FileImporter(db);
import.Import(tmpDir,RouteOptions.None,computeDependencies:true,searchPattern:"*.txt");
db.Inventory();
var s = db.GetSeriesFromTableName("instant_lapo_gh");
var expectedFlags = new string[] { "", "", "", "+", "", "", "", "-" };
for (int i = 0; i < s.Count; i++)
{
Assert.AreEqual(expectedFlags[i], s[i].Flag, " flag not expected ");
}
s = db.GetSeriesFromTableName("instant_lapo_q");
s.Read();
Assert.IsTrue(s.Count > 0, "No flow data computed lapo");
s.WriteToConsole(true);
// computed flows should be: 2 2 2 10 2 2 1
expectedFlags = new string[]{"","","","+","","","","-"}; //q>=1 and q<= 5
for (int i = 0; i < s.Count; i++)
{
Assert.AreEqual(expectedFlags[i], s[i].Flag.Trim()," Flag check on Flow (Q) ");
}
SeriesExpressionParser.Debug = false;
}
public void ImportParameterWithUnderscore()
{
Logger.OnLogEvent += Logger_OnLogEvent;
var fn1 = FileUtility.GetTempFileName(".pdb");
Console.WriteLine(fn1);
var svr = new SQLiteServer(fn1);
var db = new TimeSeriesDatabase(svr, Reclamation.TimeSeries.Parser.LookupOption.TableName,false);
var tmpDir = TestRatingTableDependency.CopyTestDecodesFileToTempDirectory("instant_20150708152901.txt");
FileImporter import = new FileImporter(db);
import.Import(tmpDir, RouteOptions.None, computeDependencies: true, searchPattern: "*.txt");
db.Inventory();
//Assert.IsFalse(anyErrors);
}
public void wrdo()
{
Logger.EnableLogger();
var dir = FileUtility.GetTempPath();
var testDir = Path.Combine(Globals.TestDataPath, "wrdo");
FileUtility.CopyFiles(testDir, dir,"*.dat",true,true);
var c = new Series("daily_wrdo_mm"); //needed to
//c.Expression = "DailyAverage(instant_wrdo_obm,92)";
c.TimeInterval = TimeInterval.Daily;
c.Add("10-1-2014",43.27);
c.Add("10-2-2014",48.53);
c.Add("10-3-2014",52.33);
c.Add("10-4-2014",54.63);
c.Add("10-5-2014",53.97);
c.Add("10-6-2014",55.22);
c.Add("10-7-2014", 55.12);
db.AddSeries(c);
c = new CalculationSeries("daily_wrdo_mn");
c.Expression = "DailyMin(instant_wrdo_obn)";
c.TimeInterval = TimeInterval.Daily;
db.AddSeries(c);
c = new CalculationSeries("daily_wrdo_pc");
c.Expression = "DailyMidnight(instant_wrdo_pc)";
c.TimeInterval = TimeInterval.Daily;
c.Add("10-5-2014", 8);
db.AddSeries(c);
c = new CalculationSeries("daily_wrdo_pp");
c.Expression = "daily_wrdo_pc-daily_wrdo_pc[t-1]";
c.TimeInterval = TimeInterval.Daily;
c.Add("10-5-2014", 0);
db.AddSeries(c);
c = new CalculationSeries("daily_wrdo_pu");
c.TimeSeriesDatabase = db;
c.Expression = "DailyWaterYearRunningTotal(daily_wrdo_pp,daily_wrdo_pu)";
c.TimeInterval = TimeInterval.Daily;
c.Add("10-1-2014", 0);
c.Add("10-2-2014", 0);
c.Add("10-3-2014", 0);
c.Add("10-4-2014", 0);
c.Add("10-5-2014", 0);
c.Add("10-6-2014", 0);
db.AddSeries(c);
c = db.GetSeriesFromTableName("daily_wrdo_pu");
c.Properties.Set("DaysBack", "4");
c.Properties.Save();
c = new CalculationSeries("daily_wrdo_mx");
c.Expression = "DailyMax(instant_wrdo_obx)";
c.TimeInterval = TimeInterval.Daily;
db.AddSeries(c);
c = new CalculationSeries("daily_wrdo_ym");
c.Expression = "DailyAverage(instant_wrdo_tp,92)";
c.TimeInterval = TimeInterval.Daily;
db.AddSeries(c);
c = new CalculationSeries("daily_wrdo_wr");
c.Expression = "DailySum(instant_wrdo_ws,92)/4.0";
c.TimeInterval = TimeInterval.Daily;
db.AddSeries(c);
c = new CalculationSeries("daily_wrdo_sr");
c.Expression = "DailySum(instant_wrdo_si,92)/4.0";
c.TimeInterval = TimeInterval.Daily;
db.AddSeries(c);
c = new CalculationSeries("daily_wrdo_et");
c.SiteID = "wrdo"; // needed for expression pre-processor with %site%
c.Expression = "DailyEtKimberlyPenman(daily_%site%_mm,daily_%site%_mn,daily_%site%_mx,daily_%site%_ym,daily_%site%_wr,daily_%site%_sr,42.0125,1243.58)";
c.TimeInterval = TimeInterval.Daily;
db.AddSeries(c);
FileImporter fi = new FileImporter(db);
fi.Import(dir,RouteOptions.None,true,true,"*.dat");
var s = db.GetSeriesFromTableName("daily_wrdo_ym");
s.Read();
Assert.IsTrue(s.Count > 0, " DailyAverage(instant_wrdo_tp,92) failed!");
s = db.GetSeriesFromTableName("daily_wrdo_et");
s.Read();
Assert.AreEqual(0.16,s["10-7-2014"].Value,0.01,"Error with ET");
s = db.GetSeriesFromTableName("daily_wrdo_pu");
s.Read();
Assert.AreEqual(0,s["10-7-2014"].Value, 0.02, "Error with pu");
}
public void ImportDecodesWithMissingGageHeight()
{
FileUtility.CleanTempPath();
var fn1 = FileUtility.GetTempFileName(".pdb");
Console.WriteLine(fn1);
var svr = new SQLiteServer(fn1);
var db = new TimeSeriesDatabase(svr, Reclamation.TimeSeries.Parser.LookupOption.TableName,false);
Logger.EnableLogger();
var tmpDir = CopyTestDecodesFileToTempDirectory("decodes_mabo_missing_gh.txt");
var ratingTableFileName =CreateTempRatingTable("mabo.csv", 2.37, 2.8, x => (x*10));
var c = new CalculationSeries("instant_mabo_q");
c.Expression = "FileRatingTable(mabo_gh,\""+ratingTableFileName+"\")";
db.AddSeries(c);
FileImporter import = new FileImporter(db);
import.Import(tmpDir, RouteOptions.Outgoing, computeDependencies: true,searchPattern:"*.txt");
db.Inventory();
var s = db.GetSeriesFromTableName("instant_mabo_q");
s.Read();
Assert.IsTrue(s.CountMissing() == 0);
Assert.IsTrue(s.Count > 0, "No flow data computed");
}
static void Main(string[] argList)
{
Console.Write("HydrometServer " + Application.ProductVersion +"\n compiled: " + AssemblyUtility.CreationDate()+"\n");
Console.WriteLine("System Time ="+DateTime.Now);
Arguments args = new Arguments(argList);
var p = new OptionSet();
var cli = "";
p.Add("cli=", "interface --cli=instant|daily|monthly", x => cli = x);
try
{
p.Parse(argList);
}
catch (OptionException e)
{
Console.WriteLine(e.Message);
}
if (argList.Length == 0)
{
ShowHelp(p);
return;
}
string errorFileName = "errors.txt";
Performance perf = new Performance();
// try
{
if (args.Contains("debug"))
{
Logger.EnableLogger();
Reclamation.TimeSeries.Parser.SeriesExpressionParser.Debug = true;
}
if (args.Contains("import-rating-tables"))
{// --import-rating-tables=site_list.csv [--generateNewTables] : updates usgs,idahopower, and owrd rating tables
ApplicationTrustPolicy.TrustAll();
var cfg = args["import-rating-tables"];
if (File.Exists(cfg))
{
RatingTableDownload.UpdateRatingTables(cfg, args.Contains("generateNewTables"));
}
else
{
Console.WriteLine("Error: File not found: " + cfg);
}
return;
}
if( args.Contains("run-crop-charts"))
{
var str_yr = args["run-crop-charts"].Trim();
int year = DateTime.Now.Year;
if (str_yr != "")
year = Convert.ToInt32(str_yr);
string dir = CropDatesDataSet.GetCropOutputDirectory(year);
Logger.WriteLine("output dir = " + dir);
CropChartGenerator.CreateCropReports(year, dir, HydrometHost.PNLinux);
return;
}
var db = TimeSeriesDatabase.InitDatabase(args);
if (cli != "")
{
TimeInterval interval = TimeInterval.Irregular;
if (cli == "daily")
interval = TimeInterval.Daily;
Console.WriteLine();
HydrometServer.CommandLine.PiscesCommandLine cmd = new CommandLine.PiscesCommandLine(db,interval);
cmd.PiscesPrompt();
return;
}
if (args.Contains("error-log"))
{
errorFileName = args["error-log"];
File.AppendAllText(errorFileName,"HydrometServer.exe: Started " + DateTime.Now.ToString()+"\n");
}
string propertyFilter = "";
if (args.Contains("property-filter"))
{
propertyFilter = args["property-filter"];
}
string filter = "";
if (args.Contains("filter"))
{
filter = args["filter"];
}
if (args.Contains("inventory"))
{
db.Inventory();
}
if (args.Contains("import")) // import and process data from files
{
bool computeDependencies = args.Contains("computeDependencies");
bool computeDailyOnMidnight = args.Contains("computeDailyOnMidnight");
string searchPattern = args["import"];
if (searchPattern == "")
searchPattern = "*";
string incomingPath = ConfigurationManager.AppSettings["incoming"];
FileImporter importer = new FileImporter(db, DatabaseSaveOptions.Upsert);
importer.Import(incomingPath, RouteOptions.Outgoing,computeDependencies,computeDailyOnMidnight,searchPattern);
}
DateTime t1;
DateTime t2;
SetupDates(args, out t1, out t2);
if (args.Contains("import-hydromet-instant"))
{
File.AppendAllText(errorFileName, "begin: import-hydromet-instant " + DateTime.Now.ToString() + "\n");
ImportHydrometInstant(db, t1.AddDays(-2), t2.AddDays(1), filter, propertyFilter);
}
if (args.Contains("import-hydromet-daily"))
{
File.AppendAllText(errorFileName, "begin: import-hydromet-daily " + DateTime.Now.ToString() + "\n");
ImportHydrometDaily(db, t1.AddDays(-500), t2, filter, propertyFilter);
}
if (args.Contains("import-hydromet-monthly"))
{
File.AppendAllText(errorFileName, "begin: import-hydromet-monthly " + DateTime.Now.ToString() + "\n");
ImportHydrometMonthly(db, t1.AddYears(-5), t2.AddDays(1), filter, propertyFilter);
}
if (args.Contains("calculate-daily"))
{
DailyTimeSeriesCalculator calc = new DailyTimeSeriesCalculator(db, TimeInterval.Daily,
filter,propertyFilter);
File.AppendAllText(errorFileName, "begin: calculate-daily " + DateTime.Now.ToString() + "\n");
calc.ComputeDailyValues(t1, t2, errorFileName);
}
if (args.Contains("update-daily"))
{
string sql = "provider = '" + db.Server.SafeSqlLiteral(args["update-daily"]) + "'";
var updateList = db.GetSeriesCatalog(sql);
Console.WriteLine("Updating "+updateList.Count+" Series ");
foreach (var item in updateList)
{
try
{
Console.Write(item.Name + " ");
var s = db.GetSeries(item.id);
s.Update(t1, t2);
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
}
}
if (args.Contains("update-period-of-record"))
{
var sc = db.GetSeriesCatalog("isfolder=0");
var prop = db.GetSeriesProperties(true);
for (int i = 0; i < sc.Count; i++)
{
var s = db.GetSeries(sc[i].id);
var por = s.GetPeriodOfRecord();
s.Properties.Set("count", por.Count.ToString());
if (por.Count == 0)
{
s.Properties.Set("t1", "");
s.Properties.Set("t2", "");
}
else
{
s.Properties.Set("t1", por.T1.ToString("yyyy-MM-dd"));
s.Properties.Set("t2", por.T2.ToString("yyyy-MM-dd"));
}
Console.WriteLine(s.Name);
}
db.Server.SaveTable(prop);
}
db.Server.Cleanup();
File.AppendAllText(errorFileName, "HydrometServer.exe: Completed " + DateTime.Now.ToString() + "\n");
}
//catch (Exception e )
//{
// Logger.WriteLine(e.Message);
// File.AppendAllText(errorFileName, "Error: HydrometServer.exe: \n"+e.Message);
// // Console.ReadLine();
// throw e;
//}
var mem = GC.GetTotalMemory(true);
long mb = mem / 1024 / 1024;
Console.WriteLine("Memory Usage: " + mb.ToString() + " Mb");
perf.Report("HydrometServer: finished ");
}