public void InstantCompareLinuxToVMSCGI()
{
//http://www.usbr.gov/pn-bin/webdaycsv.pl?parameter=mddo%20ch,wcao%20q&syer=2015&smnth=4&sdy=5&eyer=2015&emnth=4&edy=5&format=2
string payload = "parameter=mddo ch,wcao q,boii Z,boii ob,&syer=2015&smnth=10&sdy=30&eyer=2015&emnth=11&edy=4&format=2";
//Program.Main(new string[] { "--cgi=instant", "--payload=?"+payload });
TimeSeriesDatabase db = TimeSeriesDatabase.InitDatabase(new Arguments(new string[] {}));
CsvTimeSeriesWriter c = new CsvTimeSeriesWriter(db);
var fn = FileUtility.GetTempFileName(".txt");
c.Run(TimeInterval.Hourly, payload, fn);
TextFile tf = new TextFile(fn);
tf.DeleteLines(0, 1);
var fnhyd0 = FileUtility.GetTempFileName(".txt");
Web.GetFile("http://www.usbr.gov/pn-bin/webdaycsv.pl?" + payload, fnhyd0);
var tf2 = new TextFile(fnhyd0);
var diff = TextFile.Compare(tf, tf2);
if (diff.Length > 0)
{
for (int i = 0; i < tf.Length; i++)
{
Console.WriteLine(tf[i]);
}
}
Assert.IsTrue(diff.Length == 0);
}
/// <summary>
/// Compares data in two usgs rdb files, ignoring comments at the beginning
/// </summary>
/// <param name="tf1"></param>
/// <param name="tf2"></param>
/// <returns>true if there are differences in the data</returns>
public static bool Diff(TextFile tf1, TextFile tf2)
{
var diff = System.Math.Abs(tf1.Length - tf2.Length);
if (diff > 5)
{
Logger.WriteLine("difference in length of files: " + diff);
return(true); //different number of lines in file
}
var idx1 = tf1.IndexOfRegex(@"^INDEP\s*SHIFT\s*DEP\s*STOR");
var idx2 = tf2.IndexOfRegex(@"^INDEP\s*SHIFT\s*DEP\s*STOR");
tf1.DeleteLines(0, idx1);
tf2.DeleteLines(0, idx2);
return(TextFile.Compare(tf1, tf2).Length != 0);
}
public static void CompareLinuxToVMSCGI(string payload, TimeInterval interval = TimeInterval.Irregular)
{
//Program.Main(new string[] { "--cgi=instant", "--payload=?"+payload });
TimeSeriesDatabase db = TimeSeriesDatabase.InitDatabase(new Arguments(new string[] {}));
WebTimeSeriesWriter c = new WebTimeSeriesWriter(db, interval, payload);
var fn = FileUtility.GetTempFileName(".txt");
Console.WriteLine("linux temp file:" + fn);
c.Run(fn);
TextFile tf = new TextFile(fn);
tf.DeleteLines(0, 1);
var fnhyd0 = FileUtility.GetTempFileName(".txt");
Console.WriteLine("vms temp file:" + fnhyd0);
string url = "http://www.usbr.gov/pn-bin/webarccsv.pl?";
if (interval == TimeInterval.Irregular || interval == TimeInterval.Hourly)
{
url = "http://www.usbr.gov/pn-bin/webdaycsv.pl?";
}
Web.GetFile(url + payload, fnhyd0);
var tf2 = new TextFile(fnhyd0);
if (!CompareHydrometData(tf, tf2))
{
// do detailed comparision.
var diff = TextFile.Compare(tf, tf2);
if (diff.Length > 0)
{
for (int i = 0; i < tf.Length; i++)
{
Console.WriteLine(tf[i]);
}
}
Assert.IsTrue(diff.Length == 0);
}
}
private static void UpdatesingleRatingTable(DataRow dRow,
bool generateNewTables,
List <string> stationUpdateList, List <string> attachments, out string attachmentRecipients)
{
string urlDownload = "";
// Define parameters to be used for this checking iteration
string cbtt = dRow["cbtt"].ToString().ToLower();
string htmlTitle = "Station: " + cbtt + "<br>\n"
+ dRow["title"].ToString() + "<br>\n"
+ DateTime.Now.ToShortDateString();
attachmentRecipients = "";
string stationID = dRow["site_id"].ToString();
var email = dRow["email"].ToString();
string agency = dRow["agency"].ToString();
Console.Write(cbtt.PadRight(8, '.') + " " + agency.ToLower().PadLeft(5));
// Check if the RDB is currently in the system
string rdbFileName = Path.Combine(hydrometRTFs, cbtt + ".rdb");
string shiftFileName = Path.Combine(Path.Combine(hydrometRTFs, "_hj_tables"), cbtt + "_hj.csv");
string qFileName = Path.Combine(Path.Combine(hydrometRTFs, "_q_tables"), cbtt + "_q.csv");
// Get full RatingTableFile
DataTable fullRatingTable;
TextFile webRdbTable;
TextFile fileRdbTable;
if (agency == "USGS")
{
usgsRatingTable = new Reclamation.TimeSeries.Usgs.UsgsRatingTable(stationID);
usgsRatingTable.CreateShiftAndFlowTablesFromWeb();
usgsRatingTable.CreateFullRatingTableFromWeb();
fullRatingTable = usgsRatingTable.fullRatingTable;
webRdbTable = usgsRatingTable.webRdbTable;
urlDownload = usgsRatingTable.downloadURL;
}
else if (agency == "OWRD")
{
var ratingTable = new Reclamation.TimeSeries.Owrd.OwrdRatingTables(stationID);
fullRatingTable = ratingTable.fullRatingTable;
webRdbTable = ratingTable.rawTable;
urlDownload = ratingTable.downloadURL;
}
else if (agency == "IDPWR")
{
var ratingTable = new Reclamation.TimeSeries.IdahoPower.IdahoPowerRatingTables(cbtt);
ratingTable.CreateFullRatingTableFromWeb();
fullRatingTable = ratingTable.fullRatingTable;
webRdbTable = ratingTable.webRdbTable;
urlDownload = ratingTable.downloadURL;
}
else
{
throw new Exception(cbtt.ToUpper() + "'s rating table from " + agency + " is not supported.");
}
// Create new RDB, files if the file does not currently exist
if (!File.Exists(rdbFileName))
{
stationUpdateList.Add(@"<a href=""" + urlDownload + @""">" + cbtt + " (" + agency + " " + stationID + ")</a> updated existing table");
if (agency == "USGS")
{
WriteHjAndQTables(shiftFileName, qFileName, usgsRatingTable);
}
WriteCsvFiles(fullRatingTable, cbtt);
Console.WriteLine(" new table");
webRdbTable.SaveAs(rdbFileName);
}
// Check the existing file for updates
else
{
// Get old RTF currently on file and copy it into temp. This is done to enable overwriting if the web file has been updated.
fileRdbTable = GetRDBTableFromFile(cbtt, hydrometRTFs);
// Compare
var diff = (TextFile.Compare(fileRdbTable, webRdbTable).Length != 0);
if (agency == "USGS")
{
diff = UsgsRatingTable.Diff(fileRdbTable, webRdbTable);
}
// Save new RTF file to repository and generate new HJ and Q tables if the file was updated
if (diff || generateNewTables)
{
stationUpdateList.Add(@"<a href=""" + urlDownload + @""">" + cbtt + " (" + agency + " " + stationID + ")</a> updated existing table");
// Copy old RDB to _Attic and save new RDB to repository
if (!generateNewTables)
{
fileRdbTable.SaveAs(Path.Combine(Path.Combine(hydrometRTFs, "_attic"), cbtt + DateTime.Now.ToString("_yyyy-MM-dd") + ".rdb"));//[JR] relies on the existence of an '_Attic' folder in the repository
webRdbTable.SaveAs(rdbFileName);
}
if (agency == "USGS")
{
WriteHjAndQTables(shiftFileName, qFileName, usgsRatingTable);
}
WriteCsvFiles(fullRatingTable, cbtt);
var htmlFile = Path.Combine(hydrometRTFs, cbtt + ".html");
if (fullRatingTable.Columns.Contains("Shift"))
{
fullRatingTable.Columns.Remove("Shift");
}
RatingTableUtility.WriteHtmlFile(fullRatingTable, htmlTitle, htmlFile);
// Define which attachments to add to the mail message if the 'email' field in the input file is not blank
if (email != "")
{
attachmentRecipients = email;
if (agency == "USGS")
{
attachments.Add(shiftFileName);
attachments.Add(qFileName);
}
attachments.Add(htmlFile);
}
Console.WriteLine(" UPDATED");
}
else
{
Console.WriteLine(" current");
}
}
//return urlDownload;
}
