static WaterML10.TimeSeriesResponseType LoadDataValueInfo(string siteCode, string varCode,
WaterML10.WaterOneFlowClient WofClient,
DateTime beginDateTime, DateTime endDateTime,
ref string queryParam, ref string errMsg)
{
string beginDT = string.Format("{0:s}", beginDateTime);
string endDT = string.Format("{0:s}", endDateTime);
errMsg = "";
queryParam = String.Format(" <{0}--{1}> ", beginDT, endDT);
Console.WriteLine("......Getting values from <{0}> to <{1}>.", beginDT, endDT);
WaterML10.TimeSeriesResponseType tsRt = null;
try
{
tsRt = WofClient.GetValuesObject(siteCode, varCode, beginDT, endDT, "");
}
catch (Exception e)
{
errMsg = "Exception@LoadData: " + e.Message;
Console.WriteLine(errMsg);
}
return(tsRt);
}
static void HandleSites_1_0(string url)
{
string network, siteCode, varCode;
int seriesCatalogID, newSCID;
string queryParam = string.Empty;
string errMsg = string.Empty;
WaterML10.WaterOneFlowClient WofClient = new WaterOneFlowClient("WaterOneFlow", url);
string[] blank = new string[0];
WaterML10.SiteInfoResponseType sitesAll = null;
try
{
// WaterML1.0
sitesAll = WofClient.GetSites(blank, "");
// WaterML1.1
//sitesAll = WofClient.GetSitesObject(blank, "");
}
catch (Exception e)
{
errMsg = "Exception@HandleSites: " + e.Message;
Console.WriteLine(errMsg);
}
if (sitesAll == null)
{
if (errMsg.Length == 0)
{
errMsg = "Info: no site was returned from the URL.";
}
Console.WriteLine(errMsg);
HQT.HandleQueryTimeseries("", "", -1, -1, -1, null, url, errMsg);
return;
}
WaterML10.SiteInfoResponseTypeSite[] sitesRt = sitesAll.site;
int nsites = sitesRt.Count();
Console.WriteLine(".........................................");
Console.WriteLine("Reading {0} sites from {1}", nsites, url);
WaterML10.seriesCatalogType[] sctAll;
#region Loop for each site
for (int i = 0; i < nsites; i++)
{
if (CPEnabled)
{
if (CP.siteIndex > i)
{
continue;
}
}
if (i % 100 == 0)
{
// Do this check and refresh for after every 100 sites
CheckAndRefreshResources();
}
network = sitesRt[i].siteInfo.siteCode[0].network;
siteCode = network + ":" + sitesRt[i].siteInfo.siteCode[0].Value;
if (CPEnabled)
{
CP.siteIndex = i;
CP.siteCode = siteCode;
SaveCheckPoint();
}
//queryParam = string.Format("QueryID = (SELECT TOP 1 QueryID FROM dbo.QueryTimeseries where SiteCode = '{0}'" +
// " ORDER BY QueryID DESC) and DATEDIFF(HOUR, QueryDateTime, GETDATE()) <= {1}",
// siteCode, NoCheckHours);
//if (OD_Utils.Exists("dbo.QueryTimeseries", queryParam, HQT.Adapter.Connection))
//{
// Console.WriteLine("..Skip Site {0} which was already checked in last {1} hours.",
// siteCode, NoCheckHours);
// continue;
//}
if (sitesRt[i].seriesCatalog != null)
{
sctAll = sitesRt[i].seriesCatalog;
}
else
{
sctAll = LoadOnesiteInfo(WofClient, url, siteCode, ref errMsg);
}
if (sctAll == null)
{
if (errMsg.Length == 0)
{
errMsg = "Error: LoadOneSiteInfo() returned NULL.";
}
Console.WriteLine(errMsg);
HQT.HandleQueryTimeseries(siteCode, "", -1, -1, -1, null, url, errMsg);
continue;
}
if ((i + 1) % 100 == 0)
{
Console.WriteLine("..Site[{0}] {1} with {2} catalogs from {3} Time {4:HH:mm:ss}.",
i, siteCode, sctAll.Count(), url, DateTime.Now);
}
else
{
Console.WriteLine("..Site[{0}] {1} with {2} catalogs Time {3:HH:mm:ss}.",
i, siteCode, sctAll.Count(), DateTime.Now);
}
// network = SRBHOS, sites[i].seriesCatalog=null
// My assumption is that
// Under that situation, need to get SiteInfoResponseType one site by one site
// Here is the one example siteCode for testing
//WaterML11.SiteInfoResponseType rt = new WaterML11.SiteInfoResponseType();
//rt = WofClient.GetSiteInfoObject("SRBHOS:RTHNet", "");
#region Loop for each seriesCatalogType
for (int j = 0; j < sctAll.Count(); j++)
{
int valueCount;
WaterML10.seriesCatalogType sct = sctAll[j];
if (sct.series == null)
{
errMsg = string.Format("Error: WaterML10.seriesCatalogType[{0}].series is NULL.", j);
HQT.HandleQueryTimeseries(siteCode, "", -1, -1, -1, null, url, errMsg);
continue;
}
#region Loop for each seriesCatalogTypeSeries
for (int k = 0; k < sct.series.Count(); k++)
{
WaterML10.seriesCatalogTypeSeries scts = sct.series[k];
WaterML10.TimeIntervalType ti = (WaterML10.TimeIntervalType)scts.variableTimeInterval;
DateTime beginDateTime = ti.beginDateTime;
DateTime endDateTime = ti.endDateTime;
string code = scts.variable.variableCode[0].Value;
varCode = network + ":" + code;
valueCount = scts.valueCount.Value;
if (valueCount <= MinValueCount)
{
Console.WriteLine(
"....Series[{0}] varCode: {1} valCount: {2}, too small, record in Query table, no further action.",
k, code, valueCount);
queryParam = String.Format(" <{0}--{1}> ", beginDateTime, endDateTime);
errMsg = "Info: value count is too small, no further action.";
HQT.HandleQueryTimeseries(siteCode, varCode, -1, -1, valueCount, null,
queryParam, errMsg);
continue;
}
// Check hiscentral/SeriesCatalog table to get seriesCatalogID.
// Note: beginDateTime and endDateTime are obtained from web service.
seriesCatalogID = newSCID = 0;
hiscentralDataSet.SeriesCatalogRow hscRow = HisSC.GetRow(siteCode, varCode, scts);
if (hscRow != null)
{
seriesCatalogID = hscRow.SeriesID;
if (hscRow.SeriesCode != null)
{
// Don't want to repeatedly print siteCode and network
int idx = hscRow.SeriesCode.IndexOf(code);
code = hscRow.SeriesCode.Substring(idx);
}
Console.WriteLine("....Series[{0}] var: {1}, ID: {2}, valCount: {3}",
k, code, seriesCatalogID, valueCount);
}
else
{
newSCID = HSC.GetOrCreateSeriesID(siteCode, varCode, scts);
Console.WriteLine("....Series[{0}] var: {1}||{2}||{3}||{4}, newID: {5}, valCount: {6}",
k, code,
// Where WaterML1.1 and WaterML1.0 differ
scts.Method.methodID,
scts.Source.sourceID,
scts.QualityControlLevel.QualityControlLevelID,
newSCID, valueCount);
}
// Update HealthQuery/DataValue table if datavalue changed
int[] hasChanged = null;
try
{
if (valueCount > 5000)
{
WaterML10.TimeSeriesResponseType[] tsRtAll = null;
tsRtAll = LoadDataValueInfoSplit(siteCode, varCode, WofClient,
valueCount, beginDateTime, endDateTime, ref queryParam, ref errMsg);
if (tsRtAll != null)
{
hasChanged = HDV.HandleDataValueInfoSplit(seriesCatalogID, newSCID, tsRtAll);
}
}
else
{
WaterML10.TimeSeriesResponseType tsRt = null;
tsRt = LoadDataValueInfo(siteCode, varCode, WofClient,
beginDateTime, endDateTime, ref queryParam, ref errMsg);
if (tsRt != null)
{
hasChanged = HDV.HandleDataValueInfo(seriesCatalogID, newSCID, tsRt);
}
}
}
catch (Exception ex)
{
errMsg = "Exception@HandleSites1: " + ex.Message;
Console.WriteLine(errMsg);
}
// Add to HealthQuery/QueryTimeseries table.
HQT.HandleQueryTimeseries(siteCode, varCode, seriesCatalogID, newSCID,
valueCount, hasChanged, queryParam, errMsg);
}
#endregion Loop for each seriesCatalogTypeSeries
// We delay inserting records to Query table to improve performance, check and do it now
if (HQT.Table.Count > Program.DbUpdateBatchSize / 2)
{
HQT.Adapter.Update(HQT.Table);
HQT.Table.Clear();
}
}
#endregion Loop for each seriesCatalogType
}
#endregion Loop for each site
if (HQT.Table.Count > 0)
{
HQT.Adapter.Update(HQT.Table);
HQT.Table.Clear();
}
} // end of HandleSites_1_0()
// Get DataValueInfo for 3 time periods: begin, middle, end.
// As corresponding to the first, mid, and last 30 data values
static WaterML10.TimeSeriesResponseType[] LoadDataValueInfoSplit(string siteCode, string varCode,
WaterML10.WaterOneFlowClient WofClient,
int valuecount, DateTime beginDateTime, DateTime endDateTime,
ref string queryParam, ref string errMsg)
{
queryParam = "";
errMsg = "";
WaterML10.TimeSeriesResponseType[] tsRtAll = new WaterML10.TimeSeriesResponseType[3];
string beginDT = string.Format("{0:s}", beginDateTime);
string endDT = string.Format("{0:s}", endDateTime);
Console.WriteLine("......Getting values from <{0}> to <{1}>.", beginDT, endDT);
DateTime[] fromDateTime = new DateTime[3];
DateTime[] toDateTime = new DateTime[3];
string fromDT, toDT;
int[] count = new int[3] {
0, 0, 0
};
// Timespan between beginDT and endDT
TimeSpan delta = endDateTime - beginDateTime;
double step0 = delta.TotalSeconds * Program.CompareRecordCount * 2 / valuecount;
// The first 30 records
fromDateTime[0] = beginDateTime;
toDateTime[0] = beginDateTime.AddSeconds(step0);
// The middle 30 records
fromDateTime[1] = beginDateTime.AddSeconds(delta.TotalSeconds / 2);
toDateTime[1] = fromDateTime[1].AddSeconds(step0);
// The last 30 records
fromDateTime[2] = endDateTime.AddSeconds(-step0);
toDateTime[2] = endDateTime;
for (int i = 0; i < 3; i++)
{
bool lastTry = false;
tsRtAll[i] = null;
double step = step0;
while (!lastTry)
{
if ((i < 2) && (DateTime.Compare(toDateTime[i], endDateTime) >= 0))
{
toDateTime[i] = endDateTime;
lastTry = true;
}
else if ((i == 2) && (DateTime.Compare(fromDateTime[i], beginDateTime) <= 0))
{
fromDateTime[i] = beginDateTime;
lastTry = true;
}
fromDT = string.Format("{0:s}", fromDateTime[i]);
toDT = string.Format("{0:s}", toDateTime[i]);
Console.WriteLine("......Getting values, Segment <{0}>: from <{1}> to <{2}>.",
i + 1, fromDT, toDT);
try
{
tsRtAll[i] = WofClient.GetValuesObject(siteCode, varCode, fromDT, toDT, "");
}
catch (Exception e)
{
errMsg = "Exception@LoadDataSplit: " + e.Message;
Console.WriteLine(errMsg);
return(null);
}
// Where WaterML1.0 and WaterML1.1 differ
if (tsRtAll[i].timeSeries.values != null)
{
count[i] = Convert.ToInt32(tsRtAll[i].timeSeries.values.count);
}
if (count[i] >= Program.CompareRecordCount)
{
queryParam += String.Format(" <{0}--{1}> ", fromDT, toDT);
break;
}
else
{
step *= 2;
if (i < 2)
{
toDateTime[i] = toDateTime[i].AddSeconds(step);
}
else
{
fromDateTime[i] = fromDateTime[i].AddSeconds(-step);
}
}
}
}
return(tsRtAll);
}
public int[] HandleDataValueInfo(int scID, int newID, WaterML10.TimeSeriesResponseType tsRt)
{
//Console.WriteLine("Parsing and inserting data value info.");
WaterML10.ValueSingleVariable dvInfo;
HealthQueryDataSet.DataValueRow row;
int[] hasChanged = new int[3] {
0, 0, 0
}; //0 or 1
int valueCount = Convert.ToInt32(tsRt.timeSeries.values.count);
if (valueCount == 1 &&
tsRt.timeSeries.values.value.ToString() == tsRt.timeSeries.variable.NoDataValue)
{
Console.WriteLine("No values in WS response.");
// there are values in SC, but not in the response from WS
}
string cond;
if (scID > 0)
{
cond = string.Format("SeriesCatalogID = {0} order by RecordID", scID);
}
else
{
cond = string.Format("NewSCID = {0} order by RecordID", newID);
}
Adapter.FillWhere(Table, cond);
int[] indices = new int[] { 0, valueCount / 2, valueCount - Program.CompareRecordCount };
for (int i = 0; i < 3; i++)
{
int recordID = i * Program.CompareRecordCount + 1;
int idx = indices[i];
int limit = Math.Min(Program.CompareRecordCount, valueCount - indices[i]);
if ((i > 0) && (indices[i] < Program.CompareRecordCount * i))
{
// Record overlap, all records are put into table, no further action
break;
}
for (int j = 0; j < limit; j++)
{
dvInfo = tsRt.timeSeries.values.value[idx + j];
if (recordID + j > Table.Rows.Count)
{
row = Table.NewDataValueRow();
row.DataValue = (double)dvInfo.Value;
row.ValueAccuracy = dvInfo.accuracyStdDev;
row.DateTimeUTC = dvInfo.dateTime.ToUniversalTime();
row.SeriesCatalogID = scID;
row.NewSCID = newID;
row.RecordID = recordID + j;
Table.AddDataValueRow(row);
hasChanged[i] = 1;
continue;
}
row = Table.ElementAt(recordID - 1 + j);
if ((row.DataValue != (double)dvInfo.Value) ||
(row.ValueAccuracy != dvInfo.accuracyStdDev) ||
(DateTime.Compare(row.DateTimeUTC, dvInfo.dateTime.ToUniversalTime()) != 0))
{
row.DataValue = (double)dvInfo.Value;
row.ValueAccuracy = dvInfo.accuracyStdDev;
row.DateTimeUTC = dvInfo.dateTime.ToUniversalTime();
hasChanged[i] = 1;
}
}
}
Adapter.Update(Table);
return(hasChanged);
}
