static WaterML10.TimeSeriesResponseType LoadDataValueInfo(string siteCode, string varCode, WaterML10.WaterOneFlowClient WofClient, DateTime beginDateTime, DateTime endDateTime, ref string queryParam, ref string errMsg) { string beginDT = string.Format("{0:s}", beginDateTime); string endDT = string.Format("{0:s}", endDateTime); errMsg = ""; queryParam = String.Format(" <{0}--{1}> ", beginDT, endDT); Console.WriteLine("......Getting values from <{0}> to <{1}>.", beginDT, endDT); WaterML10.TimeSeriesResponseType tsRt = null; try { tsRt = WofClient.GetValuesObject(siteCode, varCode, beginDT, endDT, ""); } catch (Exception e) { errMsg = "Exception@LoadData: " + e.Message; Console.WriteLine(errMsg); } return(tsRt); }
static void HandleSites_1_0(string url) { string network, siteCode, varCode; int seriesCatalogID, newSCID; string queryParam = string.Empty; string errMsg = string.Empty; WaterML10.WaterOneFlowClient WofClient = new WaterOneFlowClient("WaterOneFlow", url); string[] blank = new string[0]; WaterML10.SiteInfoResponseType sitesAll = null; try { // WaterML1.0 sitesAll = WofClient.GetSites(blank, ""); // WaterML1.1 //sitesAll = WofClient.GetSitesObject(blank, ""); } catch (Exception e) { errMsg = "Exception@HandleSites: " + e.Message; Console.WriteLine(errMsg); } if (sitesAll == null) { if (errMsg.Length == 0) { errMsg = "Info: no site was returned from the URL."; } Console.WriteLine(errMsg); HQT.HandleQueryTimeseries("", "", -1, -1, -1, null, url, errMsg); return; } WaterML10.SiteInfoResponseTypeSite[] sitesRt = sitesAll.site; int nsites = sitesRt.Count(); Console.WriteLine("........................................."); Console.WriteLine("Reading {0} sites from {1}", nsites, url); WaterML10.seriesCatalogType[] sctAll; #region Loop for each site for (int i = 0; i < nsites; i++) { if (CPEnabled) { if (CP.siteIndex > i) { continue; } } if (i % 100 == 0) { // Do this check and refresh for after every 100 sites CheckAndRefreshResources(); } network = sitesRt[i].siteInfo.siteCode[0].network; siteCode = network + ":" + sitesRt[i].siteInfo.siteCode[0].Value; if (CPEnabled) { CP.siteIndex = i; CP.siteCode = siteCode; SaveCheckPoint(); } //queryParam = string.Format("QueryID = (SELECT TOP 1 QueryID FROM dbo.QueryTimeseries where SiteCode = '{0}'" + // " ORDER BY QueryID DESC) and DATEDIFF(HOUR, QueryDateTime, GETDATE()) <= {1}", // siteCode, NoCheckHours); //if (OD_Utils.Exists("dbo.QueryTimeseries", queryParam, HQT.Adapter.Connection)) //{ // Console.WriteLine("..Skip Site {0} which was already checked in last {1} hours.", // siteCode, NoCheckHours); // continue; //} if (sitesRt[i].seriesCatalog != null) { sctAll = sitesRt[i].seriesCatalog; } else { sctAll = LoadOnesiteInfo(WofClient, url, siteCode, ref errMsg); } if (sctAll == null) { if (errMsg.Length == 0) { errMsg = "Error: LoadOneSiteInfo() returned NULL."; } Console.WriteLine(errMsg); HQT.HandleQueryTimeseries(siteCode, "", -1, -1, -1, null, url, errMsg); continue; } if ((i + 1) % 100 == 0) { Console.WriteLine("..Site[{0}] {1} with {2} catalogs from {3} Time {4:HH:mm:ss}.", i, siteCode, sctAll.Count(), url, DateTime.Now); } else { Console.WriteLine("..Site[{0}] {1} with {2} catalogs Time {3:HH:mm:ss}.", i, siteCode, sctAll.Count(), DateTime.Now); } // network = SRBHOS, sites[i].seriesCatalog=null // My assumption is that // Under that situation, need to get SiteInfoResponseType one site by one site // Here is the one example siteCode for testing //WaterML11.SiteInfoResponseType rt = new WaterML11.SiteInfoResponseType(); //rt = WofClient.GetSiteInfoObject("SRBHOS:RTHNet", ""); #region Loop for each seriesCatalogType for (int j = 0; j < sctAll.Count(); j++) { int valueCount; WaterML10.seriesCatalogType sct = sctAll[j]; if (sct.series == null) { errMsg = string.Format("Error: WaterML10.seriesCatalogType[{0}].series is NULL.", j); HQT.HandleQueryTimeseries(siteCode, "", -1, -1, -1, null, url, errMsg); continue; } #region Loop for each seriesCatalogTypeSeries for (int k = 0; k < sct.series.Count(); k++) { WaterML10.seriesCatalogTypeSeries scts = sct.series[k]; WaterML10.TimeIntervalType ti = (WaterML10.TimeIntervalType)scts.variableTimeInterval; DateTime beginDateTime = ti.beginDateTime; DateTime endDateTime = ti.endDateTime; string code = scts.variable.variableCode[0].Value; varCode = network + ":" + code; valueCount = scts.valueCount.Value; if (valueCount <= MinValueCount) { Console.WriteLine( "....Series[{0}] varCode: {1} valCount: {2}, too small, record in Query table, no further action.", k, code, valueCount); queryParam = String.Format(" <{0}--{1}> ", beginDateTime, endDateTime); errMsg = "Info: value count is too small, no further action."; HQT.HandleQueryTimeseries(siteCode, varCode, -1, -1, valueCount, null, queryParam, errMsg); continue; } // Check hiscentral/SeriesCatalog table to get seriesCatalogID. // Note: beginDateTime and endDateTime are obtained from web service. seriesCatalogID = newSCID = 0; hiscentralDataSet.SeriesCatalogRow hscRow = HisSC.GetRow(siteCode, varCode, scts); if (hscRow != null) { seriesCatalogID = hscRow.SeriesID; if (hscRow.SeriesCode != null) { // Don't want to repeatedly print siteCode and network int idx = hscRow.SeriesCode.IndexOf(code); code = hscRow.SeriesCode.Substring(idx); } Console.WriteLine("....Series[{0}] var: {1}, ID: {2}, valCount: {3}", k, code, seriesCatalogID, valueCount); } else { newSCID = HSC.GetOrCreateSeriesID(siteCode, varCode, scts); Console.WriteLine("....Series[{0}] var: {1}||{2}||{3}||{4}, newID: {5}, valCount: {6}", k, code, // Where WaterML1.1 and WaterML1.0 differ scts.Method.methodID, scts.Source.sourceID, scts.QualityControlLevel.QualityControlLevelID, newSCID, valueCount); } // Update HealthQuery/DataValue table if datavalue changed int[] hasChanged = null; try { if (valueCount > 5000) { WaterML10.TimeSeriesResponseType[] tsRtAll = null; tsRtAll = LoadDataValueInfoSplit(siteCode, varCode, WofClient, valueCount, beginDateTime, endDateTime, ref queryParam, ref errMsg); if (tsRtAll != null) { hasChanged = HDV.HandleDataValueInfoSplit(seriesCatalogID, newSCID, tsRtAll); } } else { WaterML10.TimeSeriesResponseType tsRt = null; tsRt = LoadDataValueInfo(siteCode, varCode, WofClient, beginDateTime, endDateTime, ref queryParam, ref errMsg); if (tsRt != null) { hasChanged = HDV.HandleDataValueInfo(seriesCatalogID, newSCID, tsRt); } } } catch (Exception ex) { errMsg = "Exception@HandleSites1: " + ex.Message; Console.WriteLine(errMsg); } // Add to HealthQuery/QueryTimeseries table. HQT.HandleQueryTimeseries(siteCode, varCode, seriesCatalogID, newSCID, valueCount, hasChanged, queryParam, errMsg); } #endregion Loop for each seriesCatalogTypeSeries // We delay inserting records to Query table to improve performance, check and do it now if (HQT.Table.Count > Program.DbUpdateBatchSize / 2) { HQT.Adapter.Update(HQT.Table); HQT.Table.Clear(); } } #endregion Loop for each seriesCatalogType } #endregion Loop for each site if (HQT.Table.Count > 0) { HQT.Adapter.Update(HQT.Table); HQT.Table.Clear(); } } // end of HandleSites_1_0()
// Get DataValueInfo for 3 time periods: begin, middle, end. // As corresponding to the first, mid, and last 30 data values static WaterML10.TimeSeriesResponseType[] LoadDataValueInfoSplit(string siteCode, string varCode, WaterML10.WaterOneFlowClient WofClient, int valuecount, DateTime beginDateTime, DateTime endDateTime, ref string queryParam, ref string errMsg) { queryParam = ""; errMsg = ""; WaterML10.TimeSeriesResponseType[] tsRtAll = new WaterML10.TimeSeriesResponseType[3]; string beginDT = string.Format("{0:s}", beginDateTime); string endDT = string.Format("{0:s}", endDateTime); Console.WriteLine("......Getting values from <{0}> to <{1}>.", beginDT, endDT); DateTime[] fromDateTime = new DateTime[3]; DateTime[] toDateTime = new DateTime[3]; string fromDT, toDT; int[] count = new int[3] { 0, 0, 0 }; // Timespan between beginDT and endDT TimeSpan delta = endDateTime - beginDateTime; double step0 = delta.TotalSeconds * Program.CompareRecordCount * 2 / valuecount; // The first 30 records fromDateTime[0] = beginDateTime; toDateTime[0] = beginDateTime.AddSeconds(step0); // The middle 30 records fromDateTime[1] = beginDateTime.AddSeconds(delta.TotalSeconds / 2); toDateTime[1] = fromDateTime[1].AddSeconds(step0); // The last 30 records fromDateTime[2] = endDateTime.AddSeconds(-step0); toDateTime[2] = endDateTime; for (int i = 0; i < 3; i++) { bool lastTry = false; tsRtAll[i] = null; double step = step0; while (!lastTry) { if ((i < 2) && (DateTime.Compare(toDateTime[i], endDateTime) >= 0)) { toDateTime[i] = endDateTime; lastTry = true; } else if ((i == 2) && (DateTime.Compare(fromDateTime[i], beginDateTime) <= 0)) { fromDateTime[i] = beginDateTime; lastTry = true; } fromDT = string.Format("{0:s}", fromDateTime[i]); toDT = string.Format("{0:s}", toDateTime[i]); Console.WriteLine("......Getting values, Segment <{0}>: from <{1}> to <{2}>.", i + 1, fromDT, toDT); try { tsRtAll[i] = WofClient.GetValuesObject(siteCode, varCode, fromDT, toDT, ""); } catch (Exception e) { errMsg = "Exception@LoadDataSplit: " + e.Message; Console.WriteLine(errMsg); return(null); } // Where WaterML1.0 and WaterML1.1 differ if (tsRtAll[i].timeSeries.values != null) { count[i] = Convert.ToInt32(tsRtAll[i].timeSeries.values.count); } if (count[i] >= Program.CompareRecordCount) { queryParam += String.Format(" <{0}--{1}> ", fromDT, toDT); break; } else { step *= 2; if (i < 2) { toDateTime[i] = toDateTime[i].AddSeconds(step); } else { fromDateTime[i] = fromDateTime[i].AddSeconds(-step); } } } } return(tsRtAll); }
public int[] HandleDataValueInfo(int scID, int newID, WaterML10.TimeSeriesResponseType tsRt) { //Console.WriteLine("Parsing and inserting data value info."); WaterML10.ValueSingleVariable dvInfo; HealthQueryDataSet.DataValueRow row; int[] hasChanged = new int[3] { 0, 0, 0 }; //0 or 1 int valueCount = Convert.ToInt32(tsRt.timeSeries.values.count); if (valueCount == 1 && tsRt.timeSeries.values.value.ToString() == tsRt.timeSeries.variable.NoDataValue) { Console.WriteLine("No values in WS response."); // there are values in SC, but not in the response from WS } string cond; if (scID > 0) { cond = string.Format("SeriesCatalogID = {0} order by RecordID", scID); } else { cond = string.Format("NewSCID = {0} order by RecordID", newID); } Adapter.FillWhere(Table, cond); int[] indices = new int[] { 0, valueCount / 2, valueCount - Program.CompareRecordCount }; for (int i = 0; i < 3; i++) { int recordID = i * Program.CompareRecordCount + 1; int idx = indices[i]; int limit = Math.Min(Program.CompareRecordCount, valueCount - indices[i]); if ((i > 0) && (indices[i] < Program.CompareRecordCount * i)) { // Record overlap, all records are put into table, no further action break; } for (int j = 0; j < limit; j++) { dvInfo = tsRt.timeSeries.values.value[idx + j]; if (recordID + j > Table.Rows.Count) { row = Table.NewDataValueRow(); row.DataValue = (double)dvInfo.Value; row.ValueAccuracy = dvInfo.accuracyStdDev; row.DateTimeUTC = dvInfo.dateTime.ToUniversalTime(); row.SeriesCatalogID = scID; row.NewSCID = newID; row.RecordID = recordID + j; Table.AddDataValueRow(row); hasChanged[i] = 1; continue; } row = Table.ElementAt(recordID - 1 + j); if ((row.DataValue != (double)dvInfo.Value) || (row.ValueAccuracy != dvInfo.accuracyStdDev) || (DateTime.Compare(row.DateTimeUTC, dvInfo.dateTime.ToUniversalTime()) != 0)) { row.DataValue = (double)dvInfo.Value; row.ValueAccuracy = dvInfo.accuracyStdDev; row.DateTimeUTC = dvInfo.dateTime.ToUniversalTime(); hasChanged[i] = 1; } } } Adapter.Update(Table); return(hasChanged); }