// This routine is to fill in the data what stinfo and scts have. The IDs will // be updated when handling DataValues. OD_1_1_1DataSet.SeriesCatalogRow CreateNewSeriesCatalog( string siteCode, string varCode, seriesCatalogTypeSeries scts) { OD_1_1_1DataSet.SeriesCatalogRow row = Table.NewSeriesCatalogRow(); VariableInfoType varInfo = scts.variable; row.SiteID = -1; row.SiteCode = siteCode; row.SiteName = null; row.VariableID = -1; row.VariableCode = varInfo.variableCode[0].Value; row.VariableName = varInfo.variableName; row.Speciation = varInfo.speciation; row.VariableUnitsID = Convert.ToInt32(varInfo.unit.unitCode); row.VariableUnitsName = varInfo.unit.unitName; row.SampleMedium = varInfo.sampleMedium; row.ValueType = varInfo.valueType; row.TimeSupport = varInfo.timeScale.timeSupport; row.TimeUnitsID = Convert.ToInt32(varInfo.timeScale.unit.unitCode); row.TimeUnitsName = varInfo.timeScale.unit.unitName; row.DataType = varInfo.dataType; row.GeneralCategory = varInfo.generalCategory; if (scts.method != null) { row.MethodID = scts.method.methodID; row.MethodDescription = scts.method.methodDescription; } row.SourceID = scts.source.sourceID; row.Organization = scts.source.organization; row.SourceDescription = scts.source.sourceDescription; row.SetCitationNull(); row.QualityControlLevelID = scts.qualityControlLevel.qualityControlLevelID; row.QualityControlLevelCode = scts.qualityControlLevel.qualityControlLevelCode; // This table is to track what we have in DataValues table. // We don't have anything in DataValues table at this moment. TimeIntervalType ti = (TimeIntervalType)scts.variableTimeInterval; row.BeginDateTime = ti.beginDateTime; row.EndDateTime = OD_Utils.GetDateTime(row.BeginDateTime, row.TimeUnitsID, -1); row.BeginDateTimeUTC = ti.beginDateTime.ToUniversalTime(); row.EndDateTimeUTC = row.EndDateTime.ToUniversalTime(); row.ValueCount = 0; return(row); }
static void HandleSites_1_1(string url, SqlConnection sqlConn) { string network, siteCode, varCode; string queryParam = string.Empty; string errMsg = string.Empty; int count; WaterML11.WaterOneFlowClient WofClient = new WaterOneFlowClient("WaterOneFlow", url); string[] blank = new string[0]; WaterML11.SiteInfoResponseType siteRt = null; try { siteRt = WofClient.GetSitesObject(blank, ""); } catch (Exception e) { errMsg = "Exception@HandleSites: " + e.Message; Console.WriteLine(errMsg); } if (siteRt == null) { if (errMsg.Length == 0) { errMsg = "Info: no site was returned from the URL."; } Console.WriteLine(errMsg); return; } // Write into "Sites" //siteRt = WofClient.GetSiteInfoObject(siteCode, ""); if (OD_SiteInfo.HandleSiteInfo(sqlConn, siteRt) == 0) { OD_Utils.Exit(1); } int nsites = siteRt.site.Count(); Console.WriteLine("Reading {0} sites from {1}..", nsites, url); WaterML11.seriesCatalogType[] sctAll; OD_SeriesCatalog odSC = new OD_SeriesCatalog(sqlConn); #region Loop for each site for (int i = 0; i < nsites; i++) { network = siteRt.site[i].siteInfo.siteCode[0].network; siteCode = network + ":" + siteRt.site[i].siteInfo.siteCode[0].Value; if (i % 10 == 0) { // Do this check and refresh for after every 100 sites CheckAndRefreshResources(); sqlConn = SqlConnOD; odSC.Adapter.Connection = sqlConn; } sctAll = LoadOnesiteInfo(WofClient, url, siteCode, ref errMsg); if (sctAll == null) { if (errMsg.Length == 0) { errMsg = "Error: LoadOneSiteInfo() returned NULL."; } Console.WriteLine(errMsg); continue; } if ((i + 1) % 50 == 0) { Console.WriteLine("..Site[{0}] {1} with {2} catalogs from {3} Time {4:HH:mm:ss}.", i, siteCode, sctAll.Count(), url, DateTime.Now); } else { Console.WriteLine("..Site[{0}] {1} with {2} catalogs Time {3:HH:mm:ss}.", i, siteCode, sctAll.Count(), DateTime.Now); } string sourceOrg = null; int methodID = 0; #region Loop for each seriesCatalogType for (int j = 0; j < sctAll.Count(); j++) { WaterML11.seriesCatalogType sct = sctAll[j]; if (sct.series == null) { errMsg = string.Format("Error: WaterML11.seriesCatalogType[{0}].series is NULL.", j); continue; } #region Loop for each seriesCatalogTypeSeries for (int k = 0; k < sct.series.Count(); k++) { WaterML11.seriesCatalogTypeSeries scts = sct.series[k]; WaterML11.TimeIntervalType ti = (WaterML11.TimeIntervalType)scts.variableTimeInterval; DateTime beginDateTime = ti.beginDateTime; DateTime endDateTime = ti.endDateTime; string code = scts.variable.variableCode[0].Value; varCode = network + ":" + code; //valueCount = scts.valueCount.Value; Console.WriteLine(""); Console.WriteLine("....Series[{0}] var: {1} of site[{2}]", k, code, i); // Use variable info from siteInfoResponseType // instead of from variablesResponseType // since the former includes the latter and contains more infomration if (OD_VariableInfo.HandleVariableInfo(sqlConn, scts.variable) == 0) { Console.WriteLine("Failed to insert variable. Give up!"); continue; } // Add to SeriesCatalog table if not there // Check SeriesCatalog table to get the newest data time OD_1_1_1DataSet.SeriesCatalogRow scRow = odSC.GetOrCreateSeriesCatalog( siteRt.site[i].siteInfo.siteCode[0].Value, scts.variable.variableCode[0].Value, scts); if (scRow.EndDateTime >= ti.endDateTime) { Console.WriteLine("No further action since database has most recent ending date time."); Console.WriteLine("Web service reported {0} values from <{1}> to <{2}>.", scts.valueCount.Value, ti.beginDateTime.ToString(), ti.endDateTime.ToString()); Console.WriteLine("Database has {0} values from <{1}> to <{2}>.", scRow.ValueCount, scRow.BeginDateTime.ToString(), scRow.EndDateTime.ToString()); if (ti.beginDateTime != scRow.BeginDateTime) { Console.WriteLine("WARNING: Web server has older data not in database! Please double check!"); } if (scts.valueCount.Value != scRow.ValueCount) { Console.WriteLine("WARNING: data value counts mismatch, maybe due to duplicate values from web service!"); } continue; } // Should we use UTC time? if (scRow.ValueCount > 0) { beginDateTime = OD_Utils.GetDateTime( scRow.EndDateTime, scRow.TimeUnitsID, 1); } // Update DataValue table //try //{ WaterML11.TimeSeriesResponseType tsrt = null; Console.WriteLine("......Getting {0} values from <{1}> to <{2}>", scts.valueCount.Value, beginDateTime, endDateTime); tsrt = LoadDataValueInfo(siteCode, varCode, WofClient, beginDateTime, endDateTime); if (tsrt == null) { continue; } if ((sourceOrg == null) || (!string.Equals(sourceOrg, scts.source.organization))) { // Insert a new source sourceOrg = scts.source.organization; OD_SourceInfo.HandleSourceInfo(sqlConn, siteRt.site[i].siteInfo, scts, tsrt.timeSeries[0]); } //if ((methodCode == null) || (!string.Equals(methodCode, scts.method.methodCode))) if ((methodID == 0) || (methodID != scts.method.methodID)) { // Insert a new method methodID = scts.method.methodID; OD_Methods.HandleMethodsInfo(sqlConn, scts.method); } count = OD_DataValues.HandleDataValueInfo(sqlConn, odSC, scRow, siteRt.site[i].siteInfo, scts, tsrt); Console.WriteLine(" -------->>>>>>> Inserted {0} records. Database has {1} values from <{2}> to <{3}>.", count, scRow.ValueCount, scRow.BeginDateTime.ToString(), scRow.EndDateTime.ToString()); //} //catch (Exception ex) //{ // errMsg = "Exception@HandleDataValueInfo: " + ex.Message; // Console.WriteLine(errMsg); //} } #endregion Loop for each seriesCatalogTypeSeries } #endregion Loop for each seriesCatalogType } #endregion Loop for each site } // end of HandleSites_1_1()