public void GetSiteInfo_SaveXmlFilesFlag_ReturnsSameData(string url, string siteCode) { try { var target = new WaterOneFlowClient(url, 5) { SaveXmlFiles = false }; var series1 = target.GetSiteInfo(siteCode); target.SaveXmlFiles = true; var series2 = target.GetSiteInfo(siteCode); Assert.AreEqual(series1.Count, series2.Count); for (var i = 0; i < series1.Count; i++) { var site1 = series1[i].Site; var site2 = series2[i].Site; Assert.AreEqual(site1.Code, site2.Code); Assert.AreEqual(site1.Name, site2.Name); Assert.AreEqual(series1[i].ValueCount, series2[i].ValueCount); } } catch (WebException ex) { if (ex.Status.HasFlag(WebExceptionStatus.ProtocolError) || ex.Status.HasFlag(WebExceptionStatus.Timeout)) { Assert.Inconclusive("Unable to test GetSiteInfo() from: " + url); } throw; } }
public void CanSaveMultipleSeries() { Random random = new Random(); //string url1 = @"http://his02.usu.edu/littlebearriver/cuahsi_1_0.asmx"; //string url2 = @"http://icewater.boisestate.edu/dcew2dataservices/cuahsi_1_0.asmx"; string url3 = @"http://his.crwr.utexas.edu/TXEvap/cuahsi_1_0.asmx"; MetadataCacheManagerSQL manager = TestConfig.MetadataCacheManager; WaterOneFlowClient client = new WaterOneFlowClient(url3); IList <Site> siteList = client.GetSites(); IList <SeriesMetadata> seriesList = new List <SeriesMetadata>(); DataServiceInfo service = GeDatatService(random.Next()); manager.SaveDataService(service); foreach (Site site in siteList) { IList <SeriesMetadata> seriesList1 = client.GetSiteInfo(site.Code); foreach (SeriesMetadata series in seriesList1) { seriesList.Add(series); } } foreach (SeriesMetadata series in seriesList) { manager.SaveSeries(series, service); } }
public void TestCRWR_10() { string url = @"http://his.crwr.utexas.edu/tcoonts/tcoon.asmx"; WaterOneFlowClient client = new WaterOneFlowClient(url); IList <Site> sites = client.GetSites(); Assert.Greater(sites.Count, 0); }
private void button1_Click(object sender, EventArgs e) { variablesListBox.Items.Clear(); waterOneFlowClient = new WaterOneFlowClient(urlTextbox.Text); // Update service info in the metadata database var waterOneFlowServiceInfo = waterOneFlowClient.ServiceInfo; var service = waterOneFlowServiceInfo; // Get all sites for this service IList <Site> siteList; siteList = waterOneFlowClient.GetSites(); var variableList = new List <string>(); foreach (var site in siteList) { // Get series for this site IList <SeriesMetadata> currentSeriesList; try { currentSeriesList = waterOneFlowClient.GetSiteInfo(site.Code); } catch (WebException ex) { if (ex.Response != null) { var rdr = new StreamReader(ex.Response.GetResponseStream()); rdr.ReadToEnd(); } continue; } // Save series info to metadata cache database foreach (var series in currentSeriesList) { if (!variableList.Contains(series.Variable.Name)) { var variable = series.Variable.Name; variableList.Add(variable); } } } foreach (var variable in variableList) { this.variablesListBox.Items.Add(variable, false); } if (variablesListBox.Items.Count != 0) { button1.Enabled = false; button2.Enabled = true; button3.Enabled = true; button4.Enabled = true; } }
/// <summary> /// Gets a SeriesData object from /// Todo: Migrate to separate repository specific for blob storage or table storage. /// </summary> /// <param name="seriesId"></param> /// <param name="guid"></param> /// <returns></returns> public async Task <ServerSideHydroDesktop.ObjectModel.Series> GetWaterOneFlowFromCloudCache(string seriesId, string guid, string servUrl) { WaterOneFlowClient waterOneFlow = new WaterOneFlowClient(servUrl); CloudStorageAccount csa = CUAHSIDataStorage.LogHelper.GetCUAHSIDataStorage(); CloudBlobClient client = csa.CreateCloudBlobClient(); Stream s = new MemoryStream(); await client.GetContainerReference(guid).GetBlockBlobReference(seriesId).DownloadToStreamAsync(s); return(waterOneFlow.GetValuesFromStream(s).FirstOrDefault()); }
public async Task <Tuple <Stream, IList <ServerSideHydroDesktop.ObjectModel.Series> > > SeriesAndStreamOfSeriesID(SeriesMetadata meta) { WaterOneFlowClient client = new WaterOneFlowClient(meta.ServURL); return(await client.GetValuesAndRawStreamAsync( meta.SiteCode, meta.VarCode, meta.StartDate, DateTime.UtcNow, Convert.ToInt32(10000))); }
public void TestLittleBearRiver_10() { string url = @"http://icewater.usu.edu/LittleBearRiver/cuahsi_1_0.asmx"; WaterOneFlowClient client = new WaterOneFlowClient(url); IList <Site> sites = client.GetSites(); IList <SeriesMetadata> seriesList = client.GetSiteInfo(sites[0].Code); IList <Series> downloadedData = client.GetValues(seriesList[0].Site.Code, seriesList[0].Variable.Code, DateTime.MinValue, DateTime.Now); Assert.IsNotNull(seriesList); Assert.Greater(seriesList.Count, 0); Assert.AreNotEqual(seriesList[0].Source.ToString(), "unknown"); Assert.AreNotEqual(seriesList[0].Method.ToString(), "unknown"); }
public void TestSeriesWithSamples() { string url = @"http://water.sdsc.edu/lbrsdsc/cuahsi_1_1.asmx"; WaterOneFlowClient client = new WaterOneFlowClient(url); string siteCode = "LBR11:USU-LBR-Paradise"; string variableCode = "LBR11:USU39"; IList <Series> series = client.GetValues(siteCode, variableCode, DateTime.MinValue, DateTime.Now); Assert.IsNotNull(series); Assert.Greater(series.Count, 0); Assert.AreNotEqual(series[0].Source.ToString(), "unknown"); Assert.AreNotEqual(series[0].Method.ToString(), "unknown"); }
public void TestCIMSservice() { string url = @"http://cbe.cae.drexel.edu/CIMS/cuahsi_1_1.asmx"; string siteCode = "CIMS:EE3.4"; string variableCode = "CIMS:NO3F"; WaterOneFlowClient client = new WaterOneFlowClient(url); IList <Series> series = client.GetValues(siteCode, variableCode, DateTime.Now.AddYears(-30), DateTime.Now); Assert.IsNotNull(series); Assert.Greater(series.Count, 0); Assert.AreNotEqual(series[0].Source.ToString(), "unknown"); Assert.AreNotEqual(series[0].Method.ToString(), "unknown"); }
public void TestTCOON_11() { string url = @"http://his.crwr.utexas.edu/tcoonts/tcoon.asmx"; WaterOneFlowClient client = new WaterOneFlowClient(url); IList <Site> sites = client.GetSites(); IList <SeriesMetadata> seriesList = client.GetSiteInfo(sites[0].Code); //IList<Series> downloadedData = client.GetValues(seriesList[0].Site.Code, seriesList[0].Variable.Code, DateTime.MinValue, DateTime.Now); //Assert.IsNotNull(downloadedData); //Assert.Greater(downloadedData.Count, 0); //Assert.AreNotEqual(downloadedData[0].Source.ToString(), "unknown"); //Assert.AreNotEqual(downloadedData[0].Method.ToString(), "unknown"); }
public void TestNWISDV_10() { string url = @"http://river.sdsc.edu/wateroneflow/NWIS/Data.asmx"; string siteCode = "NWISDV:13069500"; string variableCode = "NWISDV:00060/DataType=Average"; DateTime start = new DateTime(2010, 1, 1); DateTime end = new DateTime(2011, 1, 1); WaterOneFlowClient client = new WaterOneFlowClient(url); IList <Series> seriesList = client.GetValues(siteCode, variableCode, start, end); Assert.Greater(seriesList.Count, 0); Assert.Greater(seriesList[0].DataValueList.Count, 0); }
public void TestBaltimoreGW_10() { string url = @"http://his09.umbc.edu/BaltGW/cuahsi_1_0.asmx"; string siteCode = "BaltimoreGW:WOLDEAMANUEL"; string variableCode = "BaltimoreGW:WATERASL"; DateTime beginDate = new DateTime(2000, 1, 1); DateTime endDate = DateTime.Now; WaterOneFlowClient client = new WaterOneFlowClient(url); IList <Series> series = client.GetValues(siteCode, variableCode, beginDate, endDate); Assert.IsNotNull(series); Assert.Greater(series.Count, 0); Assert.AreNotEqual(series[0].Source.ToString(), "unknown"); Assert.AreNotEqual(series[0].Method.ToString(), "unknown"); }
// [TestCase(@"http://his.crwr.utexas.edu/TXEvap/cuahsi_1_0.asmx")] public void TestDownload(string url) { WaterOneFlowClient cl = new WaterOneFlowClient(url); string file2 = cl.GetSitesXML(); Assert.IsTrue(File.Exists(file2), "the GetSites file does not exist."); WaterOneFlow10Parser parser1 = new WaterOneFlow10Parser(); IList <Site> sites = parser1.ParseGetSitesXml(file2); foreach (Site site in sites) { string fullSiteCode = site.Code; string siteFile = cl.GetSiteInfoXML(fullSiteCode); Assert.IsTrue(File.Exists(siteFile), "the GetSiteInfo file does not exist."); } }
public void GetValues_SaveXmlFilesFlag_ReturnsSameData(string url, string siteCode, string varCode, string startDate, string endDate) { const string DATES_FORMAT = "MM/dd/yyyy"; var provider = CultureInfo.InvariantCulture; var target = new WaterOneFlowClient(url, 5); var start = DateTime.ParseExact(startDate, DATES_FORMAT, provider); var end = DateTime.ParseExact(endDate, DATES_FORMAT, provider); try { target.SaveXmlFiles = false; var series1 = target.GetValues(siteCode, varCode, start, end); target.SaveXmlFiles = true; var series2 = target.GetValues(siteCode, varCode, start, end); Assert.AreEqual(series1.Count, series2.Count); for (var i = 0; i < series1.Count; i++) { var site1 = series1[i].Site; var site2 = series2[i].Site; var var1 = series1[i].Variable; var var2 = series2[i].Variable; Assert.AreEqual(site1.Code, site2.Code); Assert.AreEqual(site1.Name, site2.Name); Assert.AreEqual(series1[i].ValueCount, series2[i].ValueCount); Assert.AreEqual(var1.Code, var2.Code); Assert.AreEqual(var1.Name, var2.Name); } } catch (WebException ex) { if (ex.Status.HasFlag(WebExceptionStatus.ProtocolError) || ex.Status.HasFlag(WebExceptionStatus.Timeout)) { Assert.Inconclusive("Unable to test GetValues() from: " + url); } throw; } }
/// <summary> /// Gets an instance of a WaterOneFlow client to be used with /// the URL specified in the Download info. This instance is retrieved /// from the dictionary cache or created if necessary. /// </summary> /// <param name="wsdl">The URL of the web service main page</param> /// <returns>the appropriate WaterOneFlow client</returns> private WaterOneFlowClient GetWsClientInstance(string wsdl) { WaterOneFlowClient wsClient; lock (_syncObject) { //To Access the dynamic WSDLs if (_services.ContainsKey(wsdl)) { wsClient = _services[wsdl]; } else { wsClient = new WaterOneFlowClient(wsdl); _services.Add(wsdl, wsClient); } } return(wsClient); }
/// <summary> /// Gets an instance of a WaterOneFlow client to be used with /// the URL specified in the Download info. This instance is retrieved /// from the dictionary cache or created if necessary. /// </summary> /// <param name="wsdl">The URL of the web service main page</param> /// <returns>the appropriate WaterOneFlow client</returns> private WaterOneFlowClient GetWsClientInstance(string wsdl) { WaterOneFlowClient wsClient; lock (_syncObject) { //To Access the dynamic WSDLs if (_services.ContainsKey(wsdl)) { wsClient = _services[wsdl]; } else { wsClient = new WaterOneFlowClient(wsdl, valuesPerReq: _options.NumberOfValuesPerRequest, allInOneRequest: _options.GetAllValuesInOneRequest); _services.Add(wsdl, wsClient); } } return(wsClient); }
public void CanSaveOneSeries() { Random random = new Random(); string url = @"http://his.crwr.utexas.edu/TXEvap/cuahsi_1_0.asmx"; MetadataCacheManagerSQL manager = TestConfig.MetadataCacheManager; WaterOneFlowClient client = new WaterOneFlowClient(url); IList <Site> sites = client.GetSites(); IList <SeriesMetadata> seriesList = client.GetSiteInfo(sites[0].Code); SeriesMetadata firstSeries = seriesList[0]; DataServiceInfo service = GeDatatService(random.Next()); manager.SaveDataService(service); firstSeries.DataService = service; manager.SaveSeries(firstSeries, service); }
public void TestValidURL() { string goodURL = @"http://his.crwr.utexas.edu/TXEvap/cuahsi_1_0.asmx"; WaterOneFlowClient client = new WaterOneFlowClient(goodURL); IList <Site> sites = client.GetSites(); }
private void button2_Click(object sender, EventArgs e) { variablesListBox.Enabled = false; urlTextbox.Enabled = false; titleTextbox.Enabled = false; button1.Enabled = false; button2.Enabled = false; button3.Enabled = false; button4.Enabled = false; waterOneFlowClient = new WaterOneFlowClient(urlTextbox.Text); var waterOneFlowServiceInfo = waterOneFlowClient.ServiceInfo; // Trim the query off of the URL if it still exists int index = waterOneFlowServiceInfo.EndpointURL.IndexOf("?"); if (index > -1) { waterOneFlowServiceInfo.EndpointURL = waterOneFlowServiceInfo.EndpointURL.Substring(0, index); } var serviceInfo = new DataServiceInfo(); serviceInfo.IsHarvested = false; serviceInfo.ServiceName = waterOneFlowServiceInfo.ServiceName; serviceInfo.Version = waterOneFlowServiceInfo.Version; serviceInfo.ServiceType = waterOneFlowServiceInfo.ServiceType; serviceInfo.Protocol = waterOneFlowServiceInfo.Protocol; serviceInfo.VariableCount = waterOneFlowServiceInfo.VariableCount; IList <Site> siteList; siteList = waterOneFlowClient.GetSites(); // Default extent for the service. These values are designed to be overwritten as we query sites in the service double east = -180; double west = 360; double north = -90; double south = 90; int valueCount = 0; var totalDataCartSeriesList = new List <SeriesDataCart>(); foreach (var site in siteList) { // Get series for this site IList <SeriesMetadata> currentSeriesList; var dataCartSeriesList = new List <SeriesDataCart>(); try { currentSeriesList = waterOneFlowClient.GetSiteInfo(site.Code); } catch (WebException ex) { continue; } catch (Exception ex) { continue; } // Update service extent if (site.Latitude > north) { north = site.Latitude; } if (site.Latitude < south) { south = site.Latitude; } if (site.Longitude > east) { east = site.Longitude; } if (site.Longitude < west) { west = site.Longitude; } // Save series info to metadata cache database foreach (var series in currentSeriesList) { valueCount += series.ValueCount; try { if (checkedVariables.Count != 0) { if (checkedVariables.Contains(series.Variable.Name)) { var seriesDataCart = getDataCartFromMetadata(series, waterOneFlowServiceInfo); dataCartSeriesList.Add(seriesDataCart); } } else { var seriesDataCart = getDataCartFromMetadata(series, waterOneFlowServiceInfo); dataCartSeriesList.Add(seriesDataCart); } } catch (Exception ex) { continue; } } totalDataCartSeriesList.AddRange(dataCartSeriesList); variablesListBox.Enabled = true; urlTextbox.Enabled = true; titleTextbox.Enabled = true; button1.Enabled = true; button2.Enabled = true; button3.Enabled = true; button4.Enabled = true; } // Update service info serviceInfo.IsHarvested = true; serviceInfo.HarveDateTime = DateTime.Now; serviceInfo.EastLongitude = east; serviceInfo.WestLongitude = west; serviceInfo.NorthLatitude = north; serviceInfo.SouthLatitude = south; serviceInfo.SiteCount = siteList.Count; serviceInfo.ValueCount = valueCount; serviceInfo.VariableCount = waterOneFlowServiceInfo.VariableCount; SearchResult resultFeatureSet = null; if (totalDataCartSeriesList.Count > 0) { resultFeatureSet = SearchHelper.ToFeatureSetsByDataSource(totalDataCartSeriesList); } if (resultFeatureSet != null) { //We need to reproject the Search results from WGS84 to the projection of the map. var wgs84 = KnownCoordinateSystems.Geographic.World.WGS1984; foreach (var item in resultFeatureSet.ResultItems) { item.FeatureSet.Projection = wgs84; } var layers = ShowSearchResults(resultFeatureSet); Debug.WriteLine("ShowSearchResults done."); // Unselect all layers in legend (http://hydrodesktop.codeplex.com/workitem/8559) App.Map.MapFrame.GetAllLayers().ForEach(r => r.IsSelected = false); // Select first search result layer var first = layers.FirstOrDefault().GetParentItem(); if (first != null) { first.IsSelected = true; } // Unselect "Map Layers" legend item (http://hydrodesktop.codeplex.com/workitem/8458) App.Legend.RootNodes .ForEach(delegate(ILegendItem item) { if (item.LegendText == "Map Layers") { item.IsSelected = false; } }); Debug.WriteLine("Finished."); } }
/// <summary> /// For each service described in the data grid view, adds records to the metadata cache database to describe the service /// </summary> /// <param name="servicesToAdd">List of services to add to the metadata cache database</param> /// <param name="e">Parameters from the BackgroundWorker</param> /// <returns>Parameters (task type, output message, rows that were successfully added) to be processed by a BackgroundWorker event handler</returns> private object[] AddServicesToDatabase(List <DataServiceInfo> servicesToAdd, DoWorkEventArgs e) { // Build parameters to pass to the background worker object[] parameters = new object[3]; parameters[0] = BackgroundWorkerTasks.UpdateDatabase; parameters[1] = "Operation cancelled"; List <int> rowsToSelect = new List <int> (); bgwMain.ReportProgress(0, "Getting list of existing WaterOneFlow services from database..."); // Get a list of existing URLs from the metadata cache databsae List <string> existingUrls = DatabaseOperations.GetCacheServiceUrls(true); // Add service records to the database int totalSteps = servicesToAdd.Count; int currentStep = 0; int countAlreadyExists = 0; int countInvalidService = 0; IEqualityComparer <string> comparer = new CaseInsensitiveEqualityComparer(); MetadataCacheManagerSQL cacheManager = DatabaseOperations.GetCacheManager(); for (int i = 0; i < servicesToAdd.Count; i++) { if (bgwMain.CancellationPending) { e.Cancel = true; return(parameters); } currentStep++; bgwMain.ReportProgress(100 * currentStep / totalSteps, "Checking service " + currentStep + " of " + totalSteps + "..."); DataServiceInfo serviceInfo = servicesToAdd[i]; // Check if the service already exists in the database if (existingUrls.Contains(serviceInfo.EndpointURL, comparer) == true) { countAlreadyExists += 1; continue; } // Check that the URL is for a live service if (this.mnuCheckForValidService.Checked == true) { // Attempt to create a WaterOneFlowServiceClient from the URL. If the URL is not for a WaterOneFlow service, an error is thrown in the constructor. try { WaterOneFlowClient waterOneFlowClient = new WaterOneFlowClient(serviceInfo.EndpointURL); DataServiceInfo clientServiceInfo = waterOneFlowClient.ServiceInfo; serviceInfo.ServiceName = clientServiceInfo.ServiceName; serviceInfo.Protocol = clientServiceInfo.Protocol; serviceInfo.ServiceType = clientServiceInfo.ServiceType; serviceInfo.Version = clientServiceInfo.Version; } catch { countInvalidService += 1; continue; } } // Save the service cacheManager.SaveDataService(serviceInfo); existingUrls.Add(serviceInfo.EndpointURL); rowsToSelect.Add(i); } // Prepare a message to the user string message = ""; int serviceCount = rowsToSelect.Count; if (serviceCount == 0) { message += "No new services added to metadata cache database.\n\n "; } else if (serviceCount == 1) { message += "1 new service added to metadata cache database.\n\n"; } else { message += serviceCount.ToString() + " new services added to metadata cache database.\n\n"; } if (countAlreadyExists == 1) { message += "1 service was not added because it already exists in the database.\n\n"; } else if (countAlreadyExists > 1) { message += countAlreadyExists.ToString() + " services were not added because they already exist in the database.\n\n"; } if (countInvalidService == 1) { message += "1 service was not added because it does not point to a valid WaterOneFlow service.\n\n"; } else if (countInvalidService > 1) { message += countInvalidService.ToString() + " services were not added because they do not point to a valid WaterOneFlow service.\n\n"; } if (serviceCount == 1) { message += "Remember to download metadata for this service in the Metadata Fetcher window."; } else if (serviceCount > 1) { message += "Remember to download metadata for these services in the Metadata Fetcher window."; } parameters[1] = message; parameters[2] = rowsToSelect; return(parameters); }
static void HandleSites_1_1(string url, SqlConnection sqlConn) { string network, siteCode, varCode; string queryParam = string.Empty; string errMsg = string.Empty; int count; WaterML11.WaterOneFlowClient WofClient = new WaterOneFlowClient("WaterOneFlow", url); string[] blank = new string[0]; WaterML11.SiteInfoResponseType siteRt = null; try { siteRt = WofClient.GetSitesObject(blank, ""); } catch (Exception e) { errMsg = "Exception@HandleSites: " + e.Message; Console.WriteLine(errMsg); } if (siteRt == null) { if (errMsg.Length == 0) { errMsg = "Info: no site was returned from the URL."; } Console.WriteLine(errMsg); return; } // Write into "Sites" //siteRt = WofClient.GetSiteInfoObject(siteCode, ""); if (OD_SiteInfo.HandleSiteInfo(sqlConn, siteRt) == 0) { OD_Utils.Exit(1); } int nsites = siteRt.site.Count(); Console.WriteLine("Reading {0} sites from {1}..", nsites, url); WaterML11.seriesCatalogType[] sctAll; OD_SeriesCatalog odSC = new OD_SeriesCatalog(sqlConn); #region Loop for each site for (int i = 0; i < nsites; i++) { network = siteRt.site[i].siteInfo.siteCode[0].network; siteCode = network + ":" + siteRt.site[i].siteInfo.siteCode[0].Value; if (i % 10 == 0) { // Do this check and refresh for after every 100 sites CheckAndRefreshResources(); sqlConn = SqlConnOD; odSC.Adapter.Connection = sqlConn; } sctAll = LoadOnesiteInfo(WofClient, url, siteCode, ref errMsg); if (sctAll == null) { if (errMsg.Length == 0) { errMsg = "Error: LoadOneSiteInfo() returned NULL."; } Console.WriteLine(errMsg); continue; } if ((i + 1) % 50 == 0) { Console.WriteLine("..Site[{0}] {1} with {2} catalogs from {3} Time {4:HH:mm:ss}.", i, siteCode, sctAll.Count(), url, DateTime.Now); } else { Console.WriteLine("..Site[{0}] {1} with {2} catalogs Time {3:HH:mm:ss}.", i, siteCode, sctAll.Count(), DateTime.Now); } string sourceOrg = null; int methodID = 0; #region Loop for each seriesCatalogType for (int j = 0; j < sctAll.Count(); j++) { WaterML11.seriesCatalogType sct = sctAll[j]; if (sct.series == null) { errMsg = string.Format("Error: WaterML11.seriesCatalogType[{0}].series is NULL.", j); continue; } #region Loop for each seriesCatalogTypeSeries for (int k = 0; k < sct.series.Count(); k++) { WaterML11.seriesCatalogTypeSeries scts = sct.series[k]; WaterML11.TimeIntervalType ti = (WaterML11.TimeIntervalType)scts.variableTimeInterval; DateTime beginDateTime = ti.beginDateTime; DateTime endDateTime = ti.endDateTime; string code = scts.variable.variableCode[0].Value; varCode = network + ":" + code; //valueCount = scts.valueCount.Value; Console.WriteLine(""); Console.WriteLine("....Series[{0}] var: {1} of site[{2}]", k, code, i); // Use variable info from siteInfoResponseType // instead of from variablesResponseType // since the former includes the latter and contains more infomration if (OD_VariableInfo.HandleVariableInfo(sqlConn, scts.variable) == 0) { Console.WriteLine("Failed to insert variable. Give up!"); continue; } // Add to SeriesCatalog table if not there // Check SeriesCatalog table to get the newest data time OD_1_1_1DataSet.SeriesCatalogRow scRow = odSC.GetOrCreateSeriesCatalog( siteRt.site[i].siteInfo.siteCode[0].Value, scts.variable.variableCode[0].Value, scts); if (scRow.EndDateTime >= ti.endDateTime) { Console.WriteLine("No further action since database has most recent ending date time."); Console.WriteLine("Web service reported {0} values from <{1}> to <{2}>.", scts.valueCount.Value, ti.beginDateTime.ToString(), ti.endDateTime.ToString()); Console.WriteLine("Database has {0} values from <{1}> to <{2}>.", scRow.ValueCount, scRow.BeginDateTime.ToString(), scRow.EndDateTime.ToString()); if (ti.beginDateTime != scRow.BeginDateTime) { Console.WriteLine("WARNING: Web server has older data not in database! Please double check!"); } if (scts.valueCount.Value != scRow.ValueCount) { Console.WriteLine("WARNING: data value counts mismatch, maybe due to duplicate values from web service!"); } continue; } // Should we use UTC time? if (scRow.ValueCount > 0) { beginDateTime = OD_Utils.GetDateTime( scRow.EndDateTime, scRow.TimeUnitsID, 1); } // Update DataValue table //try //{ WaterML11.TimeSeriesResponseType tsrt = null; Console.WriteLine("......Getting {0} values from <{1}> to <{2}>", scts.valueCount.Value, beginDateTime, endDateTime); tsrt = LoadDataValueInfo(siteCode, varCode, WofClient, beginDateTime, endDateTime); if (tsrt == null) { continue; } if ((sourceOrg == null) || (!string.Equals(sourceOrg, scts.source.organization))) { // Insert a new source sourceOrg = scts.source.organization; OD_SourceInfo.HandleSourceInfo(sqlConn, siteRt.site[i].siteInfo, scts, tsrt.timeSeries[0]); } //if ((methodCode == null) || (!string.Equals(methodCode, scts.method.methodCode))) if ((methodID == 0) || (methodID != scts.method.methodID)) { // Insert a new method methodID = scts.method.methodID; OD_Methods.HandleMethodsInfo(sqlConn, scts.method); } count = OD_DataValues.HandleDataValueInfo(sqlConn, odSC, scRow, siteRt.site[i].siteInfo, scts, tsrt); Console.WriteLine(" -------->>>>>>> Inserted {0} records. Database has {1} values from <{2}> to <{3}>.", count, scRow.ValueCount, scRow.BeginDateTime.ToString(), scRow.EndDateTime.ToString()); //} //catch (Exception ex) //{ // errMsg = "Exception@HandleDataValueInfo: " + ex.Message; // Console.WriteLine(errMsg); //} } #endregion Loop for each seriesCatalogTypeSeries } #endregion Loop for each seriesCatalogType } #endregion Loop for each site } // end of HandleSites_1_1()
public void TestInvalidURL() { string badURL = @"http://example.com"; WaterOneFlowClient client = new WaterOneFlowClient(badURL); }
static void HandleSites_1_0(string url) { string network, siteCode, varCode; int seriesCatalogID, newSCID; string queryParam = string.Empty; string errMsg = string.Empty; WaterML10.WaterOneFlowClient WofClient = new WaterOneFlowClient("WaterOneFlow", url); string[] blank = new string[0]; WaterML10.SiteInfoResponseType sitesAll = null; try { // WaterML1.0 sitesAll = WofClient.GetSites(blank, ""); // WaterML1.1 //sitesAll = WofClient.GetSitesObject(blank, ""); } catch (Exception e) { errMsg = "Exception@HandleSites: " + e.Message; Console.WriteLine(errMsg); } if (sitesAll == null) { if (errMsg.Length == 0) { errMsg = "Info: no site was returned from the URL."; } Console.WriteLine(errMsg); HQT.HandleQueryTimeseries("", "", -1, -1, -1, null, url, errMsg); return; } WaterML10.SiteInfoResponseTypeSite[] sitesRt = sitesAll.site; int nsites = sitesRt.Count(); Console.WriteLine("........................................."); Console.WriteLine("Reading {0} sites from {1}", nsites, url); WaterML10.seriesCatalogType[] sctAll; #region Loop for each site for (int i = 0; i < nsites; i++) { if (CPEnabled) { if (CP.siteIndex > i) { continue; } } if (i % 100 == 0) { // Do this check and refresh for after every 100 sites CheckAndRefreshResources(); } network = sitesRt[i].siteInfo.siteCode[0].network; siteCode = network + ":" + sitesRt[i].siteInfo.siteCode[0].Value; if (CPEnabled) { CP.siteIndex = i; CP.siteCode = siteCode; SaveCheckPoint(); } //queryParam = string.Format("QueryID = (SELECT TOP 1 QueryID FROM dbo.QueryTimeseries where SiteCode = '{0}'" + // " ORDER BY QueryID DESC) and DATEDIFF(HOUR, QueryDateTime, GETDATE()) <= {1}", // siteCode, NoCheckHours); //if (OD_Utils.Exists("dbo.QueryTimeseries", queryParam, HQT.Adapter.Connection)) //{ // Console.WriteLine("..Skip Site {0} which was already checked in last {1} hours.", // siteCode, NoCheckHours); // continue; //} if (sitesRt[i].seriesCatalog != null) { sctAll = sitesRt[i].seriesCatalog; } else { sctAll = LoadOnesiteInfo(WofClient, url, siteCode, ref errMsg); } if (sctAll == null) { if (errMsg.Length == 0) { errMsg = "Error: LoadOneSiteInfo() returned NULL."; } Console.WriteLine(errMsg); HQT.HandleQueryTimeseries(siteCode, "", -1, -1, -1, null, url, errMsg); continue; } if ((i + 1) % 100 == 0) { Console.WriteLine("..Site[{0}] {1} with {2} catalogs from {3} Time {4:HH:mm:ss}.", i, siteCode, sctAll.Count(), url, DateTime.Now); } else { Console.WriteLine("..Site[{0}] {1} with {2} catalogs Time {3:HH:mm:ss}.", i, siteCode, sctAll.Count(), DateTime.Now); } // network = SRBHOS, sites[i].seriesCatalog=null // My assumption is that // Under that situation, need to get SiteInfoResponseType one site by one site // Here is the one example siteCode for testing //WaterML11.SiteInfoResponseType rt = new WaterML11.SiteInfoResponseType(); //rt = WofClient.GetSiteInfoObject("SRBHOS:RTHNet", ""); #region Loop for each seriesCatalogType for (int j = 0; j < sctAll.Count(); j++) { int valueCount; WaterML10.seriesCatalogType sct = sctAll[j]; if (sct.series == null) { errMsg = string.Format("Error: WaterML10.seriesCatalogType[{0}].series is NULL.", j); HQT.HandleQueryTimeseries(siteCode, "", -1, -1, -1, null, url, errMsg); continue; } #region Loop for each seriesCatalogTypeSeries for (int k = 0; k < sct.series.Count(); k++) { WaterML10.seriesCatalogTypeSeries scts = sct.series[k]; WaterML10.TimeIntervalType ti = (WaterML10.TimeIntervalType)scts.variableTimeInterval; DateTime beginDateTime = ti.beginDateTime; DateTime endDateTime = ti.endDateTime; string code = scts.variable.variableCode[0].Value; varCode = network + ":" + code; valueCount = scts.valueCount.Value; if (valueCount <= MinValueCount) { Console.WriteLine( "....Series[{0}] varCode: {1} valCount: {2}, too small, record in Query table, no further action.", k, code, valueCount); queryParam = String.Format(" <{0}--{1}> ", beginDateTime, endDateTime); errMsg = "Info: value count is too small, no further action."; HQT.HandleQueryTimeseries(siteCode, varCode, -1, -1, valueCount, null, queryParam, errMsg); continue; } // Check hiscentral/SeriesCatalog table to get seriesCatalogID. // Note: beginDateTime and endDateTime are obtained from web service. seriesCatalogID = newSCID = 0; hiscentralDataSet.SeriesCatalogRow hscRow = HisSC.GetRow(siteCode, varCode, scts); if (hscRow != null) { seriesCatalogID = hscRow.SeriesID; if (hscRow.SeriesCode != null) { // Don't want to repeatedly print siteCode and network int idx = hscRow.SeriesCode.IndexOf(code); code = hscRow.SeriesCode.Substring(idx); } Console.WriteLine("....Series[{0}] var: {1}, ID: {2}, valCount: {3}", k, code, seriesCatalogID, valueCount); } else { newSCID = HSC.GetOrCreateSeriesID(siteCode, varCode, scts); Console.WriteLine("....Series[{0}] var: {1}||{2}||{3}||{4}, newID: {5}, valCount: {6}", k, code, // Where WaterML1.1 and WaterML1.0 differ scts.Method.methodID, scts.Source.sourceID, scts.QualityControlLevel.QualityControlLevelID, newSCID, valueCount); } // Update HealthQuery/DataValue table if datavalue changed int[] hasChanged = null; try { if (valueCount > 5000) { WaterML10.TimeSeriesResponseType[] tsRtAll = null; tsRtAll = LoadDataValueInfoSplit(siteCode, varCode, WofClient, valueCount, beginDateTime, endDateTime, ref queryParam, ref errMsg); if (tsRtAll != null) { hasChanged = HDV.HandleDataValueInfoSplit(seriesCatalogID, newSCID, tsRtAll); } } else { WaterML10.TimeSeriesResponseType tsRt = null; tsRt = LoadDataValueInfo(siteCode, varCode, WofClient, beginDateTime, endDateTime, ref queryParam, ref errMsg); if (tsRt != null) { hasChanged = HDV.HandleDataValueInfo(seriesCatalogID, newSCID, tsRt); } } } catch (Exception ex) { errMsg = "Exception@HandleSites1: " + ex.Message; Console.WriteLine(errMsg); } // Add to HealthQuery/QueryTimeseries table. HQT.HandleQueryTimeseries(siteCode, varCode, seriesCatalogID, newSCID, valueCount, hasChanged, queryParam, errMsg); } #endregion Loop for each seriesCatalogTypeSeries // We delay inserting records to Query table to improve performance, check and do it now if (HQT.Table.Count > Program.DbUpdateBatchSize / 2) { HQT.Adapter.Update(HQT.Table); HQT.Table.Clear(); } } #endregion Loop for each seriesCatalogType } #endregion Loop for each site if (HQT.Table.Count > 0) { HQT.Adapter.Update(HQT.Table); HQT.Table.Clear(); } } // end of HandleSites_1_0()
private string FetchMetadata(List <string> serviceUrls, DoWorkEventArgs e) { var totalServices = serviceUrls.Count; var currentService = 0; var seriesCount = 0; // Keeps track of how many series were successfully processed var cacheManager = DatabaseOperations.GetCacheManager(); var errors = new StringBuilder(); // Keep track of errors and report them at the end foreach (var serviceUrl in serviceUrls) { // Check for cancel if (bgwMain.CancellationPending) { e.Cancel = true; return("Operation cancelled"); } // Update progress currentService++; bgwMain.ReportProgress(100 * (currentService - 1) / totalServices, "Reading database info for service " + currentService + " of " + totalServices + "..."); // Get the service info for this item var serviceInfo = DatabaseOperations.GetDataServiceFromCache(serviceUrl); // Check for cancel if (bgwMain.CancellationPending) { e.Cancel = true; return("Operation cancelled"); } // Update progress bgwMain.ReportProgress(100 * (currentService - 1) / totalServices, "Deleting old records for service " + currentService + " of " + totalServices + "..."); // Delete existing records for this service cacheManager.DeleteRecordsForService(serviceInfo, false); // Check for cancel if (bgwMain.CancellationPending) { e.Cancel = true; return("Operation cancelled"); } // Update progress bgwMain.ReportProgress(100 * (currentService - 1) / totalServices, "Connecting to service " + currentService + " of " + totalServices + "..."); // Create a WaterOneFlow Service for this URL WaterOneFlowClient waterOneFlowClient; try { waterOneFlowClient = new WaterOneFlowClient(serviceUrl); } catch (Exception ex) { // Flag the error and continue to the next service errors.AppendLine("Could not connect to service with URL: " + serviceUrl + ".\n" + ex.Message + "\n\n"); continue; } // Check for cancel if (bgwMain.CancellationPending) { e.Cancel = true; return("Operation cancelled"); } // Update progress bgwMain.ReportProgress(100 * (currentService - 1) / totalServices, "Updating database description for service " + currentService + " of " + totalServices + "..."); // Update service info in the metadata database var waterOneFlowServiceInfo = waterOneFlowClient.ServiceInfo; serviceInfo.IsHarvested = false; serviceInfo.ServiceName = waterOneFlowServiceInfo.ServiceName; serviceInfo.Version = waterOneFlowServiceInfo.Version; serviceInfo.ServiceType = waterOneFlowServiceInfo.ServiceType; serviceInfo.Protocol = waterOneFlowServiceInfo.Protocol; serviceInfo.VariableCount = (cacheManager.GetVariablesByService((int)serviceInfo.Id)).Count; cacheManager.UpdateDataRow(serviceInfo); // Check for cancel if (bgwMain.CancellationPending) { e.Cancel = true; return("Operation cancelled"); } // Update progress bgwMain.ReportProgress(100 * (currentService - 1) / totalServices, "Downloading site list for service " + currentService + " of " + totalServices + "..."); // Get all sites for this service IList <Site> siteList; try { siteList = waterOneFlowClient.GetSites(); } catch (WebException ex) { var sr = new StreamReader(ex.Response.GetResponseStream()); sr.ReadToEnd(); // Flag the error and continue to the next service errors.AppendLine("Could not get site list from service with URL: " + serviceUrl + ".\n" + ex.Message + "\n\n"); continue; } // Check for cancel if (bgwMain.CancellationPending) { e.Cancel = true; return("Operation cancelled"); } var totalSteps = siteList.Count; var currentStep = 0; // Error tracking var siteErrorCount = 0; // Keeps track of how many errors we had while downloading site info for the current service var saveErrorCount = 0; // Keeps track of how many errors we had while saving site info for the current service var firstSiteError = ""; // Records the message from the first error we had while getting site info for the current service var firstSaveError = ""; // Records the message from the first error we had while saving site info for the current service // Default extent for the service. These values are designed to be overwritten as we query sites in the service double east = -180; double west = 360; double north = -90; double south = 90; int valueCount = 0; IList <Site> siteList1, siteList2; if (siteList.Count % 2 == 0) { siteList1 = (siteList as List <Site>).GetRange(0, siteList.Count / 2); siteList2 = (siteList as List <Site>).GetRange(siteList.Count / 2, siteList.Count / 2); } else { siteList1 = (siteList as List <Site>).GetRange(0, (siteList.Count / 2) + 1); siteList2 = (siteList as List <Site>).GetRange((siteList.Count / 2) + 1, siteList.Count / 2); } var thread1 = new Thread(() => ProcessSites(siteList1)); thread1.Start(); var thread2 = new Thread(() => ProcessSites(siteList2)); thread2.Start(); foreach (var site in siteList) { // Check for cancel if (bgwMain.CancellationPending) { e.Cancel = true; return("Operation cancelled"); } // Update progress currentStep++; bgwMain.ReportProgress(100 * currentStep / totalSteps, "Processing site " + currentStep + " of " + totalSteps + " from service " + currentService + " of " + totalServices + "..."); // Get series for this site IList <SeriesMetadata> currentSeriesList; try { currentSeriesList = waterOneFlowClient.GetSiteInfo(site.Code); } catch (WebException ex) { // Flag the error and continue to the next site siteErrorCount++; if (siteErrorCount == 1) { firstSiteError = ex.Message; } if (ex.Response != null) { var rdr = new StreamReader(ex.Response.GetResponseStream()); rdr.ReadToEnd(); } continue; } catch (Exception ex) { // Flag the error and continue to the next site siteErrorCount++; if (siteErrorCount == 1) { firstSiteError = ex.Message; } continue; } // Update service extent if (site.Latitude > north) { north = site.Latitude; } if (site.Latitude < south) { south = site.Latitude; } if (site.Longitude > east) { east = site.Longitude; } if (site.Longitude < west) { west = site.Longitude; } // Save series info to metadata cache database foreach (var series in currentSeriesList) { valueCount += series.ValueCount; // Check for cancel if (bgwMain.CancellationPending) { e.Cancel = true; return("Operation cancelled"); } try { cacheManager.SaveSeries(series, serviceInfo); } catch (Exception ex) { // Flag the error and continue to the next series saveErrorCount++; if (saveErrorCount == 1) { firstSaveError = ex.Message; } continue; } // Keep track of how many series were successfully processed seriesCount++; } } // Log errors if (siteErrorCount == 1) { errors.AppendLine("Could not get site info for 1 site in service with URL: " + serviceUrl + "\n" + firstSiteError + "\n\n"); } else if (siteErrorCount > 1) { errors.AppendLine("Could not get site info for " + siteErrorCount + " sites in service with URL: " + serviceUrl + "\n" + firstSiteError + "\n\n"); } if (saveErrorCount == 1) { errors.AppendLine("Could not save site info for 1 site in service with URL: " + serviceUrl + "\n" + firstSaveError + "\n\n"); } else if (saveErrorCount > 1) { errors.AppendLine("Could not save site info for " + siteErrorCount + " sites in service with URL: " + serviceUrl + "\n" + firstSaveError + "\n\n"); } // Check for cancel if (bgwMain.CancellationPending) { e.Cancel = true; return("Operation cancelled"); } // Update progress bgwMain.ReportProgress(100 * currentService / totalServices, "Updating harvested statistics for service " + currentService + " of " + totalServices + "..."); // Update service info serviceInfo.IsHarvested = true; serviceInfo.HarveDateTime = DateTime.Now; serviceInfo.EastLongitude = east; serviceInfo.WestLongitude = west; serviceInfo.NorthLatitude = north; serviceInfo.SouthLatitude = south; serviceInfo.SiteCount = siteList.Count; serviceInfo.ValueCount = valueCount; serviceInfo.VariableCount = (cacheManager.GetVariablesByService((int)serviceInfo.Id)).Count; cacheManager.UpdateDataRow(serviceInfo); // Updates properties like harvest datetime and service extent } // Check for cancel if (bgwMain.CancellationPending) { e.Cancel = true; return("Operation cancelled"); } // Update progress bgwMain.ReportProgress(100, "Operation complete"); // Report result var message = "Download complete. " + seriesCount + " series saved to metadata cache database."; if (errors.Length > 0) { message += "\n\nSome errors occurred during the operation:\n\n" + errors; } return(message); }