public void GetSiteInfo_SaveXmlFilesFlag_ReturnsSameData(string url, string siteCode)
{
try
{
var target = new WaterOneFlowClient(url, 5)
{
SaveXmlFiles = false
};
var series1 = target.GetSiteInfo(siteCode);
target.SaveXmlFiles = true;
var series2 = target.GetSiteInfo(siteCode);
Assert.AreEqual(series1.Count, series2.Count);
for (var i = 0; i < series1.Count; i++)
{
var site1 = series1[i].Site;
var site2 = series2[i].Site;
Assert.AreEqual(site1.Code, site2.Code);
Assert.AreEqual(site1.Name, site2.Name);
Assert.AreEqual(series1[i].ValueCount, series2[i].ValueCount);
}
}
catch (WebException ex)
{
if (ex.Status.HasFlag(WebExceptionStatus.ProtocolError) ||
ex.Status.HasFlag(WebExceptionStatus.Timeout))
{
Assert.Inconclusive("Unable to test GetSiteInfo() from: " + url);
}
throw;
}
}
public void CanSaveMultipleSeries()
{
Random random = new Random();
//string url1 = @"http://his02.usu.edu/littlebearriver/cuahsi_1_0.asmx";
//string url2 = @"http://icewater.boisestate.edu/dcew2dataservices/cuahsi_1_0.asmx";
string url3 = @"http://his.crwr.utexas.edu/TXEvap/cuahsi_1_0.asmx";
MetadataCacheManagerSQL manager = TestConfig.MetadataCacheManager;
WaterOneFlowClient client = new WaterOneFlowClient(url3);
IList <Site> siteList = client.GetSites();
IList <SeriesMetadata> seriesList = new List <SeriesMetadata>();
DataServiceInfo service = GeDatatService(random.Next());
manager.SaveDataService(service);
foreach (Site site in siteList)
{
IList <SeriesMetadata> seriesList1 = client.GetSiteInfo(site.Code);
foreach (SeriesMetadata series in seriesList1)
{
seriesList.Add(series);
}
}
foreach (SeriesMetadata series in seriesList)
{
manager.SaveSeries(series, service);
}
}
public void TestCRWR_10()
{
string url = @"http://his.crwr.utexas.edu/tcoonts/tcoon.asmx";
WaterOneFlowClient client = new WaterOneFlowClient(url);
IList <Site> sites = client.GetSites();
Assert.Greater(sites.Count, 0);
}
private void button1_Click(object sender, EventArgs e)
{
variablesListBox.Items.Clear();
waterOneFlowClient = new WaterOneFlowClient(urlTextbox.Text);
// Update service info in the metadata database
var waterOneFlowServiceInfo = waterOneFlowClient.ServiceInfo;
var service = waterOneFlowServiceInfo;
// Get all sites for this service
IList <Site> siteList;
siteList = waterOneFlowClient.GetSites();
var variableList = new List <string>();
foreach (var site in siteList)
{
// Get series for this site
IList <SeriesMetadata> currentSeriesList;
try
{
currentSeriesList = waterOneFlowClient.GetSiteInfo(site.Code);
}
catch (WebException ex)
{
if (ex.Response != null)
{
var rdr = new StreamReader(ex.Response.GetResponseStream());
rdr.ReadToEnd();
}
continue;
}
// Save series info to metadata cache database
foreach (var series in currentSeriesList)
{
if (!variableList.Contains(series.Variable.Name))
{
var variable = series.Variable.Name;
variableList.Add(variable);
}
}
}
foreach (var variable in variableList)
{
this.variablesListBox.Items.Add(variable, false);
}
if (variablesListBox.Items.Count != 0)
{
button1.Enabled = false;
button2.Enabled = true;
button3.Enabled = true;
button4.Enabled = true;
}
}
/// <summary>
/// Gets a SeriesData object from
/// Todo: Migrate to separate repository specific for blob storage or table storage.
/// </summary>
/// <param name="seriesId"></param>
/// <param name="guid"></param>
/// <returns></returns>
public async Task <ServerSideHydroDesktop.ObjectModel.Series> GetWaterOneFlowFromCloudCache(string seriesId, string guid, string servUrl)
{
WaterOneFlowClient waterOneFlow = new WaterOneFlowClient(servUrl);
CloudStorageAccount csa = CUAHSIDataStorage.LogHelper.GetCUAHSIDataStorage();
CloudBlobClient client = csa.CreateCloudBlobClient();
Stream s = new MemoryStream();
await client.GetContainerReference(guid).GetBlockBlobReference(seriesId).DownloadToStreamAsync(s);
return(waterOneFlow.GetValuesFromStream(s).FirstOrDefault());
}
public async Task <Tuple <Stream, IList <ServerSideHydroDesktop.ObjectModel.Series> > > SeriesAndStreamOfSeriesID(SeriesMetadata meta)
{
WaterOneFlowClient client = new WaterOneFlowClient(meta.ServURL);
return(await client.GetValuesAndRawStreamAsync(
meta.SiteCode,
meta.VarCode,
meta.StartDate,
DateTime.UtcNow,
Convert.ToInt32(10000)));
}
public void TestLittleBearRiver_10()
{
string url = @"http://icewater.usu.edu/LittleBearRiver/cuahsi_1_0.asmx";
WaterOneFlowClient client = new WaterOneFlowClient(url);
IList <Site> sites = client.GetSites();
IList <SeriesMetadata> seriesList = client.GetSiteInfo(sites[0].Code);
IList <Series> downloadedData = client.GetValues(seriesList[0].Site.Code, seriesList[0].Variable.Code, DateTime.MinValue, DateTime.Now);
Assert.IsNotNull(seriesList);
Assert.Greater(seriesList.Count, 0);
Assert.AreNotEqual(seriesList[0].Source.ToString(), "unknown");
Assert.AreNotEqual(seriesList[0].Method.ToString(), "unknown");
}
public void TestSeriesWithSamples()
{
string url = @"http://water.sdsc.edu/lbrsdsc/cuahsi_1_1.asmx";
WaterOneFlowClient client = new WaterOneFlowClient(url);
string siteCode = "LBR11:USU-LBR-Paradise";
string variableCode = "LBR11:USU39";
IList <Series> series = client.GetValues(siteCode, variableCode, DateTime.MinValue, DateTime.Now);
Assert.IsNotNull(series);
Assert.Greater(series.Count, 0);
Assert.AreNotEqual(series[0].Source.ToString(), "unknown");
Assert.AreNotEqual(series[0].Method.ToString(), "unknown");
}
public void TestCIMSservice()
{
string url = @"http://cbe.cae.drexel.edu/CIMS/cuahsi_1_1.asmx";
string siteCode = "CIMS:EE3.4";
string variableCode = "CIMS:NO3F";
WaterOneFlowClient client = new WaterOneFlowClient(url);
IList <Series> series = client.GetValues(siteCode, variableCode, DateTime.Now.AddYears(-30), DateTime.Now);
Assert.IsNotNull(series);
Assert.Greater(series.Count, 0);
Assert.AreNotEqual(series[0].Source.ToString(), "unknown");
Assert.AreNotEqual(series[0].Method.ToString(), "unknown");
}
public void TestTCOON_11()
{
string url = @"http://his.crwr.utexas.edu/tcoonts/tcoon.asmx";
WaterOneFlowClient client = new WaterOneFlowClient(url);
IList <Site> sites = client.GetSites();
IList <SeriesMetadata> seriesList = client.GetSiteInfo(sites[0].Code);
//IList<Series> downloadedData = client.GetValues(seriesList[0].Site.Code, seriesList[0].Variable.Code, DateTime.MinValue, DateTime.Now);
//Assert.IsNotNull(downloadedData);
//Assert.Greater(downloadedData.Count, 0);
//Assert.AreNotEqual(downloadedData[0].Source.ToString(), "unknown");
//Assert.AreNotEqual(downloadedData[0].Method.ToString(), "unknown");
}
public void TestNWISDV_10()
{
string url = @"http://river.sdsc.edu/wateroneflow/NWIS/Data.asmx";
string siteCode = "NWISDV:13069500";
string variableCode = "NWISDV:00060/DataType=Average";
DateTime start = new DateTime(2010, 1, 1);
DateTime end = new DateTime(2011, 1, 1);
WaterOneFlowClient client = new WaterOneFlowClient(url);
IList <Series> seriesList = client.GetValues(siteCode, variableCode, start, end);
Assert.Greater(seriesList.Count, 0);
Assert.Greater(seriesList[0].DataValueList.Count, 0);
}
public void TestBaltimoreGW_10()
{
string url = @"http://his09.umbc.edu/BaltGW/cuahsi_1_0.asmx";
string siteCode = "BaltimoreGW:WOLDEAMANUEL";
string variableCode = "BaltimoreGW:WATERASL";
DateTime beginDate = new DateTime(2000, 1, 1);
DateTime endDate = DateTime.Now;
WaterOneFlowClient client = new WaterOneFlowClient(url);
IList <Series> series = client.GetValues(siteCode, variableCode, beginDate, endDate);
Assert.IsNotNull(series);
Assert.Greater(series.Count, 0);
Assert.AreNotEqual(series[0].Source.ToString(), "unknown");
Assert.AreNotEqual(series[0].Method.ToString(), "unknown");
}
// [TestCase(@"http://his.crwr.utexas.edu/TXEvap/cuahsi_1_0.asmx")]
public void TestDownload(string url)
{
WaterOneFlowClient cl = new WaterOneFlowClient(url);
string file2 = cl.GetSitesXML();
Assert.IsTrue(File.Exists(file2), "the GetSites file does not exist.");
WaterOneFlow10Parser parser1 = new WaterOneFlow10Parser();
IList <Site> sites = parser1.ParseGetSitesXml(file2);
foreach (Site site in sites)
{
string fullSiteCode = site.Code;
string siteFile = cl.GetSiteInfoXML(fullSiteCode);
Assert.IsTrue(File.Exists(siteFile), "the GetSiteInfo file does not exist.");
}
}
public void GetValues_SaveXmlFilesFlag_ReturnsSameData(string url, string siteCode, string varCode,
string startDate, string endDate)
{
const string DATES_FORMAT = "MM/dd/yyyy";
var provider = CultureInfo.InvariantCulture;
var target = new WaterOneFlowClient(url, 5);
var start = DateTime.ParseExact(startDate, DATES_FORMAT, provider);
var end = DateTime.ParseExact(endDate, DATES_FORMAT, provider);
try
{
target.SaveXmlFiles = false;
var series1 = target.GetValues(siteCode, varCode, start, end);
target.SaveXmlFiles = true;
var series2 = target.GetValues(siteCode, varCode, start, end);
Assert.AreEqual(series1.Count, series2.Count);
for (var i = 0; i < series1.Count; i++)
{
var site1 = series1[i].Site;
var site2 = series2[i].Site;
var var1 = series1[i].Variable;
var var2 = series2[i].Variable;
Assert.AreEqual(site1.Code, site2.Code);
Assert.AreEqual(site1.Name, site2.Name);
Assert.AreEqual(series1[i].ValueCount, series2[i].ValueCount);
Assert.AreEqual(var1.Code, var2.Code);
Assert.AreEqual(var1.Name, var2.Name);
}
}
catch (WebException ex)
{
if (ex.Status.HasFlag(WebExceptionStatus.ProtocolError) ||
ex.Status.HasFlag(WebExceptionStatus.Timeout))
{
Assert.Inconclusive("Unable to test GetValues() from: " + url);
}
throw;
}
}
/// <summary>
/// Gets an instance of a WaterOneFlow client to be used with
/// the URL specified in the Download info. This instance is retrieved
/// from the dictionary cache or created if necessary.
/// </summary>
/// <param name="wsdl">The URL of the web service main page</param>
/// <returns>the appropriate WaterOneFlow client</returns>
private WaterOneFlowClient GetWsClientInstance(string wsdl)
{
WaterOneFlowClient wsClient;
lock (_syncObject)
{
//To Access the dynamic WSDLs
if (_services.ContainsKey(wsdl))
{
wsClient = _services[wsdl];
}
else
{
wsClient = new WaterOneFlowClient(wsdl);
_services.Add(wsdl, wsClient);
}
}
return(wsClient);
}
/// <summary>
/// Gets an instance of a WaterOneFlow client to be used with
/// the URL specified in the Download info. This instance is retrieved
/// from the dictionary cache or created if necessary.
/// </summary>
/// <param name="wsdl">The URL of the web service main page</param>
/// <returns>the appropriate WaterOneFlow client</returns>
private WaterOneFlowClient GetWsClientInstance(string wsdl)
{
WaterOneFlowClient wsClient;
lock (_syncObject)
{
//To Access the dynamic WSDLs
if (_services.ContainsKey(wsdl))
{
wsClient = _services[wsdl];
}
else
{
wsClient = new WaterOneFlowClient(wsdl,
valuesPerReq: _options.NumberOfValuesPerRequest,
allInOneRequest: _options.GetAllValuesInOneRequest);
_services.Add(wsdl, wsClient);
}
}
return(wsClient);
}
public void CanSaveOneSeries()
{
Random random = new Random();
string url = @"http://his.crwr.utexas.edu/TXEvap/cuahsi_1_0.asmx";
MetadataCacheManagerSQL manager = TestConfig.MetadataCacheManager;
WaterOneFlowClient client = new WaterOneFlowClient(url);
IList <Site> sites = client.GetSites();
IList <SeriesMetadata> seriesList = client.GetSiteInfo(sites[0].Code);
SeriesMetadata firstSeries = seriesList[0];
DataServiceInfo service = GeDatatService(random.Next());
manager.SaveDataService(service);
firstSeries.DataService = service;
manager.SaveSeries(firstSeries, service);
}
public void TestValidURL()
{
string goodURL = @"http://his.crwr.utexas.edu/TXEvap/cuahsi_1_0.asmx";
WaterOneFlowClient client = new WaterOneFlowClient(goodURL);
IList <Site> sites = client.GetSites();
}
private void button2_Click(object sender, EventArgs e)
{
variablesListBox.Enabled = false;
urlTextbox.Enabled = false;
titleTextbox.Enabled = false;
button1.Enabled = false;
button2.Enabled = false;
button3.Enabled = false;
button4.Enabled = false;
waterOneFlowClient = new WaterOneFlowClient(urlTextbox.Text);
var waterOneFlowServiceInfo = waterOneFlowClient.ServiceInfo;
// Trim the query off of the URL if it still exists
int index = waterOneFlowServiceInfo.EndpointURL.IndexOf("?");
if (index > -1)
{
waterOneFlowServiceInfo.EndpointURL = waterOneFlowServiceInfo.EndpointURL.Substring(0, index);
}
var serviceInfo = new DataServiceInfo();
serviceInfo.IsHarvested = false;
serviceInfo.ServiceName = waterOneFlowServiceInfo.ServiceName;
serviceInfo.Version = waterOneFlowServiceInfo.Version;
serviceInfo.ServiceType = waterOneFlowServiceInfo.ServiceType;
serviceInfo.Protocol = waterOneFlowServiceInfo.Protocol;
serviceInfo.VariableCount = waterOneFlowServiceInfo.VariableCount;
IList <Site> siteList;
siteList = waterOneFlowClient.GetSites();
// Default extent for the service. These values are designed to be overwritten as we query sites in the service
double east = -180;
double west = 360;
double north = -90;
double south = 90;
int valueCount = 0;
var totalDataCartSeriesList = new List <SeriesDataCart>();
foreach (var site in siteList)
{
// Get series for this site
IList <SeriesMetadata> currentSeriesList;
var dataCartSeriesList = new List <SeriesDataCart>();
try
{
currentSeriesList = waterOneFlowClient.GetSiteInfo(site.Code);
}
catch (WebException ex)
{
continue;
}
catch (Exception ex)
{
continue;
}
// Update service extent
if (site.Latitude > north)
{
north = site.Latitude;
}
if (site.Latitude < south)
{
south = site.Latitude;
}
if (site.Longitude > east)
{
east = site.Longitude;
}
if (site.Longitude < west)
{
west = site.Longitude;
}
// Save series info to metadata cache database
foreach (var series in currentSeriesList)
{
valueCount += series.ValueCount;
try
{
if (checkedVariables.Count != 0)
{
if (checkedVariables.Contains(series.Variable.Name))
{
var seriesDataCart = getDataCartFromMetadata(series, waterOneFlowServiceInfo);
dataCartSeriesList.Add(seriesDataCart);
}
}
else
{
var seriesDataCart = getDataCartFromMetadata(series, waterOneFlowServiceInfo);
dataCartSeriesList.Add(seriesDataCart);
}
}
catch (Exception ex)
{
continue;
}
}
totalDataCartSeriesList.AddRange(dataCartSeriesList);
variablesListBox.Enabled = true;
urlTextbox.Enabled = true;
titleTextbox.Enabled = true;
button1.Enabled = true;
button2.Enabled = true;
button3.Enabled = true;
button4.Enabled = true;
}
// Update service info
serviceInfo.IsHarvested = true;
serviceInfo.HarveDateTime = DateTime.Now;
serviceInfo.EastLongitude = east;
serviceInfo.WestLongitude = west;
serviceInfo.NorthLatitude = north;
serviceInfo.SouthLatitude = south;
serviceInfo.SiteCount = siteList.Count;
serviceInfo.ValueCount = valueCount;
serviceInfo.VariableCount = waterOneFlowServiceInfo.VariableCount;
SearchResult resultFeatureSet = null;
if (totalDataCartSeriesList.Count > 0)
{
resultFeatureSet = SearchHelper.ToFeatureSetsByDataSource(totalDataCartSeriesList);
}
if (resultFeatureSet != null)
{
//We need to reproject the Search results from WGS84 to the projection of the map.
var wgs84 = KnownCoordinateSystems.Geographic.World.WGS1984;
foreach (var item in resultFeatureSet.ResultItems)
{
item.FeatureSet.Projection = wgs84;
}
var layers = ShowSearchResults(resultFeatureSet);
Debug.WriteLine("ShowSearchResults done.");
// Unselect all layers in legend (http://hydrodesktop.codeplex.com/workitem/8559)
App.Map.MapFrame.GetAllLayers().ForEach(r => r.IsSelected = false);
// Select first search result layer
var first = layers.FirstOrDefault().GetParentItem();
if (first != null)
{
first.IsSelected = true;
}
// Unselect "Map Layers" legend item (http://hydrodesktop.codeplex.com/workitem/8458)
App.Legend.RootNodes
.ForEach(delegate(ILegendItem item)
{
if (item.LegendText == "Map Layers")
{
item.IsSelected = false;
}
});
Debug.WriteLine("Finished.");
}
}
/// <summary>
/// For each service described in the data grid view, adds records to the metadata cache database to describe the service
/// </summary>
/// <param name="servicesToAdd">List of services to add to the metadata cache database</param>
/// <param name="e">Parameters from the BackgroundWorker</param>
/// <returns>Parameters (task type, output message, rows that were successfully added) to be processed by a BackgroundWorker event handler</returns>
private object[] AddServicesToDatabase(List <DataServiceInfo> servicesToAdd, DoWorkEventArgs e)
{
// Build parameters to pass to the background worker
object[] parameters = new object[3];
parameters[0] = BackgroundWorkerTasks.UpdateDatabase;
parameters[1] = "Operation cancelled";
List <int> rowsToSelect = new List <int> ();
bgwMain.ReportProgress(0, "Getting list of existing WaterOneFlow services from database...");
// Get a list of existing URLs from the metadata cache databsae
List <string> existingUrls = DatabaseOperations.GetCacheServiceUrls(true);
// Add service records to the database
int totalSteps = servicesToAdd.Count;
int currentStep = 0;
int countAlreadyExists = 0;
int countInvalidService = 0;
IEqualityComparer <string> comparer = new CaseInsensitiveEqualityComparer();
MetadataCacheManagerSQL cacheManager = DatabaseOperations.GetCacheManager();
for (int i = 0; i < servicesToAdd.Count; i++)
{
if (bgwMain.CancellationPending)
{
e.Cancel = true;
return(parameters);
}
currentStep++;
bgwMain.ReportProgress(100 * currentStep / totalSteps, "Checking service " + currentStep + " of " + totalSteps + "...");
DataServiceInfo serviceInfo = servicesToAdd[i];
// Check if the service already exists in the database
if (existingUrls.Contains(serviceInfo.EndpointURL, comparer) == true)
{
countAlreadyExists += 1;
continue;
}
// Check that the URL is for a live service
if (this.mnuCheckForValidService.Checked == true)
{
// Attempt to create a WaterOneFlowServiceClient from the URL. If the URL is not for a WaterOneFlow service, an error is thrown in the constructor.
try
{
WaterOneFlowClient waterOneFlowClient = new WaterOneFlowClient(serviceInfo.EndpointURL);
DataServiceInfo clientServiceInfo = waterOneFlowClient.ServiceInfo;
serviceInfo.ServiceName = clientServiceInfo.ServiceName;
serviceInfo.Protocol = clientServiceInfo.Protocol;
serviceInfo.ServiceType = clientServiceInfo.ServiceType;
serviceInfo.Version = clientServiceInfo.Version;
}
catch
{
countInvalidService += 1;
continue;
}
}
// Save the service
cacheManager.SaveDataService(serviceInfo);
existingUrls.Add(serviceInfo.EndpointURL);
rowsToSelect.Add(i);
}
// Prepare a message to the user
string message = "";
int serviceCount = rowsToSelect.Count;
if (serviceCount == 0)
{
message += "No new services added to metadata cache database.\n\n ";
}
else if (serviceCount == 1)
{
message += "1 new service added to metadata cache database.\n\n";
}
else
{
message += serviceCount.ToString() + " new services added to metadata cache database.\n\n";
}
if (countAlreadyExists == 1)
{
message += "1 service was not added because it already exists in the database.\n\n";
}
else if (countAlreadyExists > 1)
{
message += countAlreadyExists.ToString() + " services were not added because they already exist in the database.\n\n";
}
if (countInvalidService == 1)
{
message += "1 service was not added because it does not point to a valid WaterOneFlow service.\n\n";
}
else if (countInvalidService > 1)
{
message += countInvalidService.ToString() + " services were not added because they do not point to a valid WaterOneFlow service.\n\n";
}
if (serviceCount == 1)
{
message += "Remember to download metadata for this service in the Metadata Fetcher window.";
}
else if (serviceCount > 1)
{
message += "Remember to download metadata for these services in the Metadata Fetcher window.";
}
parameters[1] = message;
parameters[2] = rowsToSelect;
return(parameters);
}
static void HandleSites_1_1(string url, SqlConnection sqlConn)
{
string network, siteCode, varCode;
string queryParam = string.Empty;
string errMsg = string.Empty;
int count;
WaterML11.WaterOneFlowClient WofClient = new WaterOneFlowClient("WaterOneFlow", url);
string[] blank = new string[0];
WaterML11.SiteInfoResponseType siteRt = null;
try
{
siteRt = WofClient.GetSitesObject(blank, "");
}
catch (Exception e)
{
errMsg = "Exception@HandleSites: " + e.Message;
Console.WriteLine(errMsg);
}
if (siteRt == null)
{
if (errMsg.Length == 0)
{
errMsg = "Info: no site was returned from the URL.";
}
Console.WriteLine(errMsg);
return;
}
// Write into "Sites"
//siteRt = WofClient.GetSiteInfoObject(siteCode, "");
if (OD_SiteInfo.HandleSiteInfo(sqlConn, siteRt) == 0)
{
OD_Utils.Exit(1);
}
int nsites = siteRt.site.Count();
Console.WriteLine("Reading {0} sites from {1}..", nsites, url);
WaterML11.seriesCatalogType[] sctAll;
OD_SeriesCatalog odSC = new OD_SeriesCatalog(sqlConn);
#region Loop for each site
for (int i = 0; i < nsites; i++)
{
network = siteRt.site[i].siteInfo.siteCode[0].network;
siteCode = network + ":" + siteRt.site[i].siteInfo.siteCode[0].Value;
if (i % 10 == 0)
{
// Do this check and refresh for after every 100 sites
CheckAndRefreshResources();
sqlConn = SqlConnOD;
odSC.Adapter.Connection = sqlConn;
}
sctAll = LoadOnesiteInfo(WofClient, url, siteCode, ref errMsg);
if (sctAll == null)
{
if (errMsg.Length == 0)
{
errMsg = "Error: LoadOneSiteInfo() returned NULL.";
}
Console.WriteLine(errMsg);
continue;
}
if ((i + 1) % 50 == 0)
{
Console.WriteLine("..Site[{0}] {1} with {2} catalogs from {3} Time {4:HH:mm:ss}.",
i, siteCode, sctAll.Count(), url, DateTime.Now);
}
else
{
Console.WriteLine("..Site[{0}] {1} with {2} catalogs Time {3:HH:mm:ss}.",
i, siteCode, sctAll.Count(), DateTime.Now);
}
string sourceOrg = null;
int methodID = 0;
#region Loop for each seriesCatalogType
for (int j = 0; j < sctAll.Count(); j++)
{
WaterML11.seriesCatalogType sct = sctAll[j];
if (sct.series == null)
{
errMsg = string.Format("Error: WaterML11.seriesCatalogType[{0}].series is NULL.", j);
continue;
}
#region Loop for each seriesCatalogTypeSeries
for (int k = 0; k < sct.series.Count(); k++)
{
WaterML11.seriesCatalogTypeSeries scts = sct.series[k];
WaterML11.TimeIntervalType ti = (WaterML11.TimeIntervalType)scts.variableTimeInterval;
DateTime beginDateTime = ti.beginDateTime;
DateTime endDateTime = ti.endDateTime;
string code = scts.variable.variableCode[0].Value;
varCode = network + ":" + code;
//valueCount = scts.valueCount.Value;
Console.WriteLine("");
Console.WriteLine("....Series[{0}] var: {1} of site[{2}]", k, code, i);
// Use variable info from siteInfoResponseType
// instead of from variablesResponseType
// since the former includes the latter and contains more infomration
if (OD_VariableInfo.HandleVariableInfo(sqlConn, scts.variable) == 0)
{
Console.WriteLine("Failed to insert variable. Give up!");
continue;
}
// Add to SeriesCatalog table if not there
// Check SeriesCatalog table to get the newest data time
OD_1_1_1DataSet.SeriesCatalogRow scRow = odSC.GetOrCreateSeriesCatalog(
siteRt.site[i].siteInfo.siteCode[0].Value,
scts.variable.variableCode[0].Value,
scts);
if (scRow.EndDateTime >= ti.endDateTime)
{
Console.WriteLine("No further action since database has most recent ending date time.");
Console.WriteLine("Web service reported {0} values from <{1}> to <{2}>.",
scts.valueCount.Value, ti.beginDateTime.ToString(), ti.endDateTime.ToString());
Console.WriteLine("Database has {0} values from <{1}> to <{2}>.",
scRow.ValueCount, scRow.BeginDateTime.ToString(),
scRow.EndDateTime.ToString());
if (ti.beginDateTime != scRow.BeginDateTime)
{
Console.WriteLine("WARNING: Web server has older data not in database! Please double check!");
}
if (scts.valueCount.Value != scRow.ValueCount)
{
Console.WriteLine("WARNING: data value counts mismatch, maybe due to duplicate values from web service!");
}
continue;
}
// Should we use UTC time?
if (scRow.ValueCount > 0)
{
beginDateTime = OD_Utils.GetDateTime(
scRow.EndDateTime, scRow.TimeUnitsID, 1);
}
// Update DataValue table
//try
//{
WaterML11.TimeSeriesResponseType tsrt = null;
Console.WriteLine("......Getting {0} values from <{1}> to <{2}>",
scts.valueCount.Value, beginDateTime, endDateTime);
tsrt = LoadDataValueInfo(siteCode, varCode, WofClient,
beginDateTime, endDateTime);
if (tsrt == null)
{
continue;
}
if ((sourceOrg == null) || (!string.Equals(sourceOrg, scts.source.organization)))
{
// Insert a new source
sourceOrg = scts.source.organization;
OD_SourceInfo.HandleSourceInfo(sqlConn,
siteRt.site[i].siteInfo, scts, tsrt.timeSeries[0]);
}
//if ((methodCode == null) || (!string.Equals(methodCode, scts.method.methodCode)))
if ((methodID == 0) || (methodID != scts.method.methodID))
{
// Insert a new method
methodID = scts.method.methodID;
OD_Methods.HandleMethodsInfo(sqlConn, scts.method);
}
count = OD_DataValues.HandleDataValueInfo(sqlConn,
odSC, scRow, siteRt.site[i].siteInfo, scts, tsrt);
Console.WriteLine(" -------->>>>>>> Inserted {0} records. Database has {1} values from <{2}> to <{3}>.",
count, scRow.ValueCount, scRow.BeginDateTime.ToString(),
scRow.EndDateTime.ToString());
//}
//catch (Exception ex)
//{
// errMsg = "Exception@HandleDataValueInfo: " + ex.Message;
// Console.WriteLine(errMsg);
//}
}
#endregion Loop for each seriesCatalogTypeSeries
}
#endregion Loop for each seriesCatalogType
}
#endregion Loop for each site
} // end of HandleSites_1_1()
public void TestInvalidURL()
{
string badURL = @"http://example.com";
WaterOneFlowClient client = new WaterOneFlowClient(badURL);
}
static void HandleSites_1_0(string url)
{
string network, siteCode, varCode;
int seriesCatalogID, newSCID;
string queryParam = string.Empty;
string errMsg = string.Empty;
WaterML10.WaterOneFlowClient WofClient = new WaterOneFlowClient("WaterOneFlow", url);
string[] blank = new string[0];
WaterML10.SiteInfoResponseType sitesAll = null;
try
{
// WaterML1.0
sitesAll = WofClient.GetSites(blank, "");
// WaterML1.1
//sitesAll = WofClient.GetSitesObject(blank, "");
}
catch (Exception e)
{
errMsg = "Exception@HandleSites: " + e.Message;
Console.WriteLine(errMsg);
}
if (sitesAll == null)
{
if (errMsg.Length == 0)
{
errMsg = "Info: no site was returned from the URL.";
}
Console.WriteLine(errMsg);
HQT.HandleQueryTimeseries("", "", -1, -1, -1, null, url, errMsg);
return;
}
WaterML10.SiteInfoResponseTypeSite[] sitesRt = sitesAll.site;
int nsites = sitesRt.Count();
Console.WriteLine(".........................................");
Console.WriteLine("Reading {0} sites from {1}", nsites, url);
WaterML10.seriesCatalogType[] sctAll;
#region Loop for each site
for (int i = 0; i < nsites; i++)
{
if (CPEnabled)
{
if (CP.siteIndex > i)
{
continue;
}
}
if (i % 100 == 0)
{
// Do this check and refresh for after every 100 sites
CheckAndRefreshResources();
}
network = sitesRt[i].siteInfo.siteCode[0].network;
siteCode = network + ":" + sitesRt[i].siteInfo.siteCode[0].Value;
if (CPEnabled)
{
CP.siteIndex = i;
CP.siteCode = siteCode;
SaveCheckPoint();
}
//queryParam = string.Format("QueryID = (SELECT TOP 1 QueryID FROM dbo.QueryTimeseries where SiteCode = '{0}'" +
// " ORDER BY QueryID DESC) and DATEDIFF(HOUR, QueryDateTime, GETDATE()) <= {1}",
// siteCode, NoCheckHours);
//if (OD_Utils.Exists("dbo.QueryTimeseries", queryParam, HQT.Adapter.Connection))
//{
// Console.WriteLine("..Skip Site {0} which was already checked in last {1} hours.",
// siteCode, NoCheckHours);
// continue;
//}
if (sitesRt[i].seriesCatalog != null)
{
sctAll = sitesRt[i].seriesCatalog;
}
else
{
sctAll = LoadOnesiteInfo(WofClient, url, siteCode, ref errMsg);
}
if (sctAll == null)
{
if (errMsg.Length == 0)
{
errMsg = "Error: LoadOneSiteInfo() returned NULL.";
}
Console.WriteLine(errMsg);
HQT.HandleQueryTimeseries(siteCode, "", -1, -1, -1, null, url, errMsg);
continue;
}
if ((i + 1) % 100 == 0)
{
Console.WriteLine("..Site[{0}] {1} with {2} catalogs from {3} Time {4:HH:mm:ss}.",
i, siteCode, sctAll.Count(), url, DateTime.Now);
}
else
{
Console.WriteLine("..Site[{0}] {1} with {2} catalogs Time {3:HH:mm:ss}.",
i, siteCode, sctAll.Count(), DateTime.Now);
}
// network = SRBHOS, sites[i].seriesCatalog=null
// My assumption is that
// Under that situation, need to get SiteInfoResponseType one site by one site
// Here is the one example siteCode for testing
//WaterML11.SiteInfoResponseType rt = new WaterML11.SiteInfoResponseType();
//rt = WofClient.GetSiteInfoObject("SRBHOS:RTHNet", "");
#region Loop for each seriesCatalogType
for (int j = 0; j < sctAll.Count(); j++)
{
int valueCount;
WaterML10.seriesCatalogType sct = sctAll[j];
if (sct.series == null)
{
errMsg = string.Format("Error: WaterML10.seriesCatalogType[{0}].series is NULL.", j);
HQT.HandleQueryTimeseries(siteCode, "", -1, -1, -1, null, url, errMsg);
continue;
}
#region Loop for each seriesCatalogTypeSeries
for (int k = 0; k < sct.series.Count(); k++)
{
WaterML10.seriesCatalogTypeSeries scts = sct.series[k];
WaterML10.TimeIntervalType ti = (WaterML10.TimeIntervalType)scts.variableTimeInterval;
DateTime beginDateTime = ti.beginDateTime;
DateTime endDateTime = ti.endDateTime;
string code = scts.variable.variableCode[0].Value;
varCode = network + ":" + code;
valueCount = scts.valueCount.Value;
if (valueCount <= MinValueCount)
{
Console.WriteLine(
"....Series[{0}] varCode: {1} valCount: {2}, too small, record in Query table, no further action.",
k, code, valueCount);
queryParam = String.Format(" <{0}--{1}> ", beginDateTime, endDateTime);
errMsg = "Info: value count is too small, no further action.";
HQT.HandleQueryTimeseries(siteCode, varCode, -1, -1, valueCount, null,
queryParam, errMsg);
continue;
}
// Check hiscentral/SeriesCatalog table to get seriesCatalogID.
// Note: beginDateTime and endDateTime are obtained from web service.
seriesCatalogID = newSCID = 0;
hiscentralDataSet.SeriesCatalogRow hscRow = HisSC.GetRow(siteCode, varCode, scts);
if (hscRow != null)
{
seriesCatalogID = hscRow.SeriesID;
if (hscRow.SeriesCode != null)
{
// Don't want to repeatedly print siteCode and network
int idx = hscRow.SeriesCode.IndexOf(code);
code = hscRow.SeriesCode.Substring(idx);
}
Console.WriteLine("....Series[{0}] var: {1}, ID: {2}, valCount: {3}",
k, code, seriesCatalogID, valueCount);
}
else
{
newSCID = HSC.GetOrCreateSeriesID(siteCode, varCode, scts);
Console.WriteLine("....Series[{0}] var: {1}||{2}||{3}||{4}, newID: {5}, valCount: {6}",
k, code,
// Where WaterML1.1 and WaterML1.0 differ
scts.Method.methodID,
scts.Source.sourceID,
scts.QualityControlLevel.QualityControlLevelID,
newSCID, valueCount);
}
// Update HealthQuery/DataValue table if datavalue changed
int[] hasChanged = null;
try
{
if (valueCount > 5000)
{
WaterML10.TimeSeriesResponseType[] tsRtAll = null;
tsRtAll = LoadDataValueInfoSplit(siteCode, varCode, WofClient,
valueCount, beginDateTime, endDateTime, ref queryParam, ref errMsg);
if (tsRtAll != null)
{
hasChanged = HDV.HandleDataValueInfoSplit(seriesCatalogID, newSCID, tsRtAll);
}
}
else
{
WaterML10.TimeSeriesResponseType tsRt = null;
tsRt = LoadDataValueInfo(siteCode, varCode, WofClient,
beginDateTime, endDateTime, ref queryParam, ref errMsg);
if (tsRt != null)
{
hasChanged = HDV.HandleDataValueInfo(seriesCatalogID, newSCID, tsRt);
}
}
}
catch (Exception ex)
{
errMsg = "Exception@HandleSites1: " + ex.Message;
Console.WriteLine(errMsg);
}
// Add to HealthQuery/QueryTimeseries table.
HQT.HandleQueryTimeseries(siteCode, varCode, seriesCatalogID, newSCID,
valueCount, hasChanged, queryParam, errMsg);
}
#endregion Loop for each seriesCatalogTypeSeries
// We delay inserting records to Query table to improve performance, check and do it now
if (HQT.Table.Count > Program.DbUpdateBatchSize / 2)
{
HQT.Adapter.Update(HQT.Table);
HQT.Table.Clear();
}
}
#endregion Loop for each seriesCatalogType
}
#endregion Loop for each site
if (HQT.Table.Count > 0)
{
HQT.Adapter.Update(HQT.Table);
HQT.Table.Clear();
}
} // end of HandleSites_1_0()
private string FetchMetadata(List <string> serviceUrls, DoWorkEventArgs e)
{
var totalServices = serviceUrls.Count;
var currentService = 0;
var seriesCount = 0; // Keeps track of how many series were successfully processed
var cacheManager = DatabaseOperations.GetCacheManager();
var errors = new StringBuilder(); // Keep track of errors and report them at the end
foreach (var serviceUrl in serviceUrls)
{
// Check for cancel
if (bgwMain.CancellationPending)
{
e.Cancel = true;
return("Operation cancelled");
}
// Update progress
currentService++;
bgwMain.ReportProgress(100 * (currentService - 1) / totalServices, "Reading database info for service " + currentService + " of " + totalServices + "...");
// Get the service info for this item
var serviceInfo = DatabaseOperations.GetDataServiceFromCache(serviceUrl);
// Check for cancel
if (bgwMain.CancellationPending)
{
e.Cancel = true;
return("Operation cancelled");
}
// Update progress
bgwMain.ReportProgress(100 * (currentService - 1) / totalServices, "Deleting old records for service " + currentService + " of " + totalServices + "...");
// Delete existing records for this service
cacheManager.DeleteRecordsForService(serviceInfo, false);
// Check for cancel
if (bgwMain.CancellationPending)
{
e.Cancel = true;
return("Operation cancelled");
}
// Update progress
bgwMain.ReportProgress(100 * (currentService - 1) / totalServices, "Connecting to service " + currentService + " of " + totalServices + "...");
// Create a WaterOneFlow Service for this URL
WaterOneFlowClient waterOneFlowClient;
try
{
waterOneFlowClient = new WaterOneFlowClient(serviceUrl);
}
catch (Exception ex)
{
// Flag the error and continue to the next service
errors.AppendLine("Could not connect to service with URL: " + serviceUrl + ".\n" + ex.Message + "\n\n");
continue;
}
// Check for cancel
if (bgwMain.CancellationPending)
{
e.Cancel = true;
return("Operation cancelled");
}
// Update progress
bgwMain.ReportProgress(100 * (currentService - 1) / totalServices, "Updating database description for service " + currentService + " of " + totalServices + "...");
// Update service info in the metadata database
var waterOneFlowServiceInfo = waterOneFlowClient.ServiceInfo;
serviceInfo.IsHarvested = false;
serviceInfo.ServiceName = waterOneFlowServiceInfo.ServiceName;
serviceInfo.Version = waterOneFlowServiceInfo.Version;
serviceInfo.ServiceType = waterOneFlowServiceInfo.ServiceType;
serviceInfo.Protocol = waterOneFlowServiceInfo.Protocol;
serviceInfo.VariableCount = (cacheManager.GetVariablesByService((int)serviceInfo.Id)).Count;
cacheManager.UpdateDataRow(serviceInfo);
// Check for cancel
if (bgwMain.CancellationPending)
{
e.Cancel = true;
return("Operation cancelled");
}
// Update progress
bgwMain.ReportProgress(100 * (currentService - 1) / totalServices, "Downloading site list for service " + currentService + " of " + totalServices + "...");
// Get all sites for this service
IList <Site> siteList;
try
{
siteList = waterOneFlowClient.GetSites();
}
catch (WebException ex)
{
var sr = new StreamReader(ex.Response.GetResponseStream());
sr.ReadToEnd();
// Flag the error and continue to the next service
errors.AppendLine("Could not get site list from service with URL: " + serviceUrl + ".\n" + ex.Message + "\n\n");
continue;
}
// Check for cancel
if (bgwMain.CancellationPending)
{
e.Cancel = true;
return("Operation cancelled");
}
var totalSteps = siteList.Count;
var currentStep = 0;
// Error tracking
var siteErrorCount = 0; // Keeps track of how many errors we had while downloading site info for the current service
var saveErrorCount = 0; // Keeps track of how many errors we had while saving site info for the current service
var firstSiteError = ""; // Records the message from the first error we had while getting site info for the current service
var firstSaveError = ""; // Records the message from the first error we had while saving site info for the current service
// Default extent for the service. These values are designed to be overwritten as we query sites in the service
double east = -180;
double west = 360;
double north = -90;
double south = 90;
int valueCount = 0;
IList <Site> siteList1, siteList2;
if (siteList.Count % 2 == 0)
{
siteList1 = (siteList as List <Site>).GetRange(0, siteList.Count / 2);
siteList2 = (siteList as List <Site>).GetRange(siteList.Count / 2, siteList.Count / 2);
}
else
{
siteList1 = (siteList as List <Site>).GetRange(0, (siteList.Count / 2) + 1);
siteList2 = (siteList as List <Site>).GetRange((siteList.Count / 2) + 1, siteList.Count / 2);
}
var thread1 = new Thread(() => ProcessSites(siteList1));
thread1.Start();
var thread2 = new Thread(() => ProcessSites(siteList2));
thread2.Start();
foreach (var site in siteList)
{
// Check for cancel
if (bgwMain.CancellationPending)
{
e.Cancel = true;
return("Operation cancelled");
}
// Update progress
currentStep++;
bgwMain.ReportProgress(100 * currentStep / totalSteps,
"Processing site " + currentStep + " of " + totalSteps +
" from service " + currentService + " of " + totalServices + "...");
// Get series for this site
IList <SeriesMetadata> currentSeriesList;
try
{
currentSeriesList = waterOneFlowClient.GetSiteInfo(site.Code);
}
catch (WebException ex)
{
// Flag the error and continue to the next site
siteErrorCount++;
if (siteErrorCount == 1)
{
firstSiteError = ex.Message;
}
if (ex.Response != null)
{
var rdr = new StreamReader(ex.Response.GetResponseStream());
rdr.ReadToEnd();
}
continue;
}
catch (Exception ex)
{
// Flag the error and continue to the next site
siteErrorCount++;
if (siteErrorCount == 1)
{
firstSiteError = ex.Message;
}
continue;
}
// Update service extent
if (site.Latitude > north)
{
north = site.Latitude;
}
if (site.Latitude < south)
{
south = site.Latitude;
}
if (site.Longitude > east)
{
east = site.Longitude;
}
if (site.Longitude < west)
{
west = site.Longitude;
}
// Save series info to metadata cache database
foreach (var series in currentSeriesList)
{
valueCount += series.ValueCount;
// Check for cancel
if (bgwMain.CancellationPending)
{
e.Cancel = true;
return("Operation cancelled");
}
try
{
cacheManager.SaveSeries(series, serviceInfo);
}
catch (Exception ex)
{
// Flag the error and continue to the next series
saveErrorCount++;
if (saveErrorCount == 1)
{
firstSaveError = ex.Message;
}
continue;
}
// Keep track of how many series were successfully processed
seriesCount++;
}
}
// Log errors
if (siteErrorCount == 1)
{
errors.AppendLine("Could not get site info for 1 site in service with URL: " + serviceUrl +
"\n" + firstSiteError + "\n\n");
}
else if (siteErrorCount > 1)
{
errors.AppendLine("Could not get site info for " + siteErrorCount + " sites in service with URL: " + serviceUrl +
"\n" + firstSiteError + "\n\n");
}
if (saveErrorCount == 1)
{
errors.AppendLine("Could not save site info for 1 site in service with URL: " + serviceUrl +
"\n" + firstSaveError + "\n\n");
}
else if (saveErrorCount > 1)
{
errors.AppendLine("Could not save site info for " + siteErrorCount + " sites in service with URL: " + serviceUrl +
"\n" + firstSaveError + "\n\n");
}
// Check for cancel
if (bgwMain.CancellationPending)
{
e.Cancel = true;
return("Operation cancelled");
}
// Update progress
bgwMain.ReportProgress(100 * currentService / totalServices,
"Updating harvested statistics for service " + currentService + " of " + totalServices + "...");
// Update service info
serviceInfo.IsHarvested = true;
serviceInfo.HarveDateTime = DateTime.Now;
serviceInfo.EastLongitude = east;
serviceInfo.WestLongitude = west;
serviceInfo.NorthLatitude = north;
serviceInfo.SouthLatitude = south;
serviceInfo.SiteCount = siteList.Count;
serviceInfo.ValueCount = valueCount;
serviceInfo.VariableCount = (cacheManager.GetVariablesByService((int)serviceInfo.Id)).Count;
cacheManager.UpdateDataRow(serviceInfo); // Updates properties like harvest datetime and service extent
}
// Check for cancel
if (bgwMain.CancellationPending)
{
e.Cancel = true;
return("Operation cancelled");
}
// Update progress
bgwMain.ReportProgress(100, "Operation complete");
// Report result
var message = "Download complete. " + seriesCount + " series saved to metadata cache database.";
if (errors.Length > 0)
{
message += "\n\nSome errors occurred during the operation:\n\n" + errors;
}
return(message);
}
