public static IDataHubSourceWrapper CreateDataAccessWrapper(TargetSiteConfiguration targetSiteConfig, FiltersDefinition filters = null)
{
var target_uri = targetSiteConfig.GetDataAccessUri();
var target_creds = targetSiteConfig.GetDataAccessNetworkCredentials();
if (target_creds == null)
{
log.WarnFormat("Credentials are not set, target sites' services requiring credentials for data access will fail!");
}
if (target_uri.Host == "catalogue.onda-dias.eu")
{
return(new OndaDiasWrapper(new Uri(string.Format("https://catalogue.onda-dias.eu/dias-catalogue")), (NetworkCredential)target_creds, targetSiteConfig.Storage.ToOpenStackStorageSettings()));
}
if (target_uri.Host == "finder.creodias.eu")
{
if (targetSiteConfig.Data.Url != null)
{
return(new CreoDiasWrapper(target_creds, osUrl: targetSiteConfig.Data.Url, openStackStorageSettings: targetSiteConfig.Storage.ToOpenStackStorageSettings()));
}
return(new CreoDiasWrapper(target_creds, openStackStorageSettings: targetSiteConfig.Storage.ToOpenStackStorageSettings()));
}
if (target_uri.Host.Contains("mundiwebservices.com"))
{
var mundiDiasWrapper = new MundiDiasWrapper(target_creds, openStackStorageSettings: targetSiteConfig.Storage.ToOpenStackStorageSettings());
mundiDiasWrapper.S3KeyId = targetSiteConfig.Data.S3KeyId;
mundiDiasWrapper.S3SecretKey = targetSiteConfig.Data.S3SecretKey;
return(mundiDiasWrapper);
}
if (target_uri.Host.Contains("sobloo.eu"))
{
var soblooDiasWrapper = new SoblooDiasWrapper(target_creds);
soblooDiasWrapper.S3StorageSettings = targetSiteConfig.Storage.ToS3StorageSettings();
return(soblooDiasWrapper);
}
if (target_uri.Host.EndsWith("copernicus.eu") || target_uri.AbsolutePath.EndsWith("/dhus"))
{
if (filters != null && filters.GetFilters().Any(f => f.Key == "archiveStatus"))
{
UriBuilder urib = new UriBuilder(target_uri);
urib.Path += target_uri.AbsolutePath.Replace("/search", "").TrimEnd('/').EndsWith("odata/v1") ? "" : "/odata/v1";
return(new DHuSWrapper(urib.Uri, (NetworkCredential)target_creds));
}
return(new DHuSWrapper(target_uri, (NetworkCredential)target_creds));
}
return(null);
}
protected virtual IEnumerable <IMetric> AnalyzeResults(IOpenSearchResultCollection results, FiltersDefinition fd)
{
List <IMetric> metrics = new List <IMetric>();
long errors = 0;
if (results.Items.Count() > 0)
{
log.DebugFormat("[{1}] Validating {0} result items...", results.Items.Count(), Task.CurrentId);
}
foreach (var item in results.Items)
{
bool error = false;
foreach (var filterDefinition in fd.GetFilters())
{
if (filterDefinition.ItemValidator != null)
{
if (filterDefinition.ItemValidator.Invoke(item))
{
continue;
}
error = true;
log.WarnFormat("[{2}] Non expected item {0} with filter {1}", item.Identifier, filterDefinition.Label, Task.CurrentId);
}
if (filterDefinition.ResultsValidator != null)
{
if (filterDefinition.ResultsValidator.Invoke(results))
{
continue;
}
error = true;
log.WarnFormat("[{2}] Non expected results {0} with filter {1}", results.Identifier, filterDefinition.Label, Task.CurrentId);
}
}
if (error)
{
errors++;
}
}
metrics.Add(new LongMetric(MetricName.wrongResultsCount, errors, "#"));
return(metrics);
}
protected override IEnumerable <IMetric> AnalyzeResults(IOpenSearchResultCollection results, FiltersDefinition fd)
{
List <IMetric> metrics = new List <IMetric>();
long errors = 0;
long validatedResults = 0;
log.DebugFormat("[{1}] Validating and Analyzing {0} result items...", results.Items.Count(), Task.CurrentId);
if (results.Count == 0 && results.TotalResults > 0)
{
log.WarnFormat("[{2}] < results inconsistency, {0} entries whilst total results is {1}. Skipping analysis", results.Count, results.TotalResults, Task.CurrentId);
metrics.Add(new ExceptionMetric(new Exception(
string.Format("results inconsistency, {0} entries whilst total results is {1}. Skipping analysis",
results.Count, results.TotalResults))));
return(metrics);
}
List <IMetric> _testCaseMetrics = new List <IMetric>();
int i = int.Parse(fd.Filters.FirstOrDefault(f => f.Key == "queryId").Value);
var os = queryFiltersTuple[i].Reference.Target.CreateOpenSearchableEntity();
List <double> avaLatencies = new List <double>();
foreach (IOpenSearchResultItem item in results.Items)
{
foreach (var filterDefinition in fd.GetFilters())
{
if (filterDefinition.ItemValidator != null)
{
if (filterDefinition.ItemValidator.Invoke(item))
{
continue;
}
log.WarnFormat("[{2}] Non expected item {0} with filter {1}", item.Identifier, filterDefinition.Label, Task.CurrentId);
errors++;
}
if (filterDefinition.ResultsValidator != null)
{
if (filterDefinition.ResultsValidator.Invoke(results))
{
continue;
}
log.WarnFormat("[{2}] Non expected results {0} with filter {1}", results.Identifier, filterDefinition.Label, Task.CurrentId);
errors++;
}
}
IOpenSearchResultItem ref_item = FindReferenceItem(item, os);
if (ref_item == null)
{
log.WarnFormat("item {0} not found in reference target", item.Identifier);
continue;
}
DateTimeOffset onlineDate = item.PublishDate;
var onlineDateStrings = item.ElementExtensions.ReadElementExtensions <string>("onlineDate", "http://www.terradue.com/");
if (onlineDateStrings != null && onlineDateStrings.Count() > 0)
{
DateTimeOffset.TryParse(onlineDateStrings.FirstOrDefault(), out onlineDate);
}
avaLatencies.Add(onlineDate.Subtract(ref_item.PublishDate).TotalSeconds);
validatedResults++;
}
metrics.Add(new LongMetric(MetricName.wrongResultsCount, errors, "#"));
metrics.Add(new LongMetric(MetricName.totalValidatedResults, validatedResults, "#"));
metrics.Add(new LongMetric(MetricName.totalReadResults, results.Items.Count(), "#"));
if (avaLatencies.Count() > 0)
{
metrics.Add(new LongMetric(MetricName.avgDataAvailabilityLatency, (long)avaLatencies.Average(), "sec"));
metrics.Add(new LongMetric(MetricName.maxDataAvailabilityLatency, (long)avaLatencies.Max(), "sec"));
}
else
{
metrics.Add(new LongMetric(MetricName.avgDataOperationalLatency, -1, "sec"));
metrics.Add(new LongMetric(MetricName.maxDataOperationalLatency, -1, "sec"));
}
return(metrics);
}
protected override IEnumerable <IMetric> AnalyzeResults(IOpenSearchResultCollection results, FiltersDefinition fd)
{
List <IMetric> metrics = new List <IMetric>();
long wrongResults = 0;
long validatedResults = 0;
log.DebugFormat("[{1}] Validating and Analyzing {0} result items...", results.Items.Count(), Task.CurrentId);
if (results.Count == 0 && results.TotalResults > 0)
{
log.WarnFormat("[{2}] < results inconsistency, {0} entries whilst total results is {1}. Skipping analysis", results.Count, results.TotalResults, Task.CurrentId);
metrics.Add(new ExceptionMetric(new Exception(
string.Format("results inconsistency, {0} entries whilst total results is {1}. Skipping analysis",
results.Count, results.TotalResults))));
return(metrics);
}
List <double> opsLatencies = new List <double>();
foreach (IOpenSearchResultItem item in results.Items)
{
foreach (var filterDefinition in fd.GetFilters())
{
if (filterDefinition.ItemValidator != null)
{
if (filterDefinition.ItemValidator.Invoke(item))
{
continue;
}
log.WarnFormat("[{2}] Non expected item {0} with filter {1}", item.Identifier, filterDefinition.Label, Task.CurrentId);
wrongResults++;
}
if (filterDefinition.ResultsValidator != null)
{
if (filterDefinition.ResultsValidator.Invoke(results))
{
continue;
}
log.WarnFormat("[{2}] Non expected results {0} with filter {1}", results.Identifier, filterDefinition.Label, Task.CurrentId);
wrongResults++;
}
}
opsLatencies.Add(item.PublishDate.Subtract(item.FindStartDate()).TotalSeconds);
validatedResults++;
}
metrics.Add(new LongMetric(MetricName.wrongResultsCount, wrongResults, "#"));
metrics.Add(new LongMetric(MetricName.totalValidatedResults, validatedResults, "#"));
metrics.Add(new LongMetric(MetricName.totalReadResults, results.Items.Count(), "#"));
if (opsLatencies.Count() > 0)
{
metrics.Add(new LongMetric(MetricName.avgDataOperationalLatency, (long)opsLatencies.Average(), "sec"));
metrics.Add(new LongMetric(MetricName.maxDataOperationalLatency, (long)opsLatencies.Max(), "sec"));
}
else
{
metrics.Add(new LongMetric(MetricName.avgDataOperationalLatency, -1, "sec"));
metrics.Add(new LongMetric(MetricName.maxDataOperationalLatency, -1, "sec"));
}
return(metrics);
}
