public static IDataHubSourceWrapper CreateDataAccessWrapper(TargetSiteConfiguration targetSiteConfig, FiltersDefinition filters = null) { var target_uri = targetSiteConfig.GetDataAccessUri(); var target_creds = targetSiteConfig.GetDataAccessNetworkCredentials(); if (target_creds == null) { log.WarnFormat("Credentials are not set, target sites' services requiring credentials for data access will fail!"); } if (target_uri.Host == "catalogue.onda-dias.eu") { return(new OndaDiasWrapper(new Uri(string.Format("https://catalogue.onda-dias.eu/dias-catalogue")), (NetworkCredential)target_creds, targetSiteConfig.Storage.ToOpenStackStorageSettings())); } if (target_uri.Host == "finder.creodias.eu") { if (targetSiteConfig.Data.Url != null) { return(new CreoDiasWrapper(target_creds, osUrl: targetSiteConfig.Data.Url, openStackStorageSettings: targetSiteConfig.Storage.ToOpenStackStorageSettings())); } return(new CreoDiasWrapper(target_creds, openStackStorageSettings: targetSiteConfig.Storage.ToOpenStackStorageSettings())); } if (target_uri.Host.Contains("mundiwebservices.com")) { var mundiDiasWrapper = new MundiDiasWrapper(target_creds, openStackStorageSettings: targetSiteConfig.Storage.ToOpenStackStorageSettings()); mundiDiasWrapper.S3KeyId = targetSiteConfig.Data.S3KeyId; mundiDiasWrapper.S3SecretKey = targetSiteConfig.Data.S3SecretKey; return(mundiDiasWrapper); } if (target_uri.Host.Contains("sobloo.eu")) { var soblooDiasWrapper = new SoblooDiasWrapper(target_creds); soblooDiasWrapper.S3StorageSettings = targetSiteConfig.Storage.ToS3StorageSettings(); return(soblooDiasWrapper); } if (target_uri.Host.EndsWith("copernicus.eu") || target_uri.AbsolutePath.EndsWith("/dhus")) { if (filters != null && filters.GetFilters().Any(f => f.Key == "archiveStatus")) { UriBuilder urib = new UriBuilder(target_uri); urib.Path += target_uri.AbsolutePath.Replace("/search", "").TrimEnd('/').EndsWith("odata/v1") ? "" : "/odata/v1"; return(new DHuSWrapper(urib.Uri, (NetworkCredential)target_creds)); } return(new DHuSWrapper(target_uri, (NetworkCredential)target_creds)); } return(null); }
protected virtual IEnumerable <IMetric> AnalyzeResults(IOpenSearchResultCollection results, FiltersDefinition fd) { List <IMetric> metrics = new List <IMetric>(); long errors = 0; if (results.Items.Count() > 0) { log.DebugFormat("[{1}] Validating {0} result items...", results.Items.Count(), Task.CurrentId); } foreach (var item in results.Items) { bool error = false; foreach (var filterDefinition in fd.GetFilters()) { if (filterDefinition.ItemValidator != null) { if (filterDefinition.ItemValidator.Invoke(item)) { continue; } error = true; log.WarnFormat("[{2}] Non expected item {0} with filter {1}", item.Identifier, filterDefinition.Label, Task.CurrentId); } if (filterDefinition.ResultsValidator != null) { if (filterDefinition.ResultsValidator.Invoke(results)) { continue; } error = true; log.WarnFormat("[{2}] Non expected results {0} with filter {1}", results.Identifier, filterDefinition.Label, Task.CurrentId); } } if (error) { errors++; } } metrics.Add(new LongMetric(MetricName.wrongResultsCount, errors, "#")); return(metrics); }
protected override IEnumerable <IMetric> AnalyzeResults(IOpenSearchResultCollection results, FiltersDefinition fd) { List <IMetric> metrics = new List <IMetric>(); long errors = 0; long validatedResults = 0; log.DebugFormat("[{1}] Validating and Analyzing {0} result items...", results.Items.Count(), Task.CurrentId); if (results.Count == 0 && results.TotalResults > 0) { log.WarnFormat("[{2}] < results inconsistency, {0} entries whilst total results is {1}. Skipping analysis", results.Count, results.TotalResults, Task.CurrentId); metrics.Add(new ExceptionMetric(new Exception( string.Format("results inconsistency, {0} entries whilst total results is {1}. Skipping analysis", results.Count, results.TotalResults)))); return(metrics); } List <IMetric> _testCaseMetrics = new List <IMetric>(); int i = int.Parse(fd.Filters.FirstOrDefault(f => f.Key == "queryId").Value); var os = queryFiltersTuple[i].Reference.Target.CreateOpenSearchableEntity(); List <double> avaLatencies = new List <double>(); foreach (IOpenSearchResultItem item in results.Items) { foreach (var filterDefinition in fd.GetFilters()) { if (filterDefinition.ItemValidator != null) { if (filterDefinition.ItemValidator.Invoke(item)) { continue; } log.WarnFormat("[{2}] Non expected item {0} with filter {1}", item.Identifier, filterDefinition.Label, Task.CurrentId); errors++; } if (filterDefinition.ResultsValidator != null) { if (filterDefinition.ResultsValidator.Invoke(results)) { continue; } log.WarnFormat("[{2}] Non expected results {0} with filter {1}", results.Identifier, filterDefinition.Label, Task.CurrentId); errors++; } } IOpenSearchResultItem ref_item = FindReferenceItem(item, os); if (ref_item == null) { log.WarnFormat("item {0} not found in reference target", item.Identifier); continue; } DateTimeOffset onlineDate = item.PublishDate; var onlineDateStrings = item.ElementExtensions.ReadElementExtensions <string>("onlineDate", "http://www.terradue.com/"); if (onlineDateStrings != null && onlineDateStrings.Count() > 0) { DateTimeOffset.TryParse(onlineDateStrings.FirstOrDefault(), out onlineDate); } avaLatencies.Add(onlineDate.Subtract(ref_item.PublishDate).TotalSeconds); validatedResults++; } metrics.Add(new LongMetric(MetricName.wrongResultsCount, errors, "#")); metrics.Add(new LongMetric(MetricName.totalValidatedResults, validatedResults, "#")); metrics.Add(new LongMetric(MetricName.totalReadResults, results.Items.Count(), "#")); if (avaLatencies.Count() > 0) { metrics.Add(new LongMetric(MetricName.avgDataAvailabilityLatency, (long)avaLatencies.Average(), "sec")); metrics.Add(new LongMetric(MetricName.maxDataAvailabilityLatency, (long)avaLatencies.Max(), "sec")); } else { metrics.Add(new LongMetric(MetricName.avgDataOperationalLatency, -1, "sec")); metrics.Add(new LongMetric(MetricName.maxDataOperationalLatency, -1, "sec")); } return(metrics); }
protected override IEnumerable <IMetric> AnalyzeResults(IOpenSearchResultCollection results, FiltersDefinition fd) { List <IMetric> metrics = new List <IMetric>(); long wrongResults = 0; long validatedResults = 0; log.DebugFormat("[{1}] Validating and Analyzing {0} result items...", results.Items.Count(), Task.CurrentId); if (results.Count == 0 && results.TotalResults > 0) { log.WarnFormat("[{2}] < results inconsistency, {0} entries whilst total results is {1}. Skipping analysis", results.Count, results.TotalResults, Task.CurrentId); metrics.Add(new ExceptionMetric(new Exception( string.Format("results inconsistency, {0} entries whilst total results is {1}. Skipping analysis", results.Count, results.TotalResults)))); return(metrics); } List <double> opsLatencies = new List <double>(); foreach (IOpenSearchResultItem item in results.Items) { foreach (var filterDefinition in fd.GetFilters()) { if (filterDefinition.ItemValidator != null) { if (filterDefinition.ItemValidator.Invoke(item)) { continue; } log.WarnFormat("[{2}] Non expected item {0} with filter {1}", item.Identifier, filterDefinition.Label, Task.CurrentId); wrongResults++; } if (filterDefinition.ResultsValidator != null) { if (filterDefinition.ResultsValidator.Invoke(results)) { continue; } log.WarnFormat("[{2}] Non expected results {0} with filter {1}", results.Identifier, filterDefinition.Label, Task.CurrentId); wrongResults++; } } opsLatencies.Add(item.PublishDate.Subtract(item.FindStartDate()).TotalSeconds); validatedResults++; } metrics.Add(new LongMetric(MetricName.wrongResultsCount, wrongResults, "#")); metrics.Add(new LongMetric(MetricName.totalValidatedResults, validatedResults, "#")); metrics.Add(new LongMetric(MetricName.totalReadResults, results.Items.Count(), "#")); if (opsLatencies.Count() > 0) { metrics.Add(new LongMetric(MetricName.avgDataOperationalLatency, (long)opsLatencies.Average(), "sec")); metrics.Add(new LongMetric(MetricName.maxDataOperationalLatency, (long)opsLatencies.Max(), "sec")); } else { metrics.Add(new LongMetric(MetricName.avgDataOperationalLatency, -1, "sec")); metrics.Add(new LongMetric(MetricName.maxDataOperationalLatency, -1, "sec")); } return(metrics); }