private IOpenSearchResultItem FindReferenceItem(IOpenSearchResultItem item, IOpenSearchable os) { FiltersDefinition filters = new FiltersDefinition(item.Identifier); filters.AddFilter("uid", "{http://a9.com/-/opensearch/extensions/geo/1.0/}uid", item.Identifier, item.Identifier, null, null); var result = ose.Query(os, filters.GetNameValueCollection()); return(result.Items.FirstOrDefault()); }
internal IOpenSearchable CreateOpenSearchableEntity(FiltersDefinition filters = null, int maxRetries = 3) { OpenSearchableFactorySettings ossettings = new OpenSearchableFactorySettings(ose) { Credentials = Wrapper.Settings.Credentials, MaxRetries = maxRetries }; IDataHubSourceWrapper wrapper = CreateDataAccessWrapper(TargetSiteConfig, filters); wrapper.Settings.MaxRetries = 3; return(wrapper.CreateOpenSearchable(ossettings)); }
protected virtual List <IOpenSearchResultItem> FindItemsToDownload(int count = 20) { List <IOpenSearchResultItem> forcedFoundItems = new List <IOpenSearchResultItem>(); TestCase201 tc = new TestCase201(log, target, 1, Mission.GenerateExistingDataDictionary(target), out forcedFoundItems); var filters = new FiltersDefinition("all"); filters.AddFilter("missionName", "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform", "Sentinel-1", "Sentinel-1", null, null); filters.AddFilter("productType", "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType", "GRD", "GRD", null, null); filters.AddFilter("archiveStatus", "{http://a9.com/-/opensearch/extensions/eo/1.0/}statusSubType", "online", "Online", null, null); filters.AddFilter("productFormat", "{http://a9.com/-/opensearch/extensions/eo/1.0/}productFormat", "zip", "ZIP", null, null); filters.AddFilter("count", "{http://a9.com/-/spec/opensearch/1.1/}count", count > 50 ? count.ToString() : "50", "", null, null); tc.MakeQuery(target.CreateOpenSearchableEntity(), filters); return(forcedFoundItems); }
public TargetAndFiltersDefinition(TargetSiteWrapper target, FiltersDefinition filtersDefinition) { this.target = target; this.filtersDefinition = filtersDefinition; }
protected override IEnumerable <IMetric> AnalyzeResults(IOpenSearchResultCollection results, FiltersDefinition fd) { List <IMetric> metrics = new List <IMetric>(); long errors = 0; long validatedResults = 0; log.DebugFormat("[{1}] Validating and Analyzing {0} result items...", results.Items.Count(), Task.CurrentId); if (results.Count == 0 && results.TotalResults > 0) { log.WarnFormat("[{2}] < results inconsistency, {0} entries whilst total results is {1}. Skipping analysis", results.Count, results.TotalResults, Task.CurrentId); metrics.Add(new ExceptionMetric(new Exception( string.Format("results inconsistency, {0} entries whilst total results is {1}. Skipping analysis", results.Count, results.TotalResults)))); return(metrics); } List <IMetric> _testCaseMetrics = new List <IMetric>(); int i = int.Parse(fd.Filters.FirstOrDefault(f => f.Key == "queryId").Value); var os = queryFiltersTuple[i].Reference.Target.CreateOpenSearchableEntity(); List <double> avaLatencies = new List <double>(); foreach (IOpenSearchResultItem item in results.Items) { foreach (var filterDefinition in fd.GetFilters()) { if (filterDefinition.ItemValidator != null) { if (filterDefinition.ItemValidator.Invoke(item)) { continue; } log.WarnFormat("[{2}] Non expected item {0} with filter {1}", item.Identifier, filterDefinition.Label, Task.CurrentId); errors++; } if (filterDefinition.ResultsValidator != null) { if (filterDefinition.ResultsValidator.Invoke(results)) { continue; } log.WarnFormat("[{2}] Non expected results {0} with filter {1}", results.Identifier, filterDefinition.Label, Task.CurrentId); errors++; } } IOpenSearchResultItem ref_item = FindReferenceItem(item, os); if (ref_item == null) { log.WarnFormat("item {0} not found in reference target", item.Identifier); continue; } DateTimeOffset onlineDate = item.PublishDate; var onlineDateStrings = item.ElementExtensions.ReadElementExtensions <string>("onlineDate", "http://www.terradue.com/"); if (onlineDateStrings != null && onlineDateStrings.Count() > 0) { DateTimeOffset.TryParse(onlineDateStrings.FirstOrDefault(), out onlineDate); } avaLatencies.Add(onlineDate.Subtract(ref_item.PublishDate).TotalSeconds); validatedResults++; } metrics.Add(new LongMetric(MetricName.wrongResultsCount, errors, "#")); metrics.Add(new LongMetric(MetricName.totalValidatedResults, validatedResults, "#")); metrics.Add(new LongMetric(MetricName.totalReadResults, results.Items.Count(), "#")); if (avaLatencies.Count() > 0) { metrics.Add(new LongMetric(MetricName.avgDataAvailabilityLatency, (long)avaLatencies.Average(), "sec")); metrics.Add(new LongMetric(MetricName.maxDataAvailabilityLatency, (long)avaLatencies.Max(), "sec")); } else { metrics.Add(new LongMetric(MetricName.avgDataOperationalLatency, -1, "sec")); metrics.Add(new LongMetric(MetricName.maxDataOperationalLatency, -1, "sec")); } return(metrics); }
protected TestUnitResult Download(IAssetAccess enclosureAccess) { if (enclosureAccess == null) { return(null); } List <IMetric> metrics = new List <IMetric>(); metrics.Add(new StringMetric(MetricName.url, enclosureAccess.Uri.ToString(), "")); int i = 0; int taskId = Task.CurrentId.Value; DateTimeOffset timeStart = DateTimeOffset.UtcNow; metrics.Add(new DateTimeMetric(MetricName.startTime, timeStart, "dateTime")); long totalSize = Convert.ToInt64(enclosureAccess.TotalSize); log.DebugFormat("[{1}] Method #{3} {2}: GET {0} ({4}) ...", enclosureAccess.Uri, taskId, enclosureAccess.AccessMethod, i + 1, BytesToString(totalSize)); var requests = enclosureAccess.GetDownloadRequests(); long[] respTime = new long[requests.Count()]; long totalByteCounter = 0; TestUnitResultStatus tcrStatus = TestUnitResultStatus.Complete; if (requests == null || requests.Count() == 0) { log.DebugFormat("[{1}] No download request for {0}. Skipping test unit.", enclosureAccess.Uri, taskId); tcrStatus = TestUnitResultStatus.NotStarted; } Stopwatch stopWatchDownloadElaspedTime = new Stopwatch(); metrics.Add(new LongMetric(MetricName.beginGetResponseTime, DateTimeOffset.UtcNow.Ticks, "ticks")); int j = 0; foreach (var request in requests) { if (Configuration.Current.Global.TestMode && totalByteCounter >= 10485760 * 2) { // Test Breaker at 20MB ) break; } j++; log.DebugFormat("[{1}] File #{0}/{3}: GET {2} ...", j, taskId, request.RequestUri, requests.Count()); Stopwatch sw = new Stopwatch(); sw.Start(); var downloadTask = request.GetResponseAsync().ContinueWith(resp => { sw.Stop(); respTime[i] = sw.ElapsedMilliseconds; using (ITransferResponse response = resp.Result) { log.InfoFormat("[{1}] > {3} Status Code {0} ({2}ms)", response.StatusCode, taskId, respTime[i], enclosureAccess.AccessMethod); metrics.Add(new StringMetric(MetricName.httpStatusCode, string.Format("{0}:{1}", (int)response.StatusCode, response.StatusDescription), "")); if (!(response.StatusCode == TransferStatusCode.OK || response.StatusCode == TransferStatusCode.Accepted)) { log.DebugFormat("[{0}] < Not OK. Exception: {1}", taskId, response.StatusDescription); metrics.Add(new ExceptionMetric(new Exception( string.Format("[{0}] < Not OK. Exception: {1}", taskId, response.StatusDescription)))); } else { using (var responseStream = response.GetResponseStream()) { if (response.StatusCode == TransferStatusCode.OK && !response.ContentType.StartsWith("text/html")) { string hsize = "unknown"; if (response.ContentLength > 0) { hsize = BytesToString(response.ContentLength); if (totalSize == 0) { totalSize = response.ContentLength; } } log.InfoFormat("[{1}] < Content Type {2} Length {0}", hsize, taskId, response.ContentType); byte[] buf = new byte[4096]; long fileSize = response.ContentLength; double progessPct = 0; long fileByteCounter = 0; long intermediateCounter = 0; stopWatchDownloadElaspedTime.Start(); int nbByteRead = responseStream.Read(buf, 0, 4096); while (nbByteRead > 0) { totalByteCounter += nbByteRead; fileByteCounter += nbByteRead; intermediateCounter += nbByteRead; double totalPct = ((double)totalByteCounter / totalSize) * 100; double filePct = ((double)fileByteCounter / fileSize) * 100; string hpct = ""; if ((fileSize > 0 && Math.Abs(progessPct - totalPct) > 1) || intermediateCounter >= 10485760) { if (fileSize > 0) { progessPct = totalPct; hpct = string.Format(" (file #{2}:{1}% / total:{0}%)", totalPct.ToString("F1"), filePct.ToString("F1"), j); } double bytespersec = (double)totalByteCounter / ((double)stopWatchDownloadElaspedTime.ElapsedMilliseconds / 1000); log.DebugFormat("[{3}] {0} downloaded{1} [{2}/s]", BytesToString(totalByteCounter), hpct, BytesToString((long)bytespersec), taskId); intermediateCounter = 0; } nbByteRead = responseStream.Read(buf, 0, 4096); if (Configuration.Current.Global.TestMode && totalByteCounter >= 10485760 * 2) { // Test Breaker at 20MB ) log.Warn("!!!!!TEST DOWNLOAD BREAKER!!!!"); break; } } stopWatchDownloadElaspedTime.Stop(); } } } } }); try { downloadTask.Wait(); } catch (Exception e) { Exception ie = e; while (ie.InnerException != null) { ie = ie.InnerException; } if (ie is WebException) { WebException we = ie as WebException; log.DebugFormat("[{0}] Error downloading {2}. {4} Error: {1}[{3}]", Task.CurrentId, we.Message, enclosureAccess.Uri, we.Status.ToString(), enclosureAccess.AccessMethod); if (we.Response is HttpWebResponse) { metrics.Add(new StringMetric(MetricName.httpStatusCode, string.Format("{0}:{1}", (int)((HttpWebResponse)we.Response).StatusCode, ((HttpWebResponse)we.Response).StatusDescription), "")); } metrics.Add(new ExceptionMetric(we)); } else { log.DebugFormat("[{0}] Error during download {2}. Exception: {1}", Task.CurrentId, ie.Message, enclosureAccess.Uri); log.Debug(ie.StackTrace); metrics.Add(new ExceptionMetric(ie)); } break; } i++; } DateTimeOffset timeStop = DateTimeOffset.UtcNow; metrics.Add(new DateTimeMetric(MetricName.endTime, timeStop, "dateTime")); metrics.Add(new LongMetric(MetricName.endGetResponseTime, DateTime.UtcNow.Ticks, "ticks")); if (respTime.Count() > 0) { metrics.Add(new LongMetric(MetricName.responseTime, Convert.ToInt64(respTime.Average()), "ms")); } else { metrics.Add(new LongMetric(MetricName.responseTime, 0, "ms")); } metrics.Add(new LongMetric(MetricName.size, totalByteCounter, "bytes")); metrics.Add(new LongMetric(MetricName.downloadElapsedTime, stopWatchDownloadElaspedTime.ElapsedMilliseconds, "ms")); metrics.Add(new LongMetric(MetricName.maxTotalResults, 1, "#")); metrics.Add(new LongMetric(MetricName.totalReadResults, 1, "#")); metrics.Add(new LongMetric(MetricName.wrongResultsCount, 0, "#")); var tcr = new TestUnitResult(metrics, tcrStatus); if (enclosureAccess.SourceItem != null) { FiltersDefinition fd = DataHelper.GenerateFiltersDefinitionFromItem("Download", enclosureAccess.SourceItem); metrics.Add(new StringMetric(MetricName.dataCollectionDivision, fd.Label, "string")); tcr.FiltersDefinition = fd; } tcr.State = enclosureAccess; return(tcr); }
protected override IEnumerable <IMetric> AnalyzeResults(IOpenSearchResultCollection results, FiltersDefinition fd) { List <IMetric> metrics = new List <IMetric>(); long wrongResults = 0; long validatedResults = 0; log.DebugFormat("[{1}] Validating and Analyzing {0} result items...", results.Items.Count(), Task.CurrentId); if (results.Count == 0 && results.TotalResults > 0) { log.WarnFormat("[{2}] < results inconsistency, {0} entries whilst total results is {1}. Skipping analysis", results.Count, results.TotalResults, Task.CurrentId); metrics.Add(new ExceptionMetric(new Exception( string.Format("results inconsistency, {0} entries whilst total results is {1}. Skipping analysis", results.Count, results.TotalResults)))); return(metrics); } List <double> opsLatencies = new List <double>(); foreach (IOpenSearchResultItem item in results.Items) { foreach (var filterDefinition in fd.GetFilters()) { if (filterDefinition.ItemValidator != null) { if (filterDefinition.ItemValidator.Invoke(item)) { continue; } log.WarnFormat("[{2}] Non expected item {0} with filter {1}", item.Identifier, filterDefinition.Label, Task.CurrentId); wrongResults++; } if (filterDefinition.ResultsValidator != null) { if (filterDefinition.ResultsValidator.Invoke(results)) { continue; } log.WarnFormat("[{2}] Non expected results {0} with filter {1}", results.Identifier, filterDefinition.Label, Task.CurrentId); wrongResults++; } } opsLatencies.Add(item.PublishDate.Subtract(item.FindStartDate()).TotalSeconds); validatedResults++; } metrics.Add(new LongMetric(MetricName.wrongResultsCount, wrongResults, "#")); metrics.Add(new LongMetric(MetricName.totalValidatedResults, validatedResults, "#")); metrics.Add(new LongMetric(MetricName.totalReadResults, results.Items.Count(), "#")); if (opsLatencies.Count() > 0) { metrics.Add(new LongMetric(MetricName.avgDataOperationalLatency, (long)opsLatencies.Average(), "sec")); metrics.Add(new LongMetric(MetricName.maxDataOperationalLatency, (long)opsLatencies.Max(), "sec")); } else { metrics.Add(new LongMetric(MetricName.avgDataOperationalLatency, -1, "sec")); metrics.Add(new LongMetric(MetricName.maxDataOperationalLatency, -1, "sec")); } return(metrics); }
public static IDataHubSourceWrapper CreateDataAccessWrapper(TargetSiteConfiguration targetSiteConfig, FiltersDefinition filters = null) { var target_uri = targetSiteConfig.GetDataAccessUri(); var target_creds = targetSiteConfig.GetDataAccessNetworkCredentials(); if (target_creds == null) { log.WarnFormat("Credentials are not set, target sites' services requiring credentials for data access will fail!"); } if (target_uri.Host == "catalogue.onda-dias.eu") { return(new OndaDiasWrapper(new Uri(string.Format("https://catalogue.onda-dias.eu/dias-catalogue")), (NetworkCredential)target_creds, targetSiteConfig.Storage.ToOpenStackStorageSettings())); } if (target_uri.Host == "finder.creodias.eu") { if (targetSiteConfig.Data.Url != null) { return(new CreoDiasWrapper(target_creds, osUrl: targetSiteConfig.Data.Url, openStackStorageSettings: targetSiteConfig.Storage.ToOpenStackStorageSettings())); } return(new CreoDiasWrapper(target_creds, openStackStorageSettings: targetSiteConfig.Storage.ToOpenStackStorageSettings())); } if (target_uri.Host.Contains("mundiwebservices.com")) { var mundiDiasWrapper = new MundiDiasWrapper(target_creds, openStackStorageSettings: targetSiteConfig.Storage.ToOpenStackStorageSettings()); mundiDiasWrapper.S3KeyId = targetSiteConfig.Data.S3KeyId; mundiDiasWrapper.S3SecretKey = targetSiteConfig.Data.S3SecretKey; return(mundiDiasWrapper); } if (target_uri.Host.Contains("sobloo.eu")) { var soblooDiasWrapper = new SoblooDiasWrapper(target_creds); soblooDiasWrapper.S3StorageSettings = targetSiteConfig.Storage.ToS3StorageSettings(); return(soblooDiasWrapper); } if (target_uri.Host.EndsWith("copernicus.eu") || target_uri.AbsolutePath.EndsWith("/dhus")) { if (filters != null && filters.GetFilters().Any(f => f.Key == "archiveStatus")) { UriBuilder urib = new UriBuilder(target_uri); urib.Path += target_uri.AbsolutePath.Replace("/search", "").TrimEnd('/').EndsWith("odata/v1") ? "" : "/odata/v1"; return(new DHuSWrapper(urib.Uri, (NetworkCredential)target_creds)); } return(new DHuSWrapper(target_uri, (NetworkCredential)target_creds)); } return(null); }
protected virtual IEnumerable <IMetric> AnalyzeResults(IOpenSearchResultCollection results, FiltersDefinition fd) { List <IMetric> metrics = new List <IMetric>(); long errors = 0; if (results.Items.Count() > 0) { log.DebugFormat("[{1}] Validating {0} result items...", results.Items.Count(), Task.CurrentId); } foreach (var item in results.Items) { bool error = false; foreach (var filterDefinition in fd.GetFilters()) { if (filterDefinition.ItemValidator != null) { if (filterDefinition.ItemValidator.Invoke(item)) { continue; } error = true; log.WarnFormat("[{2}] Non expected item {0} with filter {1}", item.Identifier, filterDefinition.Label, Task.CurrentId); } if (filterDefinition.ResultsValidator != null) { if (filterDefinition.ResultsValidator.Invoke(results)) { continue; } error = true; log.WarnFormat("[{2}] Non expected results {0} with filter {1}", results.Identifier, filterDefinition.Label, Task.CurrentId); } } if (error) { errors++; } } metrics.Add(new LongMetric(MetricName.wrongResultsCount, errors, "#")); return(metrics); }
internal TestUnitResult MakeQuery(IOpenSearchable entity, FiltersDefinition fd) { List <IMetric> metrics = new List <IMetric>(); log.DebugFormat("[{1}] > Query {0} {2}...", fd.Name, Task.CurrentId, fd.Label); Stopwatch sw = new Stopwatch(); sw.Start(); DateTimeOffset timeStart = DateTimeOffset.UtcNow; var parameters = fd.GetNameValueCollection(); return(catalogue_task_factory.StartNew(() => { return ose.Query(entity, parameters); }).ContinueWith <TestUnitResult>(task => { var respTime = sw.ElapsedMilliseconds; Terradue.OpenSearch.Result.IOpenSearchResultCollection results = null; try { results = task.Result; } catch (AggregateException e) { log.DebugFormat("[{0}] < No results for {2}. Exception: {1}", Task.CurrentId, e.InnerException.Message, fd.Label); log.Debug(e.InnerException.StackTrace); metrics.Add(new ExceptionMetric(e.InnerException)); metrics.Add(new LongMetric(MetricName.maxTotalResults, -1, "#")); metrics.Add(new LongMetric(MetricName.totalReadResults, -1, "#")); } finally { sw.Stop(); } DateTimeOffset timeStop = DateTimeOffset.UtcNow; metrics.Add(new DateTimeMetric(MetricName.startTime, timeStart, "dateTime")); metrics.Add(new DateTimeMetric(MetricName.endTime, timeStop, "dateTime")); metrics.Add(new StringMetric(MetricName.dataCollectionDivision, fd.Label, "string")); if (results != null) { foundItems.AddRange(results.Items); long serializedSize = 0; try { serializedSize = Encoding.Default.GetBytes(results.SerializeToString()).Length; } catch { } var metricsArray = results.ElementExtensions.ReadElementExtensions <Terradue.OpenSearch.Benchmarking.Metrics>("Metrics", "http://www.terradue.com/metrics", Terradue.OpenSearch.Benchmarking.MetricFactory.Serializer); if (metricsArray == null || metricsArray.Count() == 0) { log.Warn("No query metrics found! Response Time and error rate may be biased!"); metrics.Add(new LongMetric(MetricName.responseTime, respTime, "ms")); metrics.Add(new LongMetric(MetricName.size, serializedSize, "bytes")); metrics.Add(new LongMetric(MetricName.beginGetResponseTime, timeStart.Ticks, "ticks")); metrics.Add(new LongMetric(MetricName.endGetResponseTime, timeStop.Ticks, "ticks")); } else { Terradue.OpenSearch.Benchmarking.Metrics osMetrics = metricsArray.First(); Terradue.OpenSearch.Benchmarking.Metric _sizeMetric = osMetrics.Metric.FirstOrDefault(m => m.Identifier == "size"); Terradue.OpenSearch.Benchmarking.Metric _responseTimeMetric = osMetrics.Metric.FirstOrDefault(m => m.Identifier == "responseTime"); Terradue.OpenSearch.Benchmarking.Metric _retryNumberMetric = osMetrics.Metric.FirstOrDefault(m => m.Identifier == "retryNumber"); Terradue.OpenSearch.Benchmarking.Metric _beginGetResponseTime = osMetrics.Metric.FirstOrDefault(m => m.Identifier == "beginGetResponseTime"); Terradue.OpenSearch.Benchmarking.Metric _endGetResponseTime = osMetrics.Metric.FirstOrDefault(m => m.Identifier == "endGetResponseTime"); log.DebugFormat("[{4}] < {0}/{1} entries for {6} {5}. {2}bytes in {3}ms", results.Count, results.TotalResults, _sizeMetric.Value, _responseTimeMetric.Value, Task.CurrentId, fd.Label, fd.Name); if (_responseTimeMetric != null) { metrics.Add(new LongMetric(MetricName.responseTime, Convert.ToInt64(_responseTimeMetric.Value), "ms")); } else { metrics.Add(new LongMetric(MetricName.responseTime, respTime, "ms")); } if (_sizeMetric != null) { metrics.Add(new LongMetric(MetricName.size, Convert.ToInt64(_sizeMetric.Value), "bytes")); } else { metrics.Add(new LongMetric(MetricName.size, serializedSize, "bytes")); } if (_retryNumberMetric != null) { metrics.Add(new LongMetric(MetricName.retryNumber, Convert.ToInt64(_retryNumberMetric.Value), "#")); } else { metrics.Add(new LongMetric(MetricName.retryNumber, 1, "#")); } if (_beginGetResponseTime != null && _endGetResponseTime != null) { metrics.Add(new LongMetric(MetricName.beginGetResponseTime, Convert.ToInt64(_beginGetResponseTime.Value), "ticks")); metrics.Add(new LongMetric(MetricName.endGetResponseTime, Convert.ToInt64(_endGetResponseTime.Value), "ticks")); } else { metrics.Add(new LongMetric(MetricName.beginGetResponseTime, timeStart.Ticks, "ticks")); metrics.Add(new LongMetric(MetricName.endGetResponseTime, timeStop.Ticks, "ticks")); } } metrics.Add(new LongMetric(MetricName.maxTotalResults, results.TotalResults, "#")); metrics.Add(new LongMetric(MetricName.totalReadResults, results.Count, "#")); if (results.Count == 0) { if (results.TotalResults > 0 && !ignoreEmptyResult) { log.WarnFormat("[{1}] no entries for {2} whilst {0} totalResult. This seems to be an error.", results.TotalResults, Task.CurrentId, fd.Label); metrics.Add(new ExceptionMetric(new InvalidOperationException("No entries found"))); } } else { try { Stopwatch sw3 = new Stopwatch(); sw3.Start(); metrics.AddRange(AnalyzeResults(results, fd)); sw3.Stop(); metrics.Add(new LongMetric(MetricName.analysisTime, sw3.ElapsedMilliseconds, "msec")); } catch (Exception e) { metrics.Add(new ExceptionMetric(e)); log.ErrorFormat("[{0}] < Analysis failed for results {2} : {1}", Task.CurrentId, e.Message, fd.Label); log.Debug(e.StackTrace); } } } return new TestUnitResult(metrics, fd); }).Result); }