private static string RemoveNamesFromFilterString(string filterString) { MetricFilter filter = MetricFilterExpressionParser.Parse(filterString); MetricFilter nameless = new MetricFilter() { TimeGrain = filter.TimeGrain, StartTime = filter.StartTime, EndTime = filter.EndTime }; return(ShoeboxHelper.GenerateMetricFilterString(nameless)); }
public async Task <MetricListResponse> GetMetricsAsync(string resourceUri, string filterString, CancellationToken cancellationToken) { if (resourceUri == null) { throw new ArgumentNullException("resourceUri"); } // Generate filter strings MetricFilter filter = MetricFilterExpressionParser.Parse(filterString); string filterStringNamesOnly = filter.DimensionFilters == null ? null : ShoeboxHelper.GenerateMetricDefinitionFilterString(filter.DimensionFilters.Select(df => df.Name)); // Get definitions for requested metrics IList <MetricDefinition> definitions = (await this.Client.MetricDefinitionOperations.GetMetricDefinitionsAsync( resourceUri, filterStringNamesOnly, cancellationToken).ConfigureAwait(false)).MetricDefinitionCollection.Value; // Get Metrics with definitions return(await this.GetMetricsAsync(resourceUri, filterString, definitions, cancellationToken)); }
public async Task <MetricListResponse> GetMetricsAsync(string resourceUri, string filterString, CancellationToken cancellationToken) { // Ensure exactly one '/' at the start resourceUri = '/' + resourceUri.TrimStart('/'); // Generate filter strings MetricFilter filter = MetricFilterExpressionParser.Parse(filterString); string metricFilterString = GenerateNamelessMetricFilterString(filter); string definitionFilterString = filter.DimensionFilters == null ? null : ShoeboxHelper.GenerateMetricDefinitionFilterString(filter.DimensionFilters.Select(df => df.Name)); // Get definitions for requested metrics IList <MetricDefinition> definitions = (await this.Client.MetricDefinitionOperations.GetMetricDefinitionsAsync( resourceUri, definitionFilterString, cancellationToken).ConfigureAwait(false)).MetricDefinitionCollection.Value; // Separate passthrough metrics with dimensions specified IEnumerable <MetricDefinition> passthruDefinitions = definitions.Where(d => !IsShoebox(d, filter.TimeGrain)); IEnumerable <MetricDefinition> shoeboxDefinitions = definitions.Where(d => IsShoebox(d, filter.TimeGrain)); // Get Passthru definitions List <Metric> passthruMetrics = new List <Metric>(); string invocationId = TracingAdapter.NextInvocationId.ToString(CultureInfo.InvariantCulture); this.LogStartGetMetrics(invocationId, resourceUri, filterString, passthruDefinitions); if (passthruDefinitions.Any()) { // Create new filter for passthru metrics List <MetricDimension> passthruDimensionFilters = filter.DimensionFilters == null ? new List <MetricDimension>() : filter.DimensionFilters.Where(df => passthruDefinitions.Any(d => string.Equals(d.Name.Value, df.Name, StringComparison.OrdinalIgnoreCase))).ToList(); foreach (MetricDefinition def in passthruDefinitions .Where(d => !passthruDimensionFilters.Any(pdf => string.Equals(pdf.Name, d.Name.Value, StringComparison.OrdinalIgnoreCase)))) { passthruDimensionFilters.Add(new MetricDimension() { Name = def.Name.Value }); } MetricFilter passthruFilter = new MetricFilter() { TimeGrain = filter.TimeGrain, StartTime = filter.StartTime, EndTime = filter.EndTime, DimensionFilters = passthruDimensionFilters }; // Create passthru filter string string passthruFilterString = ShoeboxHelper.GenerateMetricFilterString(passthruFilter); // Get Metrics from passthrough (hydra) client MetricListResponse passthruResponse = await this.GetMetricsInternalAsync(resourceUri, passthruFilterString, cancellationToken).ConfigureAwait(false); passthruMetrics = passthruResponse.MetricCollection.Value.ToList(); this.LogMetricCountFromResponses(invocationId, passthruMetrics.Count()); // Fill in values (resourceUri, displayName, unit) from definitions CompleteShoeboxMetrics(passthruMetrics, passthruDefinitions, resourceUri); // Add empty objects for metrics that had no values come back, ensuring a metric is returned for each definition IEnumerable <Metric> emptyMetrics = passthruDefinitions .Where(d => !passthruMetrics.Any(m => string.Equals(m.Name.Value, d.Name.Value, StringComparison.OrdinalIgnoreCase))) .Select(d => new Metric() { Name = d.Name, Unit = d.Unit, ResourceId = resourceUri, StartTime = filter.StartTime, EndTime = filter.EndTime, TimeGrain = filter.TimeGrain, MetricValues = new List <MetricValue>(), Properties = new Dictionary <string, string>() }); passthruMetrics.AddRange(emptyMetrics); } // Get Metrics by definitions MetricListResponse shoeboxResponse = await this.GetMetricsAsync(resourceUri, metricFilterString, shoeboxDefinitions, cancellationToken).ConfigureAwait(false); // Create response (merge and wrap metrics) MetricListResponse result = new MetricListResponse() { RequestId = Guid.NewGuid().ToString("D"), StatusCode = HttpStatusCode.OK, MetricCollection = new MetricCollection() { Value = passthruMetrics.Union(shoeboxResponse.MetricCollection.Value).ToList() } }; this.LogEndGetMetrics(invocationId, result); return(result); }
// Alternate method for getting metrics by passing in the definitions // TODO [davmc]: Revisit - this method cannot support dimensions public async Task <MetricListResponse> GetMetricsAsync( string resourceUri, string filterString, IEnumerable <MetricDefinition> definitions, CancellationToken cancellationToken) { MetricListResponse result; if (definitions == null) { throw new ArgumentNullException("definitions"); } string invocationId = TracingAdapter.NextInvocationId.ToString(CultureInfo.InvariantCulture); this.LogStartGetMetrics(invocationId, resourceUri, filterString, definitions); // If no definitions provided, return empty collection if (!definitions.Any()) { result = new MetricListResponse() { RequestId = Guid.NewGuid().ToString("D"), StatusCode = HttpStatusCode.OK }; this.LogEndGetMetrics(invocationId, result); return(result); } // Parse MetricFilter MetricFilter filter = MetricFilterExpressionParser.Parse(filterString); // Names not allowed in filter since the names are in the definitions if (filter.DimensionFilters != null && filter.DimensionFilters.Any()) { throw new ArgumentException("Cannot specify names (or dimensions) when MetricDefinitions are included", "filterString"); } // Ensure every definition has at least one availability matching the filter timegrain if (!definitions.All(d => d.MetricAvailabilities.Any(a => a.TimeGrain == filter.TimeGrain))) { throw new ArgumentException("Definition contains no availability for the timeGrain requested", "definitions"); } // Group definitions by location so we can make one request to each location Dictionary <MetricAvailability, MetricFilter> groups = definitions.GroupBy(d => d.MetricAvailabilities.First(a => a.TimeGrain == filter.TimeGrain), new AvailabilityComparer()).ToDictionary(g => g.Key, g => new MetricFilter() { TimeGrain = filter.TimeGrain, StartTime = filter.StartTime, EndTime = filter.EndTime, DimensionFilters = g.Select(d => new MetricDimension() { Name = d.Name.Value }) }); // Get Metrics from each location (group) IEnumerable <Task <MetricListResponse> > locationTasks = groups.Select(g => g.Key.Location == null ? this.GetMetricsInternalAsync(resourceUri, ShoeboxHelper.GenerateMetricFilterString(g.Value), cancellationToken) : ShoeboxClient.GetMetricsInternalAsync(g.Value, g.Key.Location, invocationId)); // Aggregate metrics from all groups MetricListResponse[] results = (await Task.Factory.ContinueWhenAll(locationTasks.ToArray(), tasks => tasks.Select(t => t.Result))).ToArray(); IEnumerable <Metric> metrics = results.Aggregate <MetricListResponse, IEnumerable <Metric> >( new List <Metric>(), (list, response) => list.Union(response.MetricCollection.Value)); this.LogMetricCountFromResponses(invocationId, metrics.Count()); // Fill in values (resourceUri, displayName, unit) from definitions CompleteShoeboxMetrics(metrics, definitions, resourceUri); // Add empty objects for metrics that had no values come back, ensuring a metric is returned for each definition IEnumerable <Metric> emptyMetrics = definitions .Where(d => !metrics.Any(m => string.Equals(m.Name.Value, d.Name.Value, StringComparison.OrdinalIgnoreCase))) .Select(d => new Metric() { Name = d.Name, Unit = d.Unit, ResourceId = resourceUri, StartTime = filter.StartTime, EndTime = filter.EndTime, TimeGrain = filter.TimeGrain, MetricValues = new List <MetricValue>(), Properties = new Dictionary <string, string>() }); // Create response (merge and wrap metrics) result = new MetricListResponse() { RequestId = Guid.NewGuid().ToString("D"), StatusCode = HttpStatusCode.OK, MetricCollection = new MetricCollection() { Value = metrics.Union(emptyMetrics).ToList() } }; this.LogEndGetMetrics(invocationId, result); return(result); }
// Alternate method for getting metrics by passing in the definitions public async Task <MetricListResponse> GetMetricsAsync( string resourceUri, string filterString, IEnumerable <MetricDefinition> definitions, CancellationToken cancellationToken) { if (definitions == null) { throw new ArgumentNullException("definitions"); } if (resourceUri == null) { throw new ArgumentNullException("resourceUri"); } // Remove any '/' characters from the start since these are handled by the hydra (thin) client // Don't encode Uri segments here since this will mess up the SAS retrievers (they use the resourceUri directly) resourceUri = resourceUri.TrimStart('/'); MetricListResponse result; string invocationId = TracingAdapter.NextInvocationId.ToString(CultureInfo.InvariantCulture); // If no definitions provided, return empty collection if (!definitions.Any()) { this.LogStartGetMetrics(invocationId, resourceUri, filterString, definitions); result = new MetricListResponse() { RequestId = Guid.NewGuid().ToString("D"), StatusCode = HttpStatusCode.OK, MetricCollection = new MetricCollection() { Value = new Metric[0] } }; this.LogEndGetMetrics(invocationId, result); return(result); } // Parse MetricFilter MetricFilter filter = MetricFilterExpressionParser.Parse(filterString); // Names in filter must match the names in the definitions if (filter.DimensionFilters != null && filter.DimensionFilters.Any()) { IEnumerable <string> filterNames = filter.DimensionFilters.Select(df => df.Name); IEnumerable <string> definitionNames = definitions.Select(d => d.Name.Value); IEnumerable <string> filterOnly = filterNames.Where(fn => !definitionNames.Contains(fn, StringComparer.InvariantCultureIgnoreCase)); IEnumerable <string> definitionOnly = definitionNames.Where(df => !filterNames.Contains(df, StringComparer.InvariantCultureIgnoreCase)); if (filterOnly.Any() || definitionOnly.Any()) { throw new ArgumentException("Set of names specified in filter string must match set of names in provided definitions", "filterString"); } // "Filter out" metrics with unsupported dimensions definitions = definitions.Where(d => SupportsRequestedDimensions(d, filter)); } else { filter = new MetricFilter() { TimeGrain = filter.TimeGrain, StartTime = filter.StartTime, EndTime = filter.EndTime, DimensionFilters = definitions.Select(d => new MetricDimension() { Name = d.Name.Value }) }; } // Parse out provider name and determine if provider is storage string providerName = this.GetProviderFromResourceId(resourceUri); bool isStorageProvider = string.Equals(providerName, "Microsoft.Storage", StringComparison.OrdinalIgnoreCase) || string.Equals(providerName, "Microsoft.ClassicStorage", StringComparison.OrdinalIgnoreCase); // Create supported MetricRetrievers IMetricRetriever proxyRetriever = new ProxyMetricRetriever(this); IMetricRetriever shoeboxRetriever = new ShoeboxMetricRetriever(); IMetricRetriever storageRetriever = new StorageMetricRetriever(); IMetricRetriever blobShoeboxMetricRetriever = new BlobShoeboxMetricRetriever(); IMetricRetriever emptyRetriever = EmptyMetricRetriever.Instance; // Create the selector function here so it has access to the retrievers, filter, and providerName Func <MetricDefinition, IMetricRetriever> retrieverSelector = (d) => { if (!d.MetricAvailabilities.Any()) { return(emptyRetriever); } if (isStorageProvider) { return(storageRetriever); } if (IsBlobSasMetric(d, filter.TimeGrain)) { return(blobShoeboxMetricRetriever); } if (IsTableSasMetric(d, filter.TimeGrain)) { return(shoeboxRetriever); } return(proxyRetriever); }; // Group definitions by retriever so we can make one request to each retriever IEnumerable <IGrouping <IMetricRetriever, MetricDefinition> > groups = definitions.GroupBy(retrieverSelector); // Get Metrics from each retriever (group) IEnumerable <Task <MetricListResponse> > locationTasks = groups.Select(g => g.Key.GetMetricsAsync(resourceUri, GetFilterStringForDefinitions(filter, g), g, invocationId)); // Aggregate metrics from all groups this.LogStartGetMetrics(invocationId, resourceUri, filterString, definitions); MetricListResponse[] results = (await Task.Factory.ContinueWhenAll(locationTasks.ToArray(), tasks => tasks.Select(t => t.Result))).ToArray(); IEnumerable <Metric> metrics = results.Aggregate <MetricListResponse, IEnumerable <Metric> >( new List <Metric>(), (list, response) => list.Union(response.MetricCollection.Value)); this.LogMetricCountFromResponses(invocationId, metrics.Count()); // Fill in values (resourceUri, displayName, unit) from definitions CompleteShoeboxMetrics(metrics, definitions, resourceUri); // Add empty objects for metrics that had no values come back, ensuring a metric is returned for each definition IEnumerable <Metric> emptyMetrics = (await emptyRetriever.GetMetricsAsync( resourceUri, filterString, definitions.Where(d => !metrics.Any(m => string.Equals(m.Name.Value, d.Name.Value, StringComparison.OrdinalIgnoreCase))), invocationId)).MetricCollection.Value; // Create response (merge and wrap metrics) result = new MetricListResponse() { RequestId = Guid.NewGuid().ToString("D"), StatusCode = HttpStatusCode.OK, MetricCollection = new MetricCollection() { Value = metrics.Union(emptyMetrics).ToList() } }; this.LogEndGetMetrics(invocationId, result); return(result); }
public async Task <MetricListResponse> GetMetricsAsync(string resourceUri, string filterString, CancellationToken cancellationToken) { // Ensure exactly one '/' at the start resourceUri = '/' + resourceUri.TrimStart('/'); // Generate filter strings string metricFilter = RemoveNamesFromFilterString(filterString); string definitionFilter = ShoeboxHelper.GenerateMetricDefinitionFilterString(MetricFilterExpressionParser.Parse(filterString).Names); // Get definitions for requested metrics IList <MetricDefinition> definitions = (await this.Client.MetricDefinitionOperations.GetMetricDefinitionsAsync( resourceUri, definitionFilter, cancellationToken).ConfigureAwait(false)).MetricDefinitionCollection.Value; // Get Metrics by definitions return(await this.GetMetricsAsync(resourceUri, metricFilter, definitions, cancellationToken).ConfigureAwait(false)); }
/// <summary> /// Get logs. /// </summary> /// <param name="resourceUri">The resourceUri</param> /// <param name="filterString">The filter string</param> /// <param name="definitions">The log definitions</param> /// <param name="cancellationToken">The cancellation token</param> /// <returns></returns> public async Task <LogListResponse> GetLogsAsync( string resourceUri, string filterString, IEnumerable <LogDefinition> definitions, CancellationToken cancellationToken) { if (definitions == null) { throw new ArgumentNullException("definitions"); } if (resourceUri == null) { throw new ArgumentNullException("resourceUri"); } // Ensure exactly one '/' at the start resourceUri = '/' + resourceUri.TrimStart('/'); LogListResponse result; string invocationId = TracingAdapter.NextInvocationId.ToString(CultureInfo.InvariantCulture); // If no definitions provided, return empty collection if (!definitions.Any()) { this.LogStartGetLogs(invocationId, resourceUri, filterString, definitions); result = new LogListResponse() { RequestId = Guid.NewGuid().ToString("D"), StatusCode = HttpStatusCode.OK, LogCollection = new LogCollection() { Value = new Log[0] } }; this.LogEndGetLogs(invocationId, result); return(result); } // Parse LogFilter. Reusing the metric filter. // We might consider extracting the parsing functionality to a class with a less specific name MetricFilter filter = MetricFilterExpressionParser.Parse(filterString, false); if (filter.StartTime == default(DateTime)) { throw new InvalidOperationException("startTime is required"); } if (filter.EndTime == default(DateTime)) { throw new InvalidOperationException("endTime is required"); } this.LogStartGetLogs(invocationId, resourceUri, filterString, definitions); var logsPerBlob = new Dictionary <string, Task <Log> >(StringComparer.OrdinalIgnoreCase); // We download all the relevant blobs first and then use the data later, to avoid download the same blob more than once. foreach (LogDefinition logDefinition in definitions) { foreach (BlobInfo blobInfo in logDefinition.BlobLocation.BlobInfo) { if (blobInfo.EndTime < filter.StartTime || blobInfo.StartTime >= filter.EndTime) { continue; } string blobId = GetBlobEndpoint(blobInfo); if (!logsPerBlob.ContainsKey(blobId)) { logsPerBlob.Add(blobId, FetchLogFromBlob(blobInfo.BlobUri, filter, logDefinition.Category)); } } } foreach (var task in logsPerBlob.Values) { await task; } var logsPerCategory = new Dictionary <string, Log>(); foreach (var task in logsPerBlob.Values) { Log log = task.Result; Log existingLog; if (logsPerCategory.TryGetValue(log.Category.Value, out existingLog)) { ((List <LogValue>)existingLog.Value).AddRange(log.Value); existingLog.StartTime = this.Min(log.StartTime, existingLog.StartTime); existingLog.EndTime = this.Max(log.StartTime, existingLog.StartTime); } else { logsPerCategory.Add(log.Category.Value, log); } } result = new LogListResponse { RequestId = Guid.NewGuid().ToString("D"), StatusCode = HttpStatusCode.OK, LogCollection = new LogCollection { Value = logsPerCategory.Values.ToList() } }; this.LogEndGetLogs(invocationId, result); return(result); }