private static string RemoveNamesFromFilterString(string filterString)
{
MetricFilter filter = MetricFilterExpressionParser.Parse(filterString);
MetricFilter nameless = new MetricFilter()
{
TimeGrain = filter.TimeGrain,
StartTime = filter.StartTime,
EndTime = filter.EndTime
};
return(ShoeboxHelper.GenerateMetricFilterString(nameless));
}
public async Task <MetricListResponse> GetMetricsAsync(string resourceUri, string filterString, CancellationToken cancellationToken)
{
if (resourceUri == null)
{
throw new ArgumentNullException("resourceUri");
}
// Generate filter strings
MetricFilter filter = MetricFilterExpressionParser.Parse(filterString);
string filterStringNamesOnly = filter.DimensionFilters == null ? null
: ShoeboxHelper.GenerateMetricDefinitionFilterString(filter.DimensionFilters.Select(df => df.Name));
// Get definitions for requested metrics
IList <MetricDefinition> definitions =
(await this.Client.MetricDefinitionOperations.GetMetricDefinitionsAsync(
resourceUri,
filterStringNamesOnly,
cancellationToken).ConfigureAwait(false)).MetricDefinitionCollection.Value;
// Get Metrics with definitions
return(await this.GetMetricsAsync(resourceUri, filterString, definitions, cancellationToken));
}
public async Task <MetricListResponse> GetMetricsAsync(string resourceUri, string filterString, CancellationToken cancellationToken)
{
// Ensure exactly one '/' at the start
resourceUri = '/' + resourceUri.TrimStart('/');
// Generate filter strings
MetricFilter filter = MetricFilterExpressionParser.Parse(filterString);
string metricFilterString = GenerateNamelessMetricFilterString(filter);
string definitionFilterString = filter.DimensionFilters == null ? null
: ShoeboxHelper.GenerateMetricDefinitionFilterString(filter.DimensionFilters.Select(df => df.Name));
// Get definitions for requested metrics
IList <MetricDefinition> definitions =
(await this.Client.MetricDefinitionOperations.GetMetricDefinitionsAsync(
resourceUri,
definitionFilterString,
cancellationToken).ConfigureAwait(false)).MetricDefinitionCollection.Value;
// Separate passthrough metrics with dimensions specified
IEnumerable <MetricDefinition> passthruDefinitions = definitions.Where(d => !IsShoebox(d, filter.TimeGrain));
IEnumerable <MetricDefinition> shoeboxDefinitions = definitions.Where(d => IsShoebox(d, filter.TimeGrain));
// Get Passthru definitions
List <Metric> passthruMetrics = new List <Metric>();
string invocationId = TracingAdapter.NextInvocationId.ToString(CultureInfo.InvariantCulture);
this.LogStartGetMetrics(invocationId, resourceUri, filterString, passthruDefinitions);
if (passthruDefinitions.Any())
{
// Create new filter for passthru metrics
List <MetricDimension> passthruDimensionFilters = filter.DimensionFilters == null ? new List <MetricDimension>() :
filter.DimensionFilters.Where(df => passthruDefinitions.Any(d => string.Equals(d.Name.Value, df.Name, StringComparison.OrdinalIgnoreCase))).ToList();
foreach (MetricDefinition def in passthruDefinitions
.Where(d => !passthruDimensionFilters.Any(pdf => string.Equals(pdf.Name, d.Name.Value, StringComparison.OrdinalIgnoreCase))))
{
passthruDimensionFilters.Add(new MetricDimension()
{
Name = def.Name.Value
});
}
MetricFilter passthruFilter = new MetricFilter()
{
TimeGrain = filter.TimeGrain,
StartTime = filter.StartTime,
EndTime = filter.EndTime,
DimensionFilters = passthruDimensionFilters
};
// Create passthru filter string
string passthruFilterString = ShoeboxHelper.GenerateMetricFilterString(passthruFilter);
// Get Metrics from passthrough (hydra) client
MetricListResponse passthruResponse = await this.GetMetricsInternalAsync(resourceUri, passthruFilterString, cancellationToken).ConfigureAwait(false);
passthruMetrics = passthruResponse.MetricCollection.Value.ToList();
this.LogMetricCountFromResponses(invocationId, passthruMetrics.Count());
// Fill in values (resourceUri, displayName, unit) from definitions
CompleteShoeboxMetrics(passthruMetrics, passthruDefinitions, resourceUri);
// Add empty objects for metrics that had no values come back, ensuring a metric is returned for each definition
IEnumerable <Metric> emptyMetrics = passthruDefinitions
.Where(d => !passthruMetrics.Any(m => string.Equals(m.Name.Value, d.Name.Value, StringComparison.OrdinalIgnoreCase)))
.Select(d => new Metric()
{
Name = d.Name,
Unit = d.Unit,
ResourceId = resourceUri,
StartTime = filter.StartTime,
EndTime = filter.EndTime,
TimeGrain = filter.TimeGrain,
MetricValues = new List <MetricValue>(),
Properties = new Dictionary <string, string>()
});
passthruMetrics.AddRange(emptyMetrics);
}
// Get Metrics by definitions
MetricListResponse shoeboxResponse = await this.GetMetricsAsync(resourceUri, metricFilterString, shoeboxDefinitions, cancellationToken).ConfigureAwait(false);
// Create response (merge and wrap metrics)
MetricListResponse result = new MetricListResponse()
{
RequestId = Guid.NewGuid().ToString("D"),
StatusCode = HttpStatusCode.OK,
MetricCollection = new MetricCollection()
{
Value = passthruMetrics.Union(shoeboxResponse.MetricCollection.Value).ToList()
}
};
this.LogEndGetMetrics(invocationId, result);
return(result);
}
// Alternate method for getting metrics by passing in the definitions
// TODO [davmc]: Revisit - this method cannot support dimensions
public async Task <MetricListResponse> GetMetricsAsync(
string resourceUri, string filterString, IEnumerable <MetricDefinition> definitions, CancellationToken cancellationToken)
{
MetricListResponse result;
if (definitions == null)
{
throw new ArgumentNullException("definitions");
}
string invocationId = TracingAdapter.NextInvocationId.ToString(CultureInfo.InvariantCulture);
this.LogStartGetMetrics(invocationId, resourceUri, filterString, definitions);
// If no definitions provided, return empty collection
if (!definitions.Any())
{
result = new MetricListResponse()
{
RequestId = Guid.NewGuid().ToString("D"),
StatusCode = HttpStatusCode.OK
};
this.LogEndGetMetrics(invocationId, result);
return(result);
}
// Parse MetricFilter
MetricFilter filter = MetricFilterExpressionParser.Parse(filterString);
// Names not allowed in filter since the names are in the definitions
if (filter.DimensionFilters != null && filter.DimensionFilters.Any())
{
throw new ArgumentException("Cannot specify names (or dimensions) when MetricDefinitions are included", "filterString");
}
// Ensure every definition has at least one availability matching the filter timegrain
if (!definitions.All(d => d.MetricAvailabilities.Any(a => a.TimeGrain == filter.TimeGrain)))
{
throw new ArgumentException("Definition contains no availability for the timeGrain requested", "definitions");
}
// Group definitions by location so we can make one request to each location
Dictionary <MetricAvailability, MetricFilter> groups =
definitions.GroupBy(d => d.MetricAvailabilities.First(a => a.TimeGrain == filter.TimeGrain),
new AvailabilityComparer()).ToDictionary(g => g.Key, g => new MetricFilter()
{
TimeGrain = filter.TimeGrain,
StartTime = filter.StartTime,
EndTime = filter.EndTime,
DimensionFilters = g.Select(d => new MetricDimension()
{
Name = d.Name.Value
})
});
// Get Metrics from each location (group)
IEnumerable <Task <MetricListResponse> > locationTasks = groups.Select(g => g.Key.Location == null
? this.GetMetricsInternalAsync(resourceUri, ShoeboxHelper.GenerateMetricFilterString(g.Value), cancellationToken)
: ShoeboxClient.GetMetricsInternalAsync(g.Value, g.Key.Location, invocationId));
// Aggregate metrics from all groups
MetricListResponse[] results = (await Task.Factory.ContinueWhenAll(locationTasks.ToArray(), tasks => tasks.Select(t => t.Result))).ToArray();
IEnumerable <Metric> metrics = results.Aggregate <MetricListResponse, IEnumerable <Metric> >(
new List <Metric>(), (list, response) => list.Union(response.MetricCollection.Value));
this.LogMetricCountFromResponses(invocationId, metrics.Count());
// Fill in values (resourceUri, displayName, unit) from definitions
CompleteShoeboxMetrics(metrics, definitions, resourceUri);
// Add empty objects for metrics that had no values come back, ensuring a metric is returned for each definition
IEnumerable <Metric> emptyMetrics = definitions
.Where(d => !metrics.Any(m => string.Equals(m.Name.Value, d.Name.Value, StringComparison.OrdinalIgnoreCase)))
.Select(d => new Metric()
{
Name = d.Name,
Unit = d.Unit,
ResourceId = resourceUri,
StartTime = filter.StartTime,
EndTime = filter.EndTime,
TimeGrain = filter.TimeGrain,
MetricValues = new List <MetricValue>(),
Properties = new Dictionary <string, string>()
});
// Create response (merge and wrap metrics)
result = new MetricListResponse()
{
RequestId = Guid.NewGuid().ToString("D"),
StatusCode = HttpStatusCode.OK,
MetricCollection = new MetricCollection()
{
Value = metrics.Union(emptyMetrics).ToList()
}
};
this.LogEndGetMetrics(invocationId, result);
return(result);
}
// Alternate method for getting metrics by passing in the definitions
public async Task <MetricListResponse> GetMetricsAsync(
string resourceUri, string filterString, IEnumerable <MetricDefinition> definitions, CancellationToken cancellationToken)
{
if (definitions == null)
{
throw new ArgumentNullException("definitions");
}
if (resourceUri == null)
{
throw new ArgumentNullException("resourceUri");
}
// Remove any '/' characters from the start since these are handled by the hydra (thin) client
// Don't encode Uri segments here since this will mess up the SAS retrievers (they use the resourceUri directly)
resourceUri = resourceUri.TrimStart('/');
MetricListResponse result;
string invocationId = TracingAdapter.NextInvocationId.ToString(CultureInfo.InvariantCulture);
// If no definitions provided, return empty collection
if (!definitions.Any())
{
this.LogStartGetMetrics(invocationId, resourceUri, filterString, definitions);
result = new MetricListResponse()
{
RequestId = Guid.NewGuid().ToString("D"),
StatusCode = HttpStatusCode.OK,
MetricCollection = new MetricCollection()
{
Value = new Metric[0]
}
};
this.LogEndGetMetrics(invocationId, result);
return(result);
}
// Parse MetricFilter
MetricFilter filter = MetricFilterExpressionParser.Parse(filterString);
// Names in filter must match the names in the definitions
if (filter.DimensionFilters != null && filter.DimensionFilters.Any())
{
IEnumerable <string> filterNames = filter.DimensionFilters.Select(df => df.Name);
IEnumerable <string> definitionNames = definitions.Select(d => d.Name.Value);
IEnumerable <string> filterOnly = filterNames.Where(fn => !definitionNames.Contains(fn, StringComparer.InvariantCultureIgnoreCase));
IEnumerable <string> definitionOnly = definitionNames.Where(df => !filterNames.Contains(df, StringComparer.InvariantCultureIgnoreCase));
if (filterOnly.Any() || definitionOnly.Any())
{
throw new ArgumentException("Set of names specified in filter string must match set of names in provided definitions", "filterString");
}
// "Filter out" metrics with unsupported dimensions
definitions = definitions.Where(d => SupportsRequestedDimensions(d, filter));
}
else
{
filter = new MetricFilter()
{
TimeGrain = filter.TimeGrain,
StartTime = filter.StartTime,
EndTime = filter.EndTime,
DimensionFilters = definitions.Select(d => new MetricDimension()
{
Name = d.Name.Value
})
};
}
// Parse out provider name and determine if provider is storage
string providerName = this.GetProviderFromResourceId(resourceUri);
bool isStorageProvider =
string.Equals(providerName, "Microsoft.Storage", StringComparison.OrdinalIgnoreCase) ||
string.Equals(providerName, "Microsoft.ClassicStorage", StringComparison.OrdinalIgnoreCase);
// Create supported MetricRetrievers
IMetricRetriever proxyRetriever = new ProxyMetricRetriever(this);
IMetricRetriever shoeboxRetriever = new ShoeboxMetricRetriever();
IMetricRetriever storageRetriever = new StorageMetricRetriever();
IMetricRetriever blobShoeboxMetricRetriever = new BlobShoeboxMetricRetriever();
IMetricRetriever emptyRetriever = EmptyMetricRetriever.Instance;
// Create the selector function here so it has access to the retrievers, filter, and providerName
Func <MetricDefinition, IMetricRetriever> retrieverSelector = (d) =>
{
if (!d.MetricAvailabilities.Any())
{
return(emptyRetriever);
}
if (isStorageProvider)
{
return(storageRetriever);
}
if (IsBlobSasMetric(d, filter.TimeGrain))
{
return(blobShoeboxMetricRetriever);
}
if (IsTableSasMetric(d, filter.TimeGrain))
{
return(shoeboxRetriever);
}
return(proxyRetriever);
};
// Group definitions by retriever so we can make one request to each retriever
IEnumerable <IGrouping <IMetricRetriever, MetricDefinition> > groups = definitions.GroupBy(retrieverSelector);
// Get Metrics from each retriever (group)
IEnumerable <Task <MetricListResponse> > locationTasks = groups.Select(g =>
g.Key.GetMetricsAsync(resourceUri, GetFilterStringForDefinitions(filter, g), g, invocationId));
// Aggregate metrics from all groups
this.LogStartGetMetrics(invocationId, resourceUri, filterString, definitions);
MetricListResponse[] results = (await Task.Factory.ContinueWhenAll(locationTasks.ToArray(), tasks => tasks.Select(t => t.Result))).ToArray();
IEnumerable <Metric> metrics = results.Aggregate <MetricListResponse, IEnumerable <Metric> >(
new List <Metric>(), (list, response) => list.Union(response.MetricCollection.Value));
this.LogMetricCountFromResponses(invocationId, metrics.Count());
// Fill in values (resourceUri, displayName, unit) from definitions
CompleteShoeboxMetrics(metrics, definitions, resourceUri);
// Add empty objects for metrics that had no values come back, ensuring a metric is returned for each definition
IEnumerable <Metric> emptyMetrics = (await emptyRetriever.GetMetricsAsync(
resourceUri,
filterString,
definitions.Where(d => !metrics.Any(m => string.Equals(m.Name.Value, d.Name.Value, StringComparison.OrdinalIgnoreCase))),
invocationId)).MetricCollection.Value;
// Create response (merge and wrap metrics)
result = new MetricListResponse()
{
RequestId = Guid.NewGuid().ToString("D"),
StatusCode = HttpStatusCode.OK,
MetricCollection = new MetricCollection()
{
Value = metrics.Union(emptyMetrics).ToList()
}
};
this.LogEndGetMetrics(invocationId, result);
return(result);
}
public async Task <MetricListResponse> GetMetricsAsync(string resourceUri, string filterString, CancellationToken cancellationToken)
{
// Ensure exactly one '/' at the start
resourceUri = '/' + resourceUri.TrimStart('/');
// Generate filter strings
string metricFilter = RemoveNamesFromFilterString(filterString);
string definitionFilter = ShoeboxHelper.GenerateMetricDefinitionFilterString(MetricFilterExpressionParser.Parse(filterString).Names);
// Get definitions for requested metrics
IList <MetricDefinition> definitions =
(await this.Client.MetricDefinitionOperations.GetMetricDefinitionsAsync(
resourceUri,
definitionFilter,
cancellationToken).ConfigureAwait(false)).MetricDefinitionCollection.Value;
// Get Metrics by definitions
return(await this.GetMetricsAsync(resourceUri, metricFilter, definitions, cancellationToken).ConfigureAwait(false));
}
/// <summary>
/// Get logs.
/// </summary>
/// <param name="resourceUri">The resourceUri</param>
/// <param name="filterString">The filter string</param>
/// <param name="definitions">The log definitions</param>
/// <param name="cancellationToken">The cancellation token</param>
/// <returns></returns>
public async Task <LogListResponse> GetLogsAsync(
string resourceUri,
string filterString,
IEnumerable <LogDefinition> definitions,
CancellationToken cancellationToken)
{
if (definitions == null)
{
throw new ArgumentNullException("definitions");
}
if (resourceUri == null)
{
throw new ArgumentNullException("resourceUri");
}
// Ensure exactly one '/' at the start
resourceUri = '/' + resourceUri.TrimStart('/');
LogListResponse result;
string invocationId = TracingAdapter.NextInvocationId.ToString(CultureInfo.InvariantCulture);
// If no definitions provided, return empty collection
if (!definitions.Any())
{
this.LogStartGetLogs(invocationId, resourceUri, filterString, definitions);
result = new LogListResponse()
{
RequestId = Guid.NewGuid().ToString("D"),
StatusCode = HttpStatusCode.OK,
LogCollection = new LogCollection()
{
Value = new Log[0]
}
};
this.LogEndGetLogs(invocationId, result);
return(result);
}
// Parse LogFilter. Reusing the metric filter.
// We might consider extracting the parsing functionality to a class with a less specific name
MetricFilter filter = MetricFilterExpressionParser.Parse(filterString, false);
if (filter.StartTime == default(DateTime))
{
throw new InvalidOperationException("startTime is required");
}
if (filter.EndTime == default(DateTime))
{
throw new InvalidOperationException("endTime is required");
}
this.LogStartGetLogs(invocationId, resourceUri, filterString, definitions);
var logsPerBlob = new Dictionary <string, Task <Log> >(StringComparer.OrdinalIgnoreCase);
// We download all the relevant blobs first and then use the data later, to avoid download the same blob more than once.
foreach (LogDefinition logDefinition in definitions)
{
foreach (BlobInfo blobInfo in logDefinition.BlobLocation.BlobInfo)
{
if (blobInfo.EndTime < filter.StartTime || blobInfo.StartTime >= filter.EndTime)
{
continue;
}
string blobId = GetBlobEndpoint(blobInfo);
if (!logsPerBlob.ContainsKey(blobId))
{
logsPerBlob.Add(blobId, FetchLogFromBlob(blobInfo.BlobUri, filter, logDefinition.Category));
}
}
}
foreach (var task in logsPerBlob.Values)
{
await task;
}
var logsPerCategory = new Dictionary <string, Log>();
foreach (var task in logsPerBlob.Values)
{
Log log = task.Result;
Log existingLog;
if (logsPerCategory.TryGetValue(log.Category.Value, out existingLog))
{
((List <LogValue>)existingLog.Value).AddRange(log.Value);
existingLog.StartTime = this.Min(log.StartTime, existingLog.StartTime);
existingLog.EndTime = this.Max(log.StartTime, existingLog.StartTime);
}
else
{
logsPerCategory.Add(log.Category.Value, log);
}
}
result = new LogListResponse
{
RequestId = Guid.NewGuid().ToString("D"),
StatusCode = HttpStatusCode.OK,
LogCollection = new LogCollection
{
Value = logsPerCategory.Values.ToList()
}
};
this.LogEndGetLogs(invocationId, result);
return(result);
}
