// Creates a TableQuery for each named metric and returns a dictionary mapping each name to its query
// Note: The overall start and end times are used in each query since this reduces processing and the query will still work the same on each Nday table
private static Dictionary <string, TableQuery> GenerateMetricNameQueries(IEnumerable <string> names, string partitionKey, DateTime startTime, DateTime endTime)
{
return(names.ToDictionary(name => ShoeboxHelper.TrimAndEscapeKey(name) + "__").ToDictionary(kvp => kvp.Value, kvp =>
GenerateMetricQuery(
partitionKey,
kvp.Key + (DateTime.MaxValue.Ticks - endTime.Ticks).ToString("D19"),
kvp.Key + (DateTime.MaxValue.Ticks - startTime.Ticks).ToString("D19"))));
}
private static string GenerateNamelessMetricFilterString(MetricFilter filter)
{
MetricFilter nameless = new MetricFilter()
{
TimeGrain = filter.TimeGrain,
StartTime = filter.StartTime,
EndTime = filter.EndTime
};
return(ShoeboxHelper.GenerateMetricFilterString(nameless));
}
private string GetFilterStringForDefinitions(MetricFilter filter, IEnumerable <MetricDefinition> definitions)
{
return(ShoeboxHelper.GenerateMetricFilterString(new MetricFilter()
{
TimeGrain = filter.TimeGrain,
StartTime = filter.StartTime,
EndTime = filter.EndTime,
DimensionFilters = filter.DimensionFilters.Where(df =>
definitions.Any(d => string.Equals(df.Name, d.Name.Value, StringComparison.OrdinalIgnoreCase)))
}));
}
private static string RemoveNamesFromFilterString(string filterString)
{
MetricFilter filter = MetricFilterExpressionParser.Parse(filterString);
MetricFilter nameless = new MetricFilter()
{
TimeGrain = filter.TimeGrain,
StartTime = filter.StartTime,
EndTime = filter.EndTime
};
return(ShoeboxHelper.GenerateMetricFilterString(nameless));
}
public async Task <MetricListResponse> GetMetricsAsync(string resourceUri, string filterString, CancellationToken cancellationToken)
{
// Ensure exactly one '/' at the start
resourceUri = '/' + resourceUri.TrimStart('/');
// Generate filter strings
string metricFilter = RemoveNamesFromFilterString(filterString);
string definitionFilter = ShoeboxHelper.GenerateMetricDefinitionFilterString(MetricFilterExpressionParser.Parse(filterString).Names);
// Get definitions for requested metrics
IList <MetricDefinition> definitions =
(await this.Client.MetricDefinitionOperations.GetMetricDefinitionsAsync(
resourceUri,
definitionFilter,
cancellationToken).ConfigureAwait(false)).MetricDefinitionCollection.Value;
// Get Metrics by definitions
return(await this.GetMetricsAsync(resourceUri, metricFilter, definitions, cancellationToken).ConfigureAwait(false));
}
public async Task <MetricListResponse> GetMetricsAsync(string resourceUri, string filterString, CancellationToken cancellationToken)
{
if (resourceUri == null)
{
throw new ArgumentNullException("resourceUri");
}
// Generate filter strings
MetricFilter filter = MetricFilterExpressionParser.Parse(filterString);
string filterStringNamesOnly = filter.DimensionFilters == null ? null
: ShoeboxHelper.GenerateMetricDefinitionFilterString(filter.DimensionFilters.Select(df => df.Name));
// Get definitions for requested metrics
IList <MetricDefinition> definitions =
(await this.Client.MetricDefinitionOperations.GetMetricDefinitionsAsync(
resourceUri,
filterStringNamesOnly,
cancellationToken).ConfigureAwait(false)).MetricDefinitionCollection.Value;
// Get Metrics with definitions
return(await this.GetMetricsAsync(resourceUri, filterString, definitions, cancellationToken));
}
// Generates queries for all metrics by timestamp (timestamp-name rowKey format) and filters the results to the requested metrics (if any)
// Note: Does not populate Metric fields unrelated to query (i.e. "display name", resourceUri, and properties)
private static async Task <MetricCollection> GetMetricsByTimestampAsync(MetricFilter filter, MetricLocation location, string invocationId)
{
// Find all the tables that fall partially or fully within the timerange
IEnumerable <CloudTable> tables = GetNdayTables(filter, location);
// Generate a query for the partition key and time range
TableQuery query = GenerateMetricTimestampQuery(location.PartitionKey, filter.StartTime, filter.EndTime);
// Get all the entities for the query
IEnumerable <DynamicTableEntity> entities = await GetEntitiesAsync(tables, query, invocationId).ConfigureAwait(false);
// Group entities by (encoded) name
IEnumerable <IGrouping <string, DynamicTableEntity> > groups = entities.GroupBy(entity => entity.RowKey.Substring(entity.RowKey.LastIndexOf('_') + 1));
// if names are specified, filter the results to those metrics only
if (filter.DimensionFilters != null)
{
groups = groups.Where(g => filter.DimensionFilters.Select(df => ShoeboxHelper.TrimAndEscapeKey(df.Name)).Contains(g.Key));
}
// Construct MetricCollection (list of metrics) by taking each group and converting the entities in that group to MetricValue objects
return(new MetricCollection()
{
Value =
groups.Select(g => new Metric()
{
Name = new LocalizableString()
{
Value = FindMetricName(g.Key, filter.DimensionFilters.Select(df => df.Name))
},
StartTime = filter.StartTime,
EndTime = filter.EndTime,
TimeGrain = filter.TimeGrain,
MetricValues = g.Select(ResolveMetricEntity).ToList()
}).ToList()
});
}
public async Task <MetricListResponse> GetMetricsAsync(string resourceUri, string filterString, CancellationToken cancellationToken)
{
// Ensure exactly one '/' at the start
resourceUri = '/' + resourceUri.TrimStart('/');
// Generate filter strings
MetricFilter filter = MetricFilterExpressionParser.Parse(filterString);
string metricFilterString = GenerateNamelessMetricFilterString(filter);
string definitionFilterString = filter.DimensionFilters == null ? null
: ShoeboxHelper.GenerateMetricDefinitionFilterString(filter.DimensionFilters.Select(df => df.Name));
// Get definitions for requested metrics
IList <MetricDefinition> definitions =
(await this.Client.MetricDefinitionOperations.GetMetricDefinitionsAsync(
resourceUri,
definitionFilterString,
cancellationToken).ConfigureAwait(false)).MetricDefinitionCollection.Value;
// Separate passthrough metrics with dimensions specified
IEnumerable <MetricDefinition> passthruDefinitions = definitions.Where(d => !IsShoebox(d, filter.TimeGrain));
IEnumerable <MetricDefinition> shoeboxDefinitions = definitions.Where(d => IsShoebox(d, filter.TimeGrain));
// Get Passthru definitions
List <Metric> passthruMetrics = new List <Metric>();
string invocationId = TracingAdapter.NextInvocationId.ToString(CultureInfo.InvariantCulture);
this.LogStartGetMetrics(invocationId, resourceUri, filterString, passthruDefinitions);
if (passthruDefinitions.Any())
{
// Create new filter for passthru metrics
List <MetricDimension> passthruDimensionFilters = filter.DimensionFilters == null ? new List <MetricDimension>() :
filter.DimensionFilters.Where(df => passthruDefinitions.Any(d => string.Equals(d.Name.Value, df.Name, StringComparison.OrdinalIgnoreCase))).ToList();
foreach (MetricDefinition def in passthruDefinitions
.Where(d => !passthruDimensionFilters.Any(pdf => string.Equals(pdf.Name, d.Name.Value, StringComparison.OrdinalIgnoreCase))))
{
passthruDimensionFilters.Add(new MetricDimension()
{
Name = def.Name.Value
});
}
MetricFilter passthruFilter = new MetricFilter()
{
TimeGrain = filter.TimeGrain,
StartTime = filter.StartTime,
EndTime = filter.EndTime,
DimensionFilters = passthruDimensionFilters
};
// Create passthru filter string
string passthruFilterString = ShoeboxHelper.GenerateMetricFilterString(passthruFilter);
// Get Metrics from passthrough (hydra) client
MetricListResponse passthruResponse = await this.GetMetricsInternalAsync(resourceUri, passthruFilterString, cancellationToken).ConfigureAwait(false);
passthruMetrics = passthruResponse.MetricCollection.Value.ToList();
this.LogMetricCountFromResponses(invocationId, passthruMetrics.Count());
// Fill in values (resourceUri, displayName, unit) from definitions
CompleteShoeboxMetrics(passthruMetrics, passthruDefinitions, resourceUri);
// Add empty objects for metrics that had no values come back, ensuring a metric is returned for each definition
IEnumerable <Metric> emptyMetrics = passthruDefinitions
.Where(d => !passthruMetrics.Any(m => string.Equals(m.Name.Value, d.Name.Value, StringComparison.OrdinalIgnoreCase)))
.Select(d => new Metric()
{
Name = d.Name,
Unit = d.Unit,
ResourceId = resourceUri,
StartTime = filter.StartTime,
EndTime = filter.EndTime,
TimeGrain = filter.TimeGrain,
MetricValues = new List <MetricValue>(),
Properties = new Dictionary <string, string>()
});
passthruMetrics.AddRange(emptyMetrics);
}
// Get Metrics by definitions
MetricListResponse shoeboxResponse = await this.GetMetricsAsync(resourceUri, metricFilterString, shoeboxDefinitions, cancellationToken).ConfigureAwait(false);
// Create response (merge and wrap metrics)
MetricListResponse result = new MetricListResponse()
{
RequestId = Guid.NewGuid().ToString("D"),
StatusCode = HttpStatusCode.OK,
MetricCollection = new MetricCollection()
{
Value = passthruMetrics.Union(shoeboxResponse.MetricCollection.Value).ToList()
}
};
this.LogEndGetMetrics(invocationId, result);
return(result);
}
// Alternate method for getting metrics by passing in the definitions
// TODO [davmc]: Revisit - this method cannot support dimensions
public async Task <MetricListResponse> GetMetricsAsync(
string resourceUri, string filterString, IEnumerable <MetricDefinition> definitions, CancellationToken cancellationToken)
{
MetricListResponse result;
if (definitions == null)
{
throw new ArgumentNullException("definitions");
}
string invocationId = TracingAdapter.NextInvocationId.ToString(CultureInfo.InvariantCulture);
this.LogStartGetMetrics(invocationId, resourceUri, filterString, definitions);
// If no definitions provided, return empty collection
if (!definitions.Any())
{
result = new MetricListResponse()
{
RequestId = Guid.NewGuid().ToString("D"),
StatusCode = HttpStatusCode.OK
};
this.LogEndGetMetrics(invocationId, result);
return(result);
}
// Parse MetricFilter
MetricFilter filter = MetricFilterExpressionParser.Parse(filterString);
// Names not allowed in filter since the names are in the definitions
if (filter.DimensionFilters != null && filter.DimensionFilters.Any())
{
throw new ArgumentException("Cannot specify names (or dimensions) when MetricDefinitions are included", "filterString");
}
// Ensure every definition has at least one availability matching the filter timegrain
if (!definitions.All(d => d.MetricAvailabilities.Any(a => a.TimeGrain == filter.TimeGrain)))
{
throw new ArgumentException("Definition contains no availability for the timeGrain requested", "definitions");
}
// Group definitions by location so we can make one request to each location
Dictionary <MetricAvailability, MetricFilter> groups =
definitions.GroupBy(d => d.MetricAvailabilities.First(a => a.TimeGrain == filter.TimeGrain),
new AvailabilityComparer()).ToDictionary(g => g.Key, g => new MetricFilter()
{
TimeGrain = filter.TimeGrain,
StartTime = filter.StartTime,
EndTime = filter.EndTime,
DimensionFilters = g.Select(d => new MetricDimension()
{
Name = d.Name.Value
})
});
// Get Metrics from each location (group)
IEnumerable <Task <MetricListResponse> > locationTasks = groups.Select(g => g.Key.Location == null
? this.GetMetricsInternalAsync(resourceUri, ShoeboxHelper.GenerateMetricFilterString(g.Value), cancellationToken)
: ShoeboxClient.GetMetricsInternalAsync(g.Value, g.Key.Location, invocationId));
// Aggregate metrics from all groups
MetricListResponse[] results = (await Task.Factory.ContinueWhenAll(locationTasks.ToArray(), tasks => tasks.Select(t => t.Result))).ToArray();
IEnumerable <Metric> metrics = results.Aggregate <MetricListResponse, IEnumerable <Metric> >(
new List <Metric>(), (list, response) => list.Union(response.MetricCollection.Value));
this.LogMetricCountFromResponses(invocationId, metrics.Count());
// Fill in values (resourceUri, displayName, unit) from definitions
CompleteShoeboxMetrics(metrics, definitions, resourceUri);
// Add empty objects for metrics that had no values come back, ensuring a metric is returned for each definition
IEnumerable <Metric> emptyMetrics = definitions
.Where(d => !metrics.Any(m => string.Equals(m.Name.Value, d.Name.Value, StringComparison.OrdinalIgnoreCase)))
.Select(d => new Metric()
{
Name = d.Name,
Unit = d.Unit,
ResourceId = resourceUri,
StartTime = filter.StartTime,
EndTime = filter.EndTime,
TimeGrain = filter.TimeGrain,
MetricValues = new List <MetricValue>(),
Properties = new Dictionary <string, string>()
});
// Create response (merge and wrap metrics)
result = new MetricListResponse()
{
RequestId = Guid.NewGuid().ToString("D"),
StatusCode = HttpStatusCode.OK,
MetricCollection = new MetricCollection()
{
Value = metrics.Union(emptyMetrics).ToList()
}
};
this.LogEndGetMetrics(invocationId, result);
return(result);
}
public async Task <MetricDefinitionListResponse> GetMetricDefinitionsAsync(string resourceUri, string filterString, CancellationToken cancellationToken)
{
MetricDefinitionListResponse result;
string invocationId = Tracing.NextInvocationId.ToString(CultureInfo.InvariantCulture);
this.LogStartGetMetricDefinitions(invocationId, resourceUri, filterString);
// Ensure exactly one '/' at the start
resourceUri = '/' + resourceUri.TrimStart('/');
IEnumerable <MetricDefinition> definitions;
// If no filter string, must request all metric definiitons since we don't know if we have them all
if (string.IsNullOrWhiteSpace(filterString))
{
// request all definitions
definitions = (await this.GetMetricDefinitionsInternalAsync(resourceUri, string.Empty, CancellationToken.None).ConfigureAwait(false))
.MetricDefinitionCollection.Value;
// cache definitions
this.Client.Cache[resourceUri] = definitions;
// wrap and return definitions
result = new MetricDefinitionListResponse()
{
StatusCode = HttpStatusCode.OK,
MetricDefinitionCollection = new MetricDefinitionCollection()
{
Value = definitions.ToList()
}
};
this.LogEndGetMetricDefinitions(invocationId, result);
return(result);
}
// Parse the filter and retrieve cached definitions
IEnumerable <string> names = MetricDefinitionFilterParser.Parse(filterString);
definitions = this.Client.Cache[resourceUri];
// Find the names in the filter that don't appear on any of the cached definitions
IEnumerable <string> missing = definitions == null
? names
: names.Where((n => !definitions.Any(d => string.Equals(d.Name.Value, n, StringComparison.OrdinalIgnoreCase))));
// Request any missing definitions and update cache (if any)
if (missing.Any())
{
string missingFilter = ShoeboxHelper.GenerateMetricDefinitionFilterString(missing);
// Request missing definitions
var missingDefinitions = (await this.GetMetricDefinitionsInternalAsync(resourceUri, missingFilter, cancellationToken).ConfigureAwait(false))
.MetricDefinitionCollection.Value;
// merge definitions
definitions = (definitions ?? new MetricDefinition[0]).Union(missingDefinitions);
// Store the new set of definitions
this.Client.Cache[resourceUri] = definitions;
}
// Filter out the metrics that were cached but not requested and wrap
result = new MetricDefinitionListResponse()
{
StatusCode = HttpStatusCode.OK,
MetricDefinitionCollection = new MetricDefinitionCollection()
{
Value = definitions.Where(d => names.Contains(d.Name.Value)).ToList()
}
};
this.LogEndGetMetricDefinitions(invocationId, result);
return(result);
}
// This method tries to figure out the original name of the metric from the encoded name
// Note: Will unescape the name if it is not in the list, but it will not be able to unhash it if it was hashed
private static string FindMetricName(string encodedName, IEnumerable <string> names)
{
return(names.FirstOrDefault(n => string.Equals(ShoeboxHelper.TrimAndEscapeKey(n), encodedName, StringComparison.OrdinalIgnoreCase)) ??
ShoeboxHelper.UnEscapeKey(encodedName));
}
// Generates queries for all metrics by timestamp (timestamp-name rowKey format) and filters the results to the requested metrics (if any)
// Note: Does not populate Metric fields unrelated to query (i.e. "display name", resourceUri, and properties)
private static async Task <MetricCollection> GetMetricsByTimestampAsync(MetricFilter filter, MetricLocation location, string invocationId)
{
// Find all the tables that fall partially or fully within the timerange
IEnumerable <CloudTable> tables = GetNdayTables(filter, location);
// Generate a query for the partition key and time range
TableQuery query = GenerateMetricTimestampQuery(location.PartitionKey, filter.StartTime, filter.EndTime);
// Get all the entities for the query
IEnumerable <DynamicTableEntity> entities = await GetEntitiesAsync(tables, query, invocationId).ConfigureAwait(false);
ICollection <string> dimensionFilterNames = null;
if (filter.DimensionFilters != null)
{
dimensionFilterNames = new HashSet <string>(filter.DimensionFilters.Select(df => ShoeboxHelper.TrimAndEscapeKey(df.Name)));
}
var metricWraps = new Dictionary <string, MetricWrap>();
var metrics = new List <Metric>();
// Iterate over the instances to do conversion and aggregation when needed.
foreach (var entity in entities)
{
string encodedName = GetMetricNameFromRowKeyByTimestampByMetricName(entity.RowKey);
// When there is filter, skip entities not included in the filter.
if (dimensionFilterNames != null && !dimensionFilterNames.Contains(encodedName, StringComparer.OrdinalIgnoreCase))
{
continue;
}
MetricWrap metricWrap;
if (!metricWraps.TryGetValue(encodedName, out metricWrap))
{
metricWrap = new MetricWrap
{
Metric = new Metric()
{
Name = new LocalizableString()
{
Value = encodedName
},
StartTime = filter.StartTime,
EndTime = filter.EndTime,
TimeGrain = filter.TimeGrain,
MetricValues = new List <MetricValue>()
},
InstanceMetrics = new List <MetricValue>(),
GlobalMetrics = new List <DynamicTableEntity>()
};
metricWraps[encodedName] = metricWrap;
metrics.Add(metricWrap.Metric);
}
// Skip aggregated entities
if (!IsInstanceMetric(entity.RowKey))
{
// We ignore the aggergated metrics if there are instance metrics.
if (metricWrap.InstanceMetrics.Count == 0)
{
metricWrap.GlobalMetrics.Add(entity);
}
continue;
}
MetricValue lastMetricValue = metricWrap.InstanceMetrics.LastOrDefault();
if (lastMetricValue == null)
{
metricWrap.InstanceMetrics.Add(ResolveMetricEntity(entity));
}
else
{
if (lastMetricValue.Timestamp.Ticks == GetTimestampFromIndexTimestampMetricName(entity))
{
Aggregate(lastMetricValue, entity);
}
else
{
metricWrap.InstanceMetrics.Add(ResolveMetricEntity(entity));
}
}
}
foreach (var metricWrap in metricWraps.Values)
{
// Decide whether to return the aggregation of the instance metrics on the fly or the final value in the storage account
// If there are instance metrics, the aggregation on the fly is used.
Metric metric = metricWrap.Metric;
metric.Name.Value = FindMetricName(metric.Name.Value, dimensionFilterNames);
if (metricWrap.InstanceMetrics.Count > 0)
{
foreach (var metricValue in metricWrap.InstanceMetrics)
{
metricValue.Average = metricValue.Total / metricValue.Count;
}
metric.MetricValues = metricWrap.InstanceMetrics;
}
else
{
metric.MetricValues = metricWrap.GlobalMetrics.Select(me => ResolveMetricEntity(me)).ToList();
}
}
return(new MetricCollection()
{
Value = metrics
});
}
