// Calling this based on a grouping (in SasMetricRetriever) should guarantee that it will have metric names specified (cannot have empty group)
internal override async Task <MetricListResponse> GetMetricsInternalAsync(MetricFilter filter, MetricLocation location, string invocationId)
{
if (filter == null)
{
throw new ArgumentNullException("filter");
}
if (location == null)
{
throw new ArgumentNullException("location");
}
// This is called based on the definitions no the dimension portion of the filter should never be null or empty
if (filter.DimensionFilters == null || !filter.DimensionFilters.Any())
{
throw new ArgumentNullException("filter.DimensionFilters");
}
// Separate out capacity metrics and transaction metrics into two groups
IEnumerable <string> capacityMetrics = filter.DimensionFilters.Select(df => df.Name).Where(StorageConstants.MetricNames.IsCapacityMetric);
IEnumerable <string> transactionMetrics = filter.DimensionFilters.Select(df => df.Name).Where(n => !StorageConstants.MetricNames.IsCapacityMetric(n));
List <Task <IEnumerable <Metric> > > queryTasks = new List <Task <IEnumerable <Metric> > >();
// Add task to get capacity metrics (if any)
if (capacityMetrics.Any())
{
MetricTableInfo capacityTableInfo = location.TableInfo.FirstOrDefault(ti => StorageConstants.IsCapacityMetricsTable(ti.TableName));
if (capacityTableInfo == null)
{
throw new InvalidOperationException("Definitions for capacity metrics must contain table info for capacity metrics table");
}
queryTasks.Add(GetCapacityMetricsAsync(filter, GetTableReference(location, capacityTableInfo), capacityMetrics, invocationId));
}
// Add tasks to get transaction metrics (if any)
if (transactionMetrics.Any())
{
IEnumerable <MetricTableInfo> transactionTableInfos = location.TableInfo.Where(ti => !StorageConstants.IsCapacityMetricsTable(ti.TableName));
if (!transactionTableInfos.Any())
{
throw new InvalidOperationException("Definitions for transaction metrics must contain table info for transaction metrics table");
}
queryTasks.AddRange(transactionTableInfos
.Select(info => GetTransactionMetricsAsync(filter, GetTableReference(location, info), transactionMetrics, invocationId)));
}
// Collect results and wrap
return(new MetricListResponse()
{
RequestId = invocationId,
StatusCode = HttpStatusCode.OK,
MetricCollection = new MetricCollection()
{
Value = (await CollectResultsAsync(queryTasks)).ToList()
}
});
}
// Copied from Microsoft.WindowsAzure.Management.Monitoring.ResourceProviders.Storage.Rest.V2011_12.MetricBaseController
private static TableQuery <DynamicTableEntity> GetCapacityQuery(MetricFilter filter)
{
// capacity only applies for blob service and only for a timegrain of 1 day
if (filter.TimeGrain != TimeSpan.FromDays(1))
{
return(null);
}
// since the timestamp field does not represent the actual sample period the only time vaule represented is the partitionkey
// this is basically truncated to the hr with the min zeroed out.
DateTime partitionKeyStartTime = filter.StartTime;
DateTime partitionKeyEndTime = filter.EndTime;
string startKey = partitionKeyStartTime.ToString("yyyyMMddTHH00");
string endKey = partitionKeyEndTime.ToString("yyyyMMddTHH00");
var filter1 = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.GreaterThanOrEqual, startKey);
var filter2 = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.LessThanOrEqual, endKey);
var filter3 = TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.Equal, "data");
var tableQuery = new TableQuery <DynamicTableEntity>()
.Where(TableQuery.CombineFilters(TableQuery.CombineFilters(filter1, TableOperators.And, filter2), TableOperators.And, filter3));
return(tableQuery);
}
// Calling this based on a grouping (in SasMetricRetriever) should guarantee that it will have metric names specified (cannot have empty group)
internal override async Task<MetricListResponse> GetMetricsInternalAsync(MetricFilter filter, MetricLocation location, string invocationId)
{
if (filter == null)
{
throw new ArgumentNullException("filter");
}
if (location == null)
{
throw new ArgumentNullException("location");
}
// This is called based on the definitions no the dimension portion of the filter should never be null or empty
if (filter.DimensionFilters == null || !filter.DimensionFilters.Any())
{
throw new ArgumentNullException("filter.DimensionFilters");
}
// Separate out capacity metrics and transaction metrics into two groups
IEnumerable<string> capacityMetrics = filter.DimensionFilters.Select(df => df.Name).Where(StorageConstants.MetricNames.IsCapacityMetric);
IEnumerable<string> transactionMetrics = filter.DimensionFilters.Select(df => df.Name).Where(n => !StorageConstants.MetricNames.IsCapacityMetric(n));
List<Task<IEnumerable<Metric>>> queryTasks = new List<Task<IEnumerable<Metric>>>();
// Add task to get capacity metrics (if any)
if (capacityMetrics.Any())
{
MetricTableInfo capacityTableInfo = location.TableInfo.FirstOrDefault(ti => StorageConstants.IsCapacityMetricsTable(ti.TableName));
if (capacityTableInfo == null)
{
throw new InvalidOperationException("Definitions for capacity metrics must contain table info for capacity metrics table");
}
queryTasks.Add(GetCapacityMetricsAsync(filter, GetTableReference(location, capacityTableInfo), capacityMetrics, invocationId));
}
// Add tasks to get transaction metrics (if any)
if (transactionMetrics.Any())
{
IEnumerable<MetricTableInfo> transactionTableInfos = location.TableInfo.Where(ti => !StorageConstants.IsCapacityMetricsTable(ti.TableName));
if (!transactionTableInfos.Any())
{
throw new InvalidOperationException("Definitions for transaction metrics must contain table info for transaction metrics table");
}
queryTasks.AddRange(transactionTableInfos
.Select(info => GetTransactionMetricsAsync(filter, GetTableReference(location, info), transactionMetrics, invocationId)));
}
// Collect results and wrap
return new MetricListResponse()
{
RequestId = invocationId,
StatusCode = HttpStatusCode.OK,
MetricCollection = new MetricCollection()
{
Value = (await CollectResultsAsync(queryTasks)).ToList()
}
};
}
public Builder(MetricRegistry registry)
{
this.registry = registry;
this.rateUnit = TimeUnit.Seconds;
this.durationUnit = TimeUnit.Milliseconds;
this.clock = Clock.DefaultClock;
this.filter = MetricFilters.ALL;
}
public static bool Contains(this IList <MetricAvailability> list, MetricFilter filter)
{
foreach (var item in list)
{
return(item.TimeGrain == filter.TimeGrain);
}
return(false);
}
internal Builder(MetricRegistry registry)
{
this.registry = registry;
this.output = Console.Out;
this.clock = Clock.DefaultClock;
this.rateUnit = TimeUnit.Seconds;
this.durationUnit = TimeUnit.Milliseconds;
this.filter = MetricFilters.ALL;
}
internal Builder(MetricRegistry registry)
{
this.registry = registry;
this.output = Console.Out;
this.clock = Clock.DefaultClock;
this.rateUnit = TimeUnit.Seconds;
this.durationUnit = TimeUnit.Milliseconds;
this.filter = MetricFilters.ALL;
}
internal Builder(MetricRegistry registry)
{
this.registry = registry;
this.clock = Clock.DefaultClock;
this.prefix = null;
this.rateUnit = TimeUnit.Seconds;
this.durationUnit = TimeUnit.Milliseconds;
this.filter = MetricFilters.ALL;
}
/// <summary>
/// Removes all metrics which match the given filter.
/// </summary>
/// <param name="filter">a filter</param>
public void RemoveMatching(MetricFilter filter)
{
foreach (KeyValuePair <MetricName, IMetric> pair in _metrics)
{
if (filter(pair.Key, pair.Value))
{
Remove(pair.Key);
}
}
}
public CsvReporter(MetricRegistry registry,
string directory,
TimeUnit rateUnit,
TimeUnit durationUnit,
Clock clock,
MetricFilter filter) : base(registry, "csv-reporter", filter, rateUnit, durationUnit)
{
this.directory = directory;
this.clock = clock;
}
private ConsoleReporter(MetricRegistry registry,
TextWriter output,
Clock clock,
TimeUnit rateUnit,
TimeUnit durationUnit,
MetricFilter filter) :
base(registry, "console-reporter", filter, rateUnit, durationUnit)
{
this.output = output;
this.clock = clock;
}
public CsvReporter(MetricRegistry registry,
string directory,
TimeUnit rateUnit,
TimeUnit durationUnit,
Clock clock,
MetricFilter filter) : base(registry, "csv-reporter", filter, rateUnit, durationUnit)
{
this.directory = directory;
this.clock = clock;
}
public async Task <MetricListResponse> GetMetricsAsync(string resourceId, string filterString, IEnumerable <MetricDefinition> definitions, string invocationId)
{
MetricFilter filter = MetricFilterExpressionParser.Parse(filterString);
// Grab all metric definitiions that contain the specific timegrain from the filter.
// See the extension class for implementation
var timegraindefinitions = from d in definitions
where d.MetricAvailabilities.Count > 0 &&
d.MetricAvailabilities.Contains(filter)
select d;
// Group definitions by location so we can make one request to each location
Dictionary <MetricAvailability, MetricFilter> groups =
timegraindefinitions.GroupBy(d => d.MetricAvailabilities.FirstOrDefault()).ToDictionary(g => g.Key, g => new MetricFilter()
{
TimeGrain = filter.TimeGrain,
StartTime = filter.StartTime,
EndTime = filter.EndTime,
DimensionFilters = g.Select(d =>
filter.DimensionFilters.FirstOrDefault(df => string.Equals(df.Name, d.Name.Value, StringComparison.OrdinalIgnoreCase))
?? new MetricDimension()
{
Name = d.Name.Value
})
}, new AvailabilityComparer());
// Verify all groups represent shoebox metrics
if (groups.Any(g => g.Key.Location == null))
{
throw new ArgumentException("All definitions provided to ShoeboxMetricRetriever must include location information.", "definitions");
}
// Get Metrics from each location (group)
IEnumerable <Task <MetricListResponse> > locationTasks = groups.Select(g => this.GetMetricsInternalAsync(g.Value, g.Key.Location, invocationId));
// Aggregate metrics from all groups
MetricListResponse[] results = (await Task.Factory.ContinueWhenAll(locationTasks.ToArray(), tasks => tasks.Select(t => t.Result))).ToArray();
IEnumerable <Metric> metrics = results.Aggregate <MetricListResponse, IEnumerable <Metric> >(
new List <Metric>(), (list, response) => list.Union(response.MetricCollection.Value));
// Return aggregated results (the MetricOperations class will fill in additional info from the MetricDefinitions)
return(new MetricListResponse()
{
RequestId = invocationId,
StatusCode = HttpStatusCode.OK,
MetricCollection = new MetricCollection()
{
Value = metrics.ToList()
}
});
}
private GraphiteReporter(MetricRegistry registry,
GraphiteSender graphite,
Clock clock,
String prefix,
TimeUnit rateUnit,
TimeUnit durationUnit,
MetricFilter filter) : base(registry, "graphite-reporter", filter, rateUnit, durationUnit)
{
this.graphite = graphite;
this.clock = clock;
this.prefix = MetricName.build(prefix);
}
/// <summary>
/// Creates a new <see cref="ScheduledReporter"/> instance
/// </summary>
/// <param name="registry">the <see cref="MetricRegistry"/> containing the metrics this reporter will report</param>
/// <param name="name">the reporter's name</param>
/// <param name="filter">the filter for which metrics to report</param>
/// <param name="rateUnit">a unit of time</param>
/// <param name="durationUnit">a unit of time</param>
protected ScheduledReporter(MetricRegistry registry,
string name,
MetricFilter filter,
TimeUnit rateUnit,
TimeUnit durationUnit)
{
this.registry = registry;
this.filter = filter;
this.rateFactor = rateUnit.ToSeconds(1);
this.rateUnit = calculateRateUnit(rateUnit);
this.durationFactor = 1.0 / durationUnit.ToNanos(1);
this.durationUnit = durationUnit.ToString().ToLowerInvariant();
}
/// <summary>
/// Creates a new <see cref="ScheduledReporter"/> instance
/// </summary>
/// <param name="registry">the <see cref="MetricRegistry"/> containing the metrics this reporter will report</param>
/// <param name="name">the reporter's name</param>
/// <param name="filter">the filter for which metrics to report</param>
/// <param name="rateUnit">a unit of time</param>
/// <param name="durationUnit">a unit of time</param>
protected ScheduledReporter(MetricRegistry registry,
string name,
MetricFilter filter,
TimeUnit rateUnit,
TimeUnit durationUnit)
{
this.registry = registry;
this.filter = filter;
this.rateFactor = rateUnit.ToSeconds(1);
this.rateUnit = calculateRateUnit(rateUnit);
this.durationFactor = 1.0 / durationUnit.ToNanos(1);
this.durationUnit = durationUnit.ToString().ToLowerInvariant();
}
private IDictionary <MetricName, T> getMetrics <T>(MetricFilter filter) where T : IMetric
{
MetricFilter finalFilter = filter + ((name, metric) => metric is T);
IDictionary <MetricName, T> retVal = new Dictionary <MetricName, T>();
foreach (KeyValuePair <MetricName, IMetric> kv in _metrics)
{
if (finalFilter(kv.Key, kv.Value))
{
retVal.Add(kv.Key, (T)kv.Value);
}
}
return(retVal.ToImmutableDictionary());
}
private static async Task <IEnumerable <Metric> > GetTransactionMetricsAsync(MetricFilter filter, CloudTable table, IEnumerable <string> metricNames, string invocationId)
{
// Get relevant dimensions
IEnumerable <MetricDimension> metricDimensions = filter.DimensionFilters.Where(df => metricNames.Contains(df.Name));
// Get appropriate entities from table
IEnumerable <DynamicTableEntity> entities = await GetEntitiesAsync(table, GetTransactionQuery(filter, GetOperationNameForQuery(
metricDimensions,
StorageConstants.Dimensions.ApiDimensionName,
StorageConstants.Dimensions.ApiDimensionAggregateValue)), invocationId);
// Construct Metrics and accumulate results
return(metricDimensions
.Select(md => CreateTransactionMetric(filter, md, entities))
.Aggregate <IEnumerable <Metric>, IEnumerable <Metric> >(new Metric[0], (a, b) => a.Union(b)));
}
// Copied from Microsoft.WindowsAzure.Management.Monitoring.ResourceProviders.Storage.Rest.V2011_12.MetricBaseController
private static TableQuery <DynamicTableEntity> GetTransactionQuery(MetricFilter filter, string operationName = null)
{
// storage transaction queries are only supported for 1 hr and 1 min timegrains
if (filter.TimeGrain != StorageConstants.PT1H && filter.TimeGrain != StorageConstants.PT1M)
{
return(null);
}
DateTime partitionKeyStartTime = filter.StartTime;
DateTime partitionKeyEndTime = filter.EndTime;
// start by assuming that we are querying for hr metrics
// since the timestamp field does not represent the actual sample period the only time value represented is the partitionkey
// this is basically truncated to the hr with the min zeroed out.
string startKey = partitionKeyStartTime.ToString("yyyyMMddTHH00");
string endKey = partitionKeyEndTime.ToString("yyyyMMddTHH00");
// if this is actually a minute metric request correct the partition keys and table name format
if (filter.TimeGrain == TimeSpan.FromMinutes(1))
{
startKey = partitionKeyStartTime.ToString("yyyyMMddTHHmm");
endKey = partitionKeyEndTime.ToString("yyyyMMddTHHmm");
}
string rowKey = "user;";
string rowComparison = QueryComparisons.Equal;
// If requesting a particular operation, get only that one (dimension value), otherwise get all
if (operationName == null)
{
rowComparison = QueryComparisons.GreaterThanOrEqual;
}
else
{
rowKey += operationName;
}
var filter1 = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.GreaterThanOrEqual, startKey);
var filter2 = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.LessThanOrEqual, endKey);
var filter3 = TableQuery.GenerateFilterCondition("RowKey", rowComparison, rowKey);
var tableQuery = new TableQuery <DynamicTableEntity>().Where(
TableQuery.CombineFilters(TableQuery.CombineFilters(filter1, TableOperators.And, filter2), TableOperators.And, filter3));
return(tableQuery);
}
private static async Task <IEnumerable <Metric> > GetCapacityMetricsAsync(MetricFilter filter, CloudTable table, IEnumerable <string> metricNames, string invocationId)
{
IEnumerable <DynamicTableEntity> entities = await SasMetricRetriever.GetEntitiesAsync(table, GetCapacityQuery(filter), invocationId);
return(metricNames.Select(n => new Metric()
{
Name = new LocalizableString()
{
Value = n,
LocalizedValue = n
},
StartTime = filter.StartTime,
EndTime = filter.EndTime,
TimeGrain = filter.TimeGrain,
Properties = new Dictionary <string, string>(),
MetricValues = entities.Select(entity => GetMetricValueFromEntity(entity, n)).ToList()
}));
}
public Task <MetricListResponse> GetMetricsAsync(string resourceId, string filterString, IEnumerable <MetricDefinition> definitions, string invocationId)
{
MetricFilter filter = MetricFilterExpressionParser.Parse(filterString);
return(Task.Factory.StartNew(() => new MetricListResponse()
{
RequestId = invocationId,
StatusCode = HttpStatusCode.OK,
MetricCollection = new MetricCollection()
{
Value = definitions == null ? new List <Metric>() : definitions.Select(d => new Metric()
{
Name = d.Name,
Unit = d.Unit,
ResourceId = resourceId,
StartTime = filter.StartTime,
EndTime = filter.EndTime,
TimeGrain = filter.TimeGrain,
MetricValues = new List <MetricValue>(),
Properties = new Dictionary <string, string>()
}).ToList()
}
}));
}
private static async Task<IEnumerable<Metric>> GetCapacityMetricsAsync(MetricFilter filter, CloudTable table, IEnumerable<string> metricNames, string invocationId)
{
IEnumerable<DynamicTableEntity> entities = await SasMetricRetriever.GetEntitiesAsync(
table: table,
query: GetCapacityQuery(filter),
invocationId: invocationId,
maxBatchSize: Util.MaxMetricEntities);
return metricNames.Select(n => new Metric()
{
Name = new LocalizableString()
{
Value = n,
LocalizedValue = n
},
StartTime = filter.StartTime,
EndTime = filter.EndTime,
TimeGrain = filter.TimeGrain,
Properties = new Dictionary<string, string>(),
MetricValues = entities.Select(entity => GetMetricValueFromEntity(entity, n)).ToList()
});
}
/**
* Only report metrics which match the given filter.
*
* @param filter a {@link MetricFilter}
* @return {@code this}
*/
public Builder withFilter(MetricFilter filter)
{
this.filter = filter;
return this;
}
// Copied from Microsoft.WindowsAzure.Management.Monitoring.ResourceProviders.Storage.Rest.V2011_12.MetricBaseController
private static TableQuery<DynamicTableEntity> GetCapacityQuery(MetricFilter filter)
{
// capacity only applies for blob service and only for a timegrain of 1 day
if (filter.TimeGrain != TimeSpan.FromDays(1))
{
return null;
}
// since the timestamp field does not represent the actual sample period the only time vaule represented is the partitionkey
// this is basically truncated to the hr with the min zeroed out.
DateTime partitionKeyStartTime = filter.StartTime;
DateTime partitionKeyEndTime = filter.EndTime;
string startKey = partitionKeyStartTime.ToString("yyyyMMddTHH00");
string endKey = partitionKeyEndTime.ToString("yyyyMMddTHH00");
var filter1 = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.GreaterThanOrEqual, startKey);
var filter2 = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.LessThanOrEqual, endKey);
var filter3 = TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.Equal, "data");
var tableQuery = new TableQuery<DynamicTableEntity>()
.Where(TableQuery.CombineFilters(TableQuery.CombineFilters(filter1, TableOperators.And, filter2), TableOperators.And, filter3));
return tableQuery;
}
private static IEnumerable <Metric> CreateTransactionMetric(MetricFilter filter, MetricDimension metricDimension, IEnumerable <DynamicTableEntity> entities)
{
List <Metric> metrics = new List <Metric>();
// Get supported metric dimension
MetricFilterDimension filterDimension = metricDimension.Dimensions == null
? null
: metricDimension.Dimensions.FirstOrDefault(fd =>
string.Equals(fd.Name, StorageConstants.Dimensions.ApiDimensionName, StringComparison.OrdinalIgnoreCase));
// no dimensions (or no supported dimensions) means only get aggregate values (user;All)
if (filterDimension == null)
{
metrics.Add(new Metric()
{
Name = new LocalizableString()
{
Value = metricDimension.Name,
LocalizedValue = metricDimension.Name
},
StartTime = filter.StartTime,
EndTime = filter.EndTime,
TimeGrain = filter.TimeGrain,
Properties = new Dictionary <string, string>(),
MetricValues = entities
.Where(e => string.Equals(e.RowKey, GetTransactionRowKey(StorageConstants.Dimensions.ApiDimensionAggregateValue), StringComparison.OrdinalIgnoreCase))
.Select(e => GetMetricValueFromEntity(e, metricDimension.Name)).ToList()
});
}
// Dimension specified, get samples with requested dimension value
else
{
// This is the function for filtering based on dimension value (row key)
Func <IGrouping <string, DynamicTableEntity>, bool> groupFilter;
// dimension specified, but no values means get all and group by dimension value (row key)
if (filterDimension.Values == null || !filterDimension.Values.Any())
{
// select all groups, but leave off aggregate. Each group becomes one metric
groupFilter = (entityGroup) =>
!string.Equals(entityGroup.Key, GetTransactionRowKey(StorageConstants.Dimensions.ApiDimensionName), StringComparison.OrdinalIgnoreCase);
}
else
{
// select only groups specified by dimension values
groupFilter = (entityGroup) => filterDimension.Values.Select(GetTransactionRowKey).Contains(entityGroup.Key);
}
// Construct and add the metrics to the collection to return
metrics.AddRange(entities
.GroupBy(e => e.RowKey)
.Where(groupFilter)
.Select(entityGroup => new Metric()
{
Name = new LocalizableString()
{
Value = metricDimension.Name,
LocalizedValue = metricDimension.Name
},
StartTime = filter.StartTime,
EndTime = filter.EndTime,
TimeGrain = filter.TimeGrain,
Properties = new Dictionary <string, string>(),
MetricValues = entityGroup.Select(e => GetMetricValueFromEntity(e, metricDimension.Name)).ToList()
}));
}
// return only values specified
return(metrics);
}
/// <summary> /// Retrieves the metric values from the shoebox /// </summary> /// <param name="filter">The $filter query string</param> /// <param name="location">The MetricLocation object</param> /// <param name="invocationId">The invocation id</param> /// <returns>The MetricValueListResponse</returns> // Note: Does not populate Metric fields unrelated to query (i.e. "display name", resourceUri, and properties) internal abstract Task <MetricListResponse> GetMetricsInternalAsync(MetricFilter filter, MetricLocation location, string invocationId);
private static IEnumerable<Metric> CreateTransactionMetric(MetricFilter filter, MetricDimension metricDimension, IEnumerable<DynamicTableEntity> entities)
{
List<Metric> metrics = new List<Metric>();
// Get supported metric dimension
MetricFilterDimension filterDimension = metricDimension.Dimensions == null
? null
: metricDimension.Dimensions.FirstOrDefault(fd =>
string.Equals(fd.Name, StorageConstants.Dimensions.ApiDimensionName, StringComparison.OrdinalIgnoreCase));
// no dimensions (or no supported dimensions) means only get aggregate values (user;All)
if (filterDimension == null)
{
metrics.Add(new Metric()
{
Name = new LocalizableString()
{
Value = metricDimension.Name,
LocalizedValue = metricDimension.Name
},
StartTime = filter.StartTime,
EndTime = filter.EndTime,
TimeGrain = filter.TimeGrain,
Properties = new Dictionary<string, string>(),
MetricValues = entities
.Where(e => string.Equals(e.RowKey, GetTransactionRowKey(StorageConstants.Dimensions.ApiDimensionAggregateValue), StringComparison.OrdinalIgnoreCase))
.Select(e => GetMetricValueFromEntity(e, metricDimension.Name)).ToList()
});
}
// Dimension specified, get samples with requested dimension value
else
{
// This is the function for filtering based on dimension value (row key)
Func<IGrouping<string, DynamicTableEntity>, bool> groupFilter;
// dimension specified, but no values means get all and group by dimension value (row key)
if (filterDimension.Values == null || !filterDimension.Values.Any())
{
// select all groups, but leave off aggregate. Each group becomes one metric
groupFilter = (entityGroup) =>
!string.Equals(entityGroup.Key, GetTransactionRowKey(StorageConstants.Dimensions.ApiDimensionName), StringComparison.OrdinalIgnoreCase);
}
else
{
// select only groups specified by dimension values
groupFilter = (entityGroup) => filterDimension.Values.Select(GetTransactionRowKey).Contains(entityGroup.Key);
}
// Construct and add the metrics to the collection to return
metrics.AddRange(entities
.GroupBy(e => e.RowKey)
.Where(groupFilter)
.Select(entityGroup => new Metric()
{
Name = new LocalizableString()
{
Value = metricDimension.Name,
LocalizedValue = metricDimension.Name
},
StartTime = filter.StartTime,
EndTime = filter.EndTime,
TimeGrain = filter.TimeGrain,
Properties = new Dictionary<string, string>(),
MetricValues = entityGroup.Select(e => GetMetricValueFromEntity(e, metricDimension.Name)).ToList()
}));
}
// return only values specified
return metrics;
}
private static bool IsMetricDefinitionIncluded(MetricFilter filter, MetricDefinition metricDefinition)
{
return filter == null || filter.DimensionFilters.Any(x => string.Equals(metricDefinition.Name.Value, x.Name, StringComparison.Ordinal));
}
private async Task <Dictionary <string, List <MetricValueBlob> > > FetchMetricValuesFromBlob(BlobInfo blobInfo, MetricFilter filter)
{
if (blobInfo.EndTime < filter.StartTime || blobInfo.StartTime >= filter.EndTime)
{
return(new Dictionary <string, List <MetricValueBlob> >());
}
var blob = new CloudBlockBlob(new Uri(blobInfo.BlobUri));
using (var memoryStream = new MemoryStream())
{
try
{
await blob.DownloadToStreamAsync(memoryStream);
}
catch (StorageException ex)
{
if (ex.RequestInformation.HttpStatusCode == 404)
{
return(new Dictionary <string, List <MetricValueBlob> >());
}
throw;
}
memoryStream.Seek(0, 0);
using (var streamReader = new StreamReader(memoryStream))
{
string content = await streamReader.ReadToEndAsync();
var metricBlob = JsonConvert.DeserializeObject <MetricBlob>(content);
var metricValues = metricBlob.records;
var metricsPerName = new Dictionary <string, List <MetricValueBlob> >();
foreach (var metric in metricValues)
{
if (metric.time < filter.StartTime || metric.time >= filter.EndTime)
{
continue;
}
List <MetricValueBlob> metrics;
if (!metricsPerName.TryGetValue(metric.metricName, out metrics))
{
metrics = new List <MetricValueBlob>();
metricsPerName.Add(metric.metricName, metrics);
}
metrics.Add(metric);
}
return(metricsPerName);
}
}
}
/// <summary>
/// Returns a map of all the counters in the registry and their names which match the given filter.
/// </summary>
/// <param name="filter">the metric filter to match</param>
/// <returns>the matching counters in the registry</returns>
public IDictionary <MetricName, Counter> getCounters(MetricFilter filter)
{
return(getMetrics <Counter>(filter));
}
/// <summary>
/// Returns a map of all the histograms in the registry and their names which match the given filter.
/// </summary>
/// <param name="filter">the metric filter to match</param>
/// <returns>the matching histograms in the registry</returns>
public IDictionary <MetricName, Histogram> getHistograms(MetricFilter filter)
{
return(getMetrics <Histogram>(filter));
}
/// <summary>
/// Only report metrics which match the given <see cref="MetricFilter"/>
/// </summary>
/// <param name="filter">a <see cref="MetricFilter"/></param>
/// <returns><c>this</c></returns>
public Builder withFilter(MetricFilter filter)
{
this.filter = filter;
return(this);
}
private GraphiteReporter(MetricRegistry registry,
GraphiteSender graphite,
Clock clock,
String prefix,
TimeUnit rateUnit,
TimeUnit durationUnit,
MetricFilter filter) : base(registry, "graphite-reporter", filter, rateUnit, durationUnit)
{
this.graphite = graphite;
this.clock = clock;
this.prefix = MetricName.build(prefix);
}
/// <summary>
/// Returns a map of all the gauges in the registry and their names which match the given filter.
/// </summary>
/// <param name="filter">the metric filter to match</param>
/// <returns>the matching gauges in the registry</returns>
public IDictionary <MetricName, Gauge> getGauges(MetricFilter filter)
{
return(getMetrics <Gauge>(filter));
}
internal override async Task <MetricListResponse> GetMetricsInternalAsync(MetricFilter filter, MetricLocation location, string invocationId)
{
return(await ShoeboxClient.GetMetricsInternalAsync(filter, location, invocationId));
}
internal override async Task<MetricListResponse> GetMetricsInternalAsync(MetricFilter filter, MetricLocation location, string invocationId)
{
return await ShoeboxClient.GetMetricsInternalAsync(filter, location, invocationId);
}
// Copied from Microsoft.WindowsAzure.Management.Monitoring.ResourceProviders.Storage.Rest.V2011_12.MetricBaseController
private static TableQuery<DynamicTableEntity> GetTransactionQuery(MetricFilter filter, string operationName = null)
{
// storage transaction queries are only supported for 1 hr and 1 min timegrains
if (filter.TimeGrain != StorageConstants.PT1H && filter.TimeGrain != StorageConstants.PT1M)
{
return null;
}
DateTime partitionKeyStartTime = filter.StartTime;
DateTime partitionKeyEndTime = filter.EndTime;
// start by assuming that we are querying for hr metrics
// since the timestamp field does not represent the actual sample period the only time value represented is the partitionkey
// this is basically truncated to the hr with the min zeroed out.
string startKey = partitionKeyStartTime.ToString("yyyyMMddTHH00");
string endKey = partitionKeyEndTime.ToString("yyyyMMddTHH00");
// if this is actually a minute metric request correct the partition keys and table name format
if (filter.TimeGrain == TimeSpan.FromMinutes(1))
{
startKey = partitionKeyStartTime.ToString("yyyyMMddTHHmm");
endKey = partitionKeyEndTime.ToString("yyyyMMddTHHmm");
}
string rowKey = "user;";
string rowComparison = QueryComparisons.Equal;
// If requesting a particular operation, get only that one (dimension value), otherwise get all
if (operationName == null)
{
rowComparison = QueryComparisons.GreaterThanOrEqual;
}
else
{
rowKey += operationName;
}
var filter1 = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.GreaterThanOrEqual, startKey);
var filter2 = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.LessThanOrEqual, endKey);
var filter3 = TableQuery.GenerateFilterCondition("RowKey", rowComparison, rowKey);
var tableQuery = new TableQuery<DynamicTableEntity>().Where(
TableQuery.CombineFilters(TableQuery.CombineFilters(filter1, TableOperators.And, filter2), TableOperators.And, filter3));
return tableQuery;
}
private ConsoleReporter(MetricRegistry registry,
TextWriter output,
Clock clock,
TimeUnit rateUnit,
TimeUnit durationUnit,
MetricFilter filter) :
base(registry, "console-reporter", filter, rateUnit, durationUnit)
{
this.output = output;
this.clock = clock;
}
private static bool IsMetricDefinitionIncluded(MetricFilter filter, MetricDefinition metricDefinition)
{
return(filter == null || filter.DimensionFilters.Any(x => string.Equals(metricDefinition.Name.Value, x.Name, StringComparison.Ordinal)));
}
/// <summary>
/// Returns a map of all the timers in the registry and their names which match the given filter.
/// </summary>
/// <param name="filter">the metric filter to match</param>
/// <returns>the matching timers in the registry</returns>
public IDictionary <MetricName, Timer> getTimers(MetricFilter filter)
{
return(getMetrics <Timer>(filter));
}
public async Task <MetricListResponse> GetMetricsAsync(string resourceId, string filterString, IEnumerable <MetricDefinition> definitions, string invocationId)
{
MetricFilter filter = MetricFilterExpressionParser.Parse(filterString);
var metricsPerBlob = new Dictionary <string, Task <Dictionary <string, List <MetricValueBlob> > > >(StringComparer.OrdinalIgnoreCase);
// We download all the relevant blobs first and then use the data later, to avoid download the same blob more than once.
foreach (MetricDefinition metricDefinition in definitions)
{
if (!IsMetricDefinitionIncluded(filter, metricDefinition))
{
continue;
}
foreach (MetricAvailability availability in metricDefinition.MetricAvailabilities)
{
if (availability.BlobLocation == null)
{
continue;
}
foreach (BlobInfo blobInfo in availability.BlobLocation.BlobInfo)
{
string blobId = GetBlobEndpoint(blobInfo);
if (!metricsPerBlob.ContainsKey(blobId))
{
metricsPerBlob.Add(blobId, FetchMetricValuesFromBlob(blobInfo, filter));
}
}
}
}
foreach (var task in metricsPerBlob.Values)
{
await task;
}
var result = new MetricListResponse
{
MetricCollection = new MetricCollection
{
Value = new List <Metric>()
}
};
// Populate the metrics result using the data from the blobs.
foreach (MetricDefinition metricDefinition in definitions)
{
if (!IsMetricDefinitionIncluded(filter, metricDefinition))
{
continue;
}
foreach (MetricAvailability availability in metricDefinition.MetricAvailabilities)
{
if (availability.BlobLocation == null)
{
continue;
}
var metricValues = new List <MetricValueBlob>();
foreach (BlobInfo blobInfo in availability.BlobLocation.BlobInfo)
{
string blobId = GetBlobEndpoint(blobInfo);
List <MetricValueBlob> metricsInBlob;
if (metricsPerBlob[blobId].Result.TryGetValue(metricDefinition.Name.Value, out metricsInBlob))
{
metricValues.AddRange(metricsInBlob);
}
}
var metric = new Metric
{
Name = new LocalizableString
{
Value = metricDefinition.Name.Value,
LocalizedValue = metricDefinition.Name.LocalizedValue,
},
StartTime = filter.StartTime,
EndTime = filter.EndTime,
MetricValues = GetAggregatedByTimestamp(metricValues),
TimeGrain = availability.TimeGrain,
};
result.MetricCollection.Value.Add(metric);
}
}
return(result);
}
/// <summary>
/// Retrieves the metric values from the shoebox
/// </summary>
/// <param name="filter">The $filter query string</param>
/// <param name="location">The MetricLocation object</param>
/// <param name="invocationId">The invocation id</param>
/// <returns>The MetricValueListResponse</returns>
// Note: Does not populate Metric fields unrelated to query (i.e. "display name", resourceUri, and properties)
internal MetricListResponse GetMetricsInternal(MetricFilter filter, MetricLocation location, string invocationId)
{
return(GetMetricsInternalAsync(filter, location, invocationId).Result);
}
private async Task<Dictionary<string, List<MetricValueBlob>>> FetchMetricValuesFromBlob(BlobInfo blobInfo, MetricFilter filter)
{
if (blobInfo.EndTime < filter.StartTime || blobInfo.StartTime >= filter.EndTime)
{
return new Dictionary<string, List<MetricValueBlob>>();
}
var blob = new CloudBlockBlob(new Uri(blobInfo.BlobUri));
using (var memoryStream = new MemoryStream())
{
try
{
await blob.DownloadToStreamAsync(memoryStream);
}
catch (StorageException ex)
{
if (ex.RequestInformation.HttpStatusCode == 404)
{
return new Dictionary<string, List<MetricValueBlob>>();
}
throw;
}
memoryStream.Seek(0, 0);
using (var streamReader = new StreamReader(memoryStream))
{
string content = await streamReader.ReadToEndAsync();
var metricBlob = JsonConvert.DeserializeObject<MetricBlob>(content);
var metricValues = metricBlob.records;
var metricsPerName = new Dictionary<string, List<MetricValueBlob>>();
foreach (var metric in metricValues)
{
if (metric.time < filter.StartTime || metric.time >= filter.EndTime)
{
continue;
}
List<MetricValueBlob> metrics;
if (!metricsPerName.TryGetValue(metric.metricName, out metrics))
{
metrics = new List<MetricValueBlob>();
metricsPerName.Add(metric.metricName, metrics);
}
metrics.Add(metric);
}
return metricsPerName;
}
}
}
private static async Task<IEnumerable<Metric>> GetTransactionMetricsAsync(MetricFilter filter, CloudTable table, IEnumerable<string> metricNames, string invocationId)
{
// Get relevant dimensions
IEnumerable<MetricDimension> metricDimensions = filter.DimensionFilters.Where(df => metricNames.Contains(df.Name));
// Get appropriate entities from table
IEnumerable<DynamicTableEntity> entities = await GetEntitiesAsync(
table: table,
query: GetTransactionQuery(
filter: filter,
operationName: GetOperationNameForQuery(
dimensions:metricDimensions,
dimensionName: StorageConstants.Dimensions.ApiDimensionName,
dimensionAggregateValue: StorageConstants.Dimensions.ApiDimensionAggregateValue)),
invocationId: invocationId,
maxBatchSize: Util.MaxMetricEntities);
// Construct Metrics and accumulate results
return metricDimensions
.Select(md => CreateTransactionMetric(filter, md, entities))
.Aggregate<IEnumerable<Metric>,IEnumerable<Metric>>(new Metric[0], (a, b) => a.Union(b));
}
