/// <summary>
/// Retrieves the metric values from the shoebox
/// </summary>
/// <param name="filter">The $filter query string</param>
/// <param name="location">The MetricLocation object</param>
/// <param name="invocationId">The invocation id</param>
/// <returns>The MetricValueListResponse</returns>
// Note: Does not populate Metric fields unrelated to query (i.e. "display name", resourceUri, and properties)
internal static async Task <MetricListResponse> GetMetricsInternalAsync(MetricFilter filter, MetricLocation location, string invocationId)
{
// TODO [davmc]: ShoeboxClient doesn't support dimensions
if (filter.DimensionFilters != null && filter.DimensionFilters.Any(df => df.Dimensions != null))
{
if (TracingAdapter.IsEnabled)
{
TracingAdapter.Information("InvocationId: {0}. ShoeboxClient encountered metrics with dimensions specified. These will be ignored.", invocationId);
}
// Remove dimensions from filter (The MetricFilter class has strict mutation rules used in parsing so the best way to modify it is to create a new one)
filter = new MetricFilter()
{
TimeGrain = filter.TimeGrain,
StartTime = filter.StartTime,
EndTime = filter.EndTime,
DimensionFilters = filter.DimensionFilters.Select(df => new MetricDimension()
{
Name = df.Name
})
};
}
// If metrics are requested by name, get those metrics specifically, unless too many are requested.
// If no names or too many names are provided, get all metrics and filter if necessary.
return(new MetricListResponse()
{
MetricCollection = await(filter.DimensionFilters == null || filter.DimensionFilters.Count() > MaxParallelRequestsByName
? GetMetricsByTimestampAsync(filter, location, invocationId)
: GetMetricsByNameAsync(filter, location, invocationId)).ConfigureAwait(false)
});
}
private bool SupportsRequestedDimensions(MetricDefinition definition, MetricFilter filter)
{
MetricDimension metric = filter.DimensionFilters.FirstOrDefault(df => string.Equals(df.Name, definition.Name.Value, StringComparison.OrdinalIgnoreCase));
var supportedDimensionNames = definition.Dimensions.Select(dim => dim.Name);
var supportedDimensionValues = definition.Dimensions.ToDictionary(dim => dim.Name.Value, dim => dim.Values.Select(v => v.Value));
// No dimensions specified for this metric
if (metric == null || metric.Dimensions == null)
{
return(true);
}
foreach (MetricFilterDimension dimension in metric.Dimensions)
{
// find dimension in definition
Dimension d = definition.Dimensions.FirstOrDefault(dim => string.Equals(dim.Name.Value, dimension.Name));
// Dimension name does't show up in definition
if (d == null)
{
return(false);
}
// Requested dimension has any value that don't show up in the values list for the definiton
if (dimension.Values.Any(value => !d.Values.Select(v => v.Value).Contains(value, StringComparer.OrdinalIgnoreCase)))
{
return(false);
}
}
return(true);
}
private static IEnumerable <CloudTable> GetNdayTables(MetricFilter filter, MetricLocation location)
{
// Get the tables that overlap the timerange and create a table reference for each table
return(location.TableInfo
.Where(info => info.StartTime < filter.EndTime && info.EndTime > filter.StartTime)
.Select(info => new CloudTableClient(new Uri(location.TableEndpoint), new StorageCredentials(info.SasToken)).GetTableReference(info.TableName)));
}
public static string GenerateMetricFilterString(MetricFilter filter)
{
return(string.Format(CultureInfo.InvariantCulture, "{0}timeGrain eq duration'{1}' and startTime eq {2} and endTime eq {3}",
IsNullOrEmpty(filter.DimensionFilters) ? string.Empty : "(" + GenerateMetricDimensionFilterString(filter.DimensionFilters) + ") and ",
XmlConvert.ToString(filter.TimeGrain),
filter.StartTime.ToString("O"),
filter.EndTime.ToString("O")));
}
public static string GenerateMetricFilterString(MetricFilter filter)
{
return(string.Format(CultureInfo.InvariantCulture, "{0}timeGrain eq duration'{1}' and startTime eq {2} and endTime eq {3}",
filter.Names == null || !filter.Names.Any() ? string.Empty : "(" + GenerateMetricDefinitionFilterString(filter.Names) + ") and ",
filter.TimeGrain.To8601String(),
filter.StartTime.ToString("O"),
filter.EndTime.ToString("O")));
}
public static string GenerateMetricFilterString(MetricFilter filter)
{
return string.Format(CultureInfo.InvariantCulture, "{0}timeGrain eq duration'{1}' and startTime eq {2} and endTime eq {3}",
filter.Names == null || !filter.Names.Any() ? string.Empty : "(" + GenerateMetricDefinitionFilterString(filter.Names) + ") and ",
filter.TimeGrain.To8601String(),
filter.StartTime.ToString("O"),
filter.EndTime.ToString("O"));
}
private static string GenerateNamelessMetricFilterString(MetricFilter filter)
{
MetricFilter nameless = new MetricFilter()
{
TimeGrain = filter.TimeGrain,
StartTime = filter.StartTime,
EndTime = filter.EndTime
};
return(ShoeboxHelper.GenerateMetricFilterString(nameless));
}
private string GetFilterStringForDefinitions(MetricFilter filter, IEnumerable <MetricDefinition> definitions)
{
return(ShoeboxHelper.GenerateMetricFilterString(new MetricFilter()
{
TimeGrain = filter.TimeGrain,
StartTime = filter.StartTime,
EndTime = filter.EndTime,
DimensionFilters = filter.DimensionFilters.Where(df =>
definitions.Any(d => string.Equals(df.Name, d.Name.Value, StringComparison.OrdinalIgnoreCase)))
}));
}
/// <summary>
/// Retrieves the metric values from the shoebox
/// </summary>
/// <param name="filter">The $filter query string</param>
/// <param name="location">The MetricLocation object</param>
/// <param name="invocationId">The invocation id</param>
/// <returns>The MetricValueListResponse</returns>
// Note: Does not populate Metric fields unrelated to query (i.e. "display name", resourceUri, and properties)
internal static async Task <MetricListResponse> GetMetricsInternalAsync(MetricFilter filter, MetricLocation location, string invocationId)
{
// If metrics are requested by name, get those metrics specifically, unless too many are requested.
// If no names or too many names are provided, get all metrics and filter if necessary.
return(new MetricListResponse()
{
MetricCollection = await(filter.Names == null || filter.Names.Count() > MaxParallelRequestsByName
? GetMetricsByTimestampAsync(filter, location, invocationId)
: GetMetricsByNameAsync(filter, location, invocationId)).ConfigureAwait(false)
});
}
private static string RemoveNamesFromFilterString(string filterString)
{
MetricFilter filter = MetricFilterExpressionParser.Parse(filterString);
MetricFilter nameless = new MetricFilter()
{
TimeGrain = filter.TimeGrain,
StartTime = filter.StartTime,
EndTime = filter.EndTime
};
return(ShoeboxHelper.GenerateMetricFilterString(nameless));
}
private async Task <Log> FetchLogFromBlob(string blobUri, MetricFilter filter, LocalizableString category)
{
var blob = new CloudBlockBlob(new Uri(blobUri));
using (var memoryStream = new MemoryStream())
{
try
{
await blob.DownloadToStreamAsync(memoryStream);
}
catch (StorageException ex)
{
if (ex.RequestInformation.HttpStatusCode == 404)
{
return(new Log
{
Category = new LocalizableString
{
LocalizedValue = category.LocalizedValue,
Value = category.Value
},
StartTime = filter.StartTime,
EndTime = filter.EndTime,
Value = new List <LogValue>()
});
}
throw;
}
memoryStream.Seek(0, 0);
using (var streamReader = new StreamReader(memoryStream))
{
string content = await streamReader.ReadToEndAsync();
var logBlob = JsonConvert.DeserializeObject <LogBlob>(content);
var logValues = logBlob.records.Where(x => x.Time >= filter.StartTime && x.Time < filter.EndTime).ToList();
return(new Log
{
Category = category,
StartTime = filter.StartTime,
EndTime = filter.EndTime,
Value = logValues
});
}
}
}
/// <summary>
/// Retrieves the metric values from the shoebox
/// </summary>
/// <param name="filter">The $filter query string</param>
/// <param name="location">The MetricLocation object</param>
/// <param name="invocationId">The invocation id</param>
/// <returns>The MetricValueListResponse</returns>
// Note: Does not populate Metric fields unrelated to query (i.e. "display name", resourceUri, and properties)
internal static async Task <MetricListResponse> GetMetricsInternalAsync(MetricFilter filter, MetricLocation location, string invocationId)
{
// TODO [davmc]: ShoeboxClient doesn't support dimensions
if (filter.DimensionFilters != null && filter.DimensionFilters.Any(df => df.Dimensions != null))
{
throw new ArgumentException("Shoebox client does not support dimensions", "filter");
}
// If metrics are requested by name, get those metrics specifically, unless too many are requested.
// If no names or too many names are provided, get all metrics and filter if necessary.
return(new MetricListResponse()
{
MetricCollection = await(filter.DimensionFilters == null || filter.DimensionFilters.Count() > MaxParallelRequestsByName
? GetMetricsByTimestampAsync(filter, location, invocationId)
: GetMetricsByNameAsync(filter, location, invocationId)).ConfigureAwait(false)
});
}
// Generates queries for each metric by name (name-timestamp rowKey format) at collects the results
// Note: Does not populate Metric fields unrelated to query (i.e. "display name", resourceUri, and properties)
private static async Task <MetricCollection> GetMetricsByNameAsync(MetricFilter filter, MetricLocation location, string invocationId)
{
// Create a query for each metric name
Dictionary <string, TableQuery> queries = GenerateMetricNameQueries(filter.DimensionFilters.Select(df => df.Name), location.PartitionKey,
filter.StartTime, filter.EndTime);
// Create a task for each query. Each query will correspond to one metric
IEnumerable <Task <Metric> > queryTasks = queries.Select(async kvp => await GetMetricByNameAsync(kvp.Key, kvp.Value, filter, location, invocationId).ConfigureAwait(false));
// Execute the queries in parallel and collect the results
IList <Metric> metrics = await Task.Factory.ContinueWhenAll(queryTasks.ToArray(), tasks => new List <Metric>(tasks.Select(t => t.Result))).ConfigureAwait(false);
// Wrap metrics in MetricCollectionObject
return(new MetricCollection()
{
Value = metrics
});
}
public async Task <MetricListResponse> GetMetricsAsync(string resourceUri, string filterString, CancellationToken cancellationToken)
{
if (resourceUri == null)
{
throw new ArgumentNullException("resourceUri");
}
// Generate filter strings
MetricFilter filter = MetricFilterExpressionParser.Parse(filterString);
string filterStringNamesOnly = filter.DimensionFilters == null ? null
: ShoeboxHelper.GenerateMetricDefinitionFilterString(filter.DimensionFilters.Select(df => df.Name));
// Get definitions for requested metrics
IList <MetricDefinition> definitions =
(await this.Client.MetricDefinitionOperations.GetMetricDefinitionsAsync(
resourceUri,
filterStringNamesOnly,
cancellationToken).ConfigureAwait(false)).MetricDefinitionCollection.Value;
// Get Metrics with definitions
return(await this.GetMetricsAsync(resourceUri, filterString, definitions, cancellationToken));
}
// Generates queries for all metrics by timestamp (timestamp-name rowKey format) and filters the results to the requested metrics (if any)
// Note: Does not populate Metric fields unrelated to query (i.e. "display name", resourceUri, and properties)
private static async Task <MetricCollection> GetMetricsByTimestampAsync(MetricFilter filter, MetricLocation location, string invocationId)
{
// Find all the tables that fall partially or fully within the timerange
IEnumerable <CloudTable> tables = GetNdayTables(filter, location);
// Generate a query for the partition key and time range
TableQuery query = GenerateMetricTimestampQuery(location.PartitionKey, filter.StartTime, filter.EndTime);
// Get all the entities for the query
IEnumerable <DynamicTableEntity> entities = await GetEntitiesAsync(tables, query, invocationId).ConfigureAwait(false);
// Group entities by (encoded) name
IEnumerable <IGrouping <string, DynamicTableEntity> > groups = entities.GroupBy(entity => entity.RowKey.Substring(entity.RowKey.LastIndexOf('_') + 1));
// if names are specified, filter the results to those metrics only
if (filter.DimensionFilters != null)
{
groups = groups.Where(g => filter.DimensionFilters.Select(df => ShoeboxHelper.TrimAndEscapeKey(df.Name)).Contains(g.Key));
}
// Construct MetricCollection (list of metrics) by taking each group and converting the entities in that group to MetricValue objects
return(new MetricCollection()
{
Value =
groups.Select(g => new Metric()
{
Name = new LocalizableString()
{
Value = FindMetricName(g.Key, filter.DimensionFilters.Select(df => df.Name))
},
StartTime = filter.StartTime,
EndTime = filter.EndTime,
TimeGrain = filter.TimeGrain,
MetricValues = g.Select(ResolveMetricEntity).ToList()
}).ToList()
});
}
private bool SupportsRequestedDimensions(MetricDefinition definition, MetricFilter filter)
{
MetricDimension metric = filter.DimensionFilters.FirstOrDefault(df => string.Equals(df.Name, definition.Name.Value, StringComparison.OrdinalIgnoreCase));
var supportedDimensionNames = definition.Dimensions.Select(dim => dim.Name);
var supportedDimensionValues = definition.Dimensions.ToDictionary(dim => dim.Name.Value, dim => dim.Values.Select(v => v.Value));
// No dimensions specified for this metric
if (metric == null || metric.Dimensions == null)
{
return true;
}
foreach (MetricFilterDimension dimension in metric.Dimensions)
{
// find dimension in definition
Dimension d = definition.Dimensions.FirstOrDefault(dim => string.Equals(dim.Name.Value, dimension.Name));
// Dimension name does't show up in definition
if (d == null)
{
return false;
}
// Requested dimension has any value that don't show up in the values list for the definiton
if (dimension.Values.Any(value => !d.Values.Select(v => v.Value).Contains(value, StringComparer.OrdinalIgnoreCase)))
{
return false;
}
}
return true;
}
/// <summary>
/// Creates a new Filter object from the ($filter) query string
/// </summary>
/// <param name="query">The query string</param>
/// <returns>A filter object representing the query</returns>
public static MetricFilter Parse(string query)
{
MetricFilter filter = new MetricFilter();
string[] clauses = splitOnAndRegex.Split(query);
// Grab all the top-level items (separated by ANDs), which can be either simple clauses or a parenthesized group of 'name' clauses
foreach (string clause in clauses)
{
Match nameGroup = nameGroupRegex.Match(clause);
// If it's in parentheses, it must be a group of 'name' clauses, only 1 is allowed
if (nameGroup.Success && filter.DimensionFilters == null)
{
// The capturing group strips off the parentheses, leaving the clauses separated by ORs (nested parentheses are not allowed)
string[] namesClauses = splitOnORegex.Split(nameGroup.Groups["clauses"].Captures[0].Value);
List<string> names = new List<string>();
// Parse each 'name' clause and collect the values
foreach (string nameClause in namesClauses)
{
Match match = nameClauseRegex.Match(nameClause);
if (!match.Success)
{
throw new FormatException(
"Only conditions of the form 'name.value eq <value>' are allowed inside parentheses");
}
names.Add(match.Groups["value"].Captures[0].Value);
}
// an empty group will cause this assignment to throw InvalidOperationException
filter.DimensionFilters = names.Select(n => new MetricDimension()
{
Name = n
});
}
else if (clause.Trim().StartsWith("("))
{
throw new FormatException("Parentheses Error: only one set of parentheses is allowed; " +
"If present, only (and all) constraints on 'name.value' must appear inside. " +
"No 'and' (and all 'or') operators may appear within parentheses.");
}
else
{
// It's not in a group, so it must be a simple clause
Match match = clauseRegex.Match(clause);
if (!match.Success || match.Groups["name"].Captures.Count <= 0 ||
match.Groups["value"].Captures.Count <= 0)
{
throw new FormatException(
"only conditions of the form '<name> eq <value>' are allowed, where <name> = 'timeGrain', 'startTime', 'endTime', or 'name.value'");
}
// Collect name and value
string name = match.Groups["name"].Captures[0].Value;
string value = match.Groups["value"].Captures[0].Value;
// Case sensitivity is handled in the regex
switch (name)
{
case "timeGrain":
// verify the OData duration value indicator
string prefix = "duration'";
if (value.StartsWith("duration'") && value.EndsWith("'") && value.Length > prefix.Length)
{
// Strip off prefix and end quote
filter.TimeGrain =
XmlConvert.ToTimeSpan(value.Substring(9, value.Length - prefix.Length - 1));
}
else throw new FormatException("Invalid duration value for timeGrain");
break;
case "startTime":
filter.StartTime = DateTime.Parse(value);
break;
case "endTime":
filter.EndTime = DateTime.Parse(value);
break;
case "name.value": // single name (without) parentheses is allowed, but only one
if (filter.DimensionFilters == null)
{
// verify quotes
if (value.StartsWith("'") && value.EndsWith("'") && value.Length >= 2)
{
// strip off quotes and store
filter.DimensionFilters = new List<MetricDimension>()
{
new MetricDimension()
{
Name = value.Substring(1, value.Length - 2)
}
};
}
else
{
throw new FormatException("Invalid string value for name.value");
}
}
else throw new FormatException("Multiple 'name' conditions must be within parentheses");
break;
default:
throw new FormatException(
"Condition name must be one of: 'timeGrain', 'startTime', 'endTime', or 'name.value'");
}
}
}
// Verify no missing values?
return filter;
}
private async Task<Log> FetchLogFromBlob(string blobUri, MetricFilter filter, LocalizableString category)
{
var blob = new CloudBlockBlob(new Uri(blobUri));
using (var memoryStream = new MemoryStream())
{
try
{
await blob.DownloadToStreamAsync(memoryStream);
}
catch (StorageException ex)
{
if (ex.RequestInformation.HttpStatusCode == 404)
{
return new Log
{
Category = new LocalizableString
{
LocalizedValue = category.LocalizedValue,
Value = category.Value
},
StartTime = filter.StartTime,
EndTime = filter.EndTime,
Value = new List<LogValue>()
};
}
throw;
}
memoryStream.Seek(0, 0);
using (var streamReader = new StreamReader(memoryStream))
{
string content = await streamReader.ReadToEndAsync();
var logBlob = JsonConvert.DeserializeObject<LogBlob>(content);
var logValues = logBlob.records.Where(x => x.Time >= filter.StartTime && x.Time < filter.EndTime).ToList();
return new Log
{
Category = category,
StartTime = filter.StartTime,
EndTime = filter.EndTime,
Value = logValues
};
}
}
}
// Alternate method for getting metrics by passing in the definitions
public async Task <MetricListResponse> GetMetricsAsync(
string resourceUri, string filterString, IEnumerable <MetricDefinition> definitions, CancellationToken cancellationToken)
{
if (definitions == null)
{
throw new ArgumentNullException("definitions");
}
if (resourceUri == null)
{
throw new ArgumentNullException("resourceUri");
}
// Remove any '/' characters from the start since these are handled by the hydra (thin) client
// Don't encode Uri segments here since this will mess up the SAS retrievers (they use the resourceUri directly)
resourceUri = resourceUri.TrimStart('/');
MetricListResponse result;
string invocationId = TracingAdapter.NextInvocationId.ToString(CultureInfo.InvariantCulture);
// If no definitions provided, return empty collection
if (!definitions.Any())
{
this.LogStartGetMetrics(invocationId, resourceUri, filterString, definitions);
result = new MetricListResponse()
{
RequestId = Guid.NewGuid().ToString("D"),
StatusCode = HttpStatusCode.OK,
MetricCollection = new MetricCollection()
{
Value = new Metric[0]
}
};
this.LogEndGetMetrics(invocationId, result);
return(result);
}
// Parse MetricFilter
MetricFilter filter = MetricFilterExpressionParser.Parse(filterString);
// Names in filter must match the names in the definitions
if (filter.DimensionFilters != null && filter.DimensionFilters.Any())
{
IEnumerable <string> filterNames = filter.DimensionFilters.Select(df => df.Name);
IEnumerable <string> definitionNames = definitions.Select(d => d.Name.Value);
IEnumerable <string> filterOnly = filterNames.Where(fn => !definitionNames.Contains(fn, StringComparer.InvariantCultureIgnoreCase));
IEnumerable <string> definitionOnly = definitionNames.Where(df => !filterNames.Contains(df, StringComparer.InvariantCultureIgnoreCase));
if (filterOnly.Any() || definitionOnly.Any())
{
throw new ArgumentException("Set of names specified in filter string must match set of names in provided definitions", "filterString");
}
// "Filter out" metrics with unsupported dimensions
definitions = definitions.Where(d => SupportsRequestedDimensions(d, filter));
}
else
{
filter = new MetricFilter()
{
TimeGrain = filter.TimeGrain,
StartTime = filter.StartTime,
EndTime = filter.EndTime,
DimensionFilters = definitions.Select(d => new MetricDimension()
{
Name = d.Name.Value
})
};
}
// Parse out provider name and determine if provider is storage
string providerName = this.GetProviderFromResourceId(resourceUri);
bool isStorageProvider =
string.Equals(providerName, "Microsoft.Storage", StringComparison.OrdinalIgnoreCase) ||
string.Equals(providerName, "Microsoft.ClassicStorage", StringComparison.OrdinalIgnoreCase);
// Create supported MetricRetrievers
IMetricRetriever proxyRetriever = new ProxyMetricRetriever(this);
IMetricRetriever shoeboxRetriever = new ShoeboxMetricRetriever();
IMetricRetriever storageRetriever = new StorageMetricRetriever();
IMetricRetriever blobShoeboxMetricRetriever = new BlobShoeboxMetricRetriever();
IMetricRetriever emptyRetriever = EmptyMetricRetriever.Instance;
// Create the selector function here so it has access to the retrievers, filter, and providerName
Func <MetricDefinition, IMetricRetriever> retrieverSelector = (d) =>
{
if (!d.MetricAvailabilities.Any())
{
return(emptyRetriever);
}
if (isStorageProvider)
{
return(storageRetriever);
}
if (IsBlobSasMetric(d, filter.TimeGrain))
{
return(blobShoeboxMetricRetriever);
}
if (IsTableSasMetric(d, filter.TimeGrain))
{
return(shoeboxRetriever);
}
return(proxyRetriever);
};
// Group definitions by retriever so we can make one request to each retriever
IEnumerable <IGrouping <IMetricRetriever, MetricDefinition> > groups = definitions.GroupBy(retrieverSelector);
// Get Metrics from each retriever (group)
IEnumerable <Task <MetricListResponse> > locationTasks = groups.Select(g =>
g.Key.GetMetricsAsync(resourceUri, GetFilterStringForDefinitions(filter, g), g, invocationId));
// Aggregate metrics from all groups
this.LogStartGetMetrics(invocationId, resourceUri, filterString, definitions);
MetricListResponse[] results = (await Task.Factory.ContinueWhenAll(locationTasks.ToArray(), tasks => tasks.Select(t => t.Result))).ToArray();
IEnumerable <Metric> metrics = results.Aggregate <MetricListResponse, IEnumerable <Metric> >(
new List <Metric>(), (list, response) => list.Union(response.MetricCollection.Value));
this.LogMetricCountFromResponses(invocationId, metrics.Count());
// Fill in values (resourceUri, displayName, unit) from definitions
CompleteShoeboxMetrics(metrics, definitions, resourceUri);
// Add empty objects for metrics that had no values come back, ensuring a metric is returned for each definition
IEnumerable <Metric> emptyMetrics = (await emptyRetriever.GetMetricsAsync(
resourceUri,
filterString,
definitions.Where(d => !metrics.Any(m => string.Equals(m.Name.Value, d.Name.Value, StringComparison.OrdinalIgnoreCase))),
invocationId)).MetricCollection.Value;
// Create response (merge and wrap metrics)
result = new MetricListResponse()
{
RequestId = Guid.NewGuid().ToString("D"),
StatusCode = HttpStatusCode.OK,
MetricCollection = new MetricCollection()
{
Value = metrics.Union(emptyMetrics).ToList()
}
};
this.LogEndGetMetrics(invocationId, result);
return(result);
}
// Alternate method for getting metrics by passing in the definitions
public async Task<MetricListResponse> GetMetricsAsync(
string resourceUri, string filterString, IEnumerable<MetricDefinition> definitions, CancellationToken cancellationToken)
{
if (definitions == null)
{
throw new ArgumentNullException("definitions");
}
if (resourceUri == null)
{
throw new ArgumentNullException("resourceUri");
}
// Remove any '/' characters from the start since these are handled by the hydra (thin) client
// Don't encode Uri segments here since this will mess up the SAS retrievers (they use the resourceUri directly)
resourceUri = resourceUri.TrimStart('/');
MetricListResponse result;
string invocationId = TracingAdapter.NextInvocationId.ToString(CultureInfo.InvariantCulture);
// If no definitions provided, return empty collection
if (!definitions.Any())
{
this.LogStartGetMetrics(invocationId, resourceUri, filterString, definitions);
result = new MetricListResponse()
{
RequestId = Guid.NewGuid().ToString("D"),
StatusCode = HttpStatusCode.OK,
MetricCollection = new MetricCollection()
{
Value = new Metric[0]
}
};
this.LogEndGetMetrics(invocationId, result);
return result;
}
// Parse MetricFilter
MetricFilter filter = MetricFilterExpressionParser.Parse(filterString);
// Names in filter must match the names in the definitions
if (filter.DimensionFilters != null && filter.DimensionFilters.Any())
{
IEnumerable<string> filterNames = filter.DimensionFilters.Select(df => df.Name);
IEnumerable<string> definitionNames = definitions.Select(d => d.Name.Value);
IEnumerable<string> filterOnly = filterNames.Where(fn => !definitionNames.Contains(fn, StringComparer.InvariantCultureIgnoreCase));
IEnumerable<string> definitionOnly = definitionNames.Where(df => !filterNames.Contains(df, StringComparer.InvariantCultureIgnoreCase));
if (filterOnly.Any() || definitionOnly.Any())
{
throw new ArgumentException("Set of names specified in filter string must match set of names in provided definitions", "filterString");
}
// "Filter out" metrics with unsupported dimensions
definitions = definitions.Where(d => SupportsRequestedDimensions(d, filter));
}
else
{
filter = new MetricFilter()
{
TimeGrain = filter.TimeGrain,
StartTime = filter.StartTime,
EndTime = filter.EndTime,
DimensionFilters = definitions.Select(d => new MetricDimension()
{
Name = d.Name.Value
})
};
}
// Parse out provider name and determine if provider is storage
string providerName = this.GetProviderFromResourceId(resourceUri);
bool isStorageProvider =
string.Equals(providerName, "Microsoft.Storage", StringComparison.OrdinalIgnoreCase) ||
string.Equals(providerName, "Microsoft.ClassicStorage", StringComparison.OrdinalIgnoreCase);
// Create supported MetricRetrievers
IMetricRetriever proxyRetriever = new ProxyMetricRetriever(this);
IMetricRetriever shoeboxRetriever = new ShoeboxMetricRetriever();
IMetricRetriever storageRetriever = new StorageMetricRetriever();
IMetricRetriever blobShoeboxMetricRetriever = new BlobShoeboxMetricRetriever();
IMetricRetriever emptyRetriever = EmptyMetricRetriever.Instance;
// Create the selector function here so it has access to the retrievers, filter, and providerName
Func<MetricDefinition, IMetricRetriever> retrieverSelector = (d) =>
{
if (!d.MetricAvailabilities.Any())
{
return emptyRetriever;
}
if (isStorageProvider)
{
return storageRetriever;
}
if (IsBlobSasMetric(d, filter.TimeGrain))
{
return blobShoeboxMetricRetriever;
}
if (IsTableSasMetric(d, filter.TimeGrain))
{
return shoeboxRetriever;
}
return proxyRetriever;
};
// Group definitions by retriever so we can make one request to each retriever
IEnumerable<IGrouping<IMetricRetriever, MetricDefinition>> groups = definitions.GroupBy(retrieverSelector);
// Get Metrics from each retriever (group)
IEnumerable<Task<MetricListResponse>> locationTasks = groups.Select(g =>
g.Key.GetMetricsAsync(resourceUri, GetFilterStringForDefinitions(filter, g), g, invocationId));
// Aggregate metrics from all groups
this.LogStartGetMetrics(invocationId, resourceUri, filterString, definitions);
MetricListResponse[] results = (await Task.Factory.ContinueWhenAll(locationTasks.ToArray(), tasks => tasks.Select(t => t.Result))).ToArray();
IEnumerable<Metric> metrics = results.Aggregate<MetricListResponse, IEnumerable<Metric>>(
new List<Metric>(), (list, response) => list.Union(response.MetricCollection.Value));
this.LogMetricCountFromResponses(invocationId, metrics.Count());
// Fill in values (resourceUri, displayName, unit) from definitions
CompleteShoeboxMetrics(metrics, definitions, resourceUri);
// Add empty objects for metrics that had no values come back, ensuring a metric is returned for each definition
IEnumerable<Metric> emptyMetrics = (await emptyRetriever.GetMetricsAsync(
resourceUri,
filterString,
definitions.Where(d => !metrics.Any(m => string.Equals(m.Name.Value, d.Name.Value, StringComparison.OrdinalIgnoreCase))),
invocationId)).MetricCollection.Value;
// Create response (merge and wrap metrics)
result = new MetricListResponse()
{
RequestId = Guid.NewGuid().ToString("D"),
StatusCode = HttpStatusCode.OK,
MetricCollection = new MetricCollection()
{
Value = metrics.Union(emptyMetrics).ToList()
}
};
this.LogEndGetMetrics(invocationId, result);
return result;
}
// Gets the named metric by calling the provided query on each table that overlaps the given time range
// Note: Does not populate Metric fields unrelated to query (i.e. "display name", resourceUri, and properties)
private static async Task <Metric> GetMetricAsync(string name, TableQuery query, MetricFilter filter, MetricLocation location, string invocationId)
{
// The GetEnititesAsync function provides one task that will call all the queries in parallel
return(new Metric()
{
Name = new LocalizableString()
{
Value = name
},
StartTime = filter.StartTime,
EndTime = filter.EndTime,
TimeGrain = filter.TimeGrain,
MetricValues = (await GetEntitiesAsync(GetNdayTables(filter, location), query, invocationId).ConfigureAwait(false)).Select(ResolveMetricEntity).ToList()
});
}
// Alternate method for getting metrics by passing in the definitions
// TODO [davmc]: Revisit - this method cannot support dimensions
public async Task <MetricListResponse> GetMetricsAsync(
string resourceUri, string filterString, IEnumerable <MetricDefinition> definitions, CancellationToken cancellationToken)
{
MetricListResponse result;
if (definitions == null)
{
throw new ArgumentNullException("definitions");
}
string invocationId = TracingAdapter.NextInvocationId.ToString(CultureInfo.InvariantCulture);
this.LogStartGetMetrics(invocationId, resourceUri, filterString, definitions);
// If no definitions provided, return empty collection
if (!definitions.Any())
{
result = new MetricListResponse()
{
RequestId = Guid.NewGuid().ToString("D"),
StatusCode = HttpStatusCode.OK
};
this.LogEndGetMetrics(invocationId, result);
return(result);
}
// Parse MetricFilter
MetricFilter filter = MetricFilterExpressionParser.Parse(filterString);
// Names not allowed in filter since the names are in the definitions
if (filter.DimensionFilters != null && filter.DimensionFilters.Any())
{
throw new ArgumentException("Cannot specify names (or dimensions) when MetricDefinitions are included", "filterString");
}
// Ensure every definition has at least one availability matching the filter timegrain
if (!definitions.All(d => d.MetricAvailabilities.Any(a => a.TimeGrain == filter.TimeGrain)))
{
throw new ArgumentException("Definition contains no availability for the timeGrain requested", "definitions");
}
// Group definitions by location so we can make one request to each location
Dictionary <MetricAvailability, MetricFilter> groups =
definitions.GroupBy(d => d.MetricAvailabilities.First(a => a.TimeGrain == filter.TimeGrain),
new AvailabilityComparer()).ToDictionary(g => g.Key, g => new MetricFilter()
{
TimeGrain = filter.TimeGrain,
StartTime = filter.StartTime,
EndTime = filter.EndTime,
DimensionFilters = g.Select(d => new MetricDimension()
{
Name = d.Name.Value
})
});
// Get Metrics from each location (group)
IEnumerable <Task <MetricListResponse> > locationTasks = groups.Select(g => g.Key.Location == null
? this.GetMetricsInternalAsync(resourceUri, ShoeboxHelper.GenerateMetricFilterString(g.Value), cancellationToken)
: ShoeboxClient.GetMetricsInternalAsync(g.Value, g.Key.Location, invocationId));
// Aggregate metrics from all groups
MetricListResponse[] results = (await Task.Factory.ContinueWhenAll(locationTasks.ToArray(), tasks => tasks.Select(t => t.Result))).ToArray();
IEnumerable <Metric> metrics = results.Aggregate <MetricListResponse, IEnumerable <Metric> >(
new List <Metric>(), (list, response) => list.Union(response.MetricCollection.Value));
this.LogMetricCountFromResponses(invocationId, metrics.Count());
// Fill in values (resourceUri, displayName, unit) from definitions
CompleteShoeboxMetrics(metrics, definitions, resourceUri);
// Add empty objects for metrics that had no values come back, ensuring a metric is returned for each definition
IEnumerable <Metric> emptyMetrics = definitions
.Where(d => !metrics.Any(m => string.Equals(m.Name.Value, d.Name.Value, StringComparison.OrdinalIgnoreCase)))
.Select(d => new Metric()
{
Name = d.Name,
Unit = d.Unit,
ResourceId = resourceUri,
StartTime = filter.StartTime,
EndTime = filter.EndTime,
TimeGrain = filter.TimeGrain,
MetricValues = new List <MetricValue>(),
Properties = new Dictionary <string, string>()
});
// Create response (merge and wrap metrics)
result = new MetricListResponse()
{
RequestId = Guid.NewGuid().ToString("D"),
StatusCode = HttpStatusCode.OK,
MetricCollection = new MetricCollection()
{
Value = metrics.Union(emptyMetrics).ToList()
}
};
this.LogEndGetMetrics(invocationId, result);
return(result);
}
// Gets the named metric by calling the provided query on each table that overlaps the given time range
// Note: Does not populate Metric fields unrelated to query (i.e. "display name", resourceUri, and properties)
private static async Task <Metric> GetMetricByNameAsync(string name, TableQuery query, MetricFilter filter, MetricLocation location, string invocationId)
{
Metric metric = new Metric()
{
Name = new LocalizableString()
{
Value = name
},
StartTime = filter.StartTime,
EndTime = filter.EndTime,
TimeGrain = filter.TimeGrain,
MetricValues = new List <MetricValue>()
};
var instanceMetrics = new List <MetricValue>();
var globalMetrics = new List <DynamicTableEntity>();
// The GetEnititesAsync function provides one task that will call all the queries in parallel
var entities = await GetEntitiesAsync(GetNdayTables(filter, location), query, invocationId).ConfigureAwait(false);
// Iterate over the instances to do conversion and aggregation when needed
foreach (var entity in entities)
{
// Skip aggregated entities
if (!IsInstanceMetric(entity.RowKey))
{
// We ignore the aggergated metrics if there are instance metrics.
if (instanceMetrics.Count == 0)
{
globalMetrics.Add(entity);
}
continue;
}
MetricValue lastMetricValue = instanceMetrics.LastOrDefault();
if (lastMetricValue == null)
{
instanceMetrics.Add(ResolveMetricEntity(entity));
}
else
{
if (lastMetricValue.Timestamp.Ticks == GetTimestampFromIndexMetricNameTimestamp(entity))
{
Aggregate(lastMetricValue, entity);
}
else
{
instanceMetrics.Add(ResolveMetricEntity(entity));
}
}
}
if (instanceMetrics.Count > 0)
{
foreach (var metricValue in instanceMetrics)
{
metricValue.Average = metricValue.Total / metricValue.Count;
}
metric.MetricValues = instanceMetrics;
}
else
{
metric.MetricValues = globalMetrics.Select(me => ResolveMetricEntity(me)).ToList();
}
return(metric);
}
// Generates queries for all metrics by timestamp (timestamp-name rowKey format) and filters the results to the requested metrics (if any)
// Note: Does not populate Metric fields unrelated to query (i.e. "display name", resourceUri, and properties)
private static async Task <MetricCollection> GetMetricsByTimestampAsync(MetricFilter filter, MetricLocation location, string invocationId)
{
// Find all the tables that fall partially or fully within the timerange
IEnumerable <CloudTable> tables = GetNdayTables(filter, location);
// Generate a query for the partition key and time range
TableQuery query = GenerateMetricTimestampQuery(location.PartitionKey, filter.StartTime, filter.EndTime);
// Get all the entities for the query
IEnumerable <DynamicTableEntity> entities = await GetEntitiesAsync(tables, query, invocationId).ConfigureAwait(false);
ICollection <string> dimensionFilterNames = null;
if (filter.DimensionFilters != null)
{
dimensionFilterNames = new HashSet <string>(filter.DimensionFilters.Select(df => ShoeboxHelper.TrimAndEscapeKey(df.Name)));
}
var metricWraps = new Dictionary <string, MetricWrap>();
var metrics = new List <Metric>();
// Iterate over the instances to do conversion and aggregation when needed.
foreach (var entity in entities)
{
string encodedName = GetMetricNameFromRowKeyByTimestampByMetricName(entity.RowKey);
// When there is filter, skip entities not included in the filter.
if (dimensionFilterNames != null && !dimensionFilterNames.Contains(encodedName, StringComparer.OrdinalIgnoreCase))
{
continue;
}
MetricWrap metricWrap;
if (!metricWraps.TryGetValue(encodedName, out metricWrap))
{
metricWrap = new MetricWrap
{
Metric = new Metric()
{
Name = new LocalizableString()
{
Value = encodedName
},
StartTime = filter.StartTime,
EndTime = filter.EndTime,
TimeGrain = filter.TimeGrain,
MetricValues = new List <MetricValue>()
},
InstanceMetrics = new List <MetricValue>(),
GlobalMetrics = new List <DynamicTableEntity>()
};
metricWraps[encodedName] = metricWrap;
metrics.Add(metricWrap.Metric);
}
// Skip aggregated entities
if (!IsInstanceMetric(entity.RowKey))
{
// We ignore the aggergated metrics if there are instance metrics.
if (metricWrap.InstanceMetrics.Count == 0)
{
metricWrap.GlobalMetrics.Add(entity);
}
continue;
}
MetricValue lastMetricValue = metricWrap.InstanceMetrics.LastOrDefault();
if (lastMetricValue == null)
{
metricWrap.InstanceMetrics.Add(ResolveMetricEntity(entity));
}
else
{
if (lastMetricValue.Timestamp.Ticks == GetTimestampFromIndexTimestampMetricName(entity))
{
Aggregate(lastMetricValue, entity);
}
else
{
metricWrap.InstanceMetrics.Add(ResolveMetricEntity(entity));
}
}
}
foreach (var metricWrap in metricWraps.Values)
{
// Decide whether to return the aggregation of the instance metrics on the fly or the final value in the storage account
// If there are instance metrics, the aggregation on the fly is used.
Metric metric = metricWrap.Metric;
metric.Name.Value = FindMetricName(metric.Name.Value, dimensionFilterNames);
if (metricWrap.InstanceMetrics.Count > 0)
{
foreach (var metricValue in metricWrap.InstanceMetrics)
{
metricValue.Average = metricValue.Total / metricValue.Count;
}
metric.MetricValues = metricWrap.InstanceMetrics;
}
else
{
metric.MetricValues = metricWrap.GlobalMetrics.Select(me => ResolveMetricEntity(me)).ToList();
}
}
return(new MetricCollection()
{
Value = metrics
});
}
/// <summary>
/// Creates a new Filter object from the ($filter) query string
/// </summary>
/// <param name="query">The query string</param>
/// <returns>A filter object representing the query</returns>
public static MetricFilter Parse(string query)
{
MetricFilter filter = new MetricFilter();
string[] clauses = splitOnAndRegex.Split(query);
// Grab all the top-level items (separated by ANDs), which can be either simple clauses or a parenthesized group of 'name' clauses
foreach (string clause in clauses)
{
Match nameGroup = nameGroupRegex.Match(clause);
// If it's in parentheses, it must be a group of 'name' clauses, only 1 is allowed
if (nameGroup.Success && filter.DimensionFilters == null)
{
// The capturing group strips off the parentheses, leaving the clauses separated by ORs (nested parentheses are not allowed)
string[] namesClauses = splitOnORegex.Split(nameGroup.Groups["clauses"].Captures[0].Value);
List <string> names = new List <string>();
// Parse each 'name' clause and collect the values
foreach (string nameClause in namesClauses)
{
Match match = nameClauseRegex.Match(nameClause);
if (!match.Success)
{
throw new FormatException(
"Only conditions of the form 'name.value eq <value>' are allowed inside parentheses");
}
names.Add(match.Groups["value"].Captures[0].Value);
}
// an empty group will cause this assignment to throw InvalidOperationException
filter.DimensionFilters = names.Select(n => new MetricDimension()
{
Name = n
});
}
else if (clause.Trim().StartsWith("("))
{
throw new FormatException("Parentheses Error: only one set of parentheses is allowed; " +
"If present, only (and all) constraints on 'name.value' must appear inside. " +
"No 'and' (and all 'or') operators may appear within parentheses.");
}
else
{
// It's not in a group, so it must be a simple clause
Match match = clauseRegex.Match(clause);
if (!match.Success || match.Groups["name"].Captures.Count <= 0 ||
match.Groups["value"].Captures.Count <= 0)
{
throw new FormatException(
"only conditions of the form '<name> eq <value>' are allowed, where <name> = 'timeGrain', 'startTime', 'endTime', or 'name.value'");
}
// Collect name and value
string name = match.Groups["name"].Captures[0].Value;
string value = match.Groups["value"].Captures[0].Value;
// Case sensitivity is handled in the regex
switch (name)
{
case "timeGrain":
// verify the OData duration value indicator
string prefix = "duration'";
if (value.StartsWith("duration'") && value.EndsWith("'") && value.Length > prefix.Length)
{
// Strip off prefix and end quote
filter.TimeGrain =
XmlConvert.ToTimeSpan(value.Substring(9, value.Length - prefix.Length - 1));
}
else
{
throw new FormatException("Invalid duration value for timeGrain");
}
break;
case "startTime":
filter.StartTime = DateTime.Parse(value);
break;
case "endTime":
filter.EndTime = DateTime.Parse(value);
break;
case "name.value": // single name (without) parentheses is allowed, but only one
if (filter.DimensionFilters == null)
{
// verify quotes
if (value.StartsWith("'") && value.EndsWith("'") && value.Length >= 2)
{
// strip off quotes and store
filter.DimensionFilters = new List <MetricDimension>()
{
new MetricDimension()
{
Name = value.Substring(1, value.Length - 2)
}
};
}
else
{
throw new FormatException("Invalid string value for name.value");
}
}
else
{
throw new FormatException("Multiple 'name' conditions must be within parentheses");
}
break;
default:
throw new FormatException(
"Condition name must be one of: 'timeGrain', 'startTime', 'endTime', or 'name.value'");
}
}
}
// Verify no missing values?
return(filter);
}
public async Task <MetricListResponse> GetMetricsAsync(string resourceUri, string filterString, CancellationToken cancellationToken)
{
// Ensure exactly one '/' at the start
resourceUri = '/' + resourceUri.TrimStart('/');
// Generate filter strings
MetricFilter filter = MetricFilterExpressionParser.Parse(filterString);
string metricFilterString = GenerateNamelessMetricFilterString(filter);
string definitionFilterString = filter.DimensionFilters == null ? null
: ShoeboxHelper.GenerateMetricDefinitionFilterString(filter.DimensionFilters.Select(df => df.Name));
// Get definitions for requested metrics
IList <MetricDefinition> definitions =
(await this.Client.MetricDefinitionOperations.GetMetricDefinitionsAsync(
resourceUri,
definitionFilterString,
cancellationToken).ConfigureAwait(false)).MetricDefinitionCollection.Value;
// Separate passthrough metrics with dimensions specified
IEnumerable <MetricDefinition> passthruDefinitions = definitions.Where(d => !IsShoebox(d, filter.TimeGrain));
IEnumerable <MetricDefinition> shoeboxDefinitions = definitions.Where(d => IsShoebox(d, filter.TimeGrain));
// Get Passthru definitions
List <Metric> passthruMetrics = new List <Metric>();
string invocationId = TracingAdapter.NextInvocationId.ToString(CultureInfo.InvariantCulture);
this.LogStartGetMetrics(invocationId, resourceUri, filterString, passthruDefinitions);
if (passthruDefinitions.Any())
{
// Create new filter for passthru metrics
List <MetricDimension> passthruDimensionFilters = filter.DimensionFilters == null ? new List <MetricDimension>() :
filter.DimensionFilters.Where(df => passthruDefinitions.Any(d => string.Equals(d.Name.Value, df.Name, StringComparison.OrdinalIgnoreCase))).ToList();
foreach (MetricDefinition def in passthruDefinitions
.Where(d => !passthruDimensionFilters.Any(pdf => string.Equals(pdf.Name, d.Name.Value, StringComparison.OrdinalIgnoreCase))))
{
passthruDimensionFilters.Add(new MetricDimension()
{
Name = def.Name.Value
});
}
MetricFilter passthruFilter = new MetricFilter()
{
TimeGrain = filter.TimeGrain,
StartTime = filter.StartTime,
EndTime = filter.EndTime,
DimensionFilters = passthruDimensionFilters
};
// Create passthru filter string
string passthruFilterString = ShoeboxHelper.GenerateMetricFilterString(passthruFilter);
// Get Metrics from passthrough (hydra) client
MetricListResponse passthruResponse = await this.GetMetricsInternalAsync(resourceUri, passthruFilterString, cancellationToken).ConfigureAwait(false);
passthruMetrics = passthruResponse.MetricCollection.Value.ToList();
this.LogMetricCountFromResponses(invocationId, passthruMetrics.Count());
// Fill in values (resourceUri, displayName, unit) from definitions
CompleteShoeboxMetrics(passthruMetrics, passthruDefinitions, resourceUri);
// Add empty objects for metrics that had no values come back, ensuring a metric is returned for each definition
IEnumerable <Metric> emptyMetrics = passthruDefinitions
.Where(d => !passthruMetrics.Any(m => string.Equals(m.Name.Value, d.Name.Value, StringComparison.OrdinalIgnoreCase)))
.Select(d => new Metric()
{
Name = d.Name,
Unit = d.Unit,
ResourceId = resourceUri,
StartTime = filter.StartTime,
EndTime = filter.EndTime,
TimeGrain = filter.TimeGrain,
MetricValues = new List <MetricValue>(),
Properties = new Dictionary <string, string>()
});
passthruMetrics.AddRange(emptyMetrics);
}
// Get Metrics by definitions
MetricListResponse shoeboxResponse = await this.GetMetricsAsync(resourceUri, metricFilterString, shoeboxDefinitions, cancellationToken).ConfigureAwait(false);
// Create response (merge and wrap metrics)
MetricListResponse result = new MetricListResponse()
{
RequestId = Guid.NewGuid().ToString("D"),
StatusCode = HttpStatusCode.OK,
MetricCollection = new MetricCollection()
{
Value = passthruMetrics.Union(shoeboxResponse.MetricCollection.Value).ToList()
}
};
this.LogEndGetMetrics(invocationId, result);
return(result);
}
/// <summary>
/// Retrieves the metric values from the shoebox
/// </summary>
/// <param name="filter">The $filter query string</param>
/// <param name="location">The MetricLocation object</param>
/// <param name="invocationId">The invocation id</param>
/// <returns>The MetricValueListResponse</returns>
// Note: Does not populate Metric fields unrelated to query (i.e. "display name", resourceUri, and properties)
internal static MetricListResponse GetMetricsInternal(MetricFilter filter, MetricLocation location, string invocationId)
{
return(GetMetricsInternalAsync(filter, location, invocationId).Result);
}
private string GetFilterStringForDefinitions(MetricFilter filter, IEnumerable<MetricDefinition> definitions)
{
return ShoeboxHelper.GenerateMetricFilterString(new MetricFilter()
{
TimeGrain = filter.TimeGrain,
StartTime = filter.StartTime,
EndTime = filter.EndTime,
DimensionFilters = filter.DimensionFilters.Where(df =>
definitions.Any(d => string.Equals(df.Name, d.Name.Value, StringComparison.OrdinalIgnoreCase)))
});
}
/// <summary>
/// Get logs.
/// </summary>
/// <param name="resourceUri">The resourceUri</param>
/// <param name="filterString">The filter string</param>
/// <param name="definitions">The log definitions</param>
/// <param name="cancellationToken">The cancellation token</param>
/// <returns></returns>
public async Task <LogListResponse> GetLogsAsync(
string resourceUri,
string filterString,
IEnumerable <LogDefinition> definitions,
CancellationToken cancellationToken)
{
if (definitions == null)
{
throw new ArgumentNullException("definitions");
}
if (resourceUri == null)
{
throw new ArgumentNullException("resourceUri");
}
// Ensure exactly one '/' at the start
resourceUri = '/' + resourceUri.TrimStart('/');
LogListResponse result;
string invocationId = TracingAdapter.NextInvocationId.ToString(CultureInfo.InvariantCulture);
// If no definitions provided, return empty collection
if (!definitions.Any())
{
this.LogStartGetLogs(invocationId, resourceUri, filterString, definitions);
result = new LogListResponse()
{
RequestId = Guid.NewGuid().ToString("D"),
StatusCode = HttpStatusCode.OK,
LogCollection = new LogCollection()
{
Value = new Log[0]
}
};
this.LogEndGetLogs(invocationId, result);
return(result);
}
// Parse LogFilter. Reusing the metric filter.
// We might consider extracting the parsing functionality to a class with a less specific name
MetricFilter filter = MetricFilterExpressionParser.Parse(filterString, false);
if (filter.StartTime == default(DateTime))
{
throw new InvalidOperationException("startTime is required");
}
if (filter.EndTime == default(DateTime))
{
throw new InvalidOperationException("endTime is required");
}
this.LogStartGetLogs(invocationId, resourceUri, filterString, definitions);
var logsPerBlob = new Dictionary <string, Task <Log> >(StringComparer.OrdinalIgnoreCase);
// We download all the relevant blobs first and then use the data later, to avoid download the same blob more than once.
foreach (LogDefinition logDefinition in definitions)
{
foreach (BlobInfo blobInfo in logDefinition.BlobLocation.BlobInfo)
{
if (blobInfo.EndTime < filter.StartTime || blobInfo.StartTime >= filter.EndTime)
{
continue;
}
string blobId = GetBlobEndpoint(blobInfo);
if (!logsPerBlob.ContainsKey(blobId))
{
logsPerBlob.Add(blobId, FetchLogFromBlob(blobInfo.BlobUri, filter, logDefinition.Category));
}
}
}
foreach (var task in logsPerBlob.Values)
{
await task;
}
var logsPerCategory = new Dictionary <string, Log>();
foreach (var task in logsPerBlob.Values)
{
Log log = task.Result;
Log existingLog;
if (logsPerCategory.TryGetValue(log.Category.Value, out existingLog))
{
((List <LogValue>)existingLog.Value).AddRange(log.Value);
existingLog.StartTime = this.Min(log.StartTime, existingLog.StartTime);
existingLog.EndTime = this.Max(log.StartTime, existingLog.StartTime);
}
else
{
logsPerCategory.Add(log.Category.Value, log);
}
}
result = new LogListResponse
{
RequestId = Guid.NewGuid().ToString("D"),
StatusCode = HttpStatusCode.OK,
LogCollection = new LogCollection
{
Value = logsPerCategory.Values.ToList()
}
};
this.LogEndGetLogs(invocationId, result);
return(result);
}