private IWithMetricsQueryExecute CreateMetricsQuery(AggregationType metricAggregation, TimeSpan metricsInterval, string metricFilter, string metricDimension,
int?metricLimit, IMetricDefinition metricDefinition, DateTime recordDateTime)
{
var historyStartingFromInHours = _azureMonitorIntegrationConfiguration.Value.History.StartingFromInHours;
var metricQuery = metricDefinition.DefineQuery()
.StartingFrom(recordDateTime.AddHours(-historyStartingFromInHours))
.EndsBefore(recordDateTime)
.WithAggregation(metricAggregation.ToString())
.WithInterval(metricsInterval);
var queryLimit = metricLimit ?? Defaults.MetricDefaults.Limit;
if (string.IsNullOrWhiteSpace(metricFilter) == false)
{
var filter = metricFilter.Replace("/", "%2F");
metricQuery.WithOdataFilter(filter);
metricQuery.SelectTop(queryLimit);
}
if (string.IsNullOrWhiteSpace(metricDimension) == false)
{
metricQuery.WithOdataFilter($"{metricDimension} eq '*'");
metricQuery.SelectTop(queryLimit);
}
return(metricQuery);
}
private IWithMetricsQueryExecute CreateMetricsQuery(AggregationType metricAggregation, TimeSpan metricsInterval, string metricFilter,
IMetricDefinition metricDefinition, DateTime recordDateTime)
{
var metricQuery = metricDefinition.DefineQuery()
.StartingFrom(recordDateTime.AddDays(-5))
.EndsBefore(recordDateTime)
.WithAggregation(metricAggregation.ToString())
.WithInterval(metricsInterval);
if (string.IsNullOrWhiteSpace(metricFilter) == false)
{
metricQuery.WithOdataFilter(metricFilter);
}
return(metricQuery);
}
private void Dump(IMetricDefinition metricDefinition)
{
var metricCollection = metricDefinition.DefineQuery()
.StartingFrom(DateTime.Now.AddMinutes(-20).ToUniversalTime()).EndsBefore(DateTime.Now.ToUniversalTime())
//.WithAggregation("Average")
//.WithInterval(TimeSpan.FromSeconds(5))
//.WithOdataFilter("name.value eq 'CpuPercentage'")
.Execute();
Console.WriteLine("Metrics for '" + _argsOption.ResourceId + "':");
Console.WriteLine("Namespacse: " + metricCollection.Namespace);
Console.WriteLine("Query time: " + metricCollection.Timespan);
Console.WriteLine("Time Grain: " + metricCollection.Interval);
Console.WriteLine("Cost: " + metricCollection.Cost);
foreach (var metric in metricCollection.Metrics)
{
Console.WriteLine("\tMetric: " + metric.Name.LocalizedValue);
Console.WriteLine("\tType: " + metric.Type);
Console.WriteLine("\tUnit: " + metric.Unit);
Console.WriteLine("\tTime Series: ");
foreach (var timeElement in metric.Timeseries)
{
Console.WriteLine("\t\tMetadata: ");
foreach (var metadata in timeElement.Metadatavalues)
{
Console.WriteLine("\t\t\t" + metadata.Name.LocalizedValue + ": " + metadata.Value);
}
Console.WriteLine("\t\tData: ");
foreach (var data in timeElement.Data)
{
Console.WriteLine("\t\t\t" + data.TimeStamp
+ " : (Min) " + data.Minimum
+ " : (Max) " + data.Maximum
+ " : (Avg) " + data.Average
+ " : (Total) " + data.Total
+ " : (Count) " + data.Count);
}
}
}
}
async static Task <int> ProcessMetricAsync(IAzure subApi, ISubscription subscription, IResourceGroup group, IGenericResource res, IMetricDefinition metric)
{
int m = 0;
DateTime? last = null;
MetricLastDate record = null;
lock (_ctx)
{
record = _ctx.MetricLastDates.Where(d => d.Key == metric.Id).SingleOrDefault();
}
last = record?.LastDate;
try
{
await _sem.WaitAsync();
var from = last?.AddMinutes(1.0) ?? DateTime.UtcNow.RoundMinutes().AddHours(-1.0);
var to = DateTime.UtcNow.RoundMinutes().AddMinutes(10.0);
var maxDataToGet = TimeSpan.FromHours(2.0);
if (to - from > maxDataToGet)
{
to = from + maxDataToGet;
}
if (metric.Inner.IsDimensionRequired == true)
{
var dims = metric.Inner.Dimensions;
// can ignore..
return(0);
}
var data = await metric.DefineQuery()
.StartingFrom(from)
.EndsBefore(to) // as much as possible
.WithAggregation("Average,Minimum,Maximum,Count,Total")
.WithResultType(ResultType.Data)
.WithInterval(TimeSpan.FromMinutes(1))
.ExecuteAsync();
if (data.Metrics.Count != 1 || data.Metrics[0].Timeseries.Count != 1)
{
return(0);
}
/*
* Console.WriteLine($"query from {from} to {to}: {data.Metrics[0].Timeseries[0].Data.Count()} results");
* Console.WriteLine($" min: {data.Metrics[0].Timeseries[0].Data.Min(d => d.Minimum)}");
* Console.WriteLine($" max: {data.Metrics[0].Timeseries[0].Data.Max(d => d.Maximum)}");
* Console.WriteLine($" avg: {data.Metrics[0].Timeseries[0].Data.Average(d => d.Average)}");
*/
var dubiousCutoffUtc = DateTime.UtcNow.AddMinutes(-30.0);
var pts = data.Metrics[0].Timeseries[0].Data
// from future to past
.OrderByDescending(d => d.TimeStamp)
// skip anything with no data yet
.SkipWhile(d =>
d.TimeStamp > dubiousCutoffUtc &&
(d.Average ?? 0) == 0 &&
(d.Maximum ?? 0) == 0 &&
(d.Minimum ?? 0) == 0 &&
(d.Total ?? 0) == 0 &&
(
(d.Count ?? 0) == 0 || d.TimeStamp > dubiousCutoffUtc
)
)
// and the first one with data (probably the current slice)
.Skip(1);
/*
* if (pts.Any())
* Console.WriteLine($"actual data points: {pts.Count()}, from {pts.Min(p => p.TimeStamp)} to {pts.Max(p => p.TimeStamp)}");
* else
* Console.WriteLine($"all data points dismissed");
*/
if (pts.Any())
{
lock (_logger)
{
foreach (var pt in pts.Reverse())
{
var entry = new Dictionary <string, object>(13)
{
{ "@timestamp", pt.TimeStamp.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ss.FFFFFFFZ") },
{ "lib", "Saffron" },
{ "type", "cloud" },
{ "cloud", "Azure" },
{ "sub", subscription.DisplayName },
{ "group", group.Name },
{ "resource", new Dictionary <string, object>(3)
{
{ "name", res.Name },
{ "type", res.Type },
{ "tags", res.Tags }
} },
{ "metric", new Dictionary <string, object>(2)
{
{ "name", metric.Name.Value },
{ "unit", metric.Unit }
} },
};
if (pt.Average != null)
{
entry["avg"] = pt.Average;
}
if (pt.Count != null)
{
entry["num"] = pt.Count;
}
if (pt.Maximum != null)
{
entry["max"] = pt.Maximum;
}
if (pt.Minimum != null)
{
entry["min"] = pt.Minimum;
}
if (pt.Total != null)
{
entry["sum"] = pt.Total;
}
_logger.LogAsJson(entry);
m++;
}
}
lock (_ctx)
{
if (record == null)
{
record = new MetricLastDate {
Key = metric.Id
};
_ctx.MetricLastDates.Add(record);
}
record.LastDate = pts.First().TimeStamp;
_ctx.SaveChanges();
numStats++;
long ageMs = (long)(DateTime.UtcNow - record.LastDate).TotalMilliseconds;
totalAgeMs += ageMs;
if (ageMs > oldestAgeMs)
{
oldestAgeMs = ageMs;
}
}
}
}
catch (ErrorResponseException ex)
{
// ignored
//Console.WriteLine($"Error reading {metric.Name.LocalizedValue} from {res.Name} ({res.Type}): {ex.Message} - {ex.Body.Code} {ex.Body.Message}");
}
catch (HttpRequestException ex)
{
Console.WriteLine($"HTTP error reading {metric.Name.LocalizedValue} from {res.Name} ({res.Type}): {ex.Message}");
}
catch (Exception ex)
{
Console.WriteLine($"Error reading {metric.Name.LocalizedValue} from {res.Name} ({res.Type}): {ex.Message}");
}
finally
{
_sem.Release();
}
return(m);
}
