/// <summary>
/// Writes a set of statistics to storage.
/// </summary>
/// <param name="statsCounters">Statistics to write</param>
/// <returns></returns>
public Task ReportStats(List <ICounter> statsCounters)
{
var bulkPromises = new List <Task>();
var data = new List <StatsTableData>();
foreach (ICounter count in statsCounters.Where(cs => cs.Storage == CounterStorage.LogAndTable).OrderBy(cs => cs.Name))
{
if (data.Count >= AzureTableDefaultPolicies.MAX_BULK_UPDATE_ROWS)
{
// Write intermediate batch
bulkPromises.Add(tableManager.BulkInsertTableEntries(data));
data.Clear();
}
StatsTableData statsTableEntry = PopulateStatsTableDataEntry(count);
if (statsTableEntry == null)
{
continue; // Skip blank entries
}
if (logger.IsEnabled(LogLevel.Trace))
{
logger.Trace("Preparing to bulk insert {1} stats table entry: {0}", statsTableEntry, isSilo ? "silo" : "");
}
data.Add(statsTableEntry);
}
if (data.Count > 0)
{
// Write final batch
bulkPromises.Add(tableManager.BulkInsertTableEntries(data));
}
return(Task.WhenAll(bulkPromises));
}
private StatsTableData PopulateStatsTableDataEntry(ICounter statsCounter)
{
string statValue = statsCounter.IsValueDelta ? statsCounter.GetDeltaString() : statsCounter.GetValueString();
if ("0".Equals(statValue))
{
// Skip writing empty records
return(null);
}
counter++;
var entry = new StatsTableData {
StatValue = statValue
};
// We store the statistics grouped by an hour in the same partition and sorted by reverse ticks within the partition.
// Since by default Azure table stores Entities in ascending order based on the Row Key - if we just used the current ticks
// it would return the oldest entry first.
// If we store the rows in the reverse order (some max value - current ticks), it will return the latest most recent entry first.
// More details here:
// http://gauravmantri.com/2012/02/17/effective-way-of-fetching-diagnostics-data-from-windows-azure-diagnostics-table-hint-use-partitionkey/
// https://alexandrebrisebois.wordpress.com/2014/06/16/using-time-based-partition-keys-in-azure-table-storage/
// https://stackoverflow.com/questions/1004698/how-to-truncate-milliseconds-off-of-a-net-datetime
// Our format:
// PartitionKey: DeploymentId$ReverseTimestampToTheNearestHour
// RowKey: ReverseTimestampToTheNearestSecond$Name$counter
var now = DateTime.UtcNow;
// number of ticks remaining until the year 9683
var ticks = DateTime.MaxValue.Ticks - now.Ticks;
// partition the table according to the deployment id and hour
entry.PartitionKey = string.Join("$", deploymentId, string.Format("{0:d19}", ticks - ticks % TimeSpan.TicksPerHour));
var counterStr = String.Format("{0:000000}", counter);
// order the rows latest-first in the table
entry.RowKey = string.Join("$", string.Format("{0:d19}", ticks), name, counterStr);
entry.DeploymentId = deploymentId;
entry.Time = now.ToString(DATE_TIME_FORMAT, CultureInfo.InvariantCulture);;
entry.Address = address;
entry.Name = name;
entry.HostName = myHostName;
entry.Statistic = statsCounter.Name;
entry.IsDelta = statsCounter.IsValueDelta;
return(entry);
}
private StatsTableData PopulateStatsTableDataEntry(ICounter statsCounter)
{
string statValue = statsCounter.IsValueDelta ? statsCounter.GetDeltaString() : statsCounter.GetValueString();
var entry = new StatsTableData {
StatValue = statValue
};
if ("0".Equals(entry.StatValue))
{
// Skip writing empty records
return(null);
}
counter++;
var time = DateTime.UtcNow;
var timeStr = time.ToString(DATE_FORMAT, CultureInfo.InvariantCulture);
entry.PartitionKey = deploymentId + ":" + timeStr;
var counterStr = String.Format("{0:000000}", counter);
if (isSilo)
{
entry.RowKey = name + ":" + counterStr;
}
else
{
entry.RowKey = name + ":" + clientEpoch + ":" + counterStr;
}
entry.DeploymentId = deploymentId;
entry.Time = time;
entry.Address = address;
entry.Name = name;
entry.HostName = myHostName;
entry.Statistic = statsCounter.Name;
entry.IsDelta = statsCounter.IsValueDelta;
return(entry);
}
private StatsTableData PopulateStatsTableDataEntry(ICounter statsCounter)
{
string statValue = statsCounter.IsValueDelta ? statsCounter.GetDeltaString() : statsCounter.GetValueString();
if ("0".Equals(statValue))
{
// Skip writing empty records
return null;
}
counter++;
var entry = new StatsTableData { StatValue = statValue };
// We store the statistics grouped by an hour in the same partition and sorted by reverse ticks within the partition.
// Since by default Azure table stores Entities in ascending order based on the Row Key - if we just used the current ticks
// it would return the oldest entry first.
// If we store the rows in the reverse order (some max value - current ticks), it will return the latest most recent entry first.
// More details here:
// http://gauravmantri.com/2012/02/17/effective-way-of-fetching-diagnostics-data-from-windows-azure-diagnostics-table-hint-use-partitionkey/
// https://alexandrebrisebois.wordpress.com/2014/06/16/using-time-based-partition-keys-in-azure-table-storage/
// https://stackoverflow.com/questions/1004698/how-to-truncate-milliseconds-off-of-a-net-datetime
// Our format:
// PartitionKey: DeploymentId$ReverseTimestampToTheNearestHour
// RowKey: ReverseTimestampToTheNearestSecond$Name$counter
var now = DateTime.UtcNow;
// number of ticks remaining until the year 9683
var ticks = DateTime.MaxValue.Ticks - now.Ticks;
// partition the table according to the deployment id and hour
entry.PartitionKey = string.Join("$", deploymentId, string.Format("{0:d19}", ticks - ticks % TimeSpan.TicksPerHour));
var counterStr = String.Format("{0:000000}", counter);
// order the rows latest-first in the table
entry.RowKey = string.Join("$", string.Format("{0:d19}", ticks), name, counterStr);
entry.DeploymentId = deploymentId;
entry.Time = now.ToString(DATE_TIME_FORMAT, CultureInfo.InvariantCulture); ;
entry.Address = address;
entry.Name = name;
entry.HostName = myHostName;
entry.Statistic = statsCounter.Name;
entry.IsDelta = statsCounter.IsValueDelta;
return entry;
}