static Tuple <EntryPartition, EntryPartition> GetRangeForLowerReversedDateBoundBinarySearch(
IAzureDiagnosticLogsTable wadTable,
long dateTicks,
EntryPartition searchRangeBegin,
CancellationToken cancellationToken)
{
EntryPartition datePartition = new EntryPartition(new EntryTimestamp(dateTicks));
EntryPartition lastStepPartition = datePartition;
for (int step = 1; ; step *= 2)
{
EntryPartition p = EntryPartition.Max(searchRangeBegin, datePartition.Advance(step));
if (EntryPartition.Compare(p, lastStepPartition) == 0)
{
return(null);
}
var tmp = wadTable.GetFirstEntryOlderThan(p.ToString());
if (tmp != null && tmp.EventTickCount < dateTicks)
{
return(new Tuple <EntryPartition, EntryPartition>(
new EntryPartition(tmp.PartitionKey),
lastStepPartition.Advance()));
}
lastStepPartition = p;
cancellationToken.ThrowIfCancellationRequested();
}
}
public static EntryPartition Max(EntryPartition p1, EntryPartition p2)
{
return(new EntryPartition()
{
partitionKeyTicks = Math.Max(p1.Ticks, p2.Ticks)
});
}
public static IEnumerable <IndexedAzureDiagnosticLogEntry> LoadEntriesRange(
IAzureDiagnosticLogsTable wadTable,
EntryPartition beginPartition,
EntryPartition endPartition,
int?entriesLimit,
CancellationToken cancellationToken)
{
string currentPartitionKey = null;
var currentPartitionEntries = new List <AzureDiagnosticLogEntry>();
Comparison <AzureDiagnosticLogEntry> compareEntries = (e1, e2) => Math.Sign(e1.EventTickCount - e2.EventTickCount);
foreach (var entry in wadTable.GetEntriesInRange(beginPartition.ToString(), endPartition.ToString(), entriesLimit))
{
cancellationToken.ThrowIfCancellationRequested();
if (entry.PartitionKey != currentPartitionKey)
{
currentPartitionEntries.Sort(compareEntries);
for (var i = 0; i < currentPartitionEntries.Count; ++i)
{
yield return(new IndexedAzureDiagnosticLogEntry(currentPartitionEntries[i], i));
}
currentPartitionEntries.Clear();
currentPartitionKey = entry.PartitionKey;
}
currentPartitionEntries.Add(entry);
}
currentPartitionEntries.Sort(compareEntries);
for (var i = 0; i < currentPartitionEntries.Count; ++i)
{
yield return(new IndexedAzureDiagnosticLogEntry(currentPartitionEntries[i], i));
}
}
static Tuple <EntryPartition, EntryPartition> GetRangeForLastMessageBinarySearch(IAzureDiagnosticLogsTable wadTable, DateTime utcNow, CancellationToken cancellation)
{
EntryPartition initialPartition = new EntryTimestamp(utcNow).Partition;
bool thereAreEntriesOlderThanNow = wadTable.GetFirstEntryOlderThan(initialPartition.ToString()) != null;
bool searchingForward = thereAreEntriesOlderThanNow;
EntryPartition currentPartition = initialPartition;
for (long step = searchingForward ? 1 : -1; ; step *= 2)
{
cancellation.ThrowIfCancellationRequested();
EntryPartition t = initialPartition.Advance(step);
if (EntryPartition.Compare(currentPartition, t) == 0)
{
return(null);
}
var tmp = wadTable.GetFirstEntryOlderThan(t.ToString());
if (searchingForward)
{
if (tmp == null)
{
return(new Tuple <EntryPartition, EntryPartition>(currentPartition.Advance(), t));
}
}
else
{
if (tmp != null)
{
return(new Tuple <EntryPartition, EntryPartition>(new EntryPartition(tmp.EventTickCount), currentPartition.Advance()));
}
}
currentPartition = t;
}
}
public static EntryPartition?FindLastMessagePartitionKey(IAzureDiagnosticLogsTable wadTable, DateTime utcNow, CancellationToken cancellation)
{
if (utcNow.Kind != DateTimeKind.Utc)
{
throw new ArgumentException("time must be of UTC kind", "utcNow");
}
var rangeForBinarySearch = GetRangeForLastMessageBinarySearch(wadTable, utcNow, cancellation);
if (rangeForBinarySearch == null)
{
return(null);
}
EntryPartition begin = rangeForBinarySearch.Item1;
EntryPartition end = rangeForBinarySearch.Item2;
int searchRangeDuration = EntryPartition.Distance(begin, end);
int pos = ListUtils.BinarySearch(
new ListUtils.VirtualList <int>(searchRangeDuration, i => i), 0, searchRangeDuration,
i => {
cancellation.ThrowIfCancellationRequested();
return(wadTable.GetFirstEntryOlderThan(begin.Advance(i).ToString()) != null);
});
if (pos == searchRangeDuration)
{
return(null);
}
return(begin.Advance(pos));
}
static IndexedAzureDiagnosticLogEntry FindLowerReversedDateBoundInPartition(IAzureDiagnosticLogsTable wadTable,
EntryPartition partition, long dateTicks, CancellationToken cancellation)
{
var ret = LoadEntriesRange(wadTable, partition, partition.Advance(), null, cancellation)
.LastOrDefault(e => e.Entry.EventTickCount <= dateTicks);
return(ret);
}
static IndexedAzureDiagnosticLogEntry?FindLowerReversedDateBound(
IAzureDiagnosticLogsTable wadTable,
DateTime date,
EntryPartition searchRangeBegin,
CancellationToken cancellationToken)
{
EntryTimestamp dateTimestamp = new EntryTimestamp(date);
var ret = FindLowerReversedDateBoundInPartition(wadTable, dateTimestamp.Partition, dateTimestamp.Ticks, cancellationToken);
if (ret.Entry != null)
{
return(ret);
}
var rangeForBinarySearch = GetRangeForLowerReversedDateBoundBinarySearch(wadTable, dateTimestamp.Ticks,
searchRangeBegin, cancellationToken);
if (rangeForBinarySearch == null)
{
return(null);
}
EntryPartition begin = rangeForBinarySearch.Item1;
EntryPartition end = rangeForBinarySearch.Item2;
int searchRangeDuration = (int)EntryPartition.Distance(begin, end);
int pos = ListUtils.BinarySearch(
new ListUtils.VirtualList <int>(searchRangeDuration, i => i), 0, searchRangeDuration,
timeBeingTested =>
{
cancellationToken.ThrowIfCancellationRequested();
var firstEntryFollowingTimeBeingTested = wadTable.GetFirstEntryOlderThan(begin.Advance(timeBeingTested).ToString());
bool lessThanBoundaryBeingSearched;
if (firstEntryFollowingTimeBeingTested == null)
{
lessThanBoundaryBeingSearched = false;
}
else
{
lessThanBoundaryBeingSearched = firstEntryFollowingTimeBeingTested.EventTickCount < dateTimestamp.Ticks;
}
return(lessThanBoundaryBeingSearched);
});
if (pos == searchRangeDuration)
{
return(null);
}
ret = FindLowerReversedDateBoundInPartition(wadTable, begin.Advance(pos), dateTimestamp.Ticks, cancellationToken);
if (ret.Entry != null)
{
return(ret);
}
return(null);
}
static IndexedAzureDiagnosticLogEntry?FindLowerDateBound(
IAzureDiagnosticLogsTable wadTable,
DateTime date,
EntryPartition searchRangeEnd,
CancellationToken cancellation)
{
var dateTimestamp = new EntryTimestamp(date);
var ret = LoadEntriesRange(wadTable, dateTimestamp.Partition, searchRangeEnd, null, cancellation)
.FirstOrDefault(e => e.Entry.EventTickCount >= dateTimestamp.Ticks);
return(ret.Entry != null ? ret : new IndexedAzureDiagnosticLogEntry?());
}
protected override void LiveLogListen(CancellationToken stopEvt, LiveLogXMLWriter output)
{
using (host.Trace.NewFrame)
{
try
{
if (azureConnectParams.Mode == AzureConnectionParams.LoadMode.FixedRange)
{
ReportBackgroundActivityStatus(true);
foreach (var entry in AzureDiagnosticsUtils.LoadEntriesRange(
table, new EntryPartition(azureConnectParams.From.Ticks), new EntryPartition(azureConnectParams.Till.Ticks), null, stopEvt))
{
WriteEntry(entry.Entry, output);
if (stopEvt.IsCancellationRequested)
{
return;
}
}
ReportBackgroundActivityStatus(false);
return;
}
else if (azureConnectParams.Mode == AzureConnectionParams.LoadMode.Recent)
{
ReportBackgroundActivityStatus(true);
var lastPartition = AzureDiagnosticsUtils.FindLastMessagePartitionKey(table, DateTime.UtcNow, stopEvt);
if (lastPartition.HasValue)
{
var firstPartition = new EntryPartition(lastPartition.Value.Ticks + azureConnectParams.Period.Ticks);
foreach (var entry in AzureDiagnosticsUtils.LoadEntriesRange(table, firstPartition, EntryPartition.MaxValue, null, stopEvt))
{
WriteEntry(entry.Entry, output);
stopEvt.ThrowIfCancellationRequested();
}
}
ReportBackgroundActivityStatus(false);
return;
}
}
catch (OperationCanceledException e)
{
host.Trace.Error(e, "WAD live log thread cancelled");
}
catch (Exception e)
{
host.Trace.Error(e, "WAD live log thread failed");
}
}
}
public static IndexedAzureDiagnosticLogEntry?FindDateBound(
IAzureDiagnosticLogsTable wadTable,
DateTime date,
ListUtils.ValueBound bound,
EntryPartition searchRangeBegin,
EntryPartition searchRangeEnd,
CancellationToken cancellationToken)
{
switch (bound)
{
case ListUtils.ValueBound.Lower:
return(FindLowerDateBound(wadTable, date, searchRangeEnd, cancellationToken));
case ListUtils.ValueBound.LowerReversed:
return(FindLowerReversedDateBound(wadTable, date, searchRangeBegin, cancellationToken));
default:
throw new NotImplementedException("Searching for " + bound.ToString() + " bound in Azure Diagnostics Logs is not implemented");
}
}
/// <summary>
/// Loads a range of messages from Logs Table. The range is specified by two partition keys.
/// </summary>
/// <param name="wadTable">Table to load messages from</param>
/// <param name="threads">Threads container that will store loaded threads</param>
/// <param name="beginPartitionKey">Begin of the range. Messages with PartitionKey GREATER THAN or EQUAL to <paramref name="beginPartitionKey"/> are included to the range</param>
/// <param name="endPartitionKey">End of the range. Messages with PartitionKey LESS THAN <paramref name="endPartitionKey"/> are included to the range</param>
/// <param name="entriesLimit">If specified limits the number of items to return</param>
/// <returns>Sequence of messages sorted by EventTickCount</returns>
public static IEnumerable <IMessage> LoadWADLogsTableMessagesRange(
IAzureDiagnosticLogsTable wadTable,
ILogSourceThreads threads,
EntryPartition beginPartition,
EntryPartition endPartition,
int?entriesLimit)
{
foreach (var entryAndIndex in LoadEntriesRange(wadTable, beginPartition, endPartition, entriesLimit, CancellationToken.None))
{
var entry = entryAndIndex.Entry as WADLogsTableEntry;
if (entry == null)
{
continue;
}
yield return(new Content(
new EntryPartition(entry.EventTickCount).MakeMessagePosition(entryAndIndex.IndexWithinPartition),
new EntryPartition(entry.EventTickCount).MakeMessagePosition(entryAndIndex.IndexWithinPartition + 1),
threads.GetThread(new StringSlice(string.Format("{0}-{1}", entry.Pid, entry.Tid))),
new MessageTimestamp(new DateTime(entry.EventTickCount, DateTimeKind.Utc)),
new StringSlice(entry.Message),
SeverityFlag.Info
));
}
}
public static int Distance(EntryPartition begin, EntryPartition end)
{
return(AzureDiagnosticsUtils.TicksToMinutes(end.Ticks - begin.Ticks));
}
public static int Compare(EntryPartition p1, EntryPartition p2)
{
return(Math.Sign(p1.Ticks - p2.Ticks));
}
