private static void SendLogsFromSubfolder(CloudBlobDirectory subfolder, databaseStateDictionary databaseState, OMSIngestionApi oms)
{
int nextEvent = 0;
int eventNumber = 0;
int datesCompareResult = -1;
string currentDate = null;
s_consoleTracer.TraceEvent(TraceEventType.Information, 0, "Processing sub folder: {0}", subfolder.Prefix);
string subfolderName = new DirectoryInfo(subfolder.Prefix).Name;
IEnumerable <CloudBlobDirectory> dateFolders = GetSubDirectories(subfolderName, subfolder, databaseState);
var subfolderState = databaseState[subfolderName];
string lastBlob = subfolderState.BlobName;
DateTimeOffset?lastModified = subfolderState.LastModified;
try
{
foreach (var dateFolder in dateFolders)
{
currentDate = new DirectoryInfo(dateFolder.Prefix).Name;
datesCompareResult = string.Compare(currentDate, subfolderState.Date, StringComparison.OrdinalIgnoreCase);
//current folder is older than last state
if (datesCompareResult < 0)
{
continue;
}
var tasks = new List <Task <int> >();
IEnumerable <CloudBlob> cloudBlobs = dateFolder.ListBlobs(useFlatBlobListing: true).OfType <CloudBlob>()
.Where(b => b.Name.EndsWith(".xel", StringComparison.OrdinalIgnoreCase)).ToList();
foreach (var blob in cloudBlobs)
{
string blobName = new FileInfo(blob.Name).Name;
if (datesCompareResult == 0)
{
int blobsCompareResult = string.Compare(blobName, subfolderState.BlobName, StringComparison.OrdinalIgnoreCase);
//blob is older than last state
if (blobsCompareResult < 0)
{
continue;
}
if (blobsCompareResult == 0)
{
if (blob.Properties.LastModified == subfolderState.LastModified)
{
continue;
}
eventNumber = subfolderState.EventNumber;
}
}
tasks.Add(SendBlobToOMS(blob, eventNumber, oms));
lastBlob = blobName;
lastModified = blob.Properties.LastModified;
eventNumber = 0;
}
Task.WaitAll(tasks.ToArray());
if (tasks.Count > 0)
{
nextEvent = tasks.Last().Result;
}
subfolderState.BlobName = lastBlob;
subfolderState.LastModified = lastModified;
if (datesCompareResult >= 0)
{
subfolderState.Date = currentDate;
}
subfolderState.EventNumber = nextEvent;
File.WriteAllText(StateFileName, JsonConvert.SerializeObject(StatesList));
}
s_consoleTracer.TraceEvent(TraceEventType.Information, 0, "Done processing sub folder: {0}", subfolder.Prefix);
}
catch (Exception e)
{
s_consoleTracer.TraceEvent(TraceEventType.Error, 0, "Failed processing sub folder: {0}. Reason: {1}", subfolder.Prefix, e);
UpdateFailuresLog(subfolder.Prefix, e);
}
}
private static void SendLogsFromSubfolder(CloudBlobDirectory subfolder, databaseStateDictionary databaseState, OMSIngestionApi oms)
{
int nextEvent = 0;
int eventNumber = 0;
int datesCompareResult = -1;
string lastBlob = null;
string currentDate = null;
string subfolderName = new DirectoryInfo(subfolder.Prefix).Name;
IEnumerable <CloudBlobDirectory> dateFolders = GetSubDirectories(subfolderName, subfolder, databaseState);
var subfolderState = databaseState[subfolderName];
foreach (var dateFolder in dateFolders)
{
currentDate = new DirectoryInfo(dateFolder.Prefix).Name;
datesCompareResult = string.Compare(currentDate, subfolderState.Date, StringComparison.OrdinalIgnoreCase);
//current folder is older than last state
if (datesCompareResult < 0)
{
continue;
}
var tasks = new List <Task <int> >();
IEnumerable <CloudPageBlob> pageBlobs = dateFolder.ListBlobs(useFlatBlobListing: true).OfType <CloudPageBlob>()
.Where(b => b.Name.EndsWith(".xel", StringComparison.OrdinalIgnoreCase));
foreach (var blob in pageBlobs)
{
string blobName = new FileInfo(blob.Name).Name;
s_consoleTracer.TraceEvent(TraceEventType.Information, 0, "processing: {0}{1}", dateFolder.Uri, blobName);
if (datesCompareResult == 0)
{
int blobsCompareResult = string.Compare(blobName, subfolderState.BlobName, StringComparison.OrdinalIgnoreCase);
//blob is older than last state
if (blobsCompareResult < 0)
{
continue;
}
if (blobsCompareResult == 0)
{
eventNumber = subfolderState.EventNumber;
}
}
tasks.Add(SendBlobToOMS(blob, eventNumber, oms));
lastBlob = blobName;
eventNumber = 0;
}
Task.WaitAll(tasks.ToArray());
nextEvent = tasks.Last().Result;
}
subfolderState.BlobName = lastBlob;
if (datesCompareResult >= 0)
{
subfolderState.Date = currentDate;
}
subfolderState.EventNumber = nextEvent;
}
