private static void SendLogsFromSubfolder(CloudBlobDirectory subfolder, databaseStateDictionary databaseState, OMSIngestionApi oms) { int nextEvent = 0; int eventNumber = 0; int datesCompareResult = -1; string currentDate = null; s_consoleTracer.TraceEvent(TraceEventType.Information, 0, "Processing sub folder: {0}", subfolder.Prefix); string subfolderName = new DirectoryInfo(subfolder.Prefix).Name; IEnumerable <CloudBlobDirectory> dateFolders = GetSubDirectories(subfolderName, subfolder, databaseState); var subfolderState = databaseState[subfolderName]; string lastBlob = subfolderState.BlobName; DateTimeOffset?lastModified = subfolderState.LastModified; try { foreach (var dateFolder in dateFolders) { currentDate = new DirectoryInfo(dateFolder.Prefix).Name; datesCompareResult = string.Compare(currentDate, subfolderState.Date, StringComparison.OrdinalIgnoreCase); //current folder is older than last state if (datesCompareResult < 0) { continue; } var tasks = new List <Task <int> >(); IEnumerable <CloudBlob> cloudBlobs = dateFolder.ListBlobs(useFlatBlobListing: true).OfType <CloudBlob>() .Where(b => b.Name.EndsWith(".xel", StringComparison.OrdinalIgnoreCase)).ToList(); foreach (var blob in cloudBlobs) { string blobName = new FileInfo(blob.Name).Name; if (datesCompareResult == 0) { int blobsCompareResult = string.Compare(blobName, subfolderState.BlobName, StringComparison.OrdinalIgnoreCase); //blob is older than last state if (blobsCompareResult < 0) { continue; } if (blobsCompareResult == 0) { if (blob.Properties.LastModified == subfolderState.LastModified) { continue; } eventNumber = subfolderState.EventNumber; } } tasks.Add(SendBlobToOMS(blob, eventNumber, oms)); lastBlob = blobName; lastModified = blob.Properties.LastModified; eventNumber = 0; } Task.WaitAll(tasks.ToArray()); if (tasks.Count > 0) { nextEvent = tasks.Last().Result; } subfolderState.BlobName = lastBlob; subfolderState.LastModified = lastModified; if (datesCompareResult >= 0) { subfolderState.Date = currentDate; } subfolderState.EventNumber = nextEvent; File.WriteAllText(StateFileName, JsonConvert.SerializeObject(StatesList)); } s_consoleTracer.TraceEvent(TraceEventType.Information, 0, "Done processing sub folder: {0}", subfolder.Prefix); } catch (Exception e) { s_consoleTracer.TraceEvent(TraceEventType.Error, 0, "Failed processing sub folder: {0}. Reason: {1}", subfolder.Prefix, e); UpdateFailuresLog(subfolder.Prefix, e); } }
private static void SendLogsFromSubfolder(CloudBlobDirectory subfolder, databaseStateDictionary databaseState, OMSIngestionApi oms) { int nextEvent = 0; int eventNumber = 0; int datesCompareResult = -1; string lastBlob = null; string currentDate = null; string subfolderName = new DirectoryInfo(subfolder.Prefix).Name; IEnumerable <CloudBlobDirectory> dateFolders = GetSubDirectories(subfolderName, subfolder, databaseState); var subfolderState = databaseState[subfolderName]; foreach (var dateFolder in dateFolders) { currentDate = new DirectoryInfo(dateFolder.Prefix).Name; datesCompareResult = string.Compare(currentDate, subfolderState.Date, StringComparison.OrdinalIgnoreCase); //current folder is older than last state if (datesCompareResult < 0) { continue; } var tasks = new List <Task <int> >(); IEnumerable <CloudPageBlob> pageBlobs = dateFolder.ListBlobs(useFlatBlobListing: true).OfType <CloudPageBlob>() .Where(b => b.Name.EndsWith(".xel", StringComparison.OrdinalIgnoreCase)); foreach (var blob in pageBlobs) { string blobName = new FileInfo(blob.Name).Name; s_consoleTracer.TraceEvent(TraceEventType.Information, 0, "processing: {0}{1}", dateFolder.Uri, blobName); if (datesCompareResult == 0) { int blobsCompareResult = string.Compare(blobName, subfolderState.BlobName, StringComparison.OrdinalIgnoreCase); //blob is older than last state if (blobsCompareResult < 0) { continue; } if (blobsCompareResult == 0) { eventNumber = subfolderState.EventNumber; } } tasks.Add(SendBlobToOMS(blob, eventNumber, oms)); lastBlob = blobName; eventNumber = 0; } Task.WaitAll(tasks.ToArray()); nextEvent = tasks.Last().Result; } subfolderState.BlobName = lastBlob; if (datesCompareResult >= 0) { subfolderState.Date = currentDate; } subfolderState.EventNumber = nextEvent; }