public static void StartIngestion(string storageAccountConnectionString, string blobContainerName, string omsWorkspaceId, string omsWorkspaceKey, TraceWriter logger)
{
connectionString = storageAccountConnectionString;
containerName = blobContainerName;
workspaceid = omsWorkspaceId;
workspacekey = omsWorkspaceKey;
log = logger;
var oms = new OMSIngestionApi(workspaceid, workspacekey, log);
if (CloudStorageAccount.TryParse(connectionString, out CloudStorageAccount storageAccount) == false)
{
log.Error($"Connection string can't be parsed: {connectionString}");
return;
}
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer container = blobClient.GetContainerReference(containerName);
log.Info("Sending logs to OMS");
IEnumerable <CloudBlobDirectory> servers = container.ListBlobs().OfType <CloudBlobDirectory>().ToList();
foreach (var server in servers)
{
SendLogsFromServer(server, oms);
}
log.Info($"Finished processing files, saving the state file");
File.WriteAllText(stateFileName, JsonConvert.SerializeObject(statesList));
}
public int ProcessQuery(OMSIngestionApi oms, HttpClient client, String query, TraceWriter log)
{
try
{
JsonResult <UCDDHourly> jsonResult = default(JsonResult <UCDDHourly>);
int count = 0;
var tasks = new List <Task>();
while (!string.IsNullOrEmpty(query))
{
var response = client.GetAsync(new Uri(query)).Result;
if (response.IsSuccessStatusCode)
{
var result = response.Content.ReadAsStringAsync().Result;
jsonResult = JsonConvert.DeserializeObject <JsonResult <UCDDHourly> >(result);
var jsonList = JsonConvert.SerializeObject(jsonResult.data.ToList());
tasks.Add(oms.SendOMSApiIngestionFile(jsonList));
count = count + jsonResult.data.ToList().Count;
Console.WriteLine($"Count {count}");
query = jsonResult.nextLink;
}
}
Task.WaitAll(tasks.ToArray());
log.Info($"Record Count {count}");
return(count);
}
catch (Exception e)
{
log.Info($"Failed processing. Reason: {e}");
throw e;
}
}
public static void StartIngestion(string omsWorkspaceId, string omsWorkspaceKey, TraceWriter log)
{
workspaceid = omsWorkspaceId;
workspacekey = omsWorkspaceKey;
log.Info("Sending logs to OMS");
var oms = new OMSIngestionApi(workspaceid, workspacekey, log);
Helper helper = new Helper();
HttpClient client = helper.BuildClient();
int count = helper.ProcessQuery(oms, client, helper.GetUsageQueryUrl(), log);
log.Info($"Finished processing files");
}
private static void SendLogsFromServer(CloudBlobDirectory serverDirectory, OMSIngestionApi oms)
{
log.Info($"Processing audit logs for server: {serverDirectory.Prefix}");
try
{
string serverName = new DirectoryInfo(serverDirectory.Prefix).Name;
IEnumerable <CloudBlobDirectory> databases = GetSubDirectories(serverName, serverDirectory, statesList);
foreach (var database in databases)
{
SendLogsFromDatabase(database, statesList[serverName], oms);
}
log.Info($"Done processing audit logs for server: {serverDirectory.Prefix}");
}
catch (Exception e)
{
log.Info("Failed processing audit logs for server: {serverDirectory}. Reason: {e}");
UpdateFailuresLog(serverDirectory.Prefix, e);
}
}
private static void SendLogsFromDatabase(CloudBlobDirectory databaseDirectory, serverStateDictionary serverState, OMSIngestionApi oms)
{
log.Info($"Processing audit logs for database: {databaseDirectory.Prefix}");
try
{
string databaseName = new DirectoryInfo(databaseDirectory.Prefix).Name;
IEnumerable <CloudBlobDirectory> subfolders = GetSubDirectories(databaseName, databaseDirectory, serverState);
foreach (var subfolder in subfolders)
{
SendLogsFromSubfolder(subfolder, serverState[databaseName], oms);
}
log.Info($"Done processing audit logs for database: {0}");
}
catch (Exception e)
{
log.Info($"Failed processing audit logs for database: {databaseDirectory.Prefix}. Reason: {e}");
UpdateFailuresLog(databaseDirectory.Prefix, e);
}
}
private static void SendLogsFromSubfolder(CloudBlobDirectory subfolder, databaseStateDictionary databaseState, OMSIngestionApi oms)
{
int nextEvent = 0;
int eventNumber = 0;
int datesCompareResult = -1;
string currentDate = null;
log.Info($"Processing sub folder: {subfolder.Prefix}");
string subfolderName = new DirectoryInfo(subfolder.Prefix).Name;
IEnumerable <CloudBlobDirectory> dateFolders = GetSubDirectories(subfolderName, subfolder, databaseState);
var subfolderState = databaseState[subfolderName];
string lastBlob = subfolderState.BlobName;
DateTimeOffset?lastModified = subfolderState.LastModified;
try
{
foreach (var dateFolder in dateFolders)
{
currentDate = new DirectoryInfo(dateFolder.Prefix).Name;
datesCompareResult = string.Compare(currentDate, subfolderState.Date, StringComparison.OrdinalIgnoreCase);
//current folder is older than last state
if (datesCompareResult <= 0)
{
continue;
}
var tasks = new List <Task <int> >();
IEnumerable <CloudBlob> cloudBlobs = dateFolder.ListBlobs(useFlatBlobListing: true).OfType <CloudBlob>()
.Where(b => b.Name.EndsWith(".json.gz", StringComparison.OrdinalIgnoreCase)).ToList();
foreach (var blob in cloudBlobs)
{
string blobName = new FileInfo(blob.Name).Name;
if (datesCompareResult == 0)
{
int blobsCompareResult = string.Compare(blobName, subfolderState.BlobName, StringComparison.OrdinalIgnoreCase);
//blob is older than last state
if (blobsCompareResult < 0)
{
continue;
}
if (blobsCompareResult == 0)
{
if (blob.Properties.LastModified == subfolderState.LastModified)
{
continue;
}
eventNumber = subfolderState.EventNumber;
}
}
tasks.Add(SendBlobToOMS(blob, eventNumber, oms));
lastBlob = blobName;
lastModified = blob.Properties.LastModified;
eventNumber = 0;
}
Task.WaitAll(tasks.ToArray());
if (tasks.Count > 0)
{
nextEvent = tasks.Last().Result;
}
subfolderState.BlobName = lastBlob;
subfolderState.LastModified = lastModified;
if (datesCompareResult >= 0)
{
subfolderState.Date = currentDate;
}
subfolderState.EventNumber = nextEvent;
log.Info($"Saving the state file after processing blobs{stateFileName}");
File.WriteAllText(stateFileName, JsonConvert.SerializeObject(statesList));
}
log.Info($"Done processing sub folder: {subfolder.Prefix}");
}
catch (Exception e)
{
log.Info($"Failed processing sub folder: {subfolder.Prefix}. Reason: {e}");
UpdateFailuresLog(subfolder.Prefix, e);
}
}
private static async Task <int> SendBlobToOMS(CloudBlob blob, int eventNumber, OMSIngestionApi oms)
{
RetryPolicy retryPolicy = new RetryPolicy(RetryPolicy.DefaultFixed.ErrorDetectionStrategy, DefaultRetryCount);
log.Info($"Processing: {blob.Uri}");
string fileName = Path.Combine(GetLocalStorageFolder(), Path.GetRandomFileName() + ".json.gz");
string decompressedFileName = Path.Combine(GetLocalStorageFolder(), Path.GetRandomFileName() + ".json");
try
{
OperationContext operationContext = new OperationContext();
operationContext.RequestCompleted += (sender, e) => PrintHeaders(e);
await retryPolicy.ExecuteAsync((() => blob.DownloadToFileAsync(fileName, FileMode.OpenOrCreate, null, null, operationContext)));
List <UCDDHourly> list = new List <UCDDHourly>();
var jsonSerializer = new JsonSerializer();
FileInfo gzipFileName = new FileInfo(fileName);
using (FileStream fileToDecompressAsStream = gzipFileName.OpenRead())
{
using (FileStream decompressedStream = File.Create(decompressedFileName))
{
using (GZipStream decompressionStream = new GZipStream(fileToDecompressAsStream, CompressionMode.Decompress))
{
decompressionStream.CopyTo(decompressedStream);
}
}
using (var reader = new StreamReader(decompressedFileName))
using (var jsonReader = new JsonTextReader(reader))
{
jsonReader.SupportMultipleContent = true;
while (jsonReader.Read())
{
UCDDHourly ucddHourly = jsonSerializer.Deserialize <UCDDHourly>(jsonReader);
list.Add(ucddHourly);
}
}
}
IEnumerable <List <UCDDHourly> > chunkedList = list.Chunk(10000);
foreach (List <UCDDHourly> chunk in chunkedList)
{
var jsonList = JsonConvert.SerializeObject(chunk);
await oms.SendOMSApiIngestionFile(jsonList);
eventNumber += chunk.Count;
totalLogs += chunk.Count;
}
}
catch (Exception e)
{
log.Info($"Failed processing: {blob.Uri}. Reason: {e}");
throw;
}
finally
{
try
{
File.Delete(fileName);
File.Delete(decompressedFileName);
}
catch (Exception e)
{
log.Info($"Was not able to delete file: {fileName}. Reason: {e.Message}");
}
}
log.Info($"Done processing: {blob.Uri}");
return(eventNumber);
}