/// <summary>
/// Update existing audit log data after addition
/// </summary>
/// <param name="auditLogAnalyticsData"></param>
/// <returns></returns>
public bool updateAnalyticsDataToTable(AuditLogAnalyticsDataInfo auditLogAnalyticsData)
{
// Parse the connection string and return a reference to the storage account.
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(AuditLogStorageConnectionString);
// Create the table client.
CloudTableClient tableClient = storageAccount.CreateCloudTableClient();
// Create the CloudTable object that represents the "Audit Log" table.
CloudTable auditDataLogTable = tableClient.GetTableReference(AuditLogDataTableName);
// Create the CloudTable object that represents the "Audit Log" table.
CloudTable auditOpsStatusTable = tableClient.GetTableReference(AuditLogAnalyticsTableName);
O365AuditLogOperations auditOpstableData = new O365AuditLogOperations(auditLogAnalyticsData.PartitionKey, auditLogAnalyticsData.RowPrefix);
auditOpstableData.ExcludedOperations = auditLogAnalyticsData.ExcludedOperations;
auditOpstableData.IncludedOperations = auditLogAnalyticsData.IncludedOperations;
auditOpstableData.IncludedRecords = auditLogAnalyticsData.IncludedRecords;
auditOpstableData.ExcludedRecords = auditLogAnalyticsData.ExcludedRecords;
auditOpstableData.TotalRecords = auditLogAnalyticsData.TotalRecords;
auditOpstableData.RunDate = auditLogAnalyticsData.RunDate;
auditOpstableData.StartHour = auditLogAnalyticsData.StartHour;
auditOpstableData.EndHour = auditLogAnalyticsData.EndHour;
auditOpstableData.RunHour = auditLogAnalyticsData.RunHour;
auditOpstableData.RunFrequency = auditLogAnalyticsData.RunFrequency;
auditOpstableData.LogOperationStatus = auditLogAnalyticsData.LogOperationStatus;
auditOpstableData.LogOperationSuccessful = auditLogAnalyticsData.LogOperationSuccessful;
TableOperation auditOpsinsertOperation = TableOperation.InsertOrMerge(auditOpstableData);
log.LogInformation($"Inserting Operations Data");
// Execute the insert operation.
TableResult result = auditOpsStatusTable.ExecuteAsync(auditOpsinsertOperation).GetAwaiter().GetResult();
if (result.HttpStatusCode.ToEnum <HttpStatusCode>() == HttpStatusCode.BadGateway || result.HttpStatusCode.ToEnum <HttpStatusCode>() == HttpStatusCode.BadRequest)
{
throw new Exception($"Table Update failed for Audit Log data entry. Captured error {result.HttpStatusCode} for Audit Log Analytics entry ");
}
return(true);
}
/// <summary>
/// Insert the Audit detail data now into Azure Table
/// </summary>
/// <param name="auditDetailedReports"></param>
/// <param name="auditLogDataAnalytics"></param>
/// <returns></returns>
public AuditLogAnalyticsDataInfo addDatatoAzureStore(List <AuditDetailedReport> auditDetailedReports, AuditLogAnalyticsDataInfo auditLogDataAnalytics)
{
try
{
List <string> operations = getAuditOperations();
List <string> IncludedOperations = new List <string>();
List <string> ExcludedOperations = new List <string>();
int includedRecords = 0;
int excludedRecords = 0;
// Parse the connection string and return a reference to the storage account.
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(AuditLogStorageConnectionString);
// Create the table client.
CloudTableClient tableClient = storageAccount.CreateCloudTableClient();
// Create the CloudTable object that represents the "Audit Log" table.
CloudTable auditDataLogTable = tableClient.GetTableReference(AuditLogDataTableName);
//// Add the Audit log information to Azure Table -- Primary Table to capture Audit log data
foreach (AuditDetailedReport auditDetailReport in auditDetailedReports)
{
if (operations.Contains(auditDetailReport.Operation, StringComparer.OrdinalIgnoreCase))
{
includedRecords++;
string auditlogpartitionPrefix = AuditLogPartitionKeyPrefix + "_" + getAuditShortDateStringFromUTC(auditDetailReport.CreationTime);
string auditlogrowprefix = auditDetailReport.Id;
O365AuditLogData auditLogTableData = new O365AuditLogData(auditlogpartitionPrefix, auditlogrowprefix);
O365MgmtApiEntities <AuditDetailedReport, O365AuditLogData> .Copy(auditDetailReport, auditLogTableData);
auditLogTableData.CreationTime = auditLogTableData.CreationTime.ToLocalTime();
// Create the TableOperation object that inserts the customer entity.
TableOperation auditLoginsertOperation = TableOperation.InsertOrMerge(auditLogTableData);
log.LogInformation($"Inserting Operation - {auditDetailReport.Operation} Id - {auditDetailReport.Id} ");
// Execute the insert operation.
TableResult result = auditDataLogTable.ExecuteAsync(auditLoginsertOperation).GetAwaiter().GetResult();
if (result.HttpStatusCode.ToEnum <HttpStatusCode>() == HttpStatusCode.BadGateway || result.HttpStatusCode.ToEnum <HttpStatusCode>() == HttpStatusCode.BadRequest)
{
throw new Exception($"Table Update failed for Audit Log data entry. Captured error {result.HttpStatusCode} for Operation - { auditDetailReport.Operation }, Id {auditDetailReport.Id} ");
}
if (!IncludedOperations.Contains(auditDetailReport.Operation, StringComparer.OrdinalIgnoreCase))
{
IncludedOperations.Add(auditDetailReport.Operation);
}
}
else
{
excludedRecords++;
if (!ExcludedOperations.Contains(auditDetailReport.Operation, StringComparer.OrdinalIgnoreCase))
{
ExcludedOperations.Add(auditDetailReport.Operation);
}
log.LogInformation($"Record not entered. Operation - { auditDetailReport.Operation }, Id {auditDetailReport.Id} ");
}
}
//// Capture the audit log report analytics for each run into the table for future reference -- Secondary Table for capturing analytics of each run
log.LogInformation($"\r\n Included Records count {includedRecords} and Excluded Records count {excludedRecords}");
auditLogDataAnalytics.ExcludedOperations = string.Join(",", ExcludedOperations.ToArray());
auditLogDataAnalytics.IncludedOperations = string.Join(",", IncludedOperations.ToArray());
auditLogDataAnalytics.IncludedRecords = includedRecords;
auditLogDataAnalytics.ExcludedRecords = excludedRecords;
auditLogDataAnalytics.TotalRecords = auditDetailedReports.Count;
auditLogDataAnalytics.LogOperationStatus = "Completed";
auditLogDataAnalytics.LogOperationSuccessful = true;
}
catch (Exception ex)
{
log.LogError(ex.Message);
auditLogDataAnalytics.LogOperationStatus = "Failed";
auditLogDataAnalytics.LogOperationSuccessful = false;
}
return(auditLogDataAnalytics);
}
//// Set the timer in the timer trigger to run - More Info at here - https://docs.microsoft.com/en-us/azure/azure-functions/functions-bindings-timer#c-example
//// Note: this Function App is using Azure Funciton 1.x because of SharePoint CSOM dependency but can be used on Azure Function 2.x
public static void Run([TimerTrigger("0 0 */1 * * *", RunOnStartup = true)] TimerInfo myTimer, ILogger log, TraceWriter logWriter)
{
log.LogInformation($"C# Timer trigger function executed at: {DateTime.Now}");
logWriter.Info($"C# Timer trigger function executed at: {DateTime.Now}");
#region "Variables"
//// Set the Variables. For local debugging, use localsettings.json (see below) and for Azure Function, use Configuration in Platform Features
/// for eg - local.settings.json
/// Note : Please replace all the strings with < > with the open bracket and close brackets
//{
// "IsEncrypted": false,
// "Values": {
// "AzureWebJobsStorage": "UseDevelopmentStorage=true",
// "AzureWebJobsDashboard": "UseDevelopmentStorage=true",
// "StorageConnectionString": "<connection string for Azure storage account in .net>",
// "FUNCTIONS_WORKER_RUNTIME": "dotnet",
// "AuditLogDataTable": "<Azure Table Name>",
// "AuditLogAnalyticsTable": "<O365 Audit log capture table for this function>",
// "AuditLogCSVExportLocation": "<Azure>",
// "SPUserName": "******",
// "SPUserPassword": "******",
// "AuditLogDataTablePrefix": "<Partition Key Prefix for easy retrival>",
// "AuditLogOpsTablePrefix": "<Analytics Partition Key Prefic for easy retrival>",
// "TenantID": "<Tenant ID for the App>",
// "AuditLogAuthUrl": "https://manage.office.com",
// "AzureADAppID": "<Azure App ID>",
// "AzureADAppSecret": "<Azure App secret>"
// }
//}
string TenantID = System.Environment.GetEnvironmentVariable("TenantID");
string authString = "https://login.windows.net/" + TenantID;
string SPServiceUrl = "https://manage.office.com/api/v1.0/" + TenantID + "/activity/feed/subscriptions/content";
string authUrl = System.Environment.GetEnvironmentVariable("AuditLogAuthUrl");
string clientId = System.Environment.GetEnvironmentVariable("AzureADAppID");
string clientSecret = System.Environment.GetEnvironmentVariable("AzureADAppSecret");
#endregion
try
{
//// **** Get the Authentication Token ****
var authenticationContext = new AuthenticationContext(authString, false);
authenticationContext.ExtendedLifeTimeEnabled = true;
//// Config for OAuth client credentials
ClientCredential clientCred = new ClientCredential(clientId, clientSecret);
AuthenticationResult authenticationResult = null;
Task runTask = Task.Run(async() => authenticationResult = await authenticationContext.AcquireTokenAsync(authUrl, clientCred));
runTask.Wait();
string token = authenticationResult.AccessToken;
int startCount = 160;
int endCount = 480;
int increments = (endCount - startCount) / 20;
int startHour = 0;
int endHour = 0;
for (int i = startCount / 20; i <= increments; i++)
{
endHour = i * 20;
startHour = (i + 1) * 20;
try
{
log.LogInformation($"Started pulling for now - {startHour} to now - {endHour}");
O365MgmtAPIDataService dataService = new O365MgmtAPIDataService(token, log);
AuditLogAnalyticsDataInfo auditLogAnalyticsDataInfo = dataService.getInitialAnalyticsInfo(startHour, endHour);
if (dataService.updateAnalyticsDataToTable(auditLogAnalyticsDataInfo))
{
//// Get the time zone of the destination tenant. The date and time used by Audit log service is UTC format.
TimeZoneInfo aestTimeZone = TimeZoneInfo.FindSystemTimeZoneById("AUS Eastern Standard Time");
DateTime startHourUTC = TimeZoneInfo.ConvertTimeToUtc(auditLogAnalyticsDataInfo.StartHour, aestTimeZone);
DateTime endHourUTC = TimeZoneInfo.ConvertTimeToUtc(auditLogAnalyticsDataInfo.EndHour, aestTimeZone);
string startDateString = startHourUTC.ToUniversalTime().ToString("yyyy-MM-ddTHH:00");
string endDateString = endHourUTC.ToString("yyyy-MM-ddTHH:00");
//// ****************** Step 1: Start the audit log gathering process *****************
log.LogInformation($"getting Data from {startDateString} to {endDateString}");
//// Here I am fetching SharePoint events.
string urlParameters = $"?contentType=Audit.SharePoint&startTime={startDateString}&endTime={endDateString}";
//// Initialize the audit log data information
AuditInitialDataObject auditInitialDataObject = new AuditInitialDataObject();
List <AuditDetailedReport> auditDetailReportsFinal = new List <AuditDetailedReport>();
//// Loop through the detail URI information provided by the initial data call till there is no next page to be read
do
{
auditInitialDataObject = dataService.getAuditInitalData(SPServiceUrl, urlParameters);
if (auditInitialDataObject != null)
{
if (auditInitialDataObject.AuditNextPageUri != null && auditInitialDataObject.AuditNextPageUri != "")
{
urlParameters = "?" + auditInitialDataObject.AuditNextPageUri.Split('?')[1];
}
List <AuditInitialReport> auditInitialReports = auditInitialDataObject.AuditInitialDataObj;
//// set batch size = 200. Note : Above 500 the speed decreases drastically as per my tests.
int maxCalls = 200;
int count = 0;
//// **************** Step 2: For each of the calls call the detailed data fetch api in batches. Here the batch = 200 ***********************
Parallel.ForEach(auditInitialReports, new ParallelOptions {
MaxDegreeOfParallelism = maxCalls
}, (auditInitialReport) =>
{
int loopCount = count++;
log.LogInformation("Looking at request " + loopCount);
List <AuditDetailedReport> auditDetailReports = dataService.getAuditDetailData(auditInitialReport.ContentUri);
log.LogInformation("Got Audit Detail Reports of " + auditDetailReports.Count + " for loop number " + loopCount);
foreach (AuditDetailedReport auditDetailReport in auditDetailReports)
{
auditDetailReportsFinal.Add(auditDetailReport);
}
});
}
} while (auditInitialDataObject.AuditNextPageUri != "");
log.LogInformation("Final Audit Detail Reports" + auditDetailReportsFinal.Count);
//// *************** Step 3 : Update additional properties to the Audit log data ***************************
int maxAuditUpdateCalls = 200;
Parallel.ForEach(auditDetailReportsFinal, new ParallelOptions {
MaxDegreeOfParallelism = maxAuditUpdateCalls
}, (auditDetailReport) =>
{
auditDetailReport = dataService.mapOrUpdateProperties(auditDetailReport);
});
//// **************** Step 4 : Add Data to Azure Table ****************
auditLogAnalyticsDataInfo = dataService.addDatatoAzureStore(auditDetailReportsFinal, auditLogAnalyticsDataInfo);
//// **************** Step 5 : Update the Report analytics for each Audit log run *******************
dataService.updateAnalyticsDataToTable(auditLogAnalyticsDataInfo);
}
}
catch (Exception ex1)
{
log.LogError($"error occurred {ex1.Message}");
}
}
}
catch (Exception ex)
{
log.LogError(ex.Message);
}
}
