public static void Log(Exception exception, string connstring, string schema, DateTime date)
{
var errorDataTable = O365ETL.DataTables.GetErrorDataTable();
DataRow errorRow = errorDataTable.NewRow();
errorRow["BatchID"] = date.ToString("yyyyMMddHHmmss");;
errorRow["LogText"] = exception?.ToString();
errorRow["ExecutedBy"] = Environment.UserName;
errorRow["RecordCount"] = 0;
errorRow["EventStart"] = date;
errorRow["EventEnd"] = DateTime.UtcNow;
errorRow["Status"] = exception == null ? "Success" : "Fail";
if (exception != null && exception is ApiFailure)
{
ApiFailure apiFailure = exception as ApiFailure;
errorRow["BatchResponse"] = apiFailure.MainPayload;
errorRow["IndividualResponse"] = apiFailure.DataPayload;
errorRow["APIContentUri"] = apiFailure.RawResponse.RequestMessage.RequestUri;
}
errorRow["APIStartDate"] = date;
errorRow["APIEndDate"] = date.AddDays(1);
errorDataTable.Rows.Add(errorRow);
O365ETL.SQLOperations.BulkInsert(connstring, errorDataTable, schema + "." + "batch_log");
}
public static async Task GetContentPayloadAsync(string accessToken, string tenantId, DateTime start, string connstring, string schema)
{
DateTime end = start.AddDays(1);
HttpClient client = new HttpClient();
client.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", accessToken);
string pageUri = $"https://manage.office.com/api/v1.0/{tenantId}/activity/feed/subscriptions/content?contentType=Audit.General&startTime={start.Date}&endTime={end.Date}";
do
{
HttpResponseMessage batchResponse = await client.GetAsync(pageUri);
var payloadRetrieved = await batchResponse.Content.ReadAsStringAsync();
if (!batchResponse.IsSuccessStatusCode)
{
throw new ApiFailure()
{
MainPayload = payloadRetrieved,
RawResponse = batchResponse,
};
}
var contentReturned = JsonConvert.DeserializeObject <List <Content> >(payloadRetrieved);
IEnumerable <String> values;
batchResponse.Headers.TryGetValues("nextpageUri", out values);
if (values != null)
{
pageUri = (values as string[]).FirstOrDefault();
}
else
{
pageUri = null;
}
foreach (var content in contentReturned)
{
var dataResponse = await client.GetAsync(content.contentUri);
var dataResponsePayload = await dataResponse.Content.ReadAsStringAsync();
if (!dataResponse.IsSuccessStatusCode)// check if the httpclient success or not
{
var apiFailure = new ApiFailure()
{
MainPayload = payloadRetrieved,
RawResponse = batchResponse,
DataPayload = dataResponsePayload,
};
// We will not fail if the content payload his an error
Log(apiFailure, connstring, schema, start);
}
else
{
var contentReturnedAuditLog = JsonConvert.DeserializeObject <List <AuditLogJson> >(dataResponsePayload);
content.RecordCount = O365ETL.SQLOperations.InsertAuditLog(contentReturnedAuditLog, connstring, schema, start.ToString("yyyyMMddHHmmss"), dataResponsePayload);
}
}
} while (pageUri != null); // check if page is null then quit the loop
}