public async Task BatchResponseContent_InitializeWithNoContentAsync()
{
HttpResponseMessage httpResponseMessage = new HttpResponseMessage(HttpStatusCode.BadRequest);
BatchResponseContent batchResponseContent = new BatchResponseContent(httpResponseMessage);
Dictionary <string, HttpResponseMessage> responses = await batchResponseContent.GetResponsesAsync();
HttpResponseMessage httpResponse = await batchResponseContent.GetResponseByIdAsync("1");
Assert.NotNull(responses);
Assert.Null(httpResponse);
Assert.True(responses.Count.Equals(0));
}
public async static Task BatchRequestExample(GraphServiceClient graphServiceClient, int limit)
{
var watch = new System.Diagnostics.Stopwatch();
watch.Start();
var batchRequestContent = new BatchRequestContent();
var teams = DbOperations.GetTeams(limit);
// 1. construct a Batch request
for (int i = 0; i < teams.Count; i++)
{
var requestUrl1 = graphServiceClient.Groups[teams[i].TeamId].Drive.Root
.Delta()
.Request().RequestUrl;
var request1 = new HttpRequestMessage(HttpMethod.Get, requestUrl1);
var requestStep1 = new BatchRequestStep($"{i}", request1, null);
batchRequestContent.AddBatchRequestStep(requestStep1);
}
//3. Submit request
var batchRequest = new HttpRequestMessage(HttpMethod.Post, "https://graph.microsoft.com/v1.0/$batch");
batchRequest.Content = batchRequestContent;
await graphServiceClient.AuthenticationProvider.AuthenticateRequestAsync(batchRequest);
var httpClient = new HttpClient();
var batchResponse = await httpClient.SendAsync(batchRequest);
// 3. Process response
var batchResponseContent = new BatchResponseContent(batchResponse);
var responses = await batchResponseContent.GetResponsesAsync();
foreach (var response in responses)
{
if (response.Value.IsSuccessStatusCode)
{
//Console.WriteLine();
//Console.WriteLine($"response {response.Key} - {await response.Value.Content.ReadAsStringAsync()}");
//Console.WriteLine();
Console.WriteLine($"response {response.Key}");
}
}
watch.Stop();
Console.WriteLine($"Checking Teams completed on {watch.ElapsedMilliseconds / 1000} seconds");
}
public async Task BatchResponseContent_InitializeWithEmptyResponseContentAsync()
{
string jsonResponse = "{ responses: [] }";
HttpContent content = new StringContent(jsonResponse);
HttpResponseMessage httpResponseMessage = new HttpResponseMessage(HttpStatusCode.BadRequest);
httpResponseMessage.Content = content;
BatchResponseContent batchResponseContent = new BatchResponseContent(httpResponseMessage);
Dictionary <string, HttpResponseMessage> responses = await batchResponseContent.GetResponsesAsync();
HttpResponseMessage httpResponse = await batchResponseContent.GetResponseByIdAsync("1");
Assert.NotNull(responses);
Assert.Null(httpResponse);
Assert.NotNull(batchResponseContent.Serializer);
Assert.True(responses.Count.Equals(0));
}
private async Task ParseBatchResponseAsync(IGraphServiceClient client, BatchRequestContent batchRequestContent, BatchResponseContent batchResponseContent, List <SignIn> signIns, int retryCount = 0)
{
AuditLogRootRestrictedSignInsCollectionResponse collection;
Dictionary <string, HttpResponseMessage> responses;
client.AssertNotNull(nameof(client));
batchRequestContent.AssertNotNull(nameof(batchRequestContent));
batchResponseContent.AssertNotNull(nameof(batchResponseContent));
signIns.AssertNotNull(nameof(signIns));
responses = await batchResponseContent.GetResponsesAsync().ConfigureAwait(false);
foreach (KeyValuePair <string, HttpResponseMessage> item in responses.Where(item => item.Value.IsSuccessStatusCode))
{
collection = await batchResponseContent
.GetResponseByIdAsync <AuditLogRootRestrictedSignInsCollectionResponse>(item.Key).ConfigureAwait(false);
collection.AdditionalData.TryGetValue("@odata.nextLink", out object nextPageLink);
string nextPageLinkString = nextPageLink as string;
if (!string.IsNullOrEmpty(nextPageLinkString))
{
collection.Value.InitializeNextPageRequest(client, nextPageLinkString);
}
if (collection.Value.NextPageRequest != null)
{
PagedRequests.Enqueue(collection.Value.NextPageRequest);
}
signIns.AddRange(collection.Value);
}
if (PagedRequests.Count >= 5)
{
await ProcessPagedRequestsAsync(client, signIns).ConfigureAwait(false);
}
await RetryRequestWithTransientFaultAsync(client, batchRequestContent, responses, signIns, ++retryCount).ConfigureAwait(false);
}
public async Task JsonBatchRequest()
{
string token = await GetAccessTokenUsingPasswordGrant();
HttpClient httpClient = new HttpClient();
httpClient.DefaultRequestHeaders.Add("Authorization", "Bearer " + token);
HttpRequestMessage httpRequestMessage1 = new HttpRequestMessage(HttpMethod.Get, "https://graph.microsoft.com/v1.0/me/");
String body = "{" +
"\"displayName\": \"My Notebook\"" +
"}";
HttpRequestMessage httpRequestMessage2 = new HttpRequestMessage(HttpMethod.Post, "https://graph.microsoft.com/v1.0/me/onenote/notebooks");
httpRequestMessage2.Content = new StringContent(body, Encoding.UTF8, "application/json");
BatchRequestStep requestStep1 = new BatchRequestStep("1", httpRequestMessage1, null);
BatchRequestStep requestStep2 = new BatchRequestStep("2", httpRequestMessage2, new List <string> {
"1"
});
BatchRequestContent batchRequestContent = new BatchRequestContent();
batchRequestContent.AddBatchRequestStep(requestStep1);
batchRequestContent.AddBatchRequestStep(requestStep2);
HttpResponseMessage response = await httpClient.PostAsync("https://graph.microsoft.com/v1.0/$batch", batchRequestContent);
BatchResponseContent batchResponseContent = new BatchResponseContent(response);
Dictionary <string, HttpResponseMessage> responses = await batchResponseContent.GetResponsesAsync();
HttpResponseMessage httpResponse = await batchResponseContent.GetResponseByIdAsync("1");
string nextLink = await batchResponseContent.GetNextLinkAsync();
Assert.True(responses.Count.Equals(2));
}
public async Task BatchResponseContent_GetResponsesAsync()
{
string responseJSON = "{\"responses\":"
+ "[{"
+ "\"id\": \"1\","
+ "\"status\":200,"
+ "\"headers\":{\"Cache-Control\":\"no-cache\",\"OData-Version\":\"4.0\",\"Content-Type\":\"application/json;odata.metadata=minimal;odata.streaming=true;IEEE754Compatible=false;charset=utf-8\"},"
+ "\"body\":{\"@odata.context\":\"https://graph.microsoft.com/v1.0/$metadata#users/$entity\",\"displayName\":\"MOD Administrator\",\"jobTitle\":null,\"id\":\"9f4fe8ea-7e6e-486e-b8f4-VkHdanfIomf\"}"
+ "},"
+ "{"
+ "\"id\": \"2\","
+ "\"status\":200,"
+ "\"headers\":{\"Cache-Control\":\"no-store, no-cache\",\"OData-Version\":\"4.0\",\"Content-Type\":\"application/json;odata.metadata=minimal;odata.streaming=true;IEEE754Compatible=false;charset=utf-8\"},"
+ "\"body\":{\"@odata.context\":\"https://graph.microsoft.com/v1.0/$metadata#drives/$entity\",\"createdDateTime\":\"2019-01-12T09:05:38Z\",\"description\":\"\",\"id\":\"b!random-VkHdanfIomf\",\"lastModifiedDateTime\":\"2019-03-06T06:59:04Z\",\"name\":\"OneDrive\",\"webUrl\":\"https://m365x751487-my.sharepoint.com/personal/admin_m365x751487_onmicrosoft_com/Documents\",\"driveType\":\"business\",\"createdBy\":{\"user\":{\"displayName\":\"System Account\"}},\"lastModifiedBy\":{\"user\":{\"displayName\":\"System Account\"}},\"owner\":{\"user\":{\"email\":\"[email protected]\",\"id\":\"6b4fa8ea-7e6e-486e-a8f4-d00a5b23488c\",\"displayName\":\"MOD Administrator\"}},\"quota\":{\"deleted\":0,\"remaining\":1099509670098,\"state\":\"normal\",\"total\":1099511627776,\"used\":30324}}"
+ "},"
+ "{"
+ "\"id\": \"3\","
+ "\"status\":201,"
+ "\"headers\":{\"Location\":\"https://graph.microsoft.com/v1.0/users/9f4fe8ea-7e6e-486e-a8f4-nothing-here/onenote/notebooks/1-zyz-a1c1-441a-8b41-9378jjdd2\",\"Preference-Applied\":\"odata.include-annotations=*\",\"Cache-Control\":\"no-cache\",\"OData-Version\":\"4.0\",\"Content-Type\":\"application/json;odata.metadata=minimal;odata.streaming=true;IEEE754Compatible=false;charset=utf-8\"},"
+ "\"body\":{\"@odata.context\":\"https://graph.microsoft.com/v1.0/$metadata#users('9f4fe8ea-7e6e-486e-a8f4-nothing-here')/onenote/notebooks/$entity\",\"id\":\"1-9f4fe8ea-7e6e-486e-a8f4-nothing-here\",\"self\":\"https://graph.microsoft.com/v1.0/users/9f4fe8ea-7e6e-486e-a8f4-nothing-here/onenote/notebooks/1-9f4fe8ea-7e6e-486e-a8f4-nothing-here\",\"createdDateTime\":\"2019-03-06T08:08:09Z\",\"displayName\":\"My Notebook -442293399\",\"lastModifiedDateTime\":\"2019-03-06T08:08:09Z\"}"
+ "}]}";
HttpContent content = new StringContent(responseJSON);
HttpResponseMessage httpResponseMessage = new HttpResponseMessage(HttpStatusCode.OK);
httpResponseMessage.Content = content;
BatchResponseContent batchResponseContent = new BatchResponseContent(httpResponseMessage);
Dictionary <string, HttpResponseMessage> responses = await batchResponseContent.GetResponsesAsync();
Assert.NotNull(responses);
Assert.True(responses.Count.Equals(3));
Assert.False(responses["1"].Headers.CacheControl.NoStore);
Assert.True(responses["2"].Headers.CacheControl.NoCache);
Assert.True(responses["2"].Headers.CacheControl.NoStore);
Assert.Equal(HttpStatusCode.Created, responses["3"].StatusCode);
}
private static async Task AddEventsInBatch(IConfigurationRoot config, GraphServiceClient client)
{
var events = new List <Event>();
int maxNoBatchItems = 20;
for (int i = 0; i < maxNoBatchItems * 3; i++)
{
var @event = new Event
{
Subject = "Subject" + i,
Body = new ItemBody
{
ContentType = BodyType.Html,
Content = "Content" + i
},
Start = new DateTimeTimeZone
{
DateTime = DateTime.UtcNow.AddHours(i).ToString("yyyy-MM-ddTHH:mm:ss", CultureInfo.InvariantCulture),
TimeZone = TimeZoneInfo.Utc.Id
},
End = new DateTimeTimeZone
{
DateTime = DateTime.UtcNow.AddHours(i).AddMinutes(30).ToString("yyyy-MM-ddTHH:mm:ss", CultureInfo.InvariantCulture),
TimeZone = TimeZoneInfo.Utc.Id
},
Location = new Location
{
DisplayName = "Dummy location"
}
};
events.Add(@event);
}
Console.WriteLine("Creating batches...");
List <BatchRequestContent> batches = new List <BatchRequestContent>();
var batchRequestContent = new BatchRequestContent();
foreach (Event e in events)
{
var httpRequestMessage = new HttpRequestMessage(HttpMethod.Post, $"https://graph.microsoft.com/v1.0/users/{config["CalendarEmail"]}/events")
{
Content = new StringContent(JsonConvert.SerializeObject(e), Encoding.UTF8, "application/json")
};
BatchRequestStep requestStep = new BatchRequestStep(events.IndexOf(e).ToString(), httpRequestMessage, null);
batchRequestContent.AddBatchRequestStep(requestStep);
// Max number of 20 request per batch. So we need to send out multiple batches.
if (events.IndexOf(e) > 0 && events.IndexOf(e) % maxNoBatchItems == 0)
{
batches.Add(batchRequestContent);
batchRequestContent = new BatchRequestContent();
}
}
if (batchRequestContent.BatchRequestSteps.Count < maxNoBatchItems)
{
batches.Add(batchRequestContent);
}
if (batches.Count == 0 && batchRequestContent != null)
{
batches.Add(batchRequestContent);
}
Console.WriteLine("Batches created. Press enter to submit them.");
Console.ReadLine();
Console.WriteLine("Submitting batches...");
List <string> createdEvents = new List <string>();
foreach (BatchRequestContent batch in batches)
{
BatchResponseContent response = null;
try
{
response = await client.Batch.Request().PostAsync(batch);
}
catch (Microsoft.Graph.ClientException ex)
{
Console.WriteLine(ex.Message);
}
Dictionary <string, HttpResponseMessage> responses = await response.GetResponsesAsync();
foreach (string key in responses.Keys)
{
HttpResponseMessage httpResponse = await response.GetResponseByIdAsync(key);
var responseContent = await httpResponse.Content.ReadAsStringAsync();
JObject eventResponse = JObject.Parse(responseContent);
var eventId = (string)eventResponse["id"];
if (eventId != null)
{
createdEvents.Add(eventId);
}
Console.WriteLine($"Response code: {responses[key].StatusCode}-{responses[key].ReasonPhrase}-{eventId}");
}
}
Console.WriteLine($"{events.Count} events created. Press enter to remove them from the calendar.");
Console.ReadLine();
Console.WriteLine($"Removing {createdEvents.Count} events...");
foreach (string eventId in createdEvents)
{
if (eventId != null)
{
await client.Users[config["CalendarEmail"]].Events[eventId]
.Request()
.DeleteAsync();
}
}
Console.WriteLine($"{createdEvents.Count} events where removed from calendar.");
}
public async static Task WatchTeamSite(IList <string> keys)
{
try
{
//var watch = new System.Diagnostics.Stopwatch();
//watch.Start();
var batchRequestContent = new BatchRequestContent();
// 1. construct a Batch request
foreach (var key in keys)
{
if (teamSitesDeltaLinks[key] == null)
{
teamSitesDeltaLinks[key] = new DeltaLinks()
{
LastSyncDate = DateTime.UtcNow.Ticks / 100000000,
DeltaLink = graphClient.Groups[key].Drive.Root
.Delta()
.Request()
.RequestUrl + "?$select=CreatedDateTime,Deleted,File,Folder,LastModifiedDateTime,Root,SharepointIds,Size,WebUrl"
}
}
;
var request = new HttpRequestMessage(HttpMethod.Get, teamSitesDeltaLinks[key].DeltaLink);
var requestStep = new BatchRequestStep($"{key}", request, null);
batchRequestContent.AddBatchRequestStep(requestStep);
}
//3. Submit request
var batchRequest = new HttpRequestMessage(HttpMethod.Post, "https://graph.microsoft.com/v1.0/$batch");
batchRequest.Content = batchRequestContent;
await graphClient.AuthenticationProvider.AuthenticateRequestAsync(batchRequest);
var httpClient = new HttpClient();
var batchResponse = await httpClient.SendAsync(batchRequest);
// 3. Process response
var batchResponseContent = new BatchResponseContent(batchResponse);
var responses = await batchResponseContent.GetResponsesAsync();
foreach (var response in responses)
{
if (!response.Value.IsSuccessStatusCode)
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine($"Issue on response {response.Value}");
Console.ResetColor();
continue;
}
//Console.WriteLine(response.Key);
var deltaResponse = JsonConvert.DeserializeObject <DeltaResponse>
(await response.Value.Content.ReadAsStringAsync());
teamSitesDeltaLinks[response.Key].DeltaLink = deltaResponse.DeltaLink;
if (!firstCall)
{
foreach (var drive in deltaResponse.DriveItems)
{
await ProcessChangesAsync(drive, teamSitesDeltaLinks[response.Key].LastSyncDate);
}
}
}
//watch.Stop();
//Console.WriteLine($"Checking Teams completed on {watch.ElapsedMilliseconds / 1000} seconds");
libraryDeltaCalls++;
}
catch (Exception exc)
{
AddException(exc, "WatchTeamSite");
if (exc.InnerException.Message.Contains("Authentication failed"))
{
Console.WriteLine($"retry again due {exc.InnerException.Message}");
await Task.Delay(500);
await WatchTeamSite(keys);
}
else
{
Console.WriteLine(exc.Message);
Console.WriteLine(exc.InnerException.Message);
Console.WriteLine(exc.StackTrace);
}
}
}
public async Task <IList <GraphProfileInformation> > GetAzureADObjectIdsAsync(IList <string> personnelNumbers, Guid?runId)
{
if (personnelNumbers.Count == 0)
{
return(new List <GraphProfileInformation>());
}
HttpStatusCode[] httpsStatusCodesWithRetryAfterHeader =
{
HttpStatusCode.TooManyRequests, // 429
HttpStatusCode.ServiceUnavailable // 503
};
var retryAfterPolicy = Policy
.HandleResult <HttpResponseMessage>(result =>
httpsStatusCodesWithRetryAfterHeader.Contains(result.StatusCode) &&
result.Headers?.RetryAfter != null)
.WaitAndRetryAsync(
_maxGraphServiceAttempts.MaxRetryAfterAttempts,
sleepDurationProvider: GetSleepDuration,
onRetryAsync: async(response, timeSpan, retryCount, context) =>
{
await _loggingRepository.LogMessageAsync(new LogMessage {
Message = $"Throttled by Graph for the timespan: {timeSpan}. The retry count is {retryCount}.", RunId = runId
});
});
HttpStatusCode[] httpStatusCodesWorthRetryingExponentially =
{
HttpStatusCode.InternalServerError, // 500
HttpStatusCode.BadGateway, // 502
HttpStatusCode.ServiceUnavailable, // 503
HttpStatusCode.GatewayTimeout // 504
};
var exceptionHandlingPolicy = Policy
.Handle <HttpRequestException>()
.OrResult <HttpResponseMessage>(r => httpStatusCodesWorthRetryingExponentially.Contains(r.StatusCode))
.WaitAndRetryAsync(
_maxGraphServiceAttempts.MaxExceptionHandlingAttempts,
retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)),
onRetryAsync: async(timeSpan, retryCount, context) =>
{
await _loggingRepository.LogMessageAsync(new LogMessage {
Message = $"Exponential backoff {retryCount}.", RunId = runId
});
}
);
var graphMemberIdPolicy = retryAfterPolicy.WrapAsync(exceptionHandlingPolicy);
IList <string> unprocessedPersonnelNumbers = new List <string>();
var profiles = new List <GraphProfileInformation>();
if (_cache.Count > 0)
{
foreach (var personnelNumber in personnelNumbers)
{
if (_cache.ContainsKey(personnelNumber))
{
profiles.Add(_cache[personnelNumber]);
}
else
{
unprocessedPersonnelNumbers.Add(personnelNumber);
}
}
}
else
{
unprocessedPersonnelNumbers = personnelNumbers;
}
await _loggingRepository.LogMessageAsync(new LogMessage { Message = $"{unprocessedPersonnelNumbers.Count} out of {personnelNumbers.Count} need to be retrieved from graph.", RunId = runId });
await _loggingRepository.LogMessageAsync(new LogMessage { Message = $"{_cache.Keys.Count} profiles exist in the cache.", RunId = runId });
var fields = string.Join(",", new[] { IdFieldName, PersonnelNumberFieldName, UserPrincipalNameFieldName });
// Graph currently limits batches to 10 requests per batch
// Graph currently limits the number of conditions in a $filter expression to 20
var requestsCreated = 0;
var pnQueue = new Queue <string>(unprocessedPersonnelNumbers);
var totalBatches = Math.Ceiling(unprocessedPersonnelNumbers.Count / BATCH_REQUEST_LIMIT / FILTER_CONDITION_LIMIT * 1d);
var batchCount = 0;
var jobTimer = new Stopwatch();
var batchTimer = new Stopwatch();
var batchTimes = new List <TimeSpan>();
jobTimer.Start();
while (pnQueue.Count > 0)
{
using (var batchRequestContent = new BatchRequestContent())
{
int limit = pnQueue.Count >= FILTER_CONDITION_LIMIT ? 10 : pnQueue.Count;
// add request steps
while (batchRequestContent.BatchRequestSteps.Count < BATCH_REQUEST_LIMIT && pnQueue.Count != 0)
{
var requestPersonnelNumbers = new List <string>();
limit = pnQueue.Count >= FILTER_CONDITION_LIMIT ? 10 : pnQueue.Count;
for (var i = 0; i < limit; i++)
{
requestPersonnelNumbers.Add(pnQueue.Dequeue());
}
// build filter expression
var filter = $"(onPremisesImmutableId eq '{string.Join("' or onPremisesImmutableId eq '", requestPersonnelNumbers.ToArray())}')";
var requestMessage = _graphClient.Users.Request().Filter(filter).Select(fields).WithPerRequestAuthProvider().GetHttpRequestMessage();
requestsCreated++;
batchRequestContent.AddBatchRequestStep(new BatchRequestStep($"{requestsCreated}", requestMessage, null));
}
// every 20 requests send a batch or if there are no more requests to make
batchCount++;
batchTimer.Start();
HttpResponseMessage httpResponse = null;
await graphMemberIdPolicy.ExecuteAsync(async() =>
{
var batchRequest = new HttpRequestMessage(HttpMethod.Post, "https://graph.microsoft.com/v1.0/$batch")
{
Content = batchRequestContent
};
await _graphClient.AuthenticationProvider.AuthenticateRequestAsync(batchRequest);
httpResponse = await _graphClient.HttpProvider.SendAsync(batchRequest);
await _loggingRepository.LogMessageAsync(new LogMessage
{
Message = $"Graph Response:"
+ $"\nStatusCode {httpResponse.StatusCode}",
RunId = runId
});
return(httpResponse);
});
var batchResponse = new BatchResponseContent(httpResponse);
// process each request in the batch
foreach (var response in await batchResponse.GetResponsesAsync())
{
// request was successful
if (response.Value.IsSuccessStatusCode)
{
var content = await response.Value.Content.ReadAsStringAsync();
var oDataResponse = JsonConvert.DeserializeObject <ODataResponse <List <User> > >(content);
// process each user
foreach (var user in oDataResponse.Value)
{
var profile = new GraphProfileInformation
{
Id = user.Id,
PersonnelNumber = user.OnPremisesImmutableId,
UserPrincipalName = user.UserPrincipalName
};
profiles.Add(profile);
_cache.Add(profile.PersonnelNumber, profile);
}
}
else
{
await _loggingRepository.LogMessageAsync(new LogMessage
{
Message = $"Graph Request failures:"
+ $"\nStatusCode {response.Value.StatusCode}"
+ $"\nReasonPhrase {response.Value.ReasonPhrase}"
+ $"\nRequestURI {batchRequestContent.BatchRequestSteps[response.Key].Request.RequestUri}",
RunId = runId
});
}
}
batchTimer.Stop();
batchTimes.Add(batchTimer.Elapsed);
await _loggingRepository.LogMessageAsync(new LogMessage
{
Message = $"Graph Request: {batchCount} of {totalBatches}{Environment.NewLine}"
+ $" Batch Time Elapsed: {batchTimer.ElapsedMilliseconds} ms{Environment.NewLine}"
+ $" Total Time Elapsed: {jobTimer.Elapsed}{Environment.NewLine}"
+ $" Total Profile Count: {profiles.Count}{Environment.NewLine}"
+ $" Total Users Not Found: {(personnelNumbers.Count - pnQueue.Count) - profiles.Count}{Environment.NewLine}"
+ $" Total Queue Remaining: {pnQueue.Count}{Environment.NewLine}",
RunId = runId
});
batchTimer.Reset();
}
}
jobTimer.Stop();
return(profiles);
}