public void ChangeMemberRoles(string id, IList <Member> members)
{
if (GroupMemberRequestFactory.BatchSize <= 1)
{
foreach (Member member in members)
{
this.ChangeMemberRole(id, member);
}
return;
}
try
{
this.WaitForGate();
using (PoolItem <DirectoryService> poolService = this.directoryServicePool.Take(NullValueHandling.Ignore))
{
List <ClientServiceRequest <Member> > requests = new List <ClientServiceRequest <Member> >();
foreach (Member member in members)
{
string memberKey = member.Email ?? member.Id;
Trace.WriteLine($"Queuing batch member role change for {memberKey} as {member.Role} to group {id}");
requests.Add(poolService.Item.Members.Patch(member, id, memberKey));
}
this.ProcessBatches(id, members, false, false, requests, poolService);
}
}
finally
{
this.ReleaseGate();
}
}
public IEnumerable <ContactEntry> GetContacts(string domain)
{
using (PoolItem <ContactsService> connection = this.contactsServicePool.Take())
{
string uri = ContactsQuery.CreateContactsUri(domain);
do
{
ContactsQuery request = new ContactsQuery(uri)
{
NumberToRetrieve = 1000
};
ContactsFeed result = ApiExtensions.InvokeWithRateLimit(() => connection.Item.Query(request), this.serviceName);
foreach (ContactEntry entry in result.Entries.OfType <ContactEntry>())
{
yield return(entry);
}
uri = result.NextChunk;
} while (uri != null);
}
}
public IEnumerable <CalendarResource> GetCalendars(string customerID, string fields)
{
using (PoolItem <DirectoryService> connection = this.directoryServicePool.Take(NullValueHandling.Ignore))
{
string token = null;
ResourcesResource.CalendarsResource.ListRequest request = new ResourcesResource.CalendarsResource.ListRequest(connection.Item, customerID);
if (fields != null)
{
request.MaxResults = 500;
request.Fields = fields;
}
request.PrettyPrint = false;
do
{
request.PageToken = token;
CalendarResources pageResults = request.ExecuteWithRetryOnBackoff();
if (pageResults.Items == null)
{
break;
}
foreach (CalendarResource item in pageResults.Items)
{
yield return(item);
}
token = pageResults.NextPageToken;
} while (token != null);
}
}
public IEnumerable <AclRule> GetCalendarAclRules(string customerId, string calendarId, string fields)
{
using (PoolItem <CalendarService> connection = this.calendarServicePool.Take(NullValueHandling.Ignore))
{
AclResource.ListRequest request = new AclResource.ListRequest(connection.Item, calendarId);
string token = null;
request.MaxResults = 500;
if (fields != null)
{
request.Fields = fields;
}
request.PrettyPrint = false;
do
{
request.PageToken = token;
Acl pageResults = request.ExecuteWithRetry(RetryEvents.BackoffNotFound);
if (pageResults.Items == null)
{
break;
}
foreach (AclRule item in pageResults.Items)
{
yield return(item);
}
token = pageResults.NextPageToken;
} while (token != null);
}
}
private void ProcessBatches <T>(string id, bool ignoreExistingStudent, bool ignoreMissingStudent, IList <ClientServiceRequest <T> > requests, PoolItem <ClassroomService> poolService, Func <CourseStudentRequestBatchHelper <T>, OnResponse <CoursesResource.StudentsResource> > onResponse)
{
CourseStudentRequestBatchHelper <T> batchHelper = new CourseStudentRequestBatchHelper <T>()
{
FailedStudents = new List <string>(),
Failures = new List <Exception>(),
IgnoreExistingStudent = ignoreExistingStudent,
IgnoreMissingStudent = ignoreMissingStudent,
RequestsToRetry = new Dictionary <string, ClientServiceRequest <T> >()
};
int batchCount = 0;
foreach (IEnumerable <ClientServiceRequest <T> > batch in requests.Batch(CourseStudentRequestFactory.BatchSize))
{
BatchRequest batchRequest = new BatchRequest(poolService.Item);
Trace.WriteLine($"Executing student batch {++batchCount} for course {id}");
foreach (ClientServiceRequest <T> request in batch)
{
batchHelper.Request = request;
batchRequest.Queue <CoursesResource.StudentsResource>(request, onResponse.Invoke(batchHelper));
}
batchRequest.ExecuteWithRetryOnBackoff(poolService.Item.Name);
batchHelper.BaseCount += CourseStudentRequestFactory.BatchSize;
}
if (batchHelper.RequestsToRetry.Count > 0)
{
Trace.WriteLine($"Retrying {batchHelper.RequestsToRetry} student change requests");
}
foreach (KeyValuePair <string, ClientServiceRequest <T> > request in batchHelper.RequestsToRetry)
{
try
{
request.Value.ExecuteWithRetry(RetryEvents.Backoff | RetryEvents.Timeout, RetryCount);
}
catch (GoogleApiException e)
{
if (!(ignoreMissingStudent && this.IsMissingStudentError(e.HttpStatusCode, e.Message)))
{
if (!(ignoreExistingStudent && this.IsExistingStudentError(e.HttpStatusCode, e.Message)))
{
batchHelper.Failures.Add(e);
batchHelper.FailedStudents.Add(request.Key);
}
}
}
catch (Exception ex)
{
batchHelper.Failures.Add(ex);
batchHelper.FailedStudents.Add(request.Key);
}
}
if (batchHelper.Failures.Count == 1)
{
throw batchHelper.Failures[0];
}
else if (batchHelper.Failures.Count > 1)
{
throw new AggregateCourseStudentException(id, batchHelper.FailedStudents, batchHelper.Failures);
}
}
private void PopulateGroups(string customerID, string userKey, string groupFields, bool getMembers, bool getSettings, BlockingCollection <GoogleGroup> completedGroups, bool excludeUserCreated, Regex regexFilter, int settingsThreads, int memberThreads)
{
BlockingCollection <GoogleGroup> settingsQueue = new BlockingCollection <GoogleGroup>();
BlockingCollection <GoogleGroup> membersQueue = new BlockingCollection <GoogleGroup>();
BlockingCollection <GoogleGroup> incomingGroups = new BlockingCollection <GoogleGroup>();
List <Task> tasks = new List <Task>();
if (getSettings)
{
Task t = new Task(() => this.ConsumeGroupSettingsQueue(settingsThreads, settingsQueue, completedGroups));
t.Start();
tasks.Add(t);
}
if (getMembers)
{
Task t = new Task(() => this.ConsumeGroupMembershipQueue(memberThreads, membersQueue, completedGroups));
t.Start();
tasks.Add(t);
}
Task t1 = new Task(() =>
{
using (PoolItem <DirectoryService> connection = this.directoryServicePool.Take(NullValueHandling.Ignore))
{
string token = null;
GroupsResource.ListRequest request = connection.Item.Groups.List();
request.Customer = customerID;
request.UserKey = userKey;
request.MaxResults = 200;
request.Fields = groupFields;
request.PrettyPrint = false;
do
{
request.PageToken = token;
Groups pageResults = request.ExecuteWithRetry(RetryEvents.Backoff | RetryEvents.Timeout);
if (pageResults.GroupsValue == null)
{
break;
}
foreach (Group group in pageResults.GroupsValue)
{
if (regexFilter != null)
{
if (!regexFilter.IsMatch(group.Email))
{
Trace.WriteLine($"Ignoring group that doesn't match regex filter {group.Email}");
continue;
}
}
if (excludeUserCreated)
{
if (!group.AdminCreated.HasValue || !group.AdminCreated.Value)
{
Trace.WriteLine($"Ignoring user created group {group.Email}");
continue;
}
}
GoogleGroup g = new GoogleGroup(group);
g.RequiresMembers = getMembers;
g.RequiresSettings = getSettings;
incomingGroups.Add(g);
if (getSettings)
{
settingsQueue.Add(g);
}
if (getMembers)
{
membersQueue.Add(g);
}
if (!getSettings && !getMembers)
{
completedGroups.Add(g);
}
}
token = pageResults.NextPageToken;
} while (token != null);
incomingGroups.CompleteAdding();
settingsQueue.CompleteAdding();
membersQueue.CompleteAdding();
}
});
t1.Start();
tasks.Add(t1);
Task.WhenAll(tasks).ContinueWith((a) => completedGroups.CompleteAdding());
}
public void Return(PoolItem <T> item)
{
this.items.Add(item);
}
private void ProcessBatches <T>(string id, IList <T> members, bool ignoreExistingMember, bool ignoreMissingMember, IList <ClientServiceRequest <T> > requests, PoolItem <DirectoryService> poolService)
{
List <string> failedMembers = new List <string>();
List <Exception> failures = new List <Exception>();
Dictionary <string, ClientServiceRequest <T> > requestsToRetry = new Dictionary <string, ClientServiceRequest <T> >();
int baseCount = 0;
int batchCount = 0;
foreach (IEnumerable <ClientServiceRequest <T> > batch in requests.Batch(GroupMemberRequestFactory.BatchSize))
{
BatchRequest batchRequest = new BatchRequest(poolService.Item);
Trace.WriteLine($"Executing batch {++batchCount} for group {id}");
foreach (ClientServiceRequest <T> request in batch)
{
batchRequest.Queue <MembersResource>(request,
(content, error, i, message) =>
{
int index = baseCount + i;
this.ProcessMemberResponse(id, members[index], ignoreExistingMember, ignoreMissingMember, error, message, requestsToRetry, request, failedMembers, failures);
});
}
batchRequest.ExecuteWithRetryOnBackoff(poolService.Item.Name);
baseCount += GroupMemberRequestFactory.BatchSize;
}
if (requestsToRetry.Count > 0)
{
Trace.WriteLine($"Retrying {requestsToRetry} member change requests");
}
foreach (KeyValuePair <string, ClientServiceRequest <T> > request in requestsToRetry)
{
try
{
request.Value.ExecuteWithRetryOnBackoff();
}
catch (GoogleApiException e)
{
if (!(ignoreMissingMember && this.IsMissingMemberError(e.HttpStatusCode, e.Message)))
{
if (!(ignoreExistingMember && this.IsExistingMemberError(e.HttpStatusCode, e.Message)))
{
failures.Add(e);
failedMembers.Add(request.Key);
}
}
}
catch (Exception ex)
{
failures.Add(ex);
failedMembers.Add(request.Key);
}
}
if (failures.Count == 1)
{
throw failures[0];
}
else if (failures.Count > 1)
{
throw new AggregateGroupUpdateException(id, failedMembers, failures);
}
}
private void PopulateCourses(string customerID, string userKey, bool getStudents, bool getTeachers, bool skipMembersForArchived, BlockingCollection <GoogleCourse> completedCourses, int memberThreads)
{
BlockingCollection <GoogleCourse> studentsQueue = new BlockingCollection <GoogleCourse>();
BlockingCollection <GoogleCourse> teachersQueue = new BlockingCollection <GoogleCourse>();
BlockingCollection <GoogleCourse> incomingCourses = new BlockingCollection <GoogleCourse>();
List <Task> tasks = new List <Task>();
if (getStudents)
{
Task t = new Task(() => this.ConsumeStudentsQueue(memberThreads, studentsQueue, completedCourses));
t.Start();
tasks.Add(t);
}
if (getTeachers)
{
Task t = new Task(() => this.ConsumeTeachersQueue(memberThreads, teachersQueue, completedCourses));
t.Start();
tasks.Add(t);
}
Task t1 = new Task(() =>
{
using (PoolItem <ClassroomService> connection = this.classroomServicePool.Take(NullValueHandling.Ignore))
{
string token = null;
CoursesResource.ListRequest request = new CoursesResource.ListRequest(connection.Item)
{
//PageSize = 500
};
request.PrettyPrint = false;
do
{
request.PageToken = token;
ListCoursesResponse pageResults = request.ExecuteWithRetry(RetryEvents.Backoff | RetryEvents.Timeout, RetryCount);
if (pageResults.Courses == null)
{
break;
}
if (pageResults.Courses == null)
{
break;
}
foreach (Course item in pageResults.Courses)
{
GoogleCourse c = new GoogleCourse(item);
c.RequiresStudents = getStudents && getMembersForCourse(c.Course, skipMembersForArchived);
c.RequiresTeachers = getTeachers && getMembersForCourse(c.Course, skipMembersForArchived);
incomingCourses.Add(c);
if (c.RequiresStudents)
{
studentsQueue.Add(c);
}
if (c.RequiresTeachers)
{
teachersQueue.Add(c);
}
if (!c.RequiresStudents && !c.RequiresTeachers)
{
completedCourses.Add(c);
}
}
token = pageResults.NextPageToken;
} while (token != null);
incomingCourses.CompleteAdding();
studentsQueue.CompleteAdding();
teachersQueue.CompleteAdding();
}
});
t1.Start();
tasks.Add(t1);
Task.WhenAll(tasks).ContinueWith((a) => completedCourses.CompleteAdding());
}
