public void ChangeMemberRoles(string id, IList <Member> members) { if (GroupMemberRequestFactory.BatchSize <= 1) { foreach (Member member in members) { this.ChangeMemberRole(id, member); } return; } try { this.WaitForGate(); using (PoolItem <DirectoryService> poolService = this.directoryServicePool.Take(NullValueHandling.Ignore)) { List <ClientServiceRequest <Member> > requests = new List <ClientServiceRequest <Member> >(); foreach (Member member in members) { string memberKey = member.Email ?? member.Id; Trace.WriteLine($"Queuing batch member role change for {memberKey} as {member.Role} to group {id}"); requests.Add(poolService.Item.Members.Patch(member, id, memberKey)); } this.ProcessBatches(id, members, false, false, requests, poolService); } } finally { this.ReleaseGate(); } }
public IEnumerable <ContactEntry> GetContacts(string domain) { using (PoolItem <ContactsService> connection = this.contactsServicePool.Take()) { string uri = ContactsQuery.CreateContactsUri(domain); do { ContactsQuery request = new ContactsQuery(uri) { NumberToRetrieve = 1000 }; ContactsFeed result = ApiExtensions.InvokeWithRateLimit(() => connection.Item.Query(request), this.serviceName); foreach (ContactEntry entry in result.Entries.OfType <ContactEntry>()) { yield return(entry); } uri = result.NextChunk; } while (uri != null); } }
public IEnumerable <CalendarResource> GetCalendars(string customerID, string fields) { using (PoolItem <DirectoryService> connection = this.directoryServicePool.Take(NullValueHandling.Ignore)) { string token = null; ResourcesResource.CalendarsResource.ListRequest request = new ResourcesResource.CalendarsResource.ListRequest(connection.Item, customerID); if (fields != null) { request.MaxResults = 500; request.Fields = fields; } request.PrettyPrint = false; do { request.PageToken = token; CalendarResources pageResults = request.ExecuteWithRetryOnBackoff(); if (pageResults.Items == null) { break; } foreach (CalendarResource item in pageResults.Items) { yield return(item); } token = pageResults.NextPageToken; } while (token != null); } }
public IEnumerable <AclRule> GetCalendarAclRules(string customerId, string calendarId, string fields) { using (PoolItem <CalendarService> connection = this.calendarServicePool.Take(NullValueHandling.Ignore)) { AclResource.ListRequest request = new AclResource.ListRequest(connection.Item, calendarId); string token = null; request.MaxResults = 500; if (fields != null) { request.Fields = fields; } request.PrettyPrint = false; do { request.PageToken = token; Acl pageResults = request.ExecuteWithRetry(RetryEvents.BackoffNotFound); if (pageResults.Items == null) { break; } foreach (AclRule item in pageResults.Items) { yield return(item); } token = pageResults.NextPageToken; } while (token != null); } }
private void ProcessBatches <T>(string id, bool ignoreExistingStudent, bool ignoreMissingStudent, IList <ClientServiceRequest <T> > requests, PoolItem <ClassroomService> poolService, Func <CourseStudentRequestBatchHelper <T>, OnResponse <CoursesResource.StudentsResource> > onResponse) { CourseStudentRequestBatchHelper <T> batchHelper = new CourseStudentRequestBatchHelper <T>() { FailedStudents = new List <string>(), Failures = new List <Exception>(), IgnoreExistingStudent = ignoreExistingStudent, IgnoreMissingStudent = ignoreMissingStudent, RequestsToRetry = new Dictionary <string, ClientServiceRequest <T> >() }; int batchCount = 0; foreach (IEnumerable <ClientServiceRequest <T> > batch in requests.Batch(CourseStudentRequestFactory.BatchSize)) { BatchRequest batchRequest = new BatchRequest(poolService.Item); Trace.WriteLine($"Executing student batch {++batchCount} for course {id}"); foreach (ClientServiceRequest <T> request in batch) { batchHelper.Request = request; batchRequest.Queue <CoursesResource.StudentsResource>(request, onResponse.Invoke(batchHelper)); } batchRequest.ExecuteWithRetryOnBackoff(poolService.Item.Name); batchHelper.BaseCount += CourseStudentRequestFactory.BatchSize; } if (batchHelper.RequestsToRetry.Count > 0) { Trace.WriteLine($"Retrying {batchHelper.RequestsToRetry} student change requests"); } foreach (KeyValuePair <string, ClientServiceRequest <T> > request in batchHelper.RequestsToRetry) { try { request.Value.ExecuteWithRetry(RetryEvents.Backoff | RetryEvents.Timeout, RetryCount); } catch (GoogleApiException e) { if (!(ignoreMissingStudent && this.IsMissingStudentError(e.HttpStatusCode, e.Message))) { if (!(ignoreExistingStudent && this.IsExistingStudentError(e.HttpStatusCode, e.Message))) { batchHelper.Failures.Add(e); batchHelper.FailedStudents.Add(request.Key); } } } catch (Exception ex) { batchHelper.Failures.Add(ex); batchHelper.FailedStudents.Add(request.Key); } } if (batchHelper.Failures.Count == 1) { throw batchHelper.Failures[0]; } else if (batchHelper.Failures.Count > 1) { throw new AggregateCourseStudentException(id, batchHelper.FailedStudents, batchHelper.Failures); } }
private void PopulateGroups(string customerID, string userKey, string groupFields, bool getMembers, bool getSettings, BlockingCollection <GoogleGroup> completedGroups, bool excludeUserCreated, Regex regexFilter, int settingsThreads, int memberThreads) { BlockingCollection <GoogleGroup> settingsQueue = new BlockingCollection <GoogleGroup>(); BlockingCollection <GoogleGroup> membersQueue = new BlockingCollection <GoogleGroup>(); BlockingCollection <GoogleGroup> incomingGroups = new BlockingCollection <GoogleGroup>(); List <Task> tasks = new List <Task>(); if (getSettings) { Task t = new Task(() => this.ConsumeGroupSettingsQueue(settingsThreads, settingsQueue, completedGroups)); t.Start(); tasks.Add(t); } if (getMembers) { Task t = new Task(() => this.ConsumeGroupMembershipQueue(memberThreads, membersQueue, completedGroups)); t.Start(); tasks.Add(t); } Task t1 = new Task(() => { using (PoolItem <DirectoryService> connection = this.directoryServicePool.Take(NullValueHandling.Ignore)) { string token = null; GroupsResource.ListRequest request = connection.Item.Groups.List(); request.Customer = customerID; request.UserKey = userKey; request.MaxResults = 200; request.Fields = groupFields; request.PrettyPrint = false; do { request.PageToken = token; Groups pageResults = request.ExecuteWithRetry(RetryEvents.Backoff | RetryEvents.Timeout); if (pageResults.GroupsValue == null) { break; } foreach (Group group in pageResults.GroupsValue) { if (regexFilter != null) { if (!regexFilter.IsMatch(group.Email)) { Trace.WriteLine($"Ignoring group that doesn't match regex filter {group.Email}"); continue; } } if (excludeUserCreated) { if (!group.AdminCreated.HasValue || !group.AdminCreated.Value) { Trace.WriteLine($"Ignoring user created group {group.Email}"); continue; } } GoogleGroup g = new GoogleGroup(group); g.RequiresMembers = getMembers; g.RequiresSettings = getSettings; incomingGroups.Add(g); if (getSettings) { settingsQueue.Add(g); } if (getMembers) { membersQueue.Add(g); } if (!getSettings && !getMembers) { completedGroups.Add(g); } } token = pageResults.NextPageToken; } while (token != null); incomingGroups.CompleteAdding(); settingsQueue.CompleteAdding(); membersQueue.CompleteAdding(); } }); t1.Start(); tasks.Add(t1); Task.WhenAll(tasks).ContinueWith((a) => completedGroups.CompleteAdding()); }
public void Return(PoolItem <T> item) { this.items.Add(item); }
private void ProcessBatches <T>(string id, IList <T> members, bool ignoreExistingMember, bool ignoreMissingMember, IList <ClientServiceRequest <T> > requests, PoolItem <DirectoryService> poolService) { List <string> failedMembers = new List <string>(); List <Exception> failures = new List <Exception>(); Dictionary <string, ClientServiceRequest <T> > requestsToRetry = new Dictionary <string, ClientServiceRequest <T> >(); int baseCount = 0; int batchCount = 0; foreach (IEnumerable <ClientServiceRequest <T> > batch in requests.Batch(GroupMemberRequestFactory.BatchSize)) { BatchRequest batchRequest = new BatchRequest(poolService.Item); Trace.WriteLine($"Executing batch {++batchCount} for group {id}"); foreach (ClientServiceRequest <T> request in batch) { batchRequest.Queue <MembersResource>(request, (content, error, i, message) => { int index = baseCount + i; this.ProcessMemberResponse(id, members[index], ignoreExistingMember, ignoreMissingMember, error, message, requestsToRetry, request, failedMembers, failures); }); } batchRequest.ExecuteWithRetryOnBackoff(poolService.Item.Name); baseCount += GroupMemberRequestFactory.BatchSize; } if (requestsToRetry.Count > 0) { Trace.WriteLine($"Retrying {requestsToRetry} member change requests"); } foreach (KeyValuePair <string, ClientServiceRequest <T> > request in requestsToRetry) { try { request.Value.ExecuteWithRetryOnBackoff(); } catch (GoogleApiException e) { if (!(ignoreMissingMember && this.IsMissingMemberError(e.HttpStatusCode, e.Message))) { if (!(ignoreExistingMember && this.IsExistingMemberError(e.HttpStatusCode, e.Message))) { failures.Add(e); failedMembers.Add(request.Key); } } } catch (Exception ex) { failures.Add(ex); failedMembers.Add(request.Key); } } if (failures.Count == 1) { throw failures[0]; } else if (failures.Count > 1) { throw new AggregateGroupUpdateException(id, failedMembers, failures); } }
private void PopulateCourses(string customerID, string userKey, bool getStudents, bool getTeachers, bool skipMembersForArchived, BlockingCollection <GoogleCourse> completedCourses, int memberThreads) { BlockingCollection <GoogleCourse> studentsQueue = new BlockingCollection <GoogleCourse>(); BlockingCollection <GoogleCourse> teachersQueue = new BlockingCollection <GoogleCourse>(); BlockingCollection <GoogleCourse> incomingCourses = new BlockingCollection <GoogleCourse>(); List <Task> tasks = new List <Task>(); if (getStudents) { Task t = new Task(() => this.ConsumeStudentsQueue(memberThreads, studentsQueue, completedCourses)); t.Start(); tasks.Add(t); } if (getTeachers) { Task t = new Task(() => this.ConsumeTeachersQueue(memberThreads, teachersQueue, completedCourses)); t.Start(); tasks.Add(t); } Task t1 = new Task(() => { using (PoolItem <ClassroomService> connection = this.classroomServicePool.Take(NullValueHandling.Ignore)) { string token = null; CoursesResource.ListRequest request = new CoursesResource.ListRequest(connection.Item) { //PageSize = 500 }; request.PrettyPrint = false; do { request.PageToken = token; ListCoursesResponse pageResults = request.ExecuteWithRetry(RetryEvents.Backoff | RetryEvents.Timeout, RetryCount); if (pageResults.Courses == null) { break; } if (pageResults.Courses == null) { break; } foreach (Course item in pageResults.Courses) { GoogleCourse c = new GoogleCourse(item); c.RequiresStudents = getStudents && getMembersForCourse(c.Course, skipMembersForArchived); c.RequiresTeachers = getTeachers && getMembersForCourse(c.Course, skipMembersForArchived); incomingCourses.Add(c); if (c.RequiresStudents) { studentsQueue.Add(c); } if (c.RequiresTeachers) { teachersQueue.Add(c); } if (!c.RequiresStudents && !c.RequiresTeachers) { completedCourses.Add(c); } } token = pageResults.NextPageToken; } while (token != null); incomingCourses.CompleteAdding(); studentsQueue.CompleteAdding(); teachersQueue.CompleteAdding(); } }); t1.Start(); tasks.Add(t1); Task.WhenAll(tasks).ContinueWith((a) => completedCourses.CompleteAdding()); }