public async static Task UpdateSubscribers(IGovDeliveryApiService service, IGovDeliveryContext ctx) { var localSubscribers = ctx.Subscribers.ToList(); var subscriberEnumerator = localSubscribers.GetEnumerator(); // pull x subscribers and request their data var updateTasks = Enumerable.Range(0, 5) .Select(n => { var subscriberEntity = subscriberEnumerator.Current; subscriberEnumerator.MoveNext(); return(UpdateSubscriberAsync(subscriberEntity, service, ctx)); }) .ToList(); // after each request comes back, save data, pick next eligible subscriber until none are left. while (updateTasks.Count() > 0) { var t = await Task.WhenAny(updateTasks); updateTasks.Remove(t); if (subscriberEnumerator.MoveNext()) { var subscriber = subscriberEnumerator.Current; updateTasks.Add(UpdateSubscriberAsync(subscriber, service, ctx)); } } Task.WhenAll(updateTasks).GetAwaiter().GetResult(); }
public static async Task <int> SyncSubscribers(IGovDeliveryApiService service, GovDeliveryContextFactory factory) { Console.WriteLine(" Syncing Subscribers and Subscriptions..."); await BusinessTasks.UpdateSubscribersAsync(service, factory, s => Console.WriteLine(s)); Console.WriteLine("Subscriber sync successful."); return(0); }
public static async Task <int> SyncTopics(IGovDeliveryApiService service, GovDeliveryContextFactory factory) { Console.WriteLine("Syncing Topics..."); await BusinessTasks.SyncTopics(service, factory.CreateDbContext(), s => Console.WriteLine(s)); Console.WriteLine("Topic sync Successful."); return(0); }
public static async Task <int> PerformFullSync(IGovDeliveryApiService service, GovDeliveryContextFactory factory) { Console.WriteLine("Beginning sync..."); await SyncTopics(service, factory); await SyncCategories(service, factory); await SyncSubscribers(service, factory); Console.WriteLine("Sync successful."); return(0); }
public async static Task UpdateSubscribersAsync <T>( IGovDeliveryApiService service, IGovDeliveryContextFactory <T> factory, Action <string> loggingDelegate = null ) where T : AbstractGovDeliveryContext { var ctx = factory.CreateDbContext(); var localSubscribers = ctx.Subscribers.ToList(); loggingDelegate?.Invoke($"Found {localSubscribers.Count} subscribers to update..."); var subscriberEnumerator = localSubscribers.GetEnumerator(); // pull x subscribers and request their data var updateTasks = Enumerable.Range(0, 10) .Select(n => { subscriberEnumerator.MoveNext(); var subscriberEntity = subscriberEnumerator.Current; return(UpdateSingleSubscriberAsync(subscriberEntity.Id, service, factory, loggingDelegate)); }) .ToList(); var taskCounter = 0; // after each request comes back, save data, pick next eligible subscriber until none are left. while (updateTasks.Count() > 0) { if (taskCounter % 100 == 0) { loggingDelegate?.Invoke($"Updated {taskCounter} subscribers of {localSubscribers.Count}..."); } var t = await Task.WhenAny(updateTasks); // get latest finished task updateTasks.Remove(t); // remove it from the queue if (subscriberEnumerator.MoveNext()) { var subscriber = subscriberEnumerator.Current; updateTasks.Add(UpdateSingleSubscriberAsync(subscriber.Id, service, factory, loggingDelegate)); } taskCounter++; } await Task.WhenAll(updateTasks); }
protected async static Task UpdateSubscriberAsync(Subscriber subscriber, IGovDeliveryApiService service, IGovDeliveryContext ctx) { var subscriberInfoTask = service.ReadSubscriberAsync(subscriber.Email); var subscriberTopicsTask = service.ListSubscriberTopicsAsync(subscriber.Email); var subscriberCategoriesTask = service.ListSubscriberCategoriesAsync(subscriber.Email); await Task.WhenAll(new List <Task> { subscriberInfoTask, subscriberTopicsTask, subscriberCategoriesTask }); var subscriberInfo = subscriberInfoTask.Result.Data; // Update detailed subscriber info subscriber.BulletinFrequency = (BulletinFrequency)subscriberInfo.DigestFor.Value; subscriber.GovDeliveryId = subscriberInfo.Id.Value; subscriber.Phone = subscriberInfo.Phone; subscriber.SendSubscriberUpdateNotifications = subscriberInfo.SendSubscriberUpdateNotifications.Value; // Update Category Subscriptions var subscriberCategories = subscriber .CategorySubscriptions.Select(esc => esc.Category) .ToList(); var subscriberCategoryInfo = subscriberCategoriesTask.Result.Data.Items; // existing category subscriptions - do nothing. var newCategorySubscriptions = subscriberCategoryInfo .Where(sci => !subscriberCategories.Any(sc => sc.Code == sci.CategoryCode)); foreach (var nCS in newCategorySubscriptions) { var cat = ctx.Categories.First(c => c.Code == nCS.CategoryCode); ctx.Add(new CategorySubscription { CategoryId = cat.Id, Category = cat, SubscriberId = subscriber.Id, Subscriber = subscriber }); } var deleteableCategorySubscriptions = subscriberCategories .Where(sc => !subscriberCategoryInfo.Any(sci => sci.CategoryCode == sc.Code)) .ToList(); foreach (var dSC in deleteableCategorySubscriptions) { var categorySub = ctx.CategorySubscriptions.First(cs => cs.Category.Code == dSC.Code); ctx.Remove(categorySub); } ctx.SaveChanges(); // New Topic subscriptions var subscriberTopics = subscriber .TopicSubscriptions.Select(est => est.Topic) .ToList(); var subscriberTopicInfo = subscriberTopicsTask.Result.Data.Items; var newTopicSubscriptions = subscriberTopicInfo .Where(sti => !subscriberTopics.Any(st => st.Code == sti.TopicCode)) .ToList(); foreach (var nTS in newTopicSubscriptions) { var topic = ctx.Topics.First(t => t.Code == nTS.TopicCode); ctx.Add(new TopicSubscription { Subscriber = subscriber, SubscriberId = subscriber.Id, Topic = topic, TopicId = topic.Id, }); } // deleteable topic subscriptions var deleteableTopicSubscriptions = subscriberTopics .Where(st => !subscriberTopicInfo.Any(sti => sti.TopicCode == st.Code)) .ToList(); foreach (var dts in deleteableTopicSubscriptions) { var topicSub = ctx.TopicSubscriptions.First(ts => ts.TopicId == dts.Id); ctx.Remove(topicSub); } ctx.SaveChanges(); }
public async static Task SyncTopics(IGovDeliveryApiService service, IGovDeliveryContext ctx) { var topicsResult = await service.ListTopicsAsync(); if (!topicsResult.HttpResponse.IsSuccessStatusCode) { Console.Error.WriteLine($@"Error getting Topics: {topicsResult.HttpResponse.StatusCode} - {topicsResult.HttpResponse.ReasonPhrase}"); } var numTopics = topicsResult.Data.Items != null ? topicsResult.Data.Items.Count : 0; Console.WriteLine($"Fetched {numTopics} topics."); if (numTopics > 0) { var remoteTopics = topicsResult.Data.Items .Select(i => new Topic { Id = Guid.NewGuid(), Code = i.Code, Description = i.Description.Value, Name = i.Name, ShortName = i.ShortName, WirelessEnabled = i.WirelessEnabled.Value }) .ToList(); var localTopics = ctx.Topics.ToList(); // Add new topics not present locally: var newTopics = remoteTopics .Where(rt => !localTopics.Any(lt => lt.Code == rt.Code)) .ToList(); ctx.Topics.AddRange(newTopics); ctx.SaveChanges(); // Update topics present both remotely and locally: var existingTopics = localTopics .Where(lt => remoteTopics.Any(rt => rt.Code == lt.Code)) .ToList(); foreach (var localTopic in existingTopics) { var remoteTopic = remoteTopics.First(rt => rt.Code == localTopic.Code); var topicResult = await service.ReadTopicAsync(localTopic.Code); var topicInfo = topicResult.Data; var remoteTopicCategoriesResult = await service.ListTopicCategoriesAsync(localTopic.Code); var remoteTopicCategories = remoteTopicCategoriesResult.Data.Items; localTopic.Name = topicInfo.Name; localTopic.ShortName = topicInfo.ShortName; localTopic.Description = topicInfo.Description.Value; localTopic.DefaultPagewatchResults = topicInfo.DefaultPagewatchResults.Value; localTopic.PagewatchAutosend = topicInfo.PagewatchAutosend.Value; localTopic.PagewatchEnabled = topicInfo.PagewatchSuspended.Value; localTopic.PagewatchType = (PageWatchType)topicInfo.PagewatchType.Value; localTopic.SendByEmailEnabled = topicInfo.SendByEmailEnabled.Value; localTopic.WatchTaggedContent = topicInfo.WatchTaggedContent.Value; localTopic.WirelessEnabled = topicInfo.WirelessEnabled.Value; localTopic.Pages = topicInfo.Pages.Items .Select(p => new Page { Id = Guid.NewGuid(), Url = p.Url }) .ToList(); var localTopicCategories = localTopic.TopicCategories; // new topic categories var newTopicCategories = remoteTopicCategories .Where(rtc => !localTopicCategories.Any(ltc => ltc.Category.Code == rtc.Code)) .Select(tcm => new CategoryTopic { TopicId = localTopic.Id, CategoryId = ctx.Categories.First(c => c.Code == tcm.Code).Id }) .ToList(); ctx.CategoryTopics.AddRange(newTopicCategories); var deleteableTopicCategories = localTopicCategories .Where(ltc => !remoteTopicCategories.Any(rtc => rtc.Code == ltc.Category.Code)) .ToList(); ctx.CategoryTopics.RemoveRange(deleteableTopicCategories); ctx.SaveChanges(); } // Delete all local topics not present remotely: var deletableTopics = localTopics .Where(lt => !remoteTopics.Any(rt => rt.Code == lt.Code)) .ToList(); ctx.RemoveRange(deletableTopics); ctx.SaveChanges(); } }
public async static Task SyncCategories(IGovDeliveryApiService service, IGovDeliveryContext ctx) { var categoriesResult = await service.ListCategoriesAsync(); if (!categoriesResult.HttpResponse.IsSuccessStatusCode) { Console.Error.WriteLine($@"Error getting Categories: {categoriesResult.HttpResponse.StatusCode} - {categoriesResult.HttpResponse.ReasonPhrase}"); } var numCategories = categoriesResult.Data.Items != null?categoriesResult.Data.Items.Count() : 0; Console.WriteLine($"Fetched {numCategories} categories"); if (numCategories > 0) { var remoteCategories = categoriesResult.Data.Items .Select(i => new Category { Id = Guid.NewGuid(), Code = i.Code, Description = i.Description, DefaultOpen = i.DefaultOpen.Value, AllowUserInitiatedSubscriptions = i.AllowSubscriptions.Value, Name = i.Name, ShortName = i.ShortName, }).ToList(); var localCategories = ctx.Categories.ToList(); // Add new categories: var newCategories = remoteCategories .Where(rc => !localCategories.Any(lc => lc.Code == rc.Code)) .ToList(); ctx.Categories.AddRange(newCategories); ctx.SaveChanges(); // Update existing categories: var existingCategories = localCategories .Where(lc => remoteCategories.Any(rc => rc.Code == lc.Code)) .ToList(); foreach (var localCategory in existingCategories) { var remoteCategory = remoteCategories.First(rc => rc.Code == localCategory.Code); // update category info: var fullCategoryResponse = await service.ReadCategoryAsync(remoteCategory.Code); var categoryInfo = fullCategoryResponse.Data; remoteCategory.AllowUserInitiatedSubscriptions = categoryInfo.AllowSubscriptions.Value; remoteCategory.DefaultOpen = categoryInfo.DefaultOpen.Value; remoteCategory.Description = categoryInfo.Description; remoteCategory.Name = categoryInfo.Name; remoteCategory.ShortName = categoryInfo.ShortName; var parentCategory = ctx.Categories.FirstOrDefault(c => c.Code == categoryInfo.Parent.CategoryCode); remoteCategory.ParentCategory = parentCategory; ctx.SaveChanges(); } ctx.SaveChanges(); // Delete categories not present remotely: var deletableCategories = localCategories .Where(lc => !remoteCategories.Any(rc => rc.Code == lc.Code)) .ToList(); ctx.RemoveRange(deletableCategories); ctx.SaveChanges(); } }
protected async static Task UpdateSingleSubscriberAsync <T>( Guid subscriberId, IGovDeliveryApiService service, IGovDeliveryContextFactory <T> factory, Action <string> loggingDelegte ) where T : AbstractGovDeliveryContext { var ctx = factory.CreateDbContext(); var subscriber = ctx.Subscribers.First(s => s.Id == subscriberId); if (subscriber == null || service == null || ctx == null) { return; } var subscriberInfoTask = service.ReadSubscriberAsync(subscriber.Email); var subscriberTopicsTask = service.ListSubscriberTopicsAsync(subscriber.Email); var subscriberCategoriesTask = service.ListSubscriberCategoriesAsync(subscriber.Email); await Task.WhenAll(subscriberInfoTask, subscriberTopicsTask, subscriberCategoriesTask); var subscriberInfo = (await subscriberInfoTask).Data; // Update detailed subscriber info subscriber.BulletinFrequency = (BulletinFrequency)subscriberInfo.DigestFor.Value; subscriber.GovDeliveryId = subscriberInfo.Id.Value; subscriber.Phone = subscriberInfo.Phone; if (subscriberInfo.SendSubscriberUpdateNotifications != null) { subscriber.SendSubscriberUpdateNotifications = subscriberInfo.SendSubscriberUpdateNotifications.Value; } try { await ctx.SaveChangesAsync(); } catch (Exception e) { loggingDelegte?.Invoke($@"{e.Message} {e.TargetSite}"); } // Update Category Subscriptions var subscriberCategories = ctx.CategorySubscriptions .Where(sc => sc.SubscriberId == subscriber.Id) .Select(sc => sc.Category) .ToList(); var subscriberCategoryInfo = (await subscriberCategoriesTask).Data.Items; // existing category subscriptions - do nothing. var newCategorySubscriptions = subscriberCategoryInfo .Where(sci => !subscriberCategories.Any(sc => sc.Code == sci.CategoryCode)); foreach (var nCS in newCategorySubscriptions) { var cat = ctx.Categories.First(c => c.Code == nCS.CategoryCode); ctx.CategorySubscriptions.Add(new CategorySubscription { CategoryId = cat.Id, Category = cat, SubscriberId = subscriber.Id, Subscriber = subscriber }); } var deleteableCategorySubscriptions = subscriberCategories .Where(sc => !subscriberCategoryInfo.Any(sci => sci.CategoryCode == sc.Code)) .ToList(); foreach (var dSC in deleteableCategorySubscriptions) { var categorySub = ctx.CategorySubscriptions.First(cs => cs.Category.Code == dSC.Code); ctx.CategorySubscriptions.Remove(categorySub); } await ctx.SaveChangesAsync(); // Find new Topic subscriptions var subscriberTopics = ctx.TopicSubscriptions .Where(ts => ts.SubscriberId == subscriber.Id) .Select(ts => ts.Topic) .ToList(); var subscriberTopicInfo = (await subscriberTopicsTask).Data.Items; var newTopicSubscriptions = subscriberTopicInfo .Where(sti => !subscriberTopics.Any(st => st.Code == sti.TopicCode)) .ToList(); foreach (var nTS in newTopicSubscriptions) { var topic = ctx.Topics.First(t => t.Code == nTS.TopicCode); ctx.TopicSubscriptions.Add(new TopicSubscription { Subscriber = subscriber, SubscriberId = subscriber.Id, Topic = topic, TopicId = topic.Id, }); } try { await ctx.SaveChangesAsync(); } catch (Exception e) { loggingDelegte?.Invoke($@"{e.Message} {e.TargetSite}"); } // deleteable topic subscriptions var deleteableTopicSubscriptions = subscriberTopics .Where(st => !subscriberTopicInfo.Any(sti => sti.TopicCode == st.Code)) .ToList(); foreach (var dts in deleteableTopicSubscriptions) { var topicSub = ctx.TopicSubscriptions.First(ts => ts.TopicId == dts.Id); ctx.TopicSubscriptions.Remove(topicSub); } try { await ctx.SaveChangesAsync(); } catch (Exception e) { loggingDelegte?.Invoke($@"{e.Message} {e.TargetSite}"); } }
public GovDeliveryServiceTests() { service = new MockGovDeliveryApiService(GovDeliveryServer.Staging, ACCOUNT_CODE); }
public GovDeliveryServiceTests() { service = new MockGovDeliveryApiService("foo.com", ACCOUNT_CODE); }