public static async Task RefillSearchIndex(TextWriter log) { try { log.WriteLine($"{DateTime.Now} :: Function is invoked"); var custRepo = new CustomerRepository(); var sRepo = new SearchRepository(); var tRepo = new TransactionRepository(); var customers = custRepo.GetAllCustomers(); log.WriteLine($"Total customers: {customers.Count()}"); DateTime dtLimit = DateTime.UtcNow.Date.AddDays(-Constants.DaysToKeepTransactions); DateTime dtLimitWoDay = dtLimit.AddDays(-dtLimit.Day + 1); //remove day component from DateTime DateTime dtEnd = DateTime.UtcNow.Date; DateTime dtEndWoDay = dtEnd.AddDays(-dtEnd.Day + 1); foreach (var customer in customers) { log.WriteLine($"{DateTime.Now} :: Processing customer {customer.InternalID} {customer.Name}"); for (DateTime d = dtLimitWoDay; d <= dtEndWoDay; d = d.AddMonths(1)) { string strDate = d.ToString(Constants.DateStringFormat); int dateVal = int.Parse(strDate); var res =await tRepo.GetTransactionsForCustomer(customer.InternalID.Value, dateVal); var matchingEnts = res.Where(r => r.DateTime >= dtLimit); sRepo.AddToIndex(matchingEnts.ToArray()); } } int i = 1; TableContinuationToken continuationToken = null; do { var items = tRepo.GetTransactionTotalsItemBatch(ref continuationToken); var searchItems = items.Select(item => new TotalsSearchItem(item)); sRepo.AddOrUpdateTotalsItem(searchItems.ToArray()); log.WriteLine($"{DateTime.Now} :: Added totals item batch {i++} ({searchItems.Count()} items)"); if (continuationToken != null) Thread.Sleep(500); } while (continuationToken != null); log.WriteLine($"{DateTime.Now} :: DONE"); } catch (Exception ex) { var tags = new List<string> { "RefillSearchIndex" }; new RaygunWebApiClient(ConfigurationManager.AppSettings[Constants.SettingKey_RaygunKey]) .SendInBackground(ex, tags, null); throw; } }
private static async Task ProcessTransactions(SearchRepository sRepo, TransactionRepository tRepo, Customer customer, string productID, int? dateVal) { //any transaction newer than 'dtLimit' should not be processed because it's kept as an individual search item DateTime dtLimit = DateTime.UtcNow.Date.AddDays(-Constants.DaysToKeepTransactions); int curDateVal; if(dateVal.HasValue) { curDateVal = dateVal.Value; } else { curDateVal = int.Parse(dtLimit.AddDays(-1).ToString(Constants.DateStringFormat)); } var existingItems = sRepo.GetTotalsItemsForCustomer(customer.InternalID.Value, productID, curDateVal); HashSet<string> existingItemIDs = new HashSet<string>(existingItems.Select(e => e.DocUniqueID)); Dictionary<string, int> productTotals = new Dictionary<string, int>(); var res =await tRepo.GetTransactionsForCustomer(customer.InternalID.Value, curDateVal); if(!string.IsNullOrWhiteSpace(productID)) { res = res.Where(r => r.ProductID == productID); } res = res.Where(r => r.DateTime < dtLimit); foreach (var transaction in res) { if (productTotals.ContainsKey(transaction.ProductID)) { productTotals[transaction.ProductID] = productTotals[transaction.ProductID] + transaction.Amount.Value; } else { productTotals.Add(transaction.ProductID, transaction.Amount.Value); } } var searchItems = new List<TotalsSearchItem>(); foreach (var entry in productTotals) { var newItem = new TotalsSearchItem { DocUniqueID = TotalsSearchItem.CreateUID(curDateVal, entry.Key, customer.InternalID.Value), ClientInternalID = customer.InternalID.Value.ToString(), CustomerNumber = customer.CustomerNumber, Date = curDateVal, MvaNumber = customer.MvaNumber, ProductID = entry.Key, Amount = entry.Value }; searchItems.Add(newItem); existingItemIDs.Remove(newItem.DocUniqueID); } sRepo.AddOrUpdateTotalsItem(searchItems.ToArray()); tRepo.AddOrUpdateTransactionTotalsItem(searchItems.Select(item => new TotalsItemTableEntity(item)).ToArray()); //in case a transaction was deleted, remove all totals items from search index that are no longer valid sRepo.DeleteTotalsItemWithID(existingItemIDs.ToArray()); tRepo.DeleteTransactionTotalsItemWithID(existingItemIDs.ToArray()); }