public static async Task RefillSearchIndex(TextWriter log) { try { log.WriteLine($"{DateTime.Now} :: Function is invoked"); var custRepo = new CustomerRepository(); var sRepo = new SearchRepository(); var tRepo = new TransactionRepository(); var customers = custRepo.GetAllCustomers(); log.WriteLine($"Total customers: {customers.Count()}"); DateTime dtLimit = DateTime.UtcNow.Date.AddDays(-Constants.DaysToKeepTransactions); DateTime dtLimitWoDay = dtLimit.AddDays(-dtLimit.Day + 1); //remove day component from DateTime DateTime dtEnd = DateTime.UtcNow.Date; DateTime dtEndWoDay = dtEnd.AddDays(-dtEnd.Day + 1); foreach (var customer in customers) { log.WriteLine($"{DateTime.Now} :: Processing customer {customer.InternalID} {customer.Name}"); for (DateTime d = dtLimitWoDay; d <= dtEndWoDay; d = d.AddMonths(1)) { string strDate = d.ToString(Constants.DateStringFormat); int dateVal = int.Parse(strDate); var res =await tRepo.GetTransactionsForCustomer(customer.InternalID.Value, dateVal); var matchingEnts = res.Where(r => r.DateTime >= dtLimit); sRepo.AddToIndex(matchingEnts.ToArray()); } } int i = 1; TableContinuationToken continuationToken = null; do { var items = tRepo.GetTransactionTotalsItemBatch(ref continuationToken); var searchItems = items.Select(item => new TotalsSearchItem(item)); sRepo.AddOrUpdateTotalsItem(searchItems.ToArray()); log.WriteLine($"{DateTime.Now} :: Added totals item batch {i++} ({searchItems.Count()} items)"); if (continuationToken != null) Thread.Sleep(500); } while (continuationToken != null); log.WriteLine($"{DateTime.Now} :: DONE"); } catch (Exception ex) { var tags = new List<string> { "RefillSearchIndex" }; new RaygunWebApiClient(ConfigurationManager.AppSettings[Constants.SettingKey_RaygunKey]) .SendInBackground(ex, tags, null); throw; } }
/// <summary> /// Inserts or updates (using merge operation) a batch of transaction table entities. /// Also generates IDs for items that do not have them and sets Date to DateTime.Now if it is null. /// </summary> /// <param name="batch">Batch of Transaction items to add/update</param> /// <exception cref="System.Exception">Thrown when any of the items is not valid or if adding to either table storage or search index failed.</exception> public void AddOrUpdateTransactionBatch(IEnumerable<Transaction> batch) { var tableEnts = new List<TransactionTableEntity>(); EnsureTransactionHasCustomerInternalID(batch.ToArray()); foreach (Transaction t in batch) { //also ensure that all entities are valid before any of them are committed to table storage or search index t.EnsureEntityValidForStorage(true); var ent = new TransactionTableEntity(t.ID.Value, t.Date.Value, t.InternalRef.Value) { MvaNumber = t.MvaNumber, Amount = t.Amount, Description = t.Description, CustomerNumber = t.CustomerNumber, AccountID = t.AccountID, ProductID = t.ProductID, NotInvoiceable = t.NotInvoiceable }; tableEnts.Add(ent); } //Need to group entities by partition key because table storage only allows batch operations in the scope of the same partition key var entsGroupedByPartition = tableEnts.GroupBy(t => t.PartitionKey); foreach (var entGroup in entsGroupedByPartition) { for (int i = 0; i < entGroup.Count(); i += 100) { var tableBatch = new TableBatchOperation(); foreach (var ent in entGroup.Skip(i).Take(100)) { tableBatch.Add(TableOperation.InsertOrMerge(ent)); } TransactionTable.ExecuteBatch(tableBatch); } } var dtReIndexReq = DateTime.UtcNow.Date.AddDays(-Constants.DaysToKeepTransactions); var entsToAddToIndex = tableEnts.Where(t => t.DateTime >= dtReIndexReq); var sRepo = new SearchRepository(); sRepo.AddToIndex(entsToAddToIndex.ToArray()); AddReIndexRequestIfNeeded(tableEnts.ToArray()); }