public static async Task RefillSearchIndex(TextWriter log) { try { log.WriteLine($"{DateTime.Now} :: Function is invoked"); var custRepo = new CustomerRepository(); var sRepo = new SearchRepository(); var tRepo = new TransactionRepository(); var customers = custRepo.GetAllCustomers(); log.WriteLine($"Total customers: {customers.Count()}"); DateTime dtLimit = DateTime.UtcNow.Date.AddDays(-Constants.DaysToKeepTransactions); DateTime dtLimitWoDay = dtLimit.AddDays(-dtLimit.Day + 1); //remove day component from DateTime DateTime dtEnd = DateTime.UtcNow.Date; DateTime dtEndWoDay = dtEnd.AddDays(-dtEnd.Day + 1); foreach (var customer in customers) { log.WriteLine($"{DateTime.Now} :: Processing customer {customer.InternalID} {customer.Name}"); for (DateTime d = dtLimitWoDay; d <= dtEndWoDay; d = d.AddMonths(1)) { string strDate = d.ToString(Constants.DateStringFormat); int dateVal = int.Parse(strDate); var res =await tRepo.GetTransactionsForCustomer(customer.InternalID.Value, dateVal); var matchingEnts = res.Where(r => r.DateTime >= dtLimit); sRepo.AddToIndex(matchingEnts.ToArray()); } } int i = 1; TableContinuationToken continuationToken = null; do { var items = tRepo.GetTransactionTotalsItemBatch(ref continuationToken); var searchItems = items.Select(item => new TotalsSearchItem(item)); sRepo.AddOrUpdateTotalsItem(searchItems.ToArray()); log.WriteLine($"{DateTime.Now} :: Added totals item batch {i++} ({searchItems.Count()} items)"); if (continuationToken != null) Thread.Sleep(500); } while (continuationToken != null); log.WriteLine($"{DateTime.Now} :: DONE"); } catch (Exception ex) { var tags = new List<string> { "RefillSearchIndex" }; new RaygunWebApiClient(ConfigurationManager.AppSettings[Constants.SettingKey_RaygunKey]) .SendInBackground(ex, tags, null); throw; } }
private static void AddTestData() { var transactRepo = new TransactionRepository(); var custRepo = new CustomerRepository(); var customers = custRepo.GetAllCustomers().ToList(); int productCount = 3; List<string> prodIDs = new List<string>(); for (int i = 1; i <= productCount; i++) { prodIDs.Add("Signature" + i); } DateTime minDt = new DateTime(2015, 1, 1, 0, 0, 0, DateTimeKind.Utc); DateTime maxDt = DateTime.UtcNow; TimeSpan timeSpan = maxDt - minDt; var transactionBatch = new List<Transaction>(100); Stopwatch sw = Stopwatch.StartNew(); for (int i = 0; i < 50000; i++) { string prodId = prodIDs[_rand.Next(0, productCount)]; TimeSpan randSpan = new TimeSpan(0, 0, _rand.Next(0, (int)timeSpan.TotalSeconds)); DateTime date = minDt + randSpan; var curCustomer = customers[_rand.Next(0, customers.Count)]; Guid clientId = curCustomer.InternalID.Value; int transactDuplicates = _rand.Next(0, 10) >= 9 ? 2 : 1; Transaction t = new Transaction { MvaNumber = curCustomer.MvaNumber, ExternalRef = curCustomer.ExternalIDs.First(), //InternalRef = curCustomer.InternalID, CustomerNumber = curCustomer.CustomerNumber, Date = date, Description = "test item", ProductID = prodId, Amount = transactDuplicates }; //transactRepo.AddTransactionToUpdateQueue(t); transactionBatch.Add(t); if (i % 100 == 0) { transactRepo.AddOrUpdateTransactionBatch(transactionBatch); transactionBatch.Clear(); Console.WriteLine($"{i} transactions have been added :: {sw.ElapsedMilliseconds} ms"); WriteQueueLengthAsync(); sw.Restart(); } } }
public static async Task UpdateTotals(TextWriter log) { try { log.WriteLine($"{DateTime.Now} :: Function is invoked"); List<Task> tasks = new List<Task>(); var custRepo = new CustomerRepository(); var sRepo = new SearchRepository(); var tRepo = new TransactionRepository(); var customers = custRepo.GetAllCustomers(); log.WriteLine($"Total customers: {customers.Count()}"); foreach (var customer in customers) { log.WriteLine($"{DateTime.Now} :: Processing customer {customer.InternalID} {customer.Name}"); tasks.Add(ProcessTransactions(sRepo, tRepo, customer, null, null)); } if (tasks.Any()) await Task.WhenAll(tasks); log.WriteLine($"{DateTime.Now} :: All customers processed"); sRepo.DeleteOldTransactions(); tasks = new List<Task>(); log.WriteLine($"{DateTime.Now} :: Old transactions processed"); TableContinuationToken continuationToken = null; do { var items = tRepo.GetReIndexableItemBatch(ref continuationToken); log.WriteLine($"{DateTime.Now} :: Current re-index batch items: {items.Count()}"); foreach (ReIndexTableEntity item in items) { log.WriteLine($"{DateTime.Now} :: Processing re-index item {item.RowKey}"); int reIndexDateVal = int.Parse(item.Date); if(item.CustomerInternalID.HasValue) { var reIndexCustomer = custRepo.GetCustomerByInternalID(item.CustomerInternalID.Value); if(reIndexCustomer == null) { throw new Exception($"Error while processing re-index item, customer with ID {item.CustomerInternalID.Value} not found."); } tasks.Add(ProcessTransactions(sRepo, tRepo, reIndexCustomer, item.ProductID, reIndexDateVal)); } else { //no customer ID specified, process all of them foreach (var customer in customers) { tasks.Add(ProcessTransactions(sRepo, tRepo, customer, item.ProductID, reIndexDateVal)); } } } tRepo.DeleteReIndexItemBatch(items.ToArray()); //TODO: should ignore errors that are caused by the ETag mismatch. //These can happen when an existing request was overwritten during the processing above. //The correct thing to do would be to ignore it and to process the item again next time. } while (continuationToken != null); if (tasks.Any()) await Task.WhenAll(tasks); log.WriteLine($"{DateTime.Now} :: DONE"); } catch (Exception ex) { var tags = new List<string> { "UpdateTotals" }; new RaygunWebApiClient(ConfigurationManager.AppSettings[Constants.SettingKey_RaygunKey]) .SendInBackground(ex, tags, null); throw; } }