public static async Task RefillSearchIndex(TextWriter log) { try { log.WriteLine($"{DateTime.Now} :: Function is invoked"); var custRepo = new CustomerRepository(); var sRepo = new SearchRepository(); var tRepo = new TransactionRepository(); var customers = custRepo.GetAllCustomers(); log.WriteLine($"Total customers: {customers.Count()}"); DateTime dtLimit = DateTime.UtcNow.Date.AddDays(-Constants.DaysToKeepTransactions); DateTime dtLimitWoDay = dtLimit.AddDays(-dtLimit.Day + 1); //remove day component from DateTime DateTime dtEnd = DateTime.UtcNow.Date; DateTime dtEndWoDay = dtEnd.AddDays(-dtEnd.Day + 1); foreach (var customer in customers) { log.WriteLine($"{DateTime.Now} :: Processing customer {customer.InternalID} {customer.Name}"); for (DateTime d = dtLimitWoDay; d <= dtEndWoDay; d = d.AddMonths(1)) { string strDate = d.ToString(Constants.DateStringFormat); int dateVal = int.Parse(strDate); var res =await tRepo.GetTransactionsForCustomer(customer.InternalID.Value, dateVal); var matchingEnts = res.Where(r => r.DateTime >= dtLimit); sRepo.AddToIndex(matchingEnts.ToArray()); } } int i = 1; TableContinuationToken continuationToken = null; do { var items = tRepo.GetTransactionTotalsItemBatch(ref continuationToken); var searchItems = items.Select(item => new TotalsSearchItem(item)); sRepo.AddOrUpdateTotalsItem(searchItems.ToArray()); log.WriteLine($"{DateTime.Now} :: Added totals item batch {i++} ({searchItems.Count()} items)"); if (continuationToken != null) Thread.Sleep(500); } while (continuationToken != null); log.WriteLine($"{DateTime.Now} :: DONE"); } catch (Exception ex) { var tags = new List<string> { "RefillSearchIndex" }; new RaygunWebApiClient(ConfigurationManager.AppSettings[Constants.SettingKey_RaygunKey]) .SendInBackground(ex, tags, null); throw; } }
public IHttpActionResult AddTransaction(int mvanumber, Transaction t) { if (t.ID != null) { return BadRequest(); } t.MvaNumber = mvanumber; t = new TransactionRepository().AddTransactionToUpdateQueue(t); return Ok(t); }
public IHttpActionResult AddTransactionBatch(int mvanumber, IEnumerable<Transaction> batch) { if(batch.Count() == 0 || batch.Count() > Constants.MaxTransactionBatchSizeForQueue) { return BadRequest("Invalid batch size"); } if(!batch.All(t => t.MvaNumber == mvanumber)) { return BadRequest("MVA number does not match"); } if(!batch.All(t => t.ID == null)) { return BadRequest(); } batch = new TransactionRepository().AddTransactionBatchToUpdateQueue(batch); return Ok(batch); }
public IHttpActionResult SetTransactionBilled(int mvanumber, IEnumerable<TransactionBilledItem> billedItems) { var tRepo = new TransactionRepository(); foreach (var item in billedItems) { try { tRepo.SetTransactionAsBilled(mvanumber, item.ID, item.OrderItemID); } catch(Exception ex) { return InternalServerError(new Exception($"Failed to set transaction {item.ID} as billed {ex.Message})")); } } return Ok(); }
private static void AddTestData() { var transactRepo = new TransactionRepository(); var custRepo = new CustomerRepository(); var customers = custRepo.GetAllCustomers().ToList(); int productCount = 3; List<string> prodIDs = new List<string>(); for (int i = 1; i <= productCount; i++) { prodIDs.Add("Signature" + i); } DateTime minDt = new DateTime(2015, 1, 1, 0, 0, 0, DateTimeKind.Utc); DateTime maxDt = DateTime.UtcNow; TimeSpan timeSpan = maxDt - minDt; var transactionBatch = new List<Transaction>(100); Stopwatch sw = Stopwatch.StartNew(); for (int i = 0; i < 50000; i++) { string prodId = prodIDs[_rand.Next(0, productCount)]; TimeSpan randSpan = new TimeSpan(0, 0, _rand.Next(0, (int)timeSpan.TotalSeconds)); DateTime date = minDt + randSpan; var curCustomer = customers[_rand.Next(0, customers.Count)]; Guid clientId = curCustomer.InternalID.Value; int transactDuplicates = _rand.Next(0, 10) >= 9 ? 2 : 1; Transaction t = new Transaction { MvaNumber = curCustomer.MvaNumber, ExternalRef = curCustomer.ExternalIDs.First(), //InternalRef = curCustomer.InternalID, CustomerNumber = curCustomer.CustomerNumber, Date = date, Description = "test item", ProductID = prodId, Amount = transactDuplicates }; //transactRepo.AddTransactionToUpdateQueue(t); transactionBatch.Add(t); if (i % 100 == 0) { transactRepo.AddOrUpdateTransactionBatch(transactionBatch); transactionBatch.Clear(); Console.WriteLine($"{i} transactions have been added :: {sw.ElapsedMilliseconds} ms"); WriteQueueLengthAsync(); sw.Restart(); } } }
private static async Task ProcessTransactions(SearchRepository sRepo, TransactionRepository tRepo, Customer customer, string productID, int? dateVal) { //any transaction newer than 'dtLimit' should not be processed because it's kept as an individual search item DateTime dtLimit = DateTime.UtcNow.Date.AddDays(-Constants.DaysToKeepTransactions); int curDateVal; if(dateVal.HasValue) { curDateVal = dateVal.Value; } else { curDateVal = int.Parse(dtLimit.AddDays(-1).ToString(Constants.DateStringFormat)); } var existingItems = sRepo.GetTotalsItemsForCustomer(customer.InternalID.Value, productID, curDateVal); HashSet<string> existingItemIDs = new HashSet<string>(existingItems.Select(e => e.DocUniqueID)); Dictionary<string, int> productTotals = new Dictionary<string, int>(); var res =await tRepo.GetTransactionsForCustomer(customer.InternalID.Value, curDateVal); if(!string.IsNullOrWhiteSpace(productID)) { res = res.Where(r => r.ProductID == productID); } res = res.Where(r => r.DateTime < dtLimit); foreach (var transaction in res) { if (productTotals.ContainsKey(transaction.ProductID)) { productTotals[transaction.ProductID] = productTotals[transaction.ProductID] + transaction.Amount.Value; } else { productTotals.Add(transaction.ProductID, transaction.Amount.Value); } } var searchItems = new List<TotalsSearchItem>(); foreach (var entry in productTotals) { var newItem = new TotalsSearchItem { DocUniqueID = TotalsSearchItem.CreateUID(curDateVal, entry.Key, customer.InternalID.Value), ClientInternalID = customer.InternalID.Value.ToString(), CustomerNumber = customer.CustomerNumber, Date = curDateVal, MvaNumber = customer.MvaNumber, ProductID = entry.Key, Amount = entry.Value }; searchItems.Add(newItem); existingItemIDs.Remove(newItem.DocUniqueID); } sRepo.AddOrUpdateTotalsItem(searchItems.ToArray()); tRepo.AddOrUpdateTransactionTotalsItem(searchItems.Select(item => new TotalsItemTableEntity(item)).ToArray()); //in case a transaction was deleted, remove all totals items from search index that are no longer valid sRepo.DeleteTotalsItemWithID(existingItemIDs.ToArray()); tRepo.DeleteTransactionTotalsItemWithID(existingItemIDs.ToArray()); }
public static async Task UpdateTotals(TextWriter log) { try { log.WriteLine($"{DateTime.Now} :: Function is invoked"); List<Task> tasks = new List<Task>(); var custRepo = new CustomerRepository(); var sRepo = new SearchRepository(); var tRepo = new TransactionRepository(); var customers = custRepo.GetAllCustomers(); log.WriteLine($"Total customers: {customers.Count()}"); foreach (var customer in customers) { log.WriteLine($"{DateTime.Now} :: Processing customer {customer.InternalID} {customer.Name}"); tasks.Add(ProcessTransactions(sRepo, tRepo, customer, null, null)); } if (tasks.Any()) await Task.WhenAll(tasks); log.WriteLine($"{DateTime.Now} :: All customers processed"); sRepo.DeleteOldTransactions(); tasks = new List<Task>(); log.WriteLine($"{DateTime.Now} :: Old transactions processed"); TableContinuationToken continuationToken = null; do { var items = tRepo.GetReIndexableItemBatch(ref continuationToken); log.WriteLine($"{DateTime.Now} :: Current re-index batch items: {items.Count()}"); foreach (ReIndexTableEntity item in items) { log.WriteLine($"{DateTime.Now} :: Processing re-index item {item.RowKey}"); int reIndexDateVal = int.Parse(item.Date); if(item.CustomerInternalID.HasValue) { var reIndexCustomer = custRepo.GetCustomerByInternalID(item.CustomerInternalID.Value); if(reIndexCustomer == null) { throw new Exception($"Error while processing re-index item, customer with ID {item.CustomerInternalID.Value} not found."); } tasks.Add(ProcessTransactions(sRepo, tRepo, reIndexCustomer, item.ProductID, reIndexDateVal)); } else { //no customer ID specified, process all of them foreach (var customer in customers) { tasks.Add(ProcessTransactions(sRepo, tRepo, customer, item.ProductID, reIndexDateVal)); } } } tRepo.DeleteReIndexItemBatch(items.ToArray()); //TODO: should ignore errors that are caused by the ETag mismatch. //These can happen when an existing request was overwritten during the processing above. //The correct thing to do would be to ignore it and to process the item again next time. } while (continuationToken != null); if (tasks.Any()) await Task.WhenAll(tasks); log.WriteLine($"{DateTime.Now} :: DONE"); } catch (Exception ex) { var tags = new List<string> { "UpdateTotals" }; new RaygunWebApiClient(ConfigurationManager.AppSettings[Constants.SettingKey_RaygunKey]) .SendInBackground(ex, tags, null); throw; } }