public static async Task RefillSearchIndex(TextWriter log)
{
try
{
log.WriteLine($"{DateTime.Now} :: Function is invoked");
var custRepo = new CustomerRepository();
var sRepo = new SearchRepository();
var tRepo = new TransactionRepository();
var customers = custRepo.GetAllCustomers();
log.WriteLine($"Total customers: {customers.Count()}");
DateTime dtLimit = DateTime.UtcNow.Date.AddDays(-Constants.DaysToKeepTransactions);
DateTime dtLimitWoDay = dtLimit.AddDays(-dtLimit.Day + 1); //remove day component from DateTime
DateTime dtEnd = DateTime.UtcNow.Date;
DateTime dtEndWoDay = dtEnd.AddDays(-dtEnd.Day + 1);
foreach (var customer in customers)
{
log.WriteLine($"{DateTime.Now} :: Processing customer {customer.InternalID} {customer.Name}");
for (DateTime d = dtLimitWoDay; d <= dtEndWoDay; d = d.AddMonths(1))
{
string strDate = d.ToString(Constants.DateStringFormat);
int dateVal = int.Parse(strDate);
var res =await tRepo.GetTransactionsForCustomer(customer.InternalID.Value, dateVal);
var matchingEnts = res.Where(r => r.DateTime >= dtLimit);
sRepo.AddToIndex(matchingEnts.ToArray());
}
}
int i = 1;
TableContinuationToken continuationToken = null;
do
{
var items = tRepo.GetTransactionTotalsItemBatch(ref continuationToken);
var searchItems = items.Select(item => new TotalsSearchItem(item));
sRepo.AddOrUpdateTotalsItem(searchItems.ToArray());
log.WriteLine($"{DateTime.Now} :: Added totals item batch {i++} ({searchItems.Count()} items)");
if (continuationToken != null)
Thread.Sleep(500);
}
while (continuationToken != null);
log.WriteLine($"{DateTime.Now} :: DONE");
}
catch (Exception ex)
{
var tags = new List<string> { "RefillSearchIndex" };
new RaygunWebApiClient(ConfigurationManager.AppSettings[Constants.SettingKey_RaygunKey])
.SendInBackground(ex, tags, null);
throw;
}
}
private static void AddTestData()
{
var transactRepo = new TransactionRepository();
var custRepo = new CustomerRepository();
var customers = custRepo.GetAllCustomers().ToList();
int productCount = 3;
List<string> prodIDs = new List<string>();
for (int i = 1; i <= productCount; i++)
{
prodIDs.Add("Signature" + i);
}
DateTime minDt = new DateTime(2015, 1, 1, 0, 0, 0, DateTimeKind.Utc);
DateTime maxDt = DateTime.UtcNow;
TimeSpan timeSpan = maxDt - minDt;
var transactionBatch = new List<Transaction>(100);
Stopwatch sw = Stopwatch.StartNew();
for (int i = 0; i < 50000; i++)
{
string prodId = prodIDs[_rand.Next(0, productCount)];
TimeSpan randSpan = new TimeSpan(0, 0, _rand.Next(0, (int)timeSpan.TotalSeconds));
DateTime date = minDt + randSpan;
var curCustomer = customers[_rand.Next(0, customers.Count)];
Guid clientId = curCustomer.InternalID.Value;
int transactDuplicates = _rand.Next(0, 10) >= 9 ? 2 : 1;
Transaction t = new Transaction
{
MvaNumber = curCustomer.MvaNumber,
ExternalRef = curCustomer.ExternalIDs.First(),
//InternalRef = curCustomer.InternalID,
CustomerNumber = curCustomer.CustomerNumber,
Date = date,
Description = "test item",
ProductID = prodId,
Amount = transactDuplicates
};
//transactRepo.AddTransactionToUpdateQueue(t);
transactionBatch.Add(t);
if (i % 100 == 0)
{
transactRepo.AddOrUpdateTransactionBatch(transactionBatch);
transactionBatch.Clear();
Console.WriteLine($"{i} transactions have been added :: {sw.ElapsedMilliseconds} ms");
WriteQueueLengthAsync();
sw.Restart();
}
}
}
public static async Task UpdateTotals(TextWriter log)
{
try
{
log.WriteLine($"{DateTime.Now} :: Function is invoked");
List<Task> tasks = new List<Task>();
var custRepo = new CustomerRepository();
var sRepo = new SearchRepository();
var tRepo = new TransactionRepository();
var customers = custRepo.GetAllCustomers();
log.WriteLine($"Total customers: {customers.Count()}");
foreach (var customer in customers)
{
log.WriteLine($"{DateTime.Now} :: Processing customer {customer.InternalID} {customer.Name}");
tasks.Add(ProcessTransactions(sRepo, tRepo, customer, null, null));
}
if (tasks.Any())
await Task.WhenAll(tasks);
log.WriteLine($"{DateTime.Now} :: All customers processed");
sRepo.DeleteOldTransactions();
tasks = new List<Task>();
log.WriteLine($"{DateTime.Now} :: Old transactions processed");
TableContinuationToken continuationToken = null;
do
{
var items = tRepo.GetReIndexableItemBatch(ref continuationToken);
log.WriteLine($"{DateTime.Now} :: Current re-index batch items: {items.Count()}");
foreach (ReIndexTableEntity item in items)
{
log.WriteLine($"{DateTime.Now} :: Processing re-index item {item.RowKey}");
int reIndexDateVal = int.Parse(item.Date);
if(item.CustomerInternalID.HasValue)
{
var reIndexCustomer = custRepo.GetCustomerByInternalID(item.CustomerInternalID.Value);
if(reIndexCustomer == null)
{
throw new Exception($"Error while processing re-index item, customer with ID {item.CustomerInternalID.Value} not found.");
}
tasks.Add(ProcessTransactions(sRepo, tRepo, reIndexCustomer, item.ProductID, reIndexDateVal));
}
else
{
//no customer ID specified, process all of them
foreach (var customer in customers)
{
tasks.Add(ProcessTransactions(sRepo, tRepo, customer, item.ProductID, reIndexDateVal));
}
}
}
tRepo.DeleteReIndexItemBatch(items.ToArray());
//TODO: should ignore errors that are caused by the ETag mismatch.
//These can happen when an existing request was overwritten during the processing above.
//The correct thing to do would be to ignore it and to process the item again next time.
}
while (continuationToken != null);
if (tasks.Any())
await Task.WhenAll(tasks);
log.WriteLine($"{DateTime.Now} :: DONE");
}
catch (Exception ex)
{
var tags = new List<string> { "UpdateTotals" };
new RaygunWebApiClient(ConfigurationManager.AppSettings[Constants.SettingKey_RaygunKey])
.SendInBackground(ex, tags, null);
throw;
}
}
