public static async Task RefillSearchIndex(TextWriter log)
{
try
{
log.WriteLine($"{DateTime.Now} :: Function is invoked");
var custRepo = new CustomerRepository();
var sRepo = new SearchRepository();
var tRepo = new TransactionRepository();
var customers = custRepo.GetAllCustomers();
log.WriteLine($"Total customers: {customers.Count()}");
DateTime dtLimit = DateTime.UtcNow.Date.AddDays(-Constants.DaysToKeepTransactions);
DateTime dtLimitWoDay = dtLimit.AddDays(-dtLimit.Day + 1); //remove day component from DateTime
DateTime dtEnd = DateTime.UtcNow.Date;
DateTime dtEndWoDay = dtEnd.AddDays(-dtEnd.Day + 1);
foreach (var customer in customers)
{
log.WriteLine($"{DateTime.Now} :: Processing customer {customer.InternalID} {customer.Name}");
for (DateTime d = dtLimitWoDay; d <= dtEndWoDay; d = d.AddMonths(1))
{
string strDate = d.ToString(Constants.DateStringFormat);
int dateVal = int.Parse(strDate);
var res =await tRepo.GetTransactionsForCustomer(customer.InternalID.Value, dateVal);
var matchingEnts = res.Where(r => r.DateTime >= dtLimit);
sRepo.AddToIndex(matchingEnts.ToArray());
}
}
int i = 1;
TableContinuationToken continuationToken = null;
do
{
var items = tRepo.GetTransactionTotalsItemBatch(ref continuationToken);
var searchItems = items.Select(item => new TotalsSearchItem(item));
sRepo.AddOrUpdateTotalsItem(searchItems.ToArray());
log.WriteLine($"{DateTime.Now} :: Added totals item batch {i++} ({searchItems.Count()} items)");
if (continuationToken != null)
Thread.Sleep(500);
}
while (continuationToken != null);
log.WriteLine($"{DateTime.Now} :: DONE");
}
catch (Exception ex)
{
var tags = new List<string> { "RefillSearchIndex" };
new RaygunWebApiClient(ConfigurationManager.AppSettings[Constants.SettingKey_RaygunKey])
.SendInBackground(ex, tags, null);
throw;
}
}
/// <summary>
/// Inserts or updates (using merge operation) a batch of transaction table entities.
/// Also generates IDs for items that do not have them and sets Date to DateTime.Now if it is null.
/// </summary>
/// <param name="batch">Batch of Transaction items to add/update</param>
/// <exception cref="System.Exception">Thrown when any of the items is not valid or if adding to either table storage or search index failed.</exception>
public void AddOrUpdateTransactionBatch(IEnumerable<Transaction> batch)
{
var tableEnts = new List<TransactionTableEntity>();
EnsureTransactionHasCustomerInternalID(batch.ToArray());
foreach (Transaction t in batch)
{
//also ensure that all entities are valid before any of them are committed to table storage or search index
t.EnsureEntityValidForStorage(true);
var ent = new TransactionTableEntity(t.ID.Value, t.Date.Value, t.InternalRef.Value)
{
MvaNumber = t.MvaNumber,
Amount = t.Amount,
Description = t.Description,
CustomerNumber = t.CustomerNumber,
AccountID = t.AccountID,
ProductID = t.ProductID,
NotInvoiceable = t.NotInvoiceable
};
tableEnts.Add(ent);
}
//Need to group entities by partition key because table storage only allows batch operations in the scope of the same partition key
var entsGroupedByPartition = tableEnts.GroupBy(t => t.PartitionKey);
foreach (var entGroup in entsGroupedByPartition)
{
for (int i = 0; i < entGroup.Count(); i += 100)
{
var tableBatch = new TableBatchOperation();
foreach (var ent in entGroup.Skip(i).Take(100))
{
tableBatch.Add(TableOperation.InsertOrMerge(ent));
}
TransactionTable.ExecuteBatch(tableBatch);
}
}
var dtReIndexReq = DateTime.UtcNow.Date.AddDays(-Constants.DaysToKeepTransactions);
var entsToAddToIndex = tableEnts.Where(t => t.DateTime >= dtReIndexReq);
var sRepo = new SearchRepository();
sRepo.AddToIndex(entsToAddToIndex.ToArray());
AddReIndexRequestIfNeeded(tableEnts.ToArray());
}