AddToIndex() public method

Adds transaction items to search index or updates existing items. When transaction amount is > 1, adds multiple items to search index for facet queries. Converts all entities to batches of up to 1000 to efficiently update search index. Throws exception when search index operation fails.
public AddToIndex ( ) : void
return void
示例#1
0
        public static async Task RefillSearchIndex(TextWriter log)
        {
            try
            {
                log.WriteLine($"{DateTime.Now} :: Function is invoked");

                var custRepo = new CustomerRepository();
                var sRepo = new SearchRepository();
                var tRepo = new TransactionRepository();

                var customers = custRepo.GetAllCustomers();
                log.WriteLine($"Total customers: {customers.Count()}");

                DateTime dtLimit = DateTime.UtcNow.Date.AddDays(-Constants.DaysToKeepTransactions);
                DateTime dtLimitWoDay = dtLimit.AddDays(-dtLimit.Day + 1);    //remove day component from DateTime
                DateTime dtEnd = DateTime.UtcNow.Date;
                DateTime dtEndWoDay = dtEnd.AddDays(-dtEnd.Day + 1);

                foreach (var customer in customers)
                {
                    log.WriteLine($"{DateTime.Now} :: Processing customer {customer.InternalID} {customer.Name}");

                    for (DateTime d = dtLimitWoDay; d <= dtEndWoDay; d = d.AddMonths(1))
                    {
                        string strDate = d.ToString(Constants.DateStringFormat);
                        int dateVal = int.Parse(strDate);

                        var res =await  tRepo.GetTransactionsForCustomer(customer.InternalID.Value, dateVal);
                        var matchingEnts = res.Where(r => r.DateTime >= dtLimit);

                        sRepo.AddToIndex(matchingEnts.ToArray());
                    }
                }

                int i = 1;
                TableContinuationToken continuationToken = null;
                do
                {
                    var items = tRepo.GetTransactionTotalsItemBatch(ref continuationToken);
                    var searchItems = items.Select(item => new TotalsSearchItem(item));
                    sRepo.AddOrUpdateTotalsItem(searchItems.ToArray());
                    log.WriteLine($"{DateTime.Now} :: Added totals item batch {i++} ({searchItems.Count()} items)");

                    if (continuationToken != null)
                        Thread.Sleep(500);
                }
                while (continuationToken != null);

                log.WriteLine($"{DateTime.Now} :: DONE");
            }
            catch (Exception ex)
            {
                var tags = new List<string> { "RefillSearchIndex" };
                new RaygunWebApiClient(ConfigurationManager.AppSettings[Constants.SettingKey_RaygunKey])
                    .SendInBackground(ex, tags, null);
                throw;
            }
        }
        /// <summary>
        /// Inserts or updates (using merge operation) a batch of transaction table entities.
        /// Also generates IDs for items that do not have them and sets Date to DateTime.Now if it is null.
        /// </summary>
        /// <param name="batch">Batch of Transaction items to add/update</param>
        /// <exception cref="System.Exception">Thrown when any of the items is not valid or if adding to either table storage or search index failed.</exception>
        public void AddOrUpdateTransactionBatch(IEnumerable<Transaction> batch)
        {
            var tableEnts = new List<TransactionTableEntity>();

            EnsureTransactionHasCustomerInternalID(batch.ToArray());

            foreach (Transaction t in batch)
            {
                //also ensure that all entities are valid before any of them are committed to table storage or search index
                t.EnsureEntityValidForStorage(true);

                var ent = new TransactionTableEntity(t.ID.Value, t.Date.Value, t.InternalRef.Value)
                {
                    MvaNumber = t.MvaNumber,
                    Amount = t.Amount,
                    Description = t.Description,
                    CustomerNumber = t.CustomerNumber,
                    AccountID = t.AccountID,
                    ProductID = t.ProductID,
                    NotInvoiceable = t.NotInvoiceable
                };

                tableEnts.Add(ent);
            }

            //Need to group entities by partition key because table storage only allows batch operations in the scope of the same partition key
            var entsGroupedByPartition = tableEnts.GroupBy(t => t.PartitionKey);
            foreach (var entGroup in entsGroupedByPartition)
            {
                for (int i = 0; i < entGroup.Count(); i += 100)
                {
                    var tableBatch = new TableBatchOperation();
                    foreach (var ent in entGroup.Skip(i).Take(100))
                    {
                        tableBatch.Add(TableOperation.InsertOrMerge(ent));
                    }

                    TransactionTable.ExecuteBatch(tableBatch);
                }
            }
            
            var dtReIndexReq = DateTime.UtcNow.Date.AddDays(-Constants.DaysToKeepTransactions);
            var entsToAddToIndex = tableEnts.Where(t => t.DateTime >= dtReIndexReq);

            var sRepo = new SearchRepository();
            sRepo.AddToIndex(entsToAddToIndex.ToArray());

            AddReIndexRequestIfNeeded(tableEnts.ToArray());
        }