상속: ISearchRepository
예제 #1
0
        public static async Task RefillSearchIndex(TextWriter log)
        {
            try
            {
                log.WriteLine($"{DateTime.Now} :: Function is invoked");

                var custRepo = new CustomerRepository();
                var sRepo = new SearchRepository();
                var tRepo = new TransactionRepository();

                var customers = custRepo.GetAllCustomers();
                log.WriteLine($"Total customers: {customers.Count()}");

                DateTime dtLimit = DateTime.UtcNow.Date.AddDays(-Constants.DaysToKeepTransactions);
                DateTime dtLimitWoDay = dtLimit.AddDays(-dtLimit.Day + 1);    //remove day component from DateTime
                DateTime dtEnd = DateTime.UtcNow.Date;
                DateTime dtEndWoDay = dtEnd.AddDays(-dtEnd.Day + 1);

                foreach (var customer in customers)
                {
                    log.WriteLine($"{DateTime.Now} :: Processing customer {customer.InternalID} {customer.Name}");

                    for (DateTime d = dtLimitWoDay; d <= dtEndWoDay; d = d.AddMonths(1))
                    {
                        string strDate = d.ToString(Constants.DateStringFormat);
                        int dateVal = int.Parse(strDate);

                        var res =await  tRepo.GetTransactionsForCustomer(customer.InternalID.Value, dateVal);
                        var matchingEnts = res.Where(r => r.DateTime >= dtLimit);

                        sRepo.AddToIndex(matchingEnts.ToArray());
                    }
                }

                int i = 1;
                TableContinuationToken continuationToken = null;
                do
                {
                    var items = tRepo.GetTransactionTotalsItemBatch(ref continuationToken);
                    var searchItems = items.Select(item => new TotalsSearchItem(item));
                    sRepo.AddOrUpdateTotalsItem(searchItems.ToArray());
                    log.WriteLine($"{DateTime.Now} :: Added totals item batch {i++} ({searchItems.Count()} items)");

                    if (continuationToken != null)
                        Thread.Sleep(500);
                }
                while (continuationToken != null);

                log.WriteLine($"{DateTime.Now} :: DONE");
            }
            catch (Exception ex)
            {
                var tags = new List<string> { "RefillSearchIndex" };
                new RaygunWebApiClient(ConfigurationManager.AppSettings[Constants.SettingKey_RaygunKey])
                    .SendInBackground(ex, tags, null);
                throw;
            }
        }
        public IHttpActionResult GetTransactionCountsForTopCustomers(int mvanumber, int startMonth, int startYear, int endMonth, int endYear, string productID)
        {
            string strStartDateVal = startYear.ToString("D4") + startMonth.ToString("D2");
            int startDateVal = int.Parse(strStartDateVal);

            string strEndDateVal = endYear.ToString("D4") + endMonth.ToString("D2");
            int endDateVal = int.Parse(strEndDateVal);

            var counts = new SearchRepository().GetTransactionCountsForTopCustomers(mvanumber, productID, startDateVal, endDateVal);

            return Ok(counts);
        }
        public IHttpActionResult GetPerMonthPerProductTransactionCountsForCustomer(int mvanumber, int startMonth, int startYear, int endMonth, int endYear, string customerInternalID = null)
        {
            string strStartDateVal = startYear.ToString("D4") + startMonth.ToString("D2");
            int startDateVal = int.Parse(strStartDateVal);

            string strEndDateVal = endYear.ToString("D4") + endMonth.ToString("D2");
            int endDateVal = int.Parse(strEndDateVal);

            var counts = new SearchRepository().GetTransactionCountsPerMonthPerProduct(mvanumber, customerInternalID, startDateVal, endDateVal);

            return Ok(counts);
        }
예제 #4
0
        private static async Task ProcessTransactions(SearchRepository sRepo, TransactionRepository tRepo, Customer customer, string productID, int? dateVal)
        {
            //any transaction newer than 'dtLimit' should not be processed because it's kept as an individual search item
            DateTime dtLimit = DateTime.UtcNow.Date.AddDays(-Constants.DaysToKeepTransactions);
            
            int curDateVal;
            if(dateVal.HasValue)
            {
                curDateVal = dateVal.Value;
            }
            else
            {
                curDateVal = int.Parse(dtLimit.AddDays(-1).ToString(Constants.DateStringFormat));
            }

            var existingItems = sRepo.GetTotalsItemsForCustomer(customer.InternalID.Value, productID, curDateVal);
            HashSet<string> existingItemIDs = new HashSet<string>(existingItems.Select(e => e.DocUniqueID));

            Dictionary<string, int> productTotals = new Dictionary<string, int>();

            var res =await  tRepo.GetTransactionsForCustomer(customer.InternalID.Value, curDateVal);
            if(!string.IsNullOrWhiteSpace(productID))
            {
                res = res.Where(r => r.ProductID == productID);
            }
            
            res = res.Where(r => r.DateTime < dtLimit);

            foreach (var transaction in res)
            {
                if (productTotals.ContainsKey(transaction.ProductID))
                {
                    productTotals[transaction.ProductID] = productTotals[transaction.ProductID] + transaction.Amount.Value;
                }
                else
                {
                    productTotals.Add(transaction.ProductID, transaction.Amount.Value);
                }
            }

            var searchItems = new List<TotalsSearchItem>();
            foreach (var entry in productTotals)
            {
                var newItem = new TotalsSearchItem
                {
                    DocUniqueID = TotalsSearchItem.CreateUID(curDateVal, entry.Key, customer.InternalID.Value),
                    ClientInternalID = customer.InternalID.Value.ToString(),
                    CustomerNumber = customer.CustomerNumber,
                    Date = curDateVal,
                    MvaNumber = customer.MvaNumber,
                    ProductID = entry.Key,
                    Amount = entry.Value
                };

                searchItems.Add(newItem);
                existingItemIDs.Remove(newItem.DocUniqueID);
            }

            sRepo.AddOrUpdateTotalsItem(searchItems.ToArray());
            tRepo.AddOrUpdateTransactionTotalsItem(searchItems.Select(item => new TotalsItemTableEntity(item)).ToArray());

            //in case a transaction was deleted, remove all totals items from search index that are no longer valid
            sRepo.DeleteTotalsItemWithID(existingItemIDs.ToArray());
            tRepo.DeleteTransactionTotalsItemWithID(existingItemIDs.ToArray());
        }
예제 #5
0
        public static async Task UpdateTotals(TextWriter log)
        {
            try
            {
                log.WriteLine($"{DateTime.Now} :: Function is invoked");
                List<Task> tasks = new List<Task>();

                var custRepo = new CustomerRepository();
                var sRepo = new SearchRepository();
                var tRepo = new TransactionRepository();

                var customers = custRepo.GetAllCustomers();
                log.WriteLine($"Total customers: {customers.Count()}");

                foreach (var customer in customers)
                {
                    log.WriteLine($"{DateTime.Now} :: Processing customer {customer.InternalID} {customer.Name}");

                    tasks.Add(ProcessTransactions(sRepo, tRepo, customer, null, null));
                }

                if (tasks.Any())
                    await Task.WhenAll(tasks);

                log.WriteLine($"{DateTime.Now} :: All customers processed");
                                
                sRepo.DeleteOldTransactions();
                tasks = new List<Task>();
                log.WriteLine($"{DateTime.Now} :: Old transactions processed");
                TableContinuationToken continuationToken = null;
                do
                {
                    var items = tRepo.GetReIndexableItemBatch(ref continuationToken);
                    log.WriteLine($"{DateTime.Now} :: Current re-index batch items: {items.Count()}");

                    foreach (ReIndexTableEntity item in items)
                    {
                        log.WriteLine($"{DateTime.Now} :: Processing re-index item {item.RowKey}");
                        int reIndexDateVal = int.Parse(item.Date);

                        if(item.CustomerInternalID.HasValue)
                        {
                            var reIndexCustomer = custRepo.GetCustomerByInternalID(item.CustomerInternalID.Value);
                            if(reIndexCustomer == null)
                            {
                                throw new Exception($"Error while processing re-index item, customer with ID {item.CustomerInternalID.Value} not found.");
                            }

                            tasks.Add(ProcessTransactions(sRepo, tRepo, reIndexCustomer, item.ProductID, reIndexDateVal));
                        }
                        else
                        {
                            //no customer ID specified, process all of them
                            foreach (var customer in customers)
                            {
                                tasks.Add(ProcessTransactions(sRepo, tRepo, customer, item.ProductID, reIndexDateVal));
                            }
                        }                   
                    }

                    tRepo.DeleteReIndexItemBatch(items.ToArray());
                    //TODO: should ignore errors that are caused by the ETag mismatch. 
                    //These can happen when an existing request was overwritten during the processing above.
                    //The correct thing to do would be to ignore it and to process the item again next time.
                }
                while (continuationToken != null);

                if (tasks.Any())
                    await Task.WhenAll(tasks);

                log.WriteLine($"{DateTime.Now} :: DONE");
            }
            catch (Exception ex)
            {
                var tags = new List<string> { "UpdateTotals" };
                new RaygunWebApiClient(ConfigurationManager.AppSettings[Constants.SettingKey_RaygunKey])
                    .SendInBackground(ex, tags, null);
                throw;
            }
        }
        public IHttpActionResult GetProductIDs(int mvanumber)
        {
            List<string> values = new SearchRepository().GetUniqueProductIDs(mvanumber);

            return Ok(values);
        }
        /// <summary>
        /// Inserts or updates (using merge operation) a batch of transaction table entities.
        /// Also generates IDs for items that do not have them and sets Date to DateTime.Now if it is null.
        /// </summary>
        /// <param name="batch">Batch of Transaction items to add/update</param>
        /// <exception cref="System.Exception">Thrown when any of the items is not valid or if adding to either table storage or search index failed.</exception>
        public void AddOrUpdateTransactionBatch(IEnumerable<Transaction> batch)
        {
            var tableEnts = new List<TransactionTableEntity>();

            EnsureTransactionHasCustomerInternalID(batch.ToArray());

            foreach (Transaction t in batch)
            {
                //also ensure that all entities are valid before any of them are committed to table storage or search index
                t.EnsureEntityValidForStorage(true);

                var ent = new TransactionTableEntity(t.ID.Value, t.Date.Value, t.InternalRef.Value)
                {
                    MvaNumber = t.MvaNumber,
                    Amount = t.Amount,
                    Description = t.Description,
                    CustomerNumber = t.CustomerNumber,
                    AccountID = t.AccountID,
                    ProductID = t.ProductID,
                    NotInvoiceable = t.NotInvoiceable
                };

                tableEnts.Add(ent);
            }

            //Need to group entities by partition key because table storage only allows batch operations in the scope of the same partition key
            var entsGroupedByPartition = tableEnts.GroupBy(t => t.PartitionKey);
            foreach (var entGroup in entsGroupedByPartition)
            {
                for (int i = 0; i < entGroup.Count(); i += 100)
                {
                    var tableBatch = new TableBatchOperation();
                    foreach (var ent in entGroup.Skip(i).Take(100))
                    {
                        tableBatch.Add(TableOperation.InsertOrMerge(ent));
                    }

                    TransactionTable.ExecuteBatch(tableBatch);
                }
            }
            
            var dtReIndexReq = DateTime.UtcNow.Date.AddDays(-Constants.DaysToKeepTransactions);
            var entsToAddToIndex = tableEnts.Where(t => t.DateTime >= dtReIndexReq);

            var sRepo = new SearchRepository();
            sRepo.AddToIndex(entsToAddToIndex.ToArray());

            AddReIndexRequestIfNeeded(tableEnts.ToArray());
        }