public IHttpActionResult AddCustomer(int mvanumber, Customer c)
{
c.MvaNumber = mvanumber;
c = new CustomerRepository().AddOrUpdateCustomer(c);
return Ok(c);
}
public static async Task RefillSearchIndex(TextWriter log)
{
try
{
log.WriteLine($"{DateTime.Now} :: Function is invoked");
var custRepo = new CustomerRepository();
var sRepo = new SearchRepository();
var tRepo = new TransactionRepository();
var customers = custRepo.GetAllCustomers();
log.WriteLine($"Total customers: {customers.Count()}");
DateTime dtLimit = DateTime.UtcNow.Date.AddDays(-Constants.DaysToKeepTransactions);
DateTime dtLimitWoDay = dtLimit.AddDays(-dtLimit.Day + 1); //remove day component from DateTime
DateTime dtEnd = DateTime.UtcNow.Date;
DateTime dtEndWoDay = dtEnd.AddDays(-dtEnd.Day + 1);
foreach (var customer in customers)
{
log.WriteLine($"{DateTime.Now} :: Processing customer {customer.InternalID} {customer.Name}");
for (DateTime d = dtLimitWoDay; d <= dtEndWoDay; d = d.AddMonths(1))
{
string strDate = d.ToString(Constants.DateStringFormat);
int dateVal = int.Parse(strDate);
var res =await tRepo.GetTransactionsForCustomer(customer.InternalID.Value, dateVal);
var matchingEnts = res.Where(r => r.DateTime >= dtLimit);
sRepo.AddToIndex(matchingEnts.ToArray());
}
}
int i = 1;
TableContinuationToken continuationToken = null;
do
{
var items = tRepo.GetTransactionTotalsItemBatch(ref continuationToken);
var searchItems = items.Select(item => new TotalsSearchItem(item));
sRepo.AddOrUpdateTotalsItem(searchItems.ToArray());
log.WriteLine($"{DateTime.Now} :: Added totals item batch {i++} ({searchItems.Count()} items)");
if (continuationToken != null)
Thread.Sleep(500);
}
while (continuationToken != null);
log.WriteLine($"{DateTime.Now} :: DONE");
}
catch (Exception ex)
{
var tags = new List<string> { "RefillSearchIndex" };
new RaygunWebApiClient(ConfigurationManager.AppSettings[Constants.SettingKey_RaygunKey])
.SendInBackground(ex, tags, null);
throw;
}
}
public IHttpActionResult GetCustomer(int mvanumber,string externalId)
{
if (string.IsNullOrWhiteSpace(externalId))
return BadRequest("externalID cannot be empty");
var customers = new CustomerRepository().GetCustomerByExternalID(mvanumber,externalId);
return Ok(customers);
}
public IHttpActionResult UpdateCustomer(int mvanumber, Customer c)
{
if (c.InternalID == null)
{
return BadRequest();
}
c.MvaNumber = mvanumber;
c = new CustomerRepository().AddOrUpdateCustomer(c);
return Ok(c);
}
public async Task<IHttpActionResult> GetInvoiceRows(int mvanumber, int month, int year,bool includeId=true)
{
var customers = new CustomerRepository().GetCustomersByMva(mvanumber);
string strDateVal = year.ToString("D4") + month.ToString("D2");
int dateVal = int.Parse(strDateVal);
List<InvoiceRow> rows = new List<InvoiceRow>();
//Need to use a collection that can contain only unique values because the search returns duplicate rows, when transaction amount > 1
//Dictionary key is Customer number + Product ID
Dictionary<string, List<Guid>> transactionIDsDict = new Dictionary<string, List<Guid>>();
var tItems = await new TransactionRepository().GetTransactionsForInvoicing(mvanumber, dateVal,includeId);
foreach (var tItem in tItems)
{
if (!tItem.CustomerNumber.HasValue)
{
//Not sure if we can process these
continue;
}
var productRow = rows.FirstOrDefault(r => r.ProductID == tItem.ProductID && r.EconomyCustomerNumber == tItem.CustomerNumber);
if (productRow == null)
{
productRow = new InvoiceRow()
{
Count = 0,
ProductID = tItem.ProductID,
TransactionServiceCustomerId= tItem.InternalRef==null ? tItem.InternalRef.GetValueOrDefault().ToString():tItem.AccountID,
EconomyCustomerNumber = tItem.CustomerNumber ?? 0
};
rows.Add(productRow);
transactionIDsDict.Add(tItem.CustomerNumber.ToString() + tItem.ProductID, new List<Guid>());
}
productRow.Count += tItem.Amount.Value;
Guid rowId = Guid.Parse(tItem.RowKey);
transactionIDsDict[tItem.CustomerNumber.ToString() + tItem.ProductID].Add(rowId);
}
DateTime dt = new DateTime(year, month, 1, 0, 0, 0, DateTimeKind.Utc);
List<InvoiceRow> rowsWithMinusAmounts = new List<InvoiceRow>();
//Process each previously generated transaction row.
//Add transaction IDs to each existing row and also create rows with minus amounts.
foreach (var row in rows)
{
if(includeId)
row.IDs = transactionIDsDict[row.EconomyCustomerNumber.ToString() + row.ProductID].ToArray();
rowsWithMinusAmounts.Add(row);
var custItems = customers.Where(c => c.CustomerNumber == row.EconomyCustomerNumber);
int incTotal = custItems.Sum(c => c.GetIncludedTransactionCount(row.ProductID, dt));
var minusRow = new InvoiceRow
{
Count = -Math.Min(row.Count, incTotal),
EconomyCustomerNumber = row.EconomyCustomerNumber,
ProductID = row.ProductID,
IDs = includeId ? row.IDs:null,
TransactionServiceCustomerId = row.TransactionServiceCustomerId,
};
rowsWithMinusAmounts.Add(minusRow);
}
return Ok(rowsWithMinusAmounts);
}
private static void AddTestCustomers(int customerCount)
{
var repo = new CustomerRepository();
for (int i = 1; i <= customerCount; i++)
{
Customer c = new Customer()
{
Name = $"Test {i}",
ActiveAccount = true,
InternalID = Guid.NewGuid(),
CustomerNumber = i,
ExternalIDs = new List<string> { i.ToString(), $"Test{i}"},
MvaNumber = (i / 10 + 1)
};
repo.AddOrUpdateCustomer(c);
}
}
private static void AddTestData()
{
var transactRepo = new TransactionRepository();
var custRepo = new CustomerRepository();
var customers = custRepo.GetAllCustomers().ToList();
int productCount = 3;
List<string> prodIDs = new List<string>();
for (int i = 1; i <= productCount; i++)
{
prodIDs.Add("Signature" + i);
}
DateTime minDt = new DateTime(2015, 1, 1, 0, 0, 0, DateTimeKind.Utc);
DateTime maxDt = DateTime.UtcNow;
TimeSpan timeSpan = maxDt - minDt;
var transactionBatch = new List<Transaction>(100);
Stopwatch sw = Stopwatch.StartNew();
for (int i = 0; i < 50000; i++)
{
string prodId = prodIDs[_rand.Next(0, productCount)];
TimeSpan randSpan = new TimeSpan(0, 0, _rand.Next(0, (int)timeSpan.TotalSeconds));
DateTime date = minDt + randSpan;
var curCustomer = customers[_rand.Next(0, customers.Count)];
Guid clientId = curCustomer.InternalID.Value;
int transactDuplicates = _rand.Next(0, 10) >= 9 ? 2 : 1;
Transaction t = new Transaction
{
MvaNumber = curCustomer.MvaNumber,
ExternalRef = curCustomer.ExternalIDs.First(),
//InternalRef = curCustomer.InternalID,
CustomerNumber = curCustomer.CustomerNumber,
Date = date,
Description = "test item",
ProductID = prodId,
Amount = transactDuplicates
};
//transactRepo.AddTransactionToUpdateQueue(t);
transactionBatch.Add(t);
if (i % 100 == 0)
{
transactRepo.AddOrUpdateTransactionBatch(transactionBatch);
transactionBatch.Clear();
Console.WriteLine($"{i} transactions have been added :: {sw.ElapsedMilliseconds} ms");
WriteQueueLengthAsync();
sw.Restart();
}
}
}
public async Task<IHttpActionResult> GetTransactionsForCustomerForMonth(int mvanumber, int month, int year, string customerExternalId = null, int? customerNumber = null,bool onlyInvoicable=false)
{
if (string.IsNullOrWhiteSpace(customerExternalId) && !customerNumber.HasValue)
{
return BadRequest("Either 'customerExternalId' or 'customerNumber' must have a value");
}
string strDateVal = year.ToString("D4") + month.ToString("D2");
int dateVal = int.Parse(strDateVal);
Customer customer = null;
if (!string.IsNullOrWhiteSpace(customerExternalId))
{
customer = new CustomerRepository().GetCustomerByExternalID(mvanumber, customerExternalId);
}
else
{
customer = new CustomerRepository().GetCustomersByMva(mvanumber).FirstOrDefault(c => c.CustomerNumber == customerNumber);
}
if (customer == null)
{
return InternalServerError(new Exception("Failed to resolve customer"));
}
var tableEnts =await new TransactionRepository().GetTransactionsForCustomer(customer.InternalID.Value, dateVal, onlyInvoicable);
var transactions = tableEnts.Select(r => new Transaction
{
ID = r.ID,
ProductID = r.ProductID,
Amount = r.Amount,
AccountID = r.AccountID,
Description = r.Description,
InternalRef = r.InternalRef,
MvaNumber = r.MvaNumber,
CustomerNumber = r.CustomerNumber,
Date = DateTime.ParseExact(r.Date, Constants.DateStringFormat, CultureInfo.InvariantCulture),
NotInvoiceable = r.NotInvoiceable,
ExternalRecordID = r.ExternalRecordID
});
return Ok(transactions);
}
public static async Task UpdateTotals(TextWriter log)
{
try
{
log.WriteLine($"{DateTime.Now} :: Function is invoked");
List<Task> tasks = new List<Task>();
var custRepo = new CustomerRepository();
var sRepo = new SearchRepository();
var tRepo = new TransactionRepository();
var customers = custRepo.GetAllCustomers();
log.WriteLine($"Total customers: {customers.Count()}");
foreach (var customer in customers)
{
log.WriteLine($"{DateTime.Now} :: Processing customer {customer.InternalID} {customer.Name}");
tasks.Add(ProcessTransactions(sRepo, tRepo, customer, null, null));
}
if (tasks.Any())
await Task.WhenAll(tasks);
log.WriteLine($"{DateTime.Now} :: All customers processed");
sRepo.DeleteOldTransactions();
tasks = new List<Task>();
log.WriteLine($"{DateTime.Now} :: Old transactions processed");
TableContinuationToken continuationToken = null;
do
{
var items = tRepo.GetReIndexableItemBatch(ref continuationToken);
log.WriteLine($"{DateTime.Now} :: Current re-index batch items: {items.Count()}");
foreach (ReIndexTableEntity item in items)
{
log.WriteLine($"{DateTime.Now} :: Processing re-index item {item.RowKey}");
int reIndexDateVal = int.Parse(item.Date);
if(item.CustomerInternalID.HasValue)
{
var reIndexCustomer = custRepo.GetCustomerByInternalID(item.CustomerInternalID.Value);
if(reIndexCustomer == null)
{
throw new Exception($"Error while processing re-index item, customer with ID {item.CustomerInternalID.Value} not found.");
}
tasks.Add(ProcessTransactions(sRepo, tRepo, reIndexCustomer, item.ProductID, reIndexDateVal));
}
else
{
//no customer ID specified, process all of them
foreach (var customer in customers)
{
tasks.Add(ProcessTransactions(sRepo, tRepo, customer, item.ProductID, reIndexDateVal));
}
}
}
tRepo.DeleteReIndexItemBatch(items.ToArray());
//TODO: should ignore errors that are caused by the ETag mismatch.
//These can happen when an existing request was overwritten during the processing above.
//The correct thing to do would be to ignore it and to process the item again next time.
}
while (continuationToken != null);
if (tasks.Any())
await Task.WhenAll(tasks);
log.WriteLine($"{DateTime.Now} :: DONE");
}
catch (Exception ex)
{
var tags = new List<string> { "UpdateTotals" };
new RaygunWebApiClient(ConfigurationManager.AppSettings[Constants.SettingKey_RaygunKey])
.SendInBackground(ex, tags, null);
throw;
}
}
public IHttpActionResult GetAllCustomers(int mvanumber)
{
var customers = new CustomerRepository().GetCustomersByMva(mvanumber);
return Ok(customers);
}
public IHttpActionResult Get(int mvanumber, Guid id)
{
var customers = new CustomerRepository().GetCustomerByInternalID(id);
return Ok(customers);
}
/// <summary>
/// Adds the transaction item to search index. Can also be used for updating existing items by specifying the Transaction.ID value.
/// Throws exception when item is not valid, resolving customer external ID failed or if search index operation fails.
/// </summary>
/// <param name="t"></param>
public void AddToIndex(Transaction t)
{
t.EnsureEntityValidForStorage();
Guid internalID;
if (t.InternalRef.HasValue)
{
internalID = t.InternalRef.Value;
}
else
{
Customer c = new CustomerRepository().GetCustomerByExternalID(t.MvaNumber.Value, t.ExternalRef);
if (c == null)
{
throw new Exception(string.Format("Failed to resolve customer external ID ({0}) to internal", t.ExternalRef));
}
internalID = c.InternalID.Value;
}
var tableEnt = new TransactionTableEntity(t.ID.Value, t.Date.Value, internalID)
{
Amount = t.Amount,
CustomerNumber = t.CustomerNumber,
AccountID = t.AccountID,
Description = t.Description,
MvaNumber = t.MvaNumber,
ProductID = t.ProductID
};
AddToIndex(tableEnt);
}
/// <summary>
/// Finds top 10 customers who have the highest amount of transactions for a specific product inside the time period.
/// Also returns a 11th item that contains the amount of transactions for the rest of the customers that didn't make it into the top 10.
/// </summary>
/// <param name="mvanumber">required</param>
/// <param name="productID">required</param>
/// <param name="startDateVal">required</param>
/// <param name="endDateVal">required</param>
/// <returns></returns>
public List<CustomerTransactionCount> GetTransactionCountsForTopCustomers(int mvanumber, string productID, int startDateVal, int endDateVal)
{
var customers = new CustomerRepository().GetCustomersByMva(mvanumber);
DateTime dtLimit = DateTime.UtcNow.Date.AddDays(-Constants.DaysToKeepTransactions);
List<string> filters = new List<string>();
filters.Add("mvaNumber eq " + mvanumber);
filters.Add("date ge " + startDateVal);
filters.Add("date le " + endDateVal);
filters.Add("dateTime ge " + dtLimit.ToString(Constants.ODataDateFormat));
filters.Add("productID eq '" + productID + "'");
var sp = new SearchParameters();
sp.Top = 0;
sp.Filter = string.Join(" and ", filters);
sp.Facets = new List<string> { "clientInternalID,count:1000" };
DocumentSearchResult<TransactionSearchItem> response = TransactionIndexClient.Documents.Search<TransactionSearchItem>("*", sp);
var facetItems = response.Facets.First().Value;
var counts = facetItems.Select(f => new CustomerTransactionCount
{
CustomerInternalID = f.Value.ToString(),
CustomerName = customers.FirstOrDefault(c => c.InternalID == new Guid(f.Value.ToString()))?.Name,
Total = f.Count ?? 0
}).ToList();
filters = new List<string>();
filters.Add("mvaNumber eq " + mvanumber);
filters.Add("date ge " + startDateVal);
filters.Add("date le " + endDateVal);
filters.Add("productID eq '" + productID + "'");
sp = new SearchParameters();
sp.Top = 10000;
sp.Filter = string.Join(" and ", filters);
DocumentSearchResult<TotalsSearchItem> response2 = TotalsIndexClient.Documents.Search<TotalsSearchItem>("*", sp);
while (true)
{
foreach (var res in response2.Results)
{
string customerId = res.Document.ClientInternalID;
var cItem = counts.FirstOrDefault(itm => itm.CustomerInternalID == customerId);
if (cItem == null)
{
cItem = new CustomerTransactionCount
{
CustomerInternalID = customerId,
CustomerName = customers.FirstOrDefault(c => c.InternalID == new Guid(customerId))?.Name,
};
counts.Add(cItem);
}
cItem.Total += res.Document.Amount.Value;
}
if (response2.ContinuationToken == null)
{
break;
}
response2 = TotalsIndexClient.Documents.ContinueSearch<TotalsSearchItem>(response2.ContinuationToken);
}
var orderedTotals = counts.OrderByDescending(c => c.Total);
var retVal = orderedTotals.Take(10).ToList();
retVal.Add(new CustomerTransactionCount
{
CustomerInternalID = null,
CustomerName = null,
Total = orderedTotals.Skip(10).Sum(f => f.Total)
});
return retVal;
}
private void EnsureTransactionHasCustomerInternalID(params Transaction[] transactions)
{
CustomerRepository cRepo = null;
foreach (var t in transactions)
{
if (t.InternalRef.HasValue) continue;
if(!string.IsNullOrEmpty(t.ExternalRef))
{
if (cRepo == null) cRepo = new CustomerRepository();
Customer c = cRepo.GetCustomerByExternalID(t.MvaNumber.Value, t.ExternalRef);
if (c == null)
{
throw new Exception(string.Format("Failed to resolve customer external ID ({0}) to internal", t.ExternalRef));
}
t.InternalRef = c.InternalID.Value;
}
else
{
throw new Exception("Both customer internal ID and external ID cannot be empty");
}
}
}
public async Task<List<TransactionTableEntity>> GetTransactionsForInvoicing(int mvanumber, int dateVal, bool includeId = true)
{
List<TransactionTableEntity> list = new List<TransactionTableEntity>();
var customers = new CustomerRepository().GetCustomersByMva(mvanumber).Where(x=>x.ActiveAccount);
List<Task<IEnumerable<TransactionTableEntity>>> tasks = customers.Select(customer => GetTransactionsForCustomer(customer.InternalID.Value, dateVal, true)).ToList();
if (tasks.Any())
await Task.WhenAll(tasks);
foreach (var task in tasks)
{
list.AddRange(task.Result.Where(r => (!r.NotInvoiceable.HasValue || !r.NotInvoiceable.Value) && (!r.OrderItemID.HasValue || !(r.OrderItemID.Value > 0)) ));
}
//var paralell = Parallel.ForEach(customers, async item =>
//{
// var result = await GetTransactionsForCustomer(item.InternalID.Value, dateVal, true);
// list.AddRange(result.Where(r => !r.NotInvoiceable.HasValue || !r.NotInvoiceable.Value));
//});
//do
//{
//} while (!paralell.IsCompleted);
return list;
}
