static private ImportResults ProcessImportedData(ExternalBankData import, ProcessThreadArguments args)
{
FinancialAccount assetAccount = args.Account;
FinancialAccount autoDepositAccount = args.Organization.FinancialAccounts.IncomeDonations;
int autoDepositLimit = 1000; // TODO: this.CurrentOrganization.Parameters.AutoDonationLimit;
ImportResults result = new ImportResults();
int count = 0;
int progressUpdateInterval = import.Records.Length/40;
if (progressUpdateInterval > 100)
{
progressUpdateInterval = 100;
}
foreach (ExternalBankDataRecord row in import.Records)
{
// Update progress.
count++;
if (progressUpdateInterval < 2 || count % progressUpdateInterval == 0)
{
int percent = (count*99)/import.Records.Length;
GuidCache.Set(args.Guid + "-Progress", percent);
}
// Update high- and low-water marks.
if (row.DateTime < result.EarliestTransaction)
{
result.EarliestTransaction = row.DateTime;
}
if (row.DateTime > result.LatestTransaction)
{
result.LatestTransaction = row.DateTime;
}
string importKey = row.ImportHash;
Int64 amountCents = row.TransactionNetCents;
if (amountCents == 0) // defensive programming - these _should_ be duplicated in the interpreter if no "fee" field
{
amountCents = row.TransactionGrossCents;
}
if (args.Organization.Identity == 1 && assetAccount.Identity == 1 && PilotInstallationIds.IsPilot(PilotInstallationIds.PiratePartySE))
{
// This is an ugly-as-f**k hack that sorts under the category "just bring our pilots the f**k back to operational
// status right f*****g now".
// This code can and should be safely removed once the pilot's books are closed for 2014, which should be some time mid-2015.
if (row.DateTime < new DateTime(2014,03,22))
{
result.DuplicateTransactions++;
continue;
}
}
FinancialTransaction transaction = FinancialTransaction.ImportWithStub(args.Organization.Identity, row.DateTime,
assetAccount.Identity, amountCents,
row.Description, importKey,
args.CurrentUser.Identity);
if (transaction != null)
{
// The transaction was created. Examine if the autobook criteria are true.
result.TransactionsImported++;
FinancialAccounts accounts = FinancialAccounts.FromBankTransactionTag(row.Description);
if (accounts.Count == 1)
{
// This is a labelled local donation.
Geography geography = accounts[0].AssignedGeography;
FinancialAccount localAccount = accounts[0];
transaction.AddRow(args.Organization.FinancialAccounts.IncomeDonations, -amountCents, args.CurrentUser);
transaction.AddRow(args.Organization.FinancialAccounts.CostsLocalDonationTransfers,
amountCents, args.CurrentUser);
transaction.AddRow(localAccount, -amountCents, args.CurrentUser);
PWEvents.CreateEvent(EventSource.PirateWeb, EventType.LocalDonationReceived,
args.CurrentUser.Identity, args.Organization.Identity,
geography.Identity, 0,
transaction.Identity, localAccount.Identity.ToString());
}
else if (row.Description.ToLowerInvariant().StartsWith(args.Organization.IncomingPaymentTag))
{
// Check for previously imported payment group
// TODO: MAKE FLEXIBLE - CALL PAYMENTREADERINTERFACE!
// HACK HACK HACK HACK
PaymentGroup group = PaymentGroup.FromTag(args.Organization,
"SEBGM" + DateTime.Today.Year.ToString() + // TODO: Get tags from org
row.Description.Substring(args.Organization.IncomingPaymentTag.Length).Trim());
if (group != null && group.Open)
{
// There was a previously imported and not yet closed payment group matching this transaction
// Close the payment group and match the transaction against accounts receivable
transaction.Dependency = group;
group.Open = false;
transaction.AddRow(args.Organization.FinancialAccounts.AssetsOutboundInvoices, -amountCents, args.CurrentUser);
}
}
else if (amountCents < 0)
{
// Autowithdrawal mechanisms removed, condition kept because of downstream else-if conditions
}
else if (amountCents > 0)
{
if (row.FeeCents < 0)
{
// This is always an autodeposit, if there is a fee (which is never > 0.0)
transaction.AddRow(args.Organization.FinancialAccounts.CostsBankFees, -row.FeeCents, args.CurrentUser);
transaction.AddRow(autoDepositAccount, -row.TransactionGrossCents, args.CurrentUser);
}
else if (amountCents < autoDepositLimit * 100)
{
// Book against autoDeposit account.
transaction.AddRow(autoDepositAccount, -amountCents, args.CurrentUser);
}
}
}
else
{
// Transaction was not imported; assume duplicate
result.DuplicateTransactions++;
}
}
// Import complete. Return true if the bookkeeping account matches the bank data.
Int64 databaseAccountBalanceCents = assetAccount.BalanceTotalCents;
// Subtract any transactions made after the most recent imported transaction.
// This is necessary in case of Paypal and others which continuously feed the
// bookkeeping account with new transactions; it will already have fed transactions
// beyond the end-of-file.
Int64 beyondEofCents = assetAccount.GetDeltaCents(result.LatestTransaction.AddSeconds(1), DateTime.Now.AddDays(2)); // Caution: the "AddSeconds(1)" is not foolproof, there may be other new txs on the same second.
if (databaseAccountBalanceCents - beyondEofCents == import.LatestAccountBalanceCents)
{
Payouts.AutomatchAgainstUnbalancedTransactions(args.Organization);
result.AccountBalanceMatchesBank = true;
result.BalanceMismatchCents = 0;
}
else
{
result.AccountBalanceMatchesBank = false;
result.BalanceMismatchCents = (databaseAccountBalanceCents - beyondEofCents) -
import.LatestAccountBalanceCents;
}
result.CurrencyCode = args.Organization.Currency.Code;
return result;
}
private static void ProcessUploadThread(object args)
{
string guid = ((ProcessThreadArguments) args).Guid;
Documents documents = Documents.RecentFromDescription(guid);
if (documents.Count != 1)
{
return; // abort
}
Document uploadedDoc = documents[0];
FinancialAccount account = ((ProcessThreadArguments)args).Account;
ExternalBankData externalData = new ExternalBankData();
externalData.Profile = ExternalBankDataProfile.FromIdentity(ExternalBankDataProfile.SESebId); // TODO: HACK HACK HACK HACK LOAD
using (StreamReader reader = new StreamReader(StorageRoot + uploadedDoc.ServerFileName, Encoding.GetEncoding(1252)))
{
externalData.LoadData(reader, ((ProcessThreadArguments) args).Organization);
}
_staticDataLookup[guid + "FirstTx"] = externalData.Records[0].DateTime.ToLongDateString();
_staticDataLookup[guid + "LastTx"] =
externalData.Records[externalData.Records.Length - 1].DateTime.ToLongDateString();
_staticDataLookup[guid + "TxCount"] = externalData.Records.Length.ToString("N0");
_staticDataLookup[guid + "Profile"] = externalData.Profile;
_staticDataLookup[guid + "PercentRead"] = 1;
DateTime timeWalker = externalData.Records[0].DateTime;
int currentRecordIndex = 0;
// Walk past the first identical time records, and verify that we have at least one balance record that matches
// our own for this timestamp. There may be several transactions in the master file, but at least one should have
// a post-transaction balance that matches our records, or something is much more broken.
long swarmopsCentsStart = account.GetDeltaCents(DateTime.MinValue, timeWalker.AddSeconds(1));
// At least one of the transactions for this timestamp should match.
bool foundMatch = false;
while (externalData.Records [currentRecordIndex].DateTime == timeWalker)
{
if (externalData.Records[currentRecordIndex].AccountBalanceCents == swarmopsCentsStart)
{
foundMatch = true;
// continue loop until on first record past initial conditions
}
currentRecordIndex++; // no test for array overrun in while
}
if (!foundMatch)
{
throw new InvalidOperationException("Unable to locate stable initial conditions for resynchronization");
}
// From here on out, every new timestamp should have the exact same delta in Swarmops as it has in the master file.
List<ExternalBankMismatchingDateTime> mismatchList = new List<ExternalBankMismatchingDateTime>();
while (currentRecordIndex < externalData.Records.Length)
{
DateTime lastTimestamp = timeWalker;
timeWalker = externalData.Records[currentRecordIndex].DateTime;
long swarmopsDeltaCents = account.GetDeltaCents(lastTimestamp.AddSeconds(1), timeWalker.AddSeconds(1));
// "AddSeconds" because DeltaCents operates on ">= lowbound, < highbound"
int timestampStartIndex = currentRecordIndex;
long masterDeltaCents = externalData.Records[currentRecordIndex++].TransactionNetCents;
int masterTransactionCount = 1;
while (currentRecordIndex < externalData.Records.Length &&
externalData.Records[currentRecordIndex].DateTime == timeWalker)
{
masterDeltaCents += externalData.Records[currentRecordIndex++].TransactionNetCents;
masterTransactionCount++;
}
if (masterDeltaCents != swarmopsDeltaCents)
{
// We have a mismatch. Add it to the list.
ExternalBankMismatchingDateTime newMismatch = new ExternalBankMismatchingDateTime();
newMismatch.DateTime = timeWalker;
newMismatch.MasterDeltaCents = masterDeltaCents;
newMismatch.MasterTransactionCount = masterTransactionCount;
newMismatch.SwarmopsDeltaCents = swarmopsDeltaCents;
newMismatch.SwarmopsTransactionCount = 0; // TODO
// Load transactions from both sources. First, create the interim construction object.
ExternalBankMismatchConstruction mismatchConstruction = new ExternalBankMismatchConstruction();
// Load from Master
for (int innerRecordIndex = timestampStartIndex; innerRecordIndex < currentRecordIndex; innerRecordIndex++)
{
string description = externalData.Records[innerRecordIndex].Description.Replace(" ", " ");
if (!mismatchConstruction.Master.ContainsKey(description))
{
mismatchConstruction.Master[description] =
new ExternalBankMismatchingRecordConstruction();
}
mismatchConstruction.Master[description].Cents.Add(externalData.Records[innerRecordIndex].TransactionNetCents);
mismatchConstruction.Master[description].Transactions.Add(null); // no dependencies on the master side, only on swarmops side
}
// Load from Swarmops
FinancialAccountRows swarmopsTransactionRows =
account.GetRowsFar(lastTimestamp, timeWalker); // the "select far" is a boundary < x <= boundary selector. Default is boundary <= x < boundary.
Dictionary<int, FinancialTransaction> lookupTransactions = new Dictionary<int, FinancialTransaction>();
// note all transaction IDs, then sum up per transaction
foreach (FinancialAccountRow swarmopsTransactionRow in swarmopsTransactionRows)
{
lookupTransactions[swarmopsTransactionRow.FinancialTransactionId] =
swarmopsTransactionRow.Transaction;
}
foreach (FinancialTransaction transaction in lookupTransactions.Values)
{
string description = transaction.Description.Replace(" ", " "); // for legacy compatibility with new importer
if (!mismatchConstruction.Swarmops.ContainsKey(description))
{
mismatchConstruction.Swarmops[description] =
new ExternalBankMismatchingRecordConstruction();
}
long cents = transaction[account];
if (cents != 0) // only add nonzero records
{
mismatchConstruction.Swarmops[description].Cents.Add(transaction[account]);
mismatchConstruction.Swarmops[description].Transactions.Add(transaction);
}
}
// Then, parse the intermediate construction object to the presentation-and-action object.
Dictionary <string,ExternalBankMismatchingRecordDescription> mismatchingRecordList = new Dictionary<string, ExternalBankMismatchingRecordDescription>();
foreach (string masterKey in mismatchConstruction.Master.Keys)
{
Dictionary<int, bool> checkMasterIndex = new Dictionary<int, bool>();
Dictionary<int, bool> checkSwarmopsIndex = new Dictionary<int, bool>();
// For each key and entry for each key;
// 1) locate an exact corresponding amount in swarmops records and log; failing that,
// 2) if exactly one record left in master and swarmops records, log; failing that,
// 3) log the rest of the master OR rest of swarmops records with no corresponding
// equivalent with counterpart. (May produce bad results if 2 consistent mismatches
// for every description.)
ExternalBankMismatchingRecordDescription newRecord =
new ExternalBankMismatchingRecordDescription();
newRecord.Description = masterKey;
List<long> masterCentsList = new List<long>();
List<long> swarmopsCentsList = new List<long>();
List<object> dependenciesList = new List<object>();
List<FinancialTransaction> transactionsList = new List<FinancialTransaction>();
List<ExternalBankMismatchResyncAction> actionsList = new List<ExternalBankMismatchResyncAction>();
// STEP 1 - locate all identical matches
if (mismatchConstruction.Swarmops.ContainsKey(masterKey))
{
for (int masterIndex = 0;
masterIndex < mismatchConstruction.Master[masterKey].Cents.Count;
masterIndex++)
{
// no "continue" necessary on first run-through; nothing has been checked off yet
long findMasterCents = mismatchConstruction.Master[masterKey].Cents[masterIndex];
for (int swarmopsIndex = 0;
swarmopsIndex < mismatchConstruction.Swarmops[masterKey].Cents.Count;
swarmopsIndex++)
{
if (checkSwarmopsIndex.ContainsKey(swarmopsIndex))
{
continue; // may have been checked off already in the rare case of twin identical amounts
}
if (findMasterCents == mismatchConstruction.Swarmops[masterKey].Cents[swarmopsIndex])
{
// There is a match as per case 1. Record both, mark both as used, continue.
masterCentsList.Add(findMasterCents);
swarmopsCentsList.Add(
mismatchConstruction.Swarmops[masterKey].Cents[swarmopsIndex]);
// should be equal, we're defensive here
transactionsList.Add(
mismatchConstruction.Swarmops[masterKey].Transactions[swarmopsIndex]);
dependenciesList.Add(
mismatchConstruction.Swarmops[masterKey].Transactions[swarmopsIndex].Dependency);
actionsList.Add(ExternalBankMismatchResyncAction.NoAction);
checkMasterIndex[masterIndex] = true;
checkSwarmopsIndex[swarmopsIndex] = true;
break;
}
}
}
}
// STEP 2 - if exactly one record left on both sides, connect and log as mismatching record
// TODO: improve logic to handle same number of records left on both sides
if (mismatchConstruction.Swarmops.ContainsKey(masterKey) &&
mismatchConstruction.Master[masterKey].Cents.Count - checkMasterIndex.Keys.Count == 1 &&
mismatchConstruction.Swarmops [masterKey].Cents.Count - checkSwarmopsIndex.Keys.Count == 1)
{
for (int masterIndex = 0; masterIndex < mismatchConstruction.Master[masterKey].Cents.Count; masterIndex++)
{
if (checkMasterIndex.ContainsKey(masterIndex))
{
continue; // This will fire for all but one indexes
}
long findMasterCents = mismatchConstruction.Master[masterKey].Cents[masterIndex];
for (int swarmopsIndex = 0; swarmopsIndex < mismatchConstruction.Swarmops[masterKey].Cents.Count; swarmopsIndex++)
{
if (checkSwarmopsIndex.ContainsKey(swarmopsIndex))
{
continue;
}
masterCentsList.Add(findMasterCents);
swarmopsCentsList.Add(mismatchConstruction.Swarmops[masterKey].Cents[swarmopsIndex]);
dependenciesList.Add(mismatchConstruction.Swarmops[masterKey].Transactions[swarmopsIndex].Dependency);
transactionsList.Add(mismatchConstruction.Swarmops[masterKey].Transactions[swarmopsIndex]);
actionsList.Add(ExternalBankMismatchResyncAction.RewriteSwarmops);
checkMasterIndex[masterIndex] = true;
checkSwarmopsIndex[swarmopsIndex] = true;
}
}
}
// STEP 3 - log remaining records on both sides as missing counterparts. Only one of these should fire.
// STEP 3a - log remaining on Master side
if (mismatchConstruction.Master[masterKey].Cents.Count > checkMasterIndex.Keys.Count)
{
for (int masterIndex = 0; masterIndex < mismatchConstruction.Master[masterKey].Cents.Count; masterIndex++)
{
if (checkMasterIndex.ContainsKey(masterIndex))
{
continue;
}
masterCentsList.Add(mismatchConstruction.Master[masterKey].Cents[masterIndex]);
swarmopsCentsList.Add(0); // null equivalent; invalid value
dependenciesList.Add(null);
transactionsList.Add(null);
actionsList.Add(ExternalBankMismatchResyncAction.CreateSwarmops);
checkMasterIndex[masterIndex] = true;
}
}
// STEP 3b - log remaining on Swarmops side
if (mismatchConstruction.Swarmops.ContainsKey(masterKey) && mismatchConstruction.Swarmops[masterKey].Cents.Count > checkSwarmopsIndex.Keys.Count)
{
for (int swarmopsIndex = 0; swarmopsIndex < mismatchConstruction.Swarmops[masterKey].Cents.Count; swarmopsIndex++)
{
if (checkSwarmopsIndex.ContainsKey(swarmopsIndex))
{
continue;
}
masterCentsList.Add(0); // null equivalent; invalid value
swarmopsCentsList.Add(mismatchConstruction.Swarmops[masterKey].Cents[swarmopsIndex]);
transactionsList.Add(mismatchConstruction.Swarmops[masterKey].Transactions[swarmopsIndex]);
if (mismatchConstruction.Swarmops[masterKey].Transactions[swarmopsIndex].Dependency != null)
{
dependenciesList.Add(
mismatchConstruction.Swarmops[masterKey].Transactions[swarmopsIndex].Dependency);
actionsList.Add(ExternalBankMismatchResyncAction.ManualAction); // can't auto
}
else
{
dependenciesList.Add(null);
actionsList.Add(ExternalBankMismatchResyncAction.DeleteSwarmops);
}
checkMasterIndex[swarmopsIndex] = true;
}
}
newRecord.MasterCents = masterCentsList.ToArray();
newRecord.SwarmopsCents = swarmopsCentsList.ToArray();
newRecord.ResyncActions = actionsList.ToArray();
// newRecord.TransactionDependencies = dependenciesList.ToArray();
newRecord.Transactions = transactionsList.ToArray();
mismatchingRecordList[masterKey] = newRecord;
}
// Finally, add the transactions that were (described) in Swarmops but not in Master
foreach (string swarmopsKey in mismatchConstruction.Swarmops.Keys)
{
if (!mismatchingRecordList.ContainsKey(swarmopsKey))
{
mismatchingRecordList[swarmopsKey] = new ExternalBankMismatchingRecordDescription();
mismatchingRecordList[swarmopsKey].Description = swarmopsKey;
mismatchingRecordList[swarmopsKey].SwarmopsCents =
mismatchConstruction.Swarmops[swarmopsKey].Cents.ToArray();
mismatchingRecordList[swarmopsKey].Transactions =
mismatchConstruction.Swarmops[swarmopsKey].Transactions.ToArray();
mismatchingRecordList[swarmopsKey].MasterCents =
new long[mismatchConstruction.Swarmops[swarmopsKey].Cents.Count]; // inits to zero
mismatchingRecordList[swarmopsKey].ResyncActions = new ExternalBankMismatchResyncAction[mismatchConstruction.Swarmops[swarmopsKey].Cents.Count];
for (int index = 0; index < mismatchingRecordList[swarmopsKey].ResyncActions.Length; index++)
{
mismatchingRecordList[swarmopsKey].ResyncActions[index] = ExternalBankMismatchResyncAction.DeleteSwarmops;
}
}
}
newMismatch.MismatchingRecords = mismatchingRecordList.Values.ToArray();
mismatchList.Add(newMismatch);
}
int percentProcessed = (int) (currentRecordIndex*100L/externalData.Records.Length);
lock (_staticDataLookup)
{
if (percentProcessed > 1)
{
_staticDataLookup[guid + "PercentRead"] = percentProcessed;
// for the progress bar to update async
}
if (percentProcessed > 99)
{
// Placed inside loop to have a contiguous lock block, even though it decreases performance.
// Should normally be placed just outside.
_staticDataLookup[guid + "MismatchArray"] = mismatchList.ToArray();
_staticDataLookup[guid + "Account"] = account;
}
}
}
}
private static void ProcessUploadThread(object args)
{
string guid = ((ProcessThreadArguments) args).Guid;
BankFileType fileType = ((ProcessThreadArguments) args).FileType;
Documents documents = Documents.RecentFromDescription(guid);
GuidCache.Set(guid + "-Result", ImportResultsCategory.Bad); // default - this is what happens if exception
if (documents.Count != 1)
{
return; // abort
}
Document uploadedDoc = documents[0];
try
{
FinancialAccount account = ((ProcessThreadArguments) args).Account;
ExternalBankData externalData = new ExternalBankData();
externalData.Profile = account.ExternalBankDataProfile;
if (fileType == BankFileType.AccountStatement)
{
using (StreamReader reader = uploadedDoc.GetReader(1252))
{
externalData.LoadData(reader, ((ProcessThreadArguments) args).Organization);
// catch here and set result to BAD
ImportResults results = ProcessImportedData(externalData, (ProcessThreadArguments) args);
GuidCache.Set(guid + "-ResultDetails", results);
if (results.AccountBalanceMatchesBank)
{
GuidCache.Set(guid + "-Result", ImportResultsCategory.Good);
}
else
{
GuidCache.Set(guid + "-Result", ImportResultsCategory.Questionable);
}
}
}
else if (fileType == BankFileType.PaymentDetails)
{
// Get reader factory from ExternalBankData
throw new NotImplementedException("Need to implement new flexible payment reader structure");
// IBankDataPaymentsReader paymentsReader = externalData.GetPaymentsReader();
// then read
}
}
catch (Exception e)
{
GuidCache.Set(guid + "-Exception", e.ToString());
}
finally
{
GuidCache.Set(guid + "-Progress", 100); // only here may the caller fetch the results
uploadedDoc.Delete(); // document no longer needed after processing, no matter the result
}
}
