private static void ProcessUploadThread(object args) { string guid = ((ProcessThreadArguments) args).Guid; Documents documents = Documents.RecentFromDescription(guid); if (documents.Count != 1) { return; // abort } Document uploadedDoc = documents[0]; FinancialAccount account = ((ProcessThreadArguments)args).Account; ExternalBankData externalData = new ExternalBankData(); externalData.Profile = ExternalBankDataProfile.FromIdentity(ExternalBankDataProfile.SESebId); // TODO: HACK HACK HACK HACK LOAD using (StreamReader reader = new StreamReader(StorageRoot + uploadedDoc.ServerFileName, Encoding.GetEncoding(1252))) { externalData.LoadData(reader, ((ProcessThreadArguments) args).Organization); } _staticDataLookup[guid + "FirstTx"] = externalData.Records[0].DateTime.ToLongDateString(); _staticDataLookup[guid + "LastTx"] = externalData.Records[externalData.Records.Length - 1].DateTime.ToLongDateString(); _staticDataLookup[guid + "TxCount"] = externalData.Records.Length.ToString("N0"); _staticDataLookup[guid + "Profile"] = externalData.Profile; _staticDataLookup[guid + "PercentRead"] = 1; DateTime timeWalker = externalData.Records[0].DateTime; int currentRecordIndex = 0; // Walk past the first identical time records, and verify that we have at least one balance record that matches // our own for this timestamp. There may be several transactions in the master file, but at least one should have // a post-transaction balance that matches our records, or something is much more broken. long swarmopsCentsStart = account.GetDeltaCents(DateTime.MinValue, timeWalker.AddSeconds(1)); // At least one of the transactions for this timestamp should match. bool foundMatch = false; while (externalData.Records [currentRecordIndex].DateTime == timeWalker) { if (externalData.Records[currentRecordIndex].AccountBalanceCents == swarmopsCentsStart) { foundMatch = true; // continue loop until on first record past initial conditions } currentRecordIndex++; // no test for array overrun in while } if (!foundMatch) { throw new InvalidOperationException("Unable to locate stable initial conditions for resynchronization"); } // From here on out, every new timestamp should have the exact same delta in Swarmops as it has in the master file. List<ExternalBankMismatchingDateTime> mismatchList = new List<ExternalBankMismatchingDateTime>(); while (currentRecordIndex < externalData.Records.Length) { DateTime lastTimestamp = timeWalker; timeWalker = externalData.Records[currentRecordIndex].DateTime; long swarmopsDeltaCents = account.GetDeltaCents(lastTimestamp.AddSeconds(1), timeWalker.AddSeconds(1)); // "AddSeconds" because DeltaCents operates on ">= lowbound, < highbound" int timestampStartIndex = currentRecordIndex; long masterDeltaCents = externalData.Records[currentRecordIndex++].TransactionNetCents; int masterTransactionCount = 1; while (currentRecordIndex < externalData.Records.Length && externalData.Records[currentRecordIndex].DateTime == timeWalker) { masterDeltaCents += externalData.Records[currentRecordIndex++].TransactionNetCents; masterTransactionCount++; } if (masterDeltaCents != swarmopsDeltaCents) { // We have a mismatch. Add it to the list. ExternalBankMismatchingDateTime newMismatch = new ExternalBankMismatchingDateTime(); newMismatch.DateTime = timeWalker; newMismatch.MasterDeltaCents = masterDeltaCents; newMismatch.MasterTransactionCount = masterTransactionCount; newMismatch.SwarmopsDeltaCents = swarmopsDeltaCents; newMismatch.SwarmopsTransactionCount = 0; // TODO // Load transactions from both sources. First, create the interim construction object. ExternalBankMismatchConstruction mismatchConstruction = new ExternalBankMismatchConstruction(); // Load from Master for (int innerRecordIndex = timestampStartIndex; innerRecordIndex < currentRecordIndex; innerRecordIndex++) { string description = externalData.Records[innerRecordIndex].Description.Replace(" ", " "); if (!mismatchConstruction.Master.ContainsKey(description)) { mismatchConstruction.Master[description] = new ExternalBankMismatchingRecordConstruction(); } mismatchConstruction.Master[description].Cents.Add(externalData.Records[innerRecordIndex].TransactionNetCents); mismatchConstruction.Master[description].Transactions.Add(null); // no dependencies on the master side, only on swarmops side } // Load from Swarmops FinancialAccountRows swarmopsTransactionRows = account.GetRowsFar(lastTimestamp, timeWalker); // the "select far" is a boundary < x <= boundary selector. Default is boundary <= x < boundary. Dictionary<int, FinancialTransaction> lookupTransactions = new Dictionary<int, FinancialTransaction>(); // note all transaction IDs, then sum up per transaction foreach (FinancialAccountRow swarmopsTransactionRow in swarmopsTransactionRows) { lookupTransactions[swarmopsTransactionRow.FinancialTransactionId] = swarmopsTransactionRow.Transaction; } foreach (FinancialTransaction transaction in lookupTransactions.Values) { string description = transaction.Description.Replace(" ", " "); // for legacy compatibility with new importer if (!mismatchConstruction.Swarmops.ContainsKey(description)) { mismatchConstruction.Swarmops[description] = new ExternalBankMismatchingRecordConstruction(); } long cents = transaction[account]; if (cents != 0) // only add nonzero records { mismatchConstruction.Swarmops[description].Cents.Add(transaction[account]); mismatchConstruction.Swarmops[description].Transactions.Add(transaction); } } // Then, parse the intermediate construction object to the presentation-and-action object. Dictionary <string,ExternalBankMismatchingRecordDescription> mismatchingRecordList = new Dictionary<string, ExternalBankMismatchingRecordDescription>(); foreach (string masterKey in mismatchConstruction.Master.Keys) { Dictionary<int, bool> checkMasterIndex = new Dictionary<int, bool>(); Dictionary<int, bool> checkSwarmopsIndex = new Dictionary<int, bool>(); // For each key and entry for each key; // 1) locate an exact corresponding amount in swarmops records and log; failing that, // 2) if exactly one record left in master and swarmops records, log; failing that, // 3) log the rest of the master OR rest of swarmops records with no corresponding // equivalent with counterpart. (May produce bad results if 2 consistent mismatches // for every description.) ExternalBankMismatchingRecordDescription newRecord = new ExternalBankMismatchingRecordDescription(); newRecord.Description = masterKey; List<long> masterCentsList = new List<long>(); List<long> swarmopsCentsList = new List<long>(); List<object> dependenciesList = new List<object>(); List<FinancialTransaction> transactionsList = new List<FinancialTransaction>(); List<ExternalBankMismatchResyncAction> actionsList = new List<ExternalBankMismatchResyncAction>(); // STEP 1 - locate all identical matches if (mismatchConstruction.Swarmops.ContainsKey(masterKey)) { for (int masterIndex = 0; masterIndex < mismatchConstruction.Master[masterKey].Cents.Count; masterIndex++) { // no "continue" necessary on first run-through; nothing has been checked off yet long findMasterCents = mismatchConstruction.Master[masterKey].Cents[masterIndex]; for (int swarmopsIndex = 0; swarmopsIndex < mismatchConstruction.Swarmops[masterKey].Cents.Count; swarmopsIndex++) { if (checkSwarmopsIndex.ContainsKey(swarmopsIndex)) { continue; // may have been checked off already in the rare case of twin identical amounts } if (findMasterCents == mismatchConstruction.Swarmops[masterKey].Cents[swarmopsIndex]) { // There is a match as per case 1. Record both, mark both as used, continue. masterCentsList.Add(findMasterCents); swarmopsCentsList.Add( mismatchConstruction.Swarmops[masterKey].Cents[swarmopsIndex]); // should be equal, we're defensive here transactionsList.Add( mismatchConstruction.Swarmops[masterKey].Transactions[swarmopsIndex]); dependenciesList.Add( mismatchConstruction.Swarmops[masterKey].Transactions[swarmopsIndex].Dependency); actionsList.Add(ExternalBankMismatchResyncAction.NoAction); checkMasterIndex[masterIndex] = true; checkSwarmopsIndex[swarmopsIndex] = true; break; } } } } // STEP 2 - if exactly one record left on both sides, connect and log as mismatching record // TODO: improve logic to handle same number of records left on both sides if (mismatchConstruction.Swarmops.ContainsKey(masterKey) && mismatchConstruction.Master[masterKey].Cents.Count - checkMasterIndex.Keys.Count == 1 && mismatchConstruction.Swarmops [masterKey].Cents.Count - checkSwarmopsIndex.Keys.Count == 1) { for (int masterIndex = 0; masterIndex < mismatchConstruction.Master[masterKey].Cents.Count; masterIndex++) { if (checkMasterIndex.ContainsKey(masterIndex)) { continue; // This will fire for all but one indexes } long findMasterCents = mismatchConstruction.Master[masterKey].Cents[masterIndex]; for (int swarmopsIndex = 0; swarmopsIndex < mismatchConstruction.Swarmops[masterKey].Cents.Count; swarmopsIndex++) { if (checkSwarmopsIndex.ContainsKey(swarmopsIndex)) { continue; } masterCentsList.Add(findMasterCents); swarmopsCentsList.Add(mismatchConstruction.Swarmops[masterKey].Cents[swarmopsIndex]); dependenciesList.Add(mismatchConstruction.Swarmops[masterKey].Transactions[swarmopsIndex].Dependency); transactionsList.Add(mismatchConstruction.Swarmops[masterKey].Transactions[swarmopsIndex]); actionsList.Add(ExternalBankMismatchResyncAction.RewriteSwarmops); checkMasterIndex[masterIndex] = true; checkSwarmopsIndex[swarmopsIndex] = true; } } } // STEP 3 - log remaining records on both sides as missing counterparts. Only one of these should fire. // STEP 3a - log remaining on Master side if (mismatchConstruction.Master[masterKey].Cents.Count > checkMasterIndex.Keys.Count) { for (int masterIndex = 0; masterIndex < mismatchConstruction.Master[masterKey].Cents.Count; masterIndex++) { if (checkMasterIndex.ContainsKey(masterIndex)) { continue; } masterCentsList.Add(mismatchConstruction.Master[masterKey].Cents[masterIndex]); swarmopsCentsList.Add(0); // null equivalent; invalid value dependenciesList.Add(null); transactionsList.Add(null); actionsList.Add(ExternalBankMismatchResyncAction.CreateSwarmops); checkMasterIndex[masterIndex] = true; } } // STEP 3b - log remaining on Swarmops side if (mismatchConstruction.Swarmops.ContainsKey(masterKey) && mismatchConstruction.Swarmops[masterKey].Cents.Count > checkSwarmopsIndex.Keys.Count) { for (int swarmopsIndex = 0; swarmopsIndex < mismatchConstruction.Swarmops[masterKey].Cents.Count; swarmopsIndex++) { if (checkSwarmopsIndex.ContainsKey(swarmopsIndex)) { continue; } masterCentsList.Add(0); // null equivalent; invalid value swarmopsCentsList.Add(mismatchConstruction.Swarmops[masterKey].Cents[swarmopsIndex]); transactionsList.Add(mismatchConstruction.Swarmops[masterKey].Transactions[swarmopsIndex]); if (mismatchConstruction.Swarmops[masterKey].Transactions[swarmopsIndex].Dependency != null) { dependenciesList.Add( mismatchConstruction.Swarmops[masterKey].Transactions[swarmopsIndex].Dependency); actionsList.Add(ExternalBankMismatchResyncAction.ManualAction); // can't auto } else { dependenciesList.Add(null); actionsList.Add(ExternalBankMismatchResyncAction.DeleteSwarmops); } checkMasterIndex[swarmopsIndex] = true; } } newRecord.MasterCents = masterCentsList.ToArray(); newRecord.SwarmopsCents = swarmopsCentsList.ToArray(); newRecord.ResyncActions = actionsList.ToArray(); // newRecord.TransactionDependencies = dependenciesList.ToArray(); newRecord.Transactions = transactionsList.ToArray(); mismatchingRecordList[masterKey] = newRecord; } // Finally, add the transactions that were (described) in Swarmops but not in Master foreach (string swarmopsKey in mismatchConstruction.Swarmops.Keys) { if (!mismatchingRecordList.ContainsKey(swarmopsKey)) { mismatchingRecordList[swarmopsKey] = new ExternalBankMismatchingRecordDescription(); mismatchingRecordList[swarmopsKey].Description = swarmopsKey; mismatchingRecordList[swarmopsKey].SwarmopsCents = mismatchConstruction.Swarmops[swarmopsKey].Cents.ToArray(); mismatchingRecordList[swarmopsKey].Transactions = mismatchConstruction.Swarmops[swarmopsKey].Transactions.ToArray(); mismatchingRecordList[swarmopsKey].MasterCents = new long[mismatchConstruction.Swarmops[swarmopsKey].Cents.Count]; // inits to zero mismatchingRecordList[swarmopsKey].ResyncActions = new ExternalBankMismatchResyncAction[mismatchConstruction.Swarmops[swarmopsKey].Cents.Count]; for (int index = 0; index < mismatchingRecordList[swarmopsKey].ResyncActions.Length; index++) { mismatchingRecordList[swarmopsKey].ResyncActions[index] = ExternalBankMismatchResyncAction.DeleteSwarmops; } } } newMismatch.MismatchingRecords = mismatchingRecordList.Values.ToArray(); mismatchList.Add(newMismatch); } int percentProcessed = (int) (currentRecordIndex*100L/externalData.Records.Length); lock (_staticDataLookup) { if (percentProcessed > 1) { _staticDataLookup[guid + "PercentRead"] = percentProcessed; // for the progress bar to update async } if (percentProcessed > 99) { // Placed inside loop to have a contiguous lock block, even though it decreases performance. // Should normally be placed just outside. _staticDataLookup[guid + "MismatchArray"] = mismatchList.ToArray(); _staticDataLookup[guid + "Account"] = account; } } } }
private static void ProcessUploadThread(object args) { string guid = ((ProcessThreadArguments) args).Guid; BankFileType fileType = ((ProcessThreadArguments) args).FileType; Documents documents = Documents.RecentFromDescription(guid); GuidCache.Set(guid + "-Result", ImportResultsCategory.Bad); // default - this is what happens if exception if (documents.Count != 1) { return; // abort } Document uploadedDoc = documents[0]; try { FinancialAccount account = ((ProcessThreadArguments) args).Account; ExternalBankData externalData = new ExternalBankData(); externalData.Profile = account.ExternalBankDataProfile; if (fileType == BankFileType.AccountStatement) { using (StreamReader reader = uploadedDoc.GetReader(1252)) { externalData.LoadData(reader, ((ProcessThreadArguments) args).Organization); // catch here and set result to BAD ImportResults results = ProcessImportedData(externalData, (ProcessThreadArguments) args); GuidCache.Set(guid + "-ResultDetails", results); if (results.AccountBalanceMatchesBank) { GuidCache.Set(guid + "-Result", ImportResultsCategory.Good); } else { GuidCache.Set(guid + "-Result", ImportResultsCategory.Questionable); } } } else if (fileType == BankFileType.PaymentDetails) { // Get reader factory from ExternalBankData throw new NotImplementedException("Need to implement new flexible payment reader structure"); // IBankDataPaymentsReader paymentsReader = externalData.GetPaymentsReader(); // then read } } catch (Exception e) { GuidCache.Set(guid + "-Exception", e.ToString()); } finally { GuidCache.Set(guid + "-Progress", 100); // only here may the caller fetch the results uploadedDoc.Delete(); // document no longer needed after processing, no matter the result } }