public ImportedBankData ReadData(StreamReader data, FinancialAccount account, ExternalBankDataProfile profile) { throw new NotImplementedException(); }
private static void ProcessUploadThread(object args) { string guid = ((ProcessThreadArguments)args).Guid; Documents documents = Documents.RecentFromDescription(guid); if (documents.Count != 1) { return; // abort } Document uploadedDoc = documents[0]; FinancialAccount account = ((ProcessThreadArguments)args).Account; ExternalBankData externalData = new ExternalBankData(); externalData.Profile = ExternalBankDataProfile.FromIdentity(ExternalBankDataProfile.SESebId); // TODO: HACK HACK HACK HACK LOAD using ( StreamReader reader = new StreamReader(StorageRoot + uploadedDoc.ServerFileName, Encoding.GetEncoding(1252))) { externalData.LoadData(reader, ((ProcessThreadArguments)args).Organization, account.ForeignCurrency); } _staticDataLookup[guid + "FirstTx"] = externalData.Records[0].DateTime.ToLongDateString(); _staticDataLookup[guid + "LastTx"] = externalData.Records[externalData.Records.Length - 1].DateTime.ToLongDateString(); _staticDataLookup[guid + "TxCount"] = externalData.Records.Length.ToString("N0"); _staticDataLookup[guid + "Profile"] = externalData.Profile; _staticDataLookup[guid + "PercentRead"] = 1; DateTime timeWalker = externalData.Records[0].DateTime; int currentRecordIndex = 0; // Walk past the first identical time records, and verify that we have at least one balance record that matches // our own for this timestamp. There may be several transactions in the master file, but at least one should have // a post-transaction balance that matches our records, or something is much more broken. long swarmopsCentsStart = account.GetDeltaCents(Constants.DateTimeLow, timeWalker.AddSeconds(1)); // At least one of the transactions for this timestamp should match. bool foundMatch = false; while (externalData.Records[currentRecordIndex].DateTime == timeWalker) { if (externalData.Records[currentRecordIndex].AccountBalanceCents == swarmopsCentsStart) { foundMatch = true; // continue loop until on first record past initial conditions } currentRecordIndex++; // no test for array overrun in while } if (!foundMatch) { throw new InvalidOperationException("Unable to locate stable initial conditions for resynchronization"); } // From here on out, every new timestamp should have the exact same delta in Swarmops as it has in the master file. List <ExternalBankMismatchingDateTime> mismatchList = new List <ExternalBankMismatchingDateTime>(); while (currentRecordIndex < externalData.Records.Length) { DateTime lastTimestamp = timeWalker; timeWalker = externalData.Records[currentRecordIndex].DateTime; long swarmopsDeltaCents = account.GetDeltaCents(lastTimestamp.AddSeconds(1), timeWalker.AddSeconds(1)); // "AddSeconds" because DeltaCents operates on ">= lowbound, < highbound" int timestampStartIndex = currentRecordIndex; long masterDeltaCents = externalData.Records[currentRecordIndex++].TransactionNetCents; int masterTransactionCount = 1; while (currentRecordIndex < externalData.Records.Length && externalData.Records[currentRecordIndex].DateTime == timeWalker) { masterDeltaCents += externalData.Records[currentRecordIndex++].TransactionNetCents; masterTransactionCount++; } if (masterDeltaCents != swarmopsDeltaCents) { // We have a mismatch. Add it to the list. ExternalBankMismatchingDateTime newMismatch = new ExternalBankMismatchingDateTime(); newMismatch.DateTime = timeWalker; newMismatch.MasterDeltaCents = masterDeltaCents; newMismatch.MasterTransactionCount = masterTransactionCount; newMismatch.SwarmopsDeltaCents = swarmopsDeltaCents; newMismatch.SwarmopsTransactionCount = 0; // TODO // Load transactions from both sources. First, create the interim construction object. ExternalBankMismatchConstruction mismatchConstruction = new ExternalBankMismatchConstruction(); // Load from Master for (int innerRecordIndex = timestampStartIndex; innerRecordIndex < currentRecordIndex; innerRecordIndex++) { string description = externalData.Records[innerRecordIndex].Description.Replace(" ", " "); if (!mismatchConstruction.Master.ContainsKey(description)) { mismatchConstruction.Master[description] = new ExternalBankMismatchingRecordConstruction(); } mismatchConstruction.Master[description].Cents.Add( externalData.Records[innerRecordIndex].TransactionNetCents); mismatchConstruction.Master[description].Transactions.Add(null); // no dependencies on the master side, only on swarmops side } // Load from Swarmops FinancialAccountRows swarmopsTransactionRows = account.GetRowsFar(lastTimestamp, timeWalker); // the "select far" is a boundary < x <= boundary selector. Default is boundary <= x < boundary. Dictionary <int, FinancialTransaction> lookupTransactions = new Dictionary <int, FinancialTransaction>(); // note all transaction IDs, then sum up per transaction foreach (FinancialAccountRow swarmopsTransactionRow in swarmopsTransactionRows) { lookupTransactions[swarmopsTransactionRow.FinancialTransactionId] = swarmopsTransactionRow.Transaction; } foreach (FinancialTransaction transaction in lookupTransactions.Values) { string description = transaction.Description.Replace(" ", " "); // for legacy compatibility with new importer if (!mismatchConstruction.Swarmops.ContainsKey(description)) { mismatchConstruction.Swarmops[description] = new ExternalBankMismatchingRecordConstruction(); } long cents = transaction[account]; if (cents != 0) // only add nonzero records { mismatchConstruction.Swarmops[description].Cents.Add(transaction[account]); mismatchConstruction.Swarmops[description].Transactions.Add(transaction); } } // Then, parse the intermediate construction object to the presentation-and-action object. Dictionary <string, ExternalBankMismatchingRecordDescription> mismatchingRecordList = new Dictionary <string, ExternalBankMismatchingRecordDescription>(); foreach (string masterKey in mismatchConstruction.Master.Keys) { Dictionary <int, bool> checkMasterIndex = new Dictionary <int, bool>(); Dictionary <int, bool> checkSwarmopsIndex = new Dictionary <int, bool>(); // For each key and entry for each key; // 1) locate an exact corresponding amount in swarmops records and log; failing that, // 2) if exactly one record left in master and swarmops records, log; failing that, // 3) log the rest of the master OR rest of swarmops records with no corresponding // equivalent with counterpart. (May produce bad results if 2 consistent mismatches // for every description.) ExternalBankMismatchingRecordDescription newRecord = new ExternalBankMismatchingRecordDescription(); newRecord.Description = masterKey; List <long> masterCentsList = new List <long>(); List <long> swarmopsCentsList = new List <long>(); List <object> dependenciesList = new List <object>(); List <FinancialTransaction> transactionsList = new List <FinancialTransaction>(); List <ExternalBankMismatchResyncAction> actionsList = new List <ExternalBankMismatchResyncAction>(); // STEP 1 - locate all identical matches if (mismatchConstruction.Swarmops.ContainsKey(masterKey)) { for (int masterIndex = 0; masterIndex < mismatchConstruction.Master[masterKey].Cents.Count; masterIndex++) { // no "continue" necessary on first run-through; nothing has been checked off yet long findMasterCents = mismatchConstruction.Master[masterKey].Cents[masterIndex]; for (int swarmopsIndex = 0; swarmopsIndex < mismatchConstruction.Swarmops[masterKey].Cents.Count; swarmopsIndex++) { if (checkSwarmopsIndex.ContainsKey(swarmopsIndex)) { continue; // may have been checked off already in the rare case of twin identical amounts } if (findMasterCents == mismatchConstruction.Swarmops[masterKey].Cents[swarmopsIndex]) { // There is a match as per case 1. Record both, mark both as used, continue. masterCentsList.Add(findMasterCents); swarmopsCentsList.Add( mismatchConstruction.Swarmops[masterKey].Cents[swarmopsIndex]); // should be equal, we're defensive here transactionsList.Add( mismatchConstruction.Swarmops[masterKey].Transactions[swarmopsIndex]); dependenciesList.Add( mismatchConstruction.Swarmops[masterKey].Transactions[swarmopsIndex] .Dependency); actionsList.Add(ExternalBankMismatchResyncAction.NoAction); checkMasterIndex[masterIndex] = true; checkSwarmopsIndex[swarmopsIndex] = true; break; } } } } // STEP 2 - if exactly one record left on both sides, connect and log as mismatching record // TODO: improve logic to handle same number of records left on both sides if (mismatchConstruction.Swarmops.ContainsKey(masterKey) && mismatchConstruction.Master[masterKey].Cents.Count - checkMasterIndex.Keys.Count == 1 && mismatchConstruction.Swarmops[masterKey].Cents.Count - checkSwarmopsIndex.Keys.Count == 1) { for (int masterIndex = 0; masterIndex < mismatchConstruction.Master[masterKey].Cents.Count; masterIndex++) { if (checkMasterIndex.ContainsKey(masterIndex)) { continue; // This will fire for all but one indexes } long findMasterCents = mismatchConstruction.Master[masterKey].Cents[masterIndex]; for (int swarmopsIndex = 0; swarmopsIndex < mismatchConstruction.Swarmops[masterKey].Cents.Count; swarmopsIndex++) { if (checkSwarmopsIndex.ContainsKey(swarmopsIndex)) { continue; } masterCentsList.Add(findMasterCents); swarmopsCentsList.Add(mismatchConstruction.Swarmops[masterKey].Cents[swarmopsIndex]); dependenciesList.Add( mismatchConstruction.Swarmops[masterKey].Transactions[swarmopsIndex].Dependency); transactionsList.Add( mismatchConstruction.Swarmops[masterKey].Transactions[swarmopsIndex]); actionsList.Add(ExternalBankMismatchResyncAction.RewriteSwarmops); checkMasterIndex[masterIndex] = true; checkSwarmopsIndex[swarmopsIndex] = true; } } } // STEP 3 - log remaining records on both sides as missing counterparts. Only one of these should fire. // STEP 3a - log remaining on Master side if (mismatchConstruction.Master[masterKey].Cents.Count > checkMasterIndex.Keys.Count) { for (int masterIndex = 0; masterIndex < mismatchConstruction.Master[masterKey].Cents.Count; masterIndex++) { if (checkMasterIndex.ContainsKey(masterIndex)) { continue; } masterCentsList.Add(mismatchConstruction.Master[masterKey].Cents[masterIndex]); swarmopsCentsList.Add(0); // null equivalent; invalid value dependenciesList.Add(null); transactionsList.Add(null); actionsList.Add(ExternalBankMismatchResyncAction.CreateSwarmops); checkMasterIndex[masterIndex] = true; } } // STEP 3b - log remaining on Swarmops side if (mismatchConstruction.Swarmops.ContainsKey(masterKey) && mismatchConstruction.Swarmops[masterKey].Cents.Count > checkSwarmopsIndex.Keys.Count) { for (int swarmopsIndex = 0; swarmopsIndex < mismatchConstruction.Swarmops[masterKey].Cents.Count; swarmopsIndex++) { if (checkSwarmopsIndex.ContainsKey(swarmopsIndex)) { continue; } masterCentsList.Add(0); // null equivalent; invalid value swarmopsCentsList.Add(mismatchConstruction.Swarmops[masterKey].Cents[swarmopsIndex]); transactionsList.Add( mismatchConstruction.Swarmops[masterKey].Transactions[swarmopsIndex]); if (mismatchConstruction.Swarmops[masterKey].Transactions[swarmopsIndex].Dependency != null) { dependenciesList.Add( mismatchConstruction.Swarmops[masterKey].Transactions[swarmopsIndex].Dependency); actionsList.Add(ExternalBankMismatchResyncAction.ManualAction); // can't auto } else { dependenciesList.Add(null); actionsList.Add(ExternalBankMismatchResyncAction.DeleteSwarmops); } checkMasterIndex[swarmopsIndex] = true; } } newRecord.MasterCents = masterCentsList.ToArray(); newRecord.SwarmopsCents = swarmopsCentsList.ToArray(); newRecord.ResyncActions = actionsList.ToArray(); // newRecord.TransactionDependencies = dependenciesList.ToArray(); newRecord.Transactions = transactionsList.ToArray(); mismatchingRecordList[masterKey] = newRecord; } // Finally, add the transactions that were (described) in Swarmops but not in Master foreach (string swarmopsKey in mismatchConstruction.Swarmops.Keys) { if (!mismatchingRecordList.ContainsKey(swarmopsKey)) { mismatchingRecordList[swarmopsKey] = new ExternalBankMismatchingRecordDescription(); mismatchingRecordList[swarmopsKey].Description = swarmopsKey; mismatchingRecordList[swarmopsKey].SwarmopsCents = mismatchConstruction.Swarmops[swarmopsKey].Cents.ToArray(); mismatchingRecordList[swarmopsKey].Transactions = mismatchConstruction.Swarmops[swarmopsKey].Transactions.ToArray(); mismatchingRecordList[swarmopsKey].MasterCents = new long[mismatchConstruction.Swarmops[swarmopsKey].Cents.Count]; // inits to zero mismatchingRecordList[swarmopsKey].ResyncActions = new ExternalBankMismatchResyncAction[ mismatchConstruction.Swarmops[swarmopsKey].Cents.Count]; for (int index = 0; index < mismatchingRecordList[swarmopsKey].ResyncActions.Length; index++) { mismatchingRecordList[swarmopsKey].ResyncActions[index] = ExternalBankMismatchResyncAction.DeleteSwarmops; } } } newMismatch.MismatchingRecords = mismatchingRecordList.Values.ToArray(); mismatchList.Add(newMismatch); } int percentProcessed = (int)(currentRecordIndex * 100L / externalData.Records.Length); lock (_staticDataLookup) { if (percentProcessed > 1) { _staticDataLookup[guid + "PercentRead"] = percentProcessed; // for the progress bar to update async } if (percentProcessed > 99) { // Placed inside loop to have a contiguous lock block, even though it decreases performance. // Should normally be placed just outside. _staticDataLookup[guid + "MismatchArray"] = mismatchList.ToArray(); _staticDataLookup[guid + "Account"] = account; } } } }
protected void Page_Load(object sender, EventArgs e) { this._authenticationData = GetAuthenticationDataAndCulture(); this._year = DateTime.Today.Year; string guid = Request.QueryString["Guid"]; ExternalBankMismatchingDateTime[] mismatchArray = (ExternalBankMismatchingDateTime[])Session["LedgersResync" + guid + "MismatchArray"]; ExternalBankDataProfile profile = (ExternalBankDataProfile)Session["LedgersResync" + guid + "Profile"]; if ( !this._authenticationData.CurrentUser.HasAccess(new Access( this._authenticationData.CurrentOrganization, AccessAspect.Bookkeeping, AccessType.Read))) { throw new UnauthorizedAccessException(); } // TODO: Get and cache account tree and account balances Response.ContentType = "application/json"; string currentOrganizationCurrency = CurrentOrganization.Currency.Code; string response = string.Empty; List <string> items = new List <string>(); foreach (ExternalBankMismatchingDateTime mismatch in mismatchArray) { string rowId = mismatch.DateTime.ToString("yyyyMMddHHmmss"); List <string> childItems = new List <string>(); foreach (ExternalBankMismatchingRecordDescription mismatchingRecord in mismatch.MismatchingRecords) { for (int masterIndex = 0; masterIndex < mismatchingRecord.MasterCents.Count(); masterIndex++) { string dependencyString = string.Empty; FinancialTransaction transaction = mismatchingRecord.Transactions[masterIndex]; object dependency = null; if (transaction != null) { dependency = transaction.Dependency; } if (dependency != null) { dependencyString = dependency.GetType().ToString(); int lastPeriod = dependencyString.LastIndexOf('.'); dependencyString = dependencyString.Substring(lastPeriod + 1); dependencyString += " #" + (dependency as IHasIdentity).Identity.ToString("N0"); } childItems.Add("{\"id\":\"" + rowId + childItems.Count.ToString("##0") + "\",\"rowName\":\"" + JsonSanitize(mismatchingRecord.Description) + "\",\"swarmopsData\":\"" + PrintNullableCents(currentOrganizationCurrency, mismatchingRecord.SwarmopsCents[masterIndex]) + "\",\"masterData\":\"" + PrintNullableCents(currentOrganizationCurrency, mismatchingRecord.MasterCents[masterIndex]) + "\",\"resyncAction\":\"" + JsonSanitize(mismatchingRecord.ResyncActions[masterIndex].ToString()) + "\",\"notes\":\"" + JsonSanitize(dependencyString) + "\"}"); } } string childrenString = String.Join(",", childItems.ToArray()); string rowName = mismatch.DateTime.ToString(profile.DateTimeFormatString); string swarmopsData = currentOrganizationCurrency + " " + (mismatch.SwarmopsDeltaCents / 100.0).ToString("N2"); string masterData = currentOrganizationCurrency + " " + (mismatch.MasterDeltaCents / 100.0).ToString("N2"); string notes = "Diff: " + ((mismatch.MasterDeltaCents - mismatch.SwarmopsDeltaCents) / 100.0).ToString("N2"); items.Add("{\"id\":\"" + rowId + "\",\"rowName\":\"" + rowName + "\",\"swarmopsData\":\"" + JsonSanitize(swarmopsData) + "\",\"masterData\":\"" + JsonSanitize(masterData) + "\",\"notes\":\"" + JsonSanitize(notes) + "\",\"state\":\"closed\",\"children\":[" + childrenString + "]}"); } Response.Output.WriteLine("[" + String.Join(",", items.ToArray()) + "]"); Response.End(); }
public static AjaxUploadCallResult UploadBankTransactionData(string guid, string itemId) { AuthenticationData authData = GetAuthenticationDataAndCulture(); if (!authData.Authority.HasAccess(new Access(authData.CurrentOrganization, AccessAspect.BookkeepingDetails))) { throw new UnauthorizedAccessException(); } string[] parts = itemId.Split('-'); FinancialAccount account = FinancialAccount.FromIdentity(Int32.Parse(parts[1])); if (account.OrganizationId != authData.CurrentOrganization.Identity) { throw new UnauthorizedAccessException(); } Documents documents = Documents.RecentFromDescription(guid); // Safeguard 2019-Dec-23: Abort if more than one document (code below needs hardening against concurrent-threads race conditions) if (documents.Count != 1) { throw new NotImplementedException(); } // Load documents and process them as loaded strings, one by one foreach (Document document in documents) { string documentData = document.GetReader().ReadToEnd(); ExternalBankDataProfile profile = account.ExternalBankDataProfile; ExternalBankData loadedData = new ExternalBankData(); loadedData.Profile = profile; try { loadedData.LoadData(documentData, authData.CurrentOrganization, account.Currency); } catch (Exception) { return(new AjaxUploadCallResult { Success = false, DisplayMessage = "ERROR_FILEDATAFORMAT" }); } // Start async thread to import the data to the SQL database; the caller must // check the status of the import string identifier = guid + "-" + itemId + "-" + Guid.NewGuid().ToString(); /* Thread processThread = new Thread((ThreadStart) AsyncProcesses.ImportExternalTransactionDataThreadStart); * processThread.Start(new AsyncProcesses.ImportExternalTransactionDataArgs {}); */ return(new AjaxUploadCallResult { Success = true, StillProcessing = true, Identifier = identifier }); } return(new AjaxUploadCallResult { Success = true, StillProcessing = true }); }
public ImportedBankData ReadData(StreamReader data, FinancialAccount account, ExternalBankDataProfile profile) { throw new NotImplementedException(); }