Exemple #1
0
        private static void ProcessUploadThread(object args)
        {
            string guid = ((ProcessThreadArguments)args).Guid;

            Documents documents = Documents.RecentFromDescription(guid);

            if (documents.Count != 1)
            {
                return; // abort
            }

            Document uploadedDoc = documents[0];

            FinancialAccount account = ((ProcessThreadArguments)args).Account;

            ExternalBankData externalData = new ExternalBankData();

            externalData.Profile = ExternalBankDataProfile.FromIdentity(ExternalBankDataProfile.SESebId);
            // TODO: HACK HACK HACK HACK LOAD

            using (
                StreamReader reader = new StreamReader(StorageRoot + uploadedDoc.ServerFileName,
                                                       Encoding.GetEncoding(1252)))
            {
                externalData.LoadData(reader, ((ProcessThreadArguments)args).Organization, account.ForeignCurrency);
            }

            _staticDataLookup[guid + "FirstTx"] = externalData.Records[0].DateTime.ToLongDateString();
            _staticDataLookup[guid + "LastTx"]  =
                externalData.Records[externalData.Records.Length - 1].DateTime.ToLongDateString();
            _staticDataLookup[guid + "TxCount"] = externalData.Records.Length.ToString("N0");

            _staticDataLookup[guid + "Profile"] = externalData.Profile;

            _staticDataLookup[guid + "PercentRead"] = 1;

            DateTime timeWalker         = externalData.Records[0].DateTime;
            int      currentRecordIndex = 0;

            // Walk past the first identical time records, and verify that we have at least one balance record that matches
            // our own for this timestamp. There may be several transactions in the master file, but at least one should have
            // a post-transaction balance that matches our records, or something is much more broken.

            long swarmopsCentsStart = account.GetDeltaCents(Constants.DateTimeLow, timeWalker.AddSeconds(1));

            // At least one of the transactions for this timestamp should match.

            bool foundMatch = false;

            while (externalData.Records[currentRecordIndex].DateTime == timeWalker)
            {
                if (externalData.Records[currentRecordIndex].AccountBalanceCents == swarmopsCentsStart)
                {
                    foundMatch = true;

                    // continue loop until on first record past initial conditions
                }
                currentRecordIndex++; // no test for array overrun in while
            }

            if (!foundMatch)
            {
                throw new InvalidOperationException("Unable to locate stable initial conditions for resynchronization");
            }

            // From here on out, every new timestamp should have the exact same delta in Swarmops as it has in the master file.

            List <ExternalBankMismatchingDateTime> mismatchList = new List <ExternalBankMismatchingDateTime>();

            while (currentRecordIndex < externalData.Records.Length)
            {
                DateTime lastTimestamp = timeWalker;

                timeWalker = externalData.Records[currentRecordIndex].DateTime;

                long swarmopsDeltaCents = account.GetDeltaCents(lastTimestamp.AddSeconds(1), timeWalker.AddSeconds(1));
                // "AddSeconds" because DeltaCents operates on ">= lowbound, < highbound"
                int  timestampStartIndex    = currentRecordIndex;
                long masterDeltaCents       = externalData.Records[currentRecordIndex++].TransactionNetCents;
                int  masterTransactionCount = 1;

                while (currentRecordIndex < externalData.Records.Length &&
                       externalData.Records[currentRecordIndex].DateTime == timeWalker)
                {
                    masterDeltaCents += externalData.Records[currentRecordIndex++].TransactionNetCents;
                    masterTransactionCount++;
                }

                if (masterDeltaCents != swarmopsDeltaCents)
                {
                    // We have a mismatch. Add it to the list.

                    ExternalBankMismatchingDateTime newMismatch = new ExternalBankMismatchingDateTime();
                    newMismatch.DateTime                 = timeWalker;
                    newMismatch.MasterDeltaCents         = masterDeltaCents;
                    newMismatch.MasterTransactionCount   = masterTransactionCount;
                    newMismatch.SwarmopsDeltaCents       = swarmopsDeltaCents;
                    newMismatch.SwarmopsTransactionCount = 0; // TODO

                    // Load transactions from both sources. First, create the interim construction object.

                    ExternalBankMismatchConstruction mismatchConstruction = new ExternalBankMismatchConstruction();

                    // Load from Master

                    for (int innerRecordIndex = timestampStartIndex;
                         innerRecordIndex < currentRecordIndex;
                         innerRecordIndex++)
                    {
                        string description = externalData.Records[innerRecordIndex].Description.Replace("  ", " ");

                        if (!mismatchConstruction.Master.ContainsKey(description))
                        {
                            mismatchConstruction.Master[description] =
                                new ExternalBankMismatchingRecordConstruction();
                        }

                        mismatchConstruction.Master[description].Cents.Add(
                            externalData.Records[innerRecordIndex].TransactionNetCents);
                        mismatchConstruction.Master[description].Transactions.Add(null);
                        // no dependencies on the master side, only on swarmops side
                    }

                    // Load from Swarmops

                    FinancialAccountRows swarmopsTransactionRows =
                        account.GetRowsFar(lastTimestamp, timeWalker);
                    // the "select far" is a boundary < x <= boundary selector. Default is boundary <= x < boundary.

                    Dictionary <int, FinancialTransaction> lookupTransactions =
                        new Dictionary <int, FinancialTransaction>();

                    // note all transaction IDs, then sum up per transaction

                    foreach (FinancialAccountRow swarmopsTransactionRow in swarmopsTransactionRows)
                    {
                        lookupTransactions[swarmopsTransactionRow.FinancialTransactionId] =
                            swarmopsTransactionRow.Transaction;
                    }


                    foreach (FinancialTransaction transaction in lookupTransactions.Values)
                    {
                        string description = transaction.Description.Replace("  ", " ");
                        // for legacy compatibility with new importer

                        if (!mismatchConstruction.Swarmops.ContainsKey(description))
                        {
                            mismatchConstruction.Swarmops[description] =
                                new ExternalBankMismatchingRecordConstruction();
                        }

                        long cents = transaction[account];

                        if (cents != 0) // only add nonzero records
                        {
                            mismatchConstruction.Swarmops[description].Cents.Add(transaction[account]);
                            mismatchConstruction.Swarmops[description].Transactions.Add(transaction);
                        }
                    }

                    // Then, parse the intermediate construction object to the presentation-and-action object.

                    Dictionary <string, ExternalBankMismatchingRecordDescription> mismatchingRecordList =
                        new Dictionary <string, ExternalBankMismatchingRecordDescription>();

                    foreach (string masterKey in mismatchConstruction.Master.Keys)
                    {
                        Dictionary <int, bool> checkMasterIndex   = new Dictionary <int, bool>();
                        Dictionary <int, bool> checkSwarmopsIndex = new Dictionary <int, bool>();

                        // For each key and entry for each key;

                        // 1) locate an exact corresponding amount in swarmops records and log; failing that,
                        // 2) if exactly one record left in master and swarmops records, log; failing that,
                        // 3) log the rest of the master OR rest of swarmops records with no corresponding
                        //    equivalent with counterpart. (May produce bad results if 2 consistent mismatches
                        //    for every description.)

                        ExternalBankMismatchingRecordDescription newRecord =
                            new ExternalBankMismatchingRecordDescription();
                        newRecord.Description = masterKey;

                        List <long>   masterCentsList   = new List <long>();
                        List <long>   swarmopsCentsList = new List <long>();
                        List <object> dependenciesList  = new List <object>();
                        List <FinancialTransaction>             transactionsList = new List <FinancialTransaction>();
                        List <ExternalBankMismatchResyncAction> actionsList      =
                            new List <ExternalBankMismatchResyncAction>();

                        // STEP 1 - locate all identical matches

                        if (mismatchConstruction.Swarmops.ContainsKey(masterKey))
                        {
                            for (int masterIndex = 0;
                                 masterIndex < mismatchConstruction.Master[masterKey].Cents.Count;
                                 masterIndex++)
                            {
                                // no "continue" necessary on first run-through; nothing has been checked off yet

                                long findMasterCents = mismatchConstruction.Master[masterKey].Cents[masterIndex];

                                for (int swarmopsIndex = 0;
                                     swarmopsIndex < mismatchConstruction.Swarmops[masterKey].Cents.Count;
                                     swarmopsIndex++)
                                {
                                    if (checkSwarmopsIndex.ContainsKey(swarmopsIndex))
                                    {
                                        continue;
                                        // may have been checked off already in the rare case of twin identical amounts
                                    }

                                    if (findMasterCents == mismatchConstruction.Swarmops[masterKey].Cents[swarmopsIndex])
                                    {
                                        // There is a match as per case 1. Record both, mark both as used, continue.

                                        masterCentsList.Add(findMasterCents);
                                        swarmopsCentsList.Add(
                                            mismatchConstruction.Swarmops[masterKey].Cents[swarmopsIndex]);
                                        // should be equal, we're defensive here
                                        transactionsList.Add(
                                            mismatchConstruction.Swarmops[masterKey].Transactions[swarmopsIndex]);
                                        dependenciesList.Add(
                                            mismatchConstruction.Swarmops[masterKey].Transactions[swarmopsIndex]
                                            .Dependency);
                                        actionsList.Add(ExternalBankMismatchResyncAction.NoAction);

                                        checkMasterIndex[masterIndex]     = true;
                                        checkSwarmopsIndex[swarmopsIndex] = true;

                                        break;
                                    }
                                }
                            }
                        }

                        // STEP 2 - if exactly one record left on both sides, connect and log as mismatching record

                        // TODO: improve logic to handle same number of records left on both sides

                        if (mismatchConstruction.Swarmops.ContainsKey(masterKey) &&
                            mismatchConstruction.Master[masterKey].Cents.Count - checkMasterIndex.Keys.Count == 1 &&
                            mismatchConstruction.Swarmops[masterKey].Cents.Count - checkSwarmopsIndex.Keys.Count == 1)
                        {
                            for (int masterIndex = 0;
                                 masterIndex < mismatchConstruction.Master[masterKey].Cents.Count;
                                 masterIndex++)
                            {
                                if (checkMasterIndex.ContainsKey(masterIndex))
                                {
                                    continue; // This will fire for all but one indexes
                                }

                                long findMasterCents = mismatchConstruction.Master[masterKey].Cents[masterIndex];

                                for (int swarmopsIndex = 0;
                                     swarmopsIndex < mismatchConstruction.Swarmops[masterKey].Cents.Count;
                                     swarmopsIndex++)
                                {
                                    if (checkSwarmopsIndex.ContainsKey(swarmopsIndex))
                                    {
                                        continue;
                                    }

                                    masterCentsList.Add(findMasterCents);
                                    swarmopsCentsList.Add(mismatchConstruction.Swarmops[masterKey].Cents[swarmopsIndex]);
                                    dependenciesList.Add(
                                        mismatchConstruction.Swarmops[masterKey].Transactions[swarmopsIndex].Dependency);
                                    transactionsList.Add(
                                        mismatchConstruction.Swarmops[masterKey].Transactions[swarmopsIndex]);
                                    actionsList.Add(ExternalBankMismatchResyncAction.RewriteSwarmops);

                                    checkMasterIndex[masterIndex]     = true;
                                    checkSwarmopsIndex[swarmopsIndex] = true;
                                }
                            }
                        }

                        // STEP 3 - log remaining records on both sides as missing counterparts. Only one of these should fire.

                        // STEP 3a - log remaining on Master side

                        if (mismatchConstruction.Master[masterKey].Cents.Count > checkMasterIndex.Keys.Count)
                        {
                            for (int masterIndex = 0;
                                 masterIndex < mismatchConstruction.Master[masterKey].Cents.Count;
                                 masterIndex++)
                            {
                                if (checkMasterIndex.ContainsKey(masterIndex))
                                {
                                    continue;
                                }

                                masterCentsList.Add(mismatchConstruction.Master[masterKey].Cents[masterIndex]);
                                swarmopsCentsList.Add(0);  // null equivalent; invalid value
                                dependenciesList.Add(null);
                                transactionsList.Add(null);
                                actionsList.Add(ExternalBankMismatchResyncAction.CreateSwarmops);

                                checkMasterIndex[masterIndex] = true;
                            }
                        }

                        // STEP 3b - log remaining on Swarmops side

                        if (mismatchConstruction.Swarmops.ContainsKey(masterKey) &&
                            mismatchConstruction.Swarmops[masterKey].Cents.Count > checkSwarmopsIndex.Keys.Count)
                        {
                            for (int swarmopsIndex = 0;
                                 swarmopsIndex < mismatchConstruction.Swarmops[masterKey].Cents.Count;
                                 swarmopsIndex++)
                            {
                                if (checkSwarmopsIndex.ContainsKey(swarmopsIndex))
                                {
                                    continue;
                                }

                                masterCentsList.Add(0);  // null equivalent; invalid value
                                swarmopsCentsList.Add(mismatchConstruction.Swarmops[masterKey].Cents[swarmopsIndex]);
                                transactionsList.Add(
                                    mismatchConstruction.Swarmops[masterKey].Transactions[swarmopsIndex]);

                                if (mismatchConstruction.Swarmops[masterKey].Transactions[swarmopsIndex].Dependency !=
                                    null)
                                {
                                    dependenciesList.Add(
                                        mismatchConstruction.Swarmops[masterKey].Transactions[swarmopsIndex].Dependency);
                                    actionsList.Add(ExternalBankMismatchResyncAction.ManualAction);  // can't auto
                                }
                                else
                                {
                                    dependenciesList.Add(null);
                                    actionsList.Add(ExternalBankMismatchResyncAction.DeleteSwarmops);
                                }

                                checkMasterIndex[swarmopsIndex] = true;
                            }
                        }

                        newRecord.MasterCents   = masterCentsList.ToArray();
                        newRecord.SwarmopsCents = swarmopsCentsList.ToArray();
                        newRecord.ResyncActions = actionsList.ToArray();
                        // newRecord.TransactionDependencies = dependenciesList.ToArray();
                        newRecord.Transactions = transactionsList.ToArray();

                        mismatchingRecordList[masterKey] = newRecord;
                    }

                    // Finally, add the transactions that were (described) in Swarmops but not in Master

                    foreach (string swarmopsKey in mismatchConstruction.Swarmops.Keys)
                    {
                        if (!mismatchingRecordList.ContainsKey(swarmopsKey))
                        {
                            mismatchingRecordList[swarmopsKey]             = new ExternalBankMismatchingRecordDescription();
                            mismatchingRecordList[swarmopsKey].Description = swarmopsKey;

                            mismatchingRecordList[swarmopsKey].SwarmopsCents =
                                mismatchConstruction.Swarmops[swarmopsKey].Cents.ToArray();
                            mismatchingRecordList[swarmopsKey].Transactions =
                                mismatchConstruction.Swarmops[swarmopsKey].Transactions.ToArray();
                            mismatchingRecordList[swarmopsKey].MasterCents =
                                new long[mismatchConstruction.Swarmops[swarmopsKey].Cents.Count]; // inits to zero

                            mismatchingRecordList[swarmopsKey].ResyncActions =
                                new ExternalBankMismatchResyncAction[
                                    mismatchConstruction.Swarmops[swarmopsKey].Cents.Count];
                            for (int index = 0;
                                 index < mismatchingRecordList[swarmopsKey].ResyncActions.Length;
                                 index++)
                            {
                                mismatchingRecordList[swarmopsKey].ResyncActions[index] =
                                    ExternalBankMismatchResyncAction.DeleteSwarmops;
                            }
                        }
                    }

                    newMismatch.MismatchingRecords = mismatchingRecordList.Values.ToArray();

                    mismatchList.Add(newMismatch);
                }

                int percentProcessed = (int)(currentRecordIndex * 100L / externalData.Records.Length);

                lock (_staticDataLookup)
                {
                    if (percentProcessed > 1)
                    {
                        _staticDataLookup[guid + "PercentRead"] = percentProcessed;
                        // for the progress bar to update async
                    }

                    if (percentProcessed > 99)
                    {
                        // Placed inside loop to have a contiguous lock block, even though it decreases performance.
                        // Should normally be placed just outside.
                        _staticDataLookup[guid + "MismatchArray"] = mismatchList.ToArray();
                        _staticDataLookup[guid + "Account"]       = account;
                    }
                }
            }
        }