private void sindle_sided_transaction(ImportDbContext db, Acklann.Plaid.Entity.Transaction txn) { Console.WriteLine("Creating single sided transaction"); var txn_config = config.account.FirstOrDefault(a => a.plaid_account_id == txn.AccountId); if (txn_config == null) { Console.WriteLine($"Dropping transaction; account not configured for sync: {txn.AccountId}"); // Record transaction as imported db.Transactions.Add(new ImportedTransaction { PlaidId = txn.TransactionId, FireflyId = null, }); db.SaveChanges(); return; } var is_source = txn.Amount > 0; var name = txn.Name; // TODO: fill name with PaymentInfo if non-null var transfer = new FireflyIII.Model.TransactionSplit( date: txn.Date, description: txn.Name, amount: (double)Math.Abs(txn.Amount), currencyCode: txn.CurrencyCode, externalId: txn.TransactionId, tags: txn.Categories?.ToList() ); if (is_source) { transfer.Type = TransactionSplit.TypeEnum.Withdrawal; transfer.SourceId = txn_config.firefly_account_id; transfer.DestinationName = name; } else { transfer.Type = TransactionSplit.TypeEnum.Deposit; transfer.SourceName = name; transfer.DestinationId = txn_config.firefly_account_id; } var storedtransfer = firefly.StoreTransaction(new FireflyIII.Model.Transaction(new[] { transfer }.ToList())); // Record transaction as imported db.Transactions.Add(new ImportedTransaction { PlaidId = txn.TransactionId, FireflyId = storedtransfer.Data.Id, }); db.SaveChanges(); }
static int Main(string[] unparsed_args) { Args args = Parser.Default.ParseArguments <Args>(unparsed_args) .MapResult( args => args, errors => { var first = errors.First(); if (first.Tag == ErrorType.HelpRequestedError || first.Tag == ErrorType.VersionRequestedError) { System.Environment.Exit(0); } // Apparently there isn't a way to get the tag name?? Console.WriteLine($"Unknown argument: {first.Tag}"); System.Environment.Exit(-1); return(null); }); var path = System.Environment.GetEnvironmentVariable("CONFIG_PATH") ?? System.Environment.CurrentDirectory; var config_path = Path.Combine(path, "config.toml"); if (!File.Exists(config_path)) { Console.WriteLine($"Error: config file not found. Tried '{config_path}'; do you need to set `CONFIG_PATH`?"); return(-1); } var config = Toml.ReadFile <ConnectorConfig>(config_path); if (config == null) { Console.WriteLine($"Error: invalid config file"); return(-1); } if (!VerifyConfig(config)) { Console.WriteLine($"Error: invalid config file"); return(-1); } var db_path = System.Environment.GetEnvironmentVariable("DB_PATH") ?? path; // Create DB if it doesn't exist & run migrations ImportDbContext.database_file = Path.Combine(db_path, "import-db.sqlite3"); using (var db = new ImportDbContext()) { db.Database.Migrate(); } var connector = new Connector(args, config); connector.Run(); return(0); }
private void transfer_between_two_plaid_accounts(ImportDbContext db, Acklann.Plaid.Entity.Transaction txn, Acklann.Plaid.Entity.Transaction other) { var source = txn.Amount > 0 ? txn : other; var dest = txn.Amount < 0 ? txn : other; var source_config = config.account.FirstOrDefault(a => a.plaid_account_id == source.AccountId); var dest_config = config.account.FirstOrDefault(a => a.plaid_account_id == dest.AccountId); if (source_config == null || dest_config == null) { throw new Exception($"Account not found in config: {source.AccountId} or {dest.AccountId}"); } // TODO: shrink txn names? (too verbose: "Requested transfer from... account XXXXXX0123 -> Incoming transfer from ...") var transfer = new FireflyIII.Model.TransactionSplit( date: source.Date, processDate: dest.Date, description: source.Name + " -> " + dest.Name, amount: (double)source.Amount, currencyCode: source.CurrencyCode, externalId: source.TransactionId + " -> " + dest.TransactionId, type: TransactionSplit.TypeEnum.Transfer, sourceId: source_config.firefly_account_id, destinationId: dest_config.firefly_account_id ); var storedtransfer = firefly.StoreTransaction(new FireflyIII.Model.Transaction(new[] { transfer }.ToList())); // Record both transactions as imported db.Transactions.AddRange(new[] { new ImportedTransaction { PlaidId = txn.TransactionId, FireflyId = storedtransfer.Data.Id, }, new ImportedTransaction { PlaidId = other.TransactionId, FireflyId = storedtransfer.Data.Id, } }); db.SaveChanges(); }
public async Task SyncOnce() { // TODO: Update FF3 account balances? using (var datedb = new ImportDbContext()) { var plaidtxns = new List <Acklann.Plaid.Entity.Transaction>(); if (args.ForceSync) { Console.WriteLine($"Info: force sync enabled - requesting data from the last {config.sync.max_sync_days} days"); } // Get list of unique plaid access tokens to query var plats_to_use = config.account .Select(t => t.plaid_access_token) .Where(t => !String.IsNullOrWhiteSpace(t)) .Distinct(); // Get all txns for every access token in use foreach (var token in plats_to_use) { // TODO: It's not very desirable to have access tokens in the db. It would be // better to figure out the next-poll time for each account ID and store that, // then figure out the correct poll time for each access token. var lastpoll = datedb.Poll.Where(p => p.PlaidId == token).FirstOrDefault(); var max_days = DateTime.Now - TimeSpan.FromDays(config.sync.max_sync_days); // If we haven't polled this account before, request `config.sync.max_sync_days` of data if (lastpoll == null) { lastpoll = new LastPoll(); lastpoll.PlaidId = token; lastpoll.Time = DateTime.Now - TimeSpan.FromDays(config.sync.max_sync_days); } // Override lastpoll if the force-sync flag was passed if (args.ForceSync) { lastpoll.Time = max_days; } // Throw an error and exit if the last poll was more than `sync.max_sync_days` ago if (lastpoll.Time < max_days) { Console.WriteLine($"Error: last program run was more than {config.sync.max_sync_days} days ago"); Console.WriteLine("Increase 'sync.max_sync_days' configuration value or use the '--force-sync' argument to ignore this error (and potentially miss some transactions)"); System.Environment.Exit(1); } // Handle response pagination uint request_size = 100; uint page_offset = 0; var now = DateTime.Now; var page_count = 0; var page_txn_list = new List <Acklann.Plaid.Entity.Transaction>(); do { // TODO: Fill in account info from this response and eliminate most of the `InitializeAccountData` step var plaid_txn_rsp = await this.plaid.FetchTransactionsAsync(new Acklann.Plaid.Transactions.GetTransactionsRequest { StartDate = lastpoll.Time, EndDate = now, AccessToken = token, Options = new Acklann.Plaid.Transactions.GetTransactionsRequest.PaginationOptions { Offset = page_offset, Total = request_size, }, }); page_count = plaid_txn_rsp.TransactionsReturned; page_offset += (uint)plaid_txn_rsp.Transactions.Count(); page_txn_list.AddRange(plaid_txn_rsp.Transactions); Console.WriteLine($"Fetched {page_offset}/{page_count}"); } while (page_offset < page_count); // Only record transactions that are not pending plaidtxns.AddRange(page_txn_list.Where(t => t.Pending == false)); // Next run, fetch all transactions from the last pending txn forward if (page_txn_list.Count(c => c.Pending == true) > 0) { lastpoll.Time = page_txn_list .Where(t => t.Pending == true) .Select(t => t.Date) .OrderBy(t => t) .First(); } else { lastpoll.Time = DateTime.Now; } // These updates will be saved at the end of the sync process datedb.Poll.Update(lastpoll); } // Process all txns using (var db = new ImportDbContext()) { foreach (var txn in plaidtxns) { // See if we have already processed this transaction if (db.Transactions.Any(b => b.PlaidId == txn.TransactionId)) { // Already processed; skip continue; } // Handle transfers between accounts if (txn.CategoryId == "21005000" || // transfer - credit txn.CategoryId == "21006000" || // transfer - debit txn.CategoryId == "16001000" || // payment - credit card txn.CategoryId == "21009000") // some amex are miscategorized as transfer - payroll { // Attempt to collect Transfer/Debit & Transfer/Credit pair into a single FF3 transfer transaction var others = plaidtxns.Where(t => t.Amount == -1 * txn.Amount && ((t.Date - txn.Date).Duration() < TimeSpan.FromDays(7)) && // less than a week apart t.CurrencyCode == txn.CurrencyCode && ((t.CategoryId == "21005000" && txn.CategoryId == "21006000") || (t.CategoryId == "21006000" && txn.CategoryId == "21005000") || (t.CategoryId == "16001000" && txn.CategoryId == "21009000") || (t.CategoryId == "21009000" && txn.CategoryId == "16001000")) ); if (others != null && others.Count() > 0) { if (others.Count() == 1) { // Found exactly one matching txn Console.WriteLine("Found matching txn pair"); transfer_between_two_plaid_accounts(db, txn, others.First()); continue; } else { // Found multiple possible transactions Console.WriteLine("Found multiple possible transfer pairings; creating single sided txns instead"); // Create the single sided txns here; otherwise, we'll may end up creating a pair from remaining transactions as we continue processing. sindle_sided_transaction(db, txn); foreach (var other in others) { sindle_sided_transaction(db, other); } continue; } } } // Did not find a matching txn: create single sided FF3 transaction sindle_sided_transaction(db, txn); } } // save last-polled at the very end datedb.SaveChanges(); } }
private void single_sided_transaction(ImportDbContext db, Acklann.Plaid.Entity.Transaction txn) { Console.WriteLine("Creating single sided transaction"); var txn_config = config.account.FirstOrDefault(a => a.plaid_account_id == txn.AccountId); if (txn_config == null) { Console.WriteLine($"Dropping transaction; account not configured for sync: {txn.AccountId}"); // Record transaction as imported db.Transactions.Add(new ImportedTransaction { PlaidId = txn.TransactionId, FireflyId = null, }); db.SaveChanges(); return; } if (txn.Amount == 0) { Console.WriteLine("Ignoring zero-amount transaction"); // Record transaction as imported db.Transactions.Add(new ImportedTransaction { PlaidId = txn.TransactionId, FireflyId = null, }); db.SaveChanges(); return; } var is_source = txn.Amount > 0; var name = txn.Name; if (name.Length > 255) { Console.WriteLine($"Transaction name was {name.Length} characters long. Truncating to FF3's 255 character limit."); name = name.Substring(0, 255); } // TODO: fill name with PaymentInfo if non-null var transfer = new FireflyIII.Model.TransactionSplit( date: txn.Date, description: txn.Name, amount: (double)Math.Abs(txn.Amount), currencyCode: txn.CurrencyCode, externalId: txn.TransactionId, tags: txn.Categories?.ToList() ); if (is_source) { transfer.Type = TransactionSplit.TypeEnum.Withdrawal; transfer.SourceId = txn_config.firefly_account_id; transfer.DestinationName = name; } else { transfer.Type = TransactionSplit.TypeEnum.Deposit; transfer.SourceName = name; transfer.DestinationId = txn_config.firefly_account_id; } var storedtransfer = firefly.StoreTransaction(new FireflyIII.Model.Transaction(new[] { transfer }.ToList())); if (storedtransfer == null || storedtransfer.Data == null) { Console.WriteLine($"failed to store single_sided_transaction"); Console.WriteLine($"txn: {txn.Name}, {txn.Date}, {txn.Amount} {txn.CurrencyCode}, {txn.TransactionId}"); Console.WriteLine($"transfer: {transfer}"); Console.WriteLine($"storedtransfer: {storedtransfer}"); throw new ApplicationException("failed to store transaction with firefly-iii"); } // Record transaction as imported db.Transactions.Add(new ImportedTransaction { PlaidId = txn.TransactionId, FireflyId = storedtransfer.Data.Id, }); db.SaveChanges(); }