private void AddDelta(MoneyColumnMetadataModel col, TableViewModel table, int daysDiff, string name)
{
var today = table.Values.OrderByDescending(v => v.When).FirstOrDefault();
var baseSet = table.Values.OrderByDescending(v => v.When).FirstOrDefault(v => v.When.AddDays(daysDiff) < DateTime.Now);
var todayValue = today?.Cells.FirstOrDefault(v => v.Column == col);
var baseSetValue = baseSet?.Cells.FirstOrDefault(v => v.Column == col);
if (todayValue != null && baseSetValue != null)
{
Ccy = Ccy ?? todayValue.Ccy;
IncompleteData |= todayValue.FailedToResolve.Concat(baseSetValue.FailedToResolve).Any();
var dT = todayValue.Value - baseSetValue.Value;
if (dT != null && !double.IsNaN(dT.Value) && !double.IsInfinity(dT.Value))
{
Deltas.Add((name, dT.Value));
}
else
{
IncompleteData = true;
}
}
}
public IActionResult PostData([Bind] string name, [Bind] string value, [Bind] string ccy, [Bind] string when = null)
{
try
{
if (string.IsNullOrWhiteSpace(name) ||
!double.TryParse(value.Replace(".", ",").Trim(), NumberStyles.Any, new NumberFormatInfo()
{
NumberDecimalSeparator = ","
}, out var valueParsed) ||
double.IsNaN(valueParsed) ||
double.IsInfinity(valueParsed))
{
_logger.LogError($"Failed to parse post-data: {name} / {value} / {ccy}");
return(new ContentResult
{
Content = "Bad request",
StatusCode = 400
});
}
var whendt = when == null ? DateTime.UtcNow : DateTime.Parse(when);
var existing = _objectRepository.Set <MoneyColumnMetadataModel>()
.FirstOrDefault(v => v.Provider == "API" && v.AccountName == name);
if (existing == null)
{
existing = new MoneyColumnMetadataModel("API", name)
{
UserFriendlyName = name
};
_objectRepository.Add(existing);
}
_objectRepository.Add(new MoneyStateModel
{
Column = existing,
Ccy = ccy,
Amount = valueParsed,
When = whendt
});
_logger.LogInformation($"Parsed post-data: {name} / {value} / {ccy}");
return(new ContentResult
{
Content = "OK",
StatusCode = 200
});
}
catch
{
_logger.LogError($"Failed to parse post-data: {name} / {value} / {ccy}");
return(new ContentResult
{
Content = "ERROR",
StatusCode = 500
});
}
}
public static CalculatedResult FromMoney(MoneyColumnMetadataModel h, MoneyStateModel money) => new CalculatedResult
{
Ccy = money.Ccy,
Column = h,
Money = money,
Value = money.Amount,
Tooltip = money.Amount.ToString(CultureInfo.CurrentCulture)
};
private MoneyColumnMetadata ToStream(MoneyColumnMetadataModel p0)
{
return(new MoneyColumnMetadata
{
Id = p0.Id.ToUUID(),
Function = p0.Function ?? "",
Order = p0.Order,
Provider = p0.Provider ?? "",
AccountName = p0.AccountName ?? "",
AutogenerateStatements = p0.AutogenerateStatements,
UserFriendlyName = p0.UserFriendlyName ?? ""
});
}
protected PaymentModel Statement(DateTime when, string account, string what, double amount, PaymentKind kind, string ccy,
string statementReference)
{
var column = Repository.Set <MoneyColumnMetadataModel>().FirstOrDefault(v => v.Provider == ProviderName && v.AccountName == account);
if (column == null)
{
column = new MoneyColumnMetadataModel(ProviderName, account)
{
UserFriendlyName = account
};
Repository.Add(column);
}
return(new PaymentModel(when, what, amount, kind, ccy, statementReference, column));
}
public OkResult MetadataEdit(Guid id, string userFriendlyName, string function, bool autogenerateStatements)
{
MoneyColumnMetadataModel existingModel;
if (id == Guid.Empty)
{
existingModel = new MoneyColumnMetadataModel(MoneyColumnMetadataModel.ComputedProdiver, null);
_objectRepository.Add(existingModel);
}
else
{
existingModel = _objectRepository.Set <MoneyColumnMetadataModel>().First(v => v.Id == id);
}
existingModel.Function = function ?? "";
existingModel.UserFriendlyName = userFriendlyName;
existingModel.AutogenerateStatements = autogenerateStatements;
return(Ok());
}
public IActionResult MetadataEdit(Guid id, string userFriendlyName, bool isVisible, string function, string charts)
{
MoneyColumnMetadataModel existingModel;
if (id == Guid.Empty)
{
existingModel = new MoneyColumnMetadataModel(MoneyColumnMetadataModel.ComputedProdiver, null);
_objectRepository.Add(existingModel);
}
else
{
existingModel = _objectRepository.Set <MoneyColumnMetadataModel>().First(v => v.Id == id);
}
existingModel.Function = function;
existingModel.UserFriendlyName = userFriendlyName;
existingModel.IsVisible = isVisible;
return(RedirectToAction(nameof(Index)));
}
public IActionResult PostPayment([FromBody] IEnumerable <PaymentData> postData)
{
foreach (var statementData in postData)
{
if (!statementData.IsValid())
{
_logger.LogError($"Failed to parse statement with id {statementData.Id}");
return(new ContentResult
{
Content = $"Failed to parse statement with id {statementData.Id}",
StatusCode = 400
});
}
_objectRepository.Remove <PaymentModel>(v => v.StatementReference == statementData.Id);
var column = _objectRepository.Set <MoneyColumnMetadataModel>()
.FirstOrDefault(v => v.Provider == "API" && v.AccountName == statementData.Account);
if (column == null)
{
column = new MoneyColumnMetadataModel("API", statementData.Account);
_objectRepository.Add(column);
}
var kind = statementData.Amount > 0 ? PaymentKind.Income : PaymentKind.Expense;
var pm = new PaymentModel(statementData.When.Value, statementData.What, statementData.Amount, kind,
statementData.Currency, statementData.Id, column);
_objectRepository.Add(pm);
}
_logger.LogInformation($"Added {postData.Count()} payments from API");
return(new ContentResult
{
Content = "OK",
StatusCode = 200
});
}
protected MoneyStateModel Money(String account, double amount, string ccy)
{
var existing = Repository.Set <MoneyColumnMetadataModel>()
.FirstOrDefault(v => v.Provider == ProviderName && v.AccountName == account);
if (existing == null)
{
existing = new MoneyColumnMetadataModel(ProviderName, account)
{
UserFriendlyName = account
};
Repository.Add(existing);
}
return(new MoneyStateModel
{
Column = existing,
When = DateTime.UtcNow.Date,
Ccy = ccy,
Amount = amount
});
}
public MoneyColumnMetadataJsModel(ObjectRepository repository, MoneyColumnMetadataModel model)
{
_repository = repository;
_model = model;
}
public MoneyColumnMetadataJsModel(MoneyColumnMetadataModel model)
{
_model = model;
}
public static CalculatedResult FromComputed(Dictionary <string, MoneyColumnMetadataModel> columns, MoneyColumnMetadataModel h) => new CalculatedResult
{
Column = h,
_expression = Parse(columns, h.Function)
};
public static CalculatedResult Empty(MoneyColumnMetadataModel item) => new CalculatedResult
{
Ccy = null,
Column = item,
Value = double.NaN
};
public void Scrape()
{
lock (_chrome)
{
_chrome.Reset();
var logger = _logger;
var currentState = _objectRepository.Set <MoneyStateModel>();
var scrapeConfigs = _objectRepository.Set <ScraperConfigurationModel>().ToList();
foreach (var scraperConfig in scrapeConfigs)
{
var scraper = _scrapers.FirstOrDefault(v => v.ProviderName == scraperConfig.ScraperName);
if (scraper == null)
{
logger.LogError($"Failed to find scraper {scraperConfig.ScraperName}");
continue;
}
logger.LogInformation($"Scraping {scraper.ProviderName}");
if (scraper is IStatementScraper ss)
{
try
{
var minDates = new[]
{
_objectRepository.Set <PaymentModel>()
.Where(v => v.Provider == scraper.ProviderName)
.OrderByDescending(v => v.When)
.FirstOrDefault()?.When,
_objectRepository.Set <MoneyStateModel>().OrderBy(v => v.When)
.FirstOrDefault()?.When,
_objectRepository.Set <PaymentModel>().OrderBy(v => v.When)
.FirstOrDefault()?.When
};
var lastPayment = minDates.Where(v => v != null).OrderBy(v => v).FirstOrDefault() ??
DateTime.MinValue;
if (lastPayment.AddHours(24) > DateTime.Now)
{
continue; // Let's not scrape statements too often - it's hard
}
logger.LogInformation($"Scraping statement for {scraper.ProviderName} since {lastPayment}...");
var statements = ss.ScrapeStatement(scraperConfig, _chrome, lastPayment).ToList();
logger.LogInformation($"Got statement of {statements.Count} items...");
foreach (var s in statements)
{
var existingItem = _objectRepository.Set <PaymentModel>().FirstOrDefault(v =>
v.When.Date == s.When.Date &&
Math.Abs(v.Amount - s.Amount) < 0.01 &&
v.Ccy == s.Ccy &&
v.StatementReference == null || v.StatementReference == s.StatementReference);
if (existingItem == null)
{
_objectRepository.Add(s);
}
else
{
if (existingItem.Provider == null)
{
existingItem.Provider = s.Provider;
}
if (existingItem.Account == null)
{
existingItem.Account = s.Account;
}
if (existingItem.StatementReference == null)
{
existingItem.StatementReference = s.StatementReference;
}
}
}
}
catch (Exception ex)
{
logger.LogError(ex, $"Failed to get statement for {scraper.ProviderName}...");
}
}
_chrome.Reset();
var accountCount = currentState.Where(s => s.Provider == scraper.ProviderName && s.When.Date == DateTime.UtcNow.Date.AddDays(-1))
.Select(s => s.AccountName).Distinct()
.ToList();
var todayState = currentState.Where(s => s.Provider == scraper.ProviderName && s.When.Date == DateTime.UtcNow.Date)
.Select(s => s.AccountName).Distinct()
.ToList();
var toScrape = accountCount.Count == 0 || accountCount.Except(todayState).Any();
if (toScrape)
{
try
{
logger.LogInformation("No cached items, scraping...");
var items = scraper.Scrape(scraperConfig, _chrome.Driver);
logger.LogInformation($"Found {items.Count()} items, indexing...");
foreach (var item in items)
{
logger.LogInformation($" - {item.Provider} / {item.AccountName}: {item.Amount} ({item.Ccy})");
if (!string.IsNullOrWhiteSpace(item.Provider))
{
if (item.Amount <= 0.001)
{
if (!_objectRepository.Set <MoneyStateModel>().Any(s =>
s.Provider == item.Provider &&
s.AccountName == item.AccountName &&
s.Amount > 0
))
{
continue;
}
}
if (todayState.Contains(item.AccountName))
{
continue;
}
_objectRepository.Add(item);
}
}
logger.LogInformation("Indexed...");
}
catch (Exception ex)
{
logger.LogError(ex, "There were an issue scraping");
}
}
else
{
logger.LogInformation("For today there are already scraped items, continuing...");
}
}
}
foreach (var item in _objectRepository.Set <MoneyStateModel>().GroupBy(v => v.Provider, v => v.AccountName))
{
foreach (var sub in item.Distinct())
{
var existing = _objectRepository.Set <MoneyColumnMetadataModel>()
.FirstOrDefault(v => v.Provider == item.Key && v.AccountName == sub);
if (existing == null)
{
existing = new MoneyColumnMetadataModel(item.Key, sub)
{
UserFriendlyName = sub,
IsVisible = true
};
_objectRepository.Add(existing);
}
}
}
}
public ReferenceExpression(MoneyColumnMetadataModel column)
{
_column = column;
}