void OnReloadAssetData_ReloadRtDataAndSetTimer()
{
Utils.Logger.Info("ReloadRtDataAndSetTimer() START");
m_lastRtPriceQueryTime = new Dictionary <Asset, DateTime>(); // purge out history after AssetData reload
m_highFreqParam.Assets = m_highFreqTickrs.Select(r => AssetsCache.GetAsset(r) !).ToArray();
m_midFreqParam.Assets = m_midFreqTickrs.Select(r => AssetsCache.GetAsset(r) !).ToArray();
m_lowFreqParam.Assets = AssetsCache.Assets.Where(r => r.AssetId.AssetTypeID == AssetType.Stock && (r as Stock) !.ExpirationDate == string.Empty && !m_highFreqTickrs.Contains(r.SqTicker) && !m_midFreqTickrs.Contains(r.SqTicker)).ToArray() !;
RtTimer_Elapsed(m_highFreqParam);
RtTimer_Elapsed(m_midFreqParam);
RtTimer_Elapsed(m_lowFreqParam);
Utils.Logger.Info("ReloadRtDataAndSetTimer() END");
}
// GetLastRtValue() always return data without blocking. Data might be 1 hour old or 3sec (RTH) or in 60sec (non-RTH) for m_assetIds only if there was a function call in the last 5 minutes (busyMode), but it is OK.
public IEnumerable <(AssetId32Bits SecdID, float LastValue)> GetLastRtValue(uint[] p_assetIds) // C# 7.0 adds tuple types and named tuple literals. uint[] is faster to create and more RAM efficient than linked-list<uint>
{
IEnumerable <(AssetId32Bits SecdID, float LastValue)> rtPrices = p_assetIds.Select(r =>
{
var sec = AssetsCache.GetAsset(r);
m_lastRtPriceQueryTime[sec] = DateTime.UtcNow;
DateTime lastDateTime = DateTime.MinValue;
float lastValue;
if (sec.AssetId.AssetTypeID == AssetType.BrokerNAV)
{
(lastValue, lastDateTime) = GetLastNavRtPrice((sec as BrokerNav) !);
}
else
{
lastValue = sec.LastValue;
}
return(sec.AssetId, lastValue);
});
return(rtPrices);
}
// Polling for changes 3x every day
// historical data can partially come from our Redis-Sql DB or partially from YF
static async Task <CompactFinTimeSeries <DateOnly, uint, float, uint>?> CreateDailyHist(Db p_db, User[] p_users, AssetsCache p_assetCache)
{
Utils.Logger.Info("ReloadHistoricalDataAndSetTimer() START");
Console.Write("*MemDb.DailyHist Download from YF: ");
DateTime etNow = DateTime.UtcNow.FromUtcToEt();
try
{
Dictionary <AssetId32Bits, List <Split> > potentialMissingYfSplits = await GetPotentianMissingYfSplits(p_db, p_assetCache);
var assetsDates = new Dictionary <uint, DateOnly[]>();
var assetsAdjustedCloses = new Dictionary <uint, float[]>();
foreach (var asset in p_assetCache.Assets)
{
(DateOnly[] dates, float[] adjCloses) = await GetDatesAndAdjCloses(asset, potentialMissingYfSplits);
if (adjCloses.Length != 0)
{
assetsDates[asset.AssetId] = dates;
assetsAdjustedCloses[asset.AssetId] = adjCloses;
Console.Write($"{asset.Symbol}, ");
Debug.Write($"{asset.SqTicker}, first: DateTime: {dates.First()}, Close: {adjCloses.First()}, last: DateTime: {dates.Last()}, Close: {adjCloses.Last()}"); // only writes to Console in Debug mode in vscode 'Debug Console'
}
}
// NAV assets should be grouped by user, because we create a synthetic new aggregatedNAV. This aggregate should add up the RAW UnadjustedNAV (not adding up the adjustedNAV), so we have to create it at MemDbReload.
var navAssets = p_assetCache.Assets.Where(r => r.AssetId.AssetTypeID == AssetType.BrokerNAV).Select(r => (BrokerNav)r);
var navAssetsByUser = navAssets.ToLookup(r => r.User); // ToLookup() uses User.Equals()
int nVirtualAggNavAssets = 0;
foreach (IGrouping <User?, BrokerNav>?navAssetsOfUser in navAssetsByUser)
{
AddNavAssetsOfUserToAdjCloses(navAssetsOfUser, p_assetCache, p_db, ref nVirtualAggNavAssets, assetsDates, assetsAdjustedCloses);
} // NAVs per user
List <DateOnly> mergedDates = UnionAllDates(assetsDates);
DateOnly[] mergedDatesArr = mergedDates.ToArray();
var values = MergeCloses(assetsDates, assetsAdjustedCloses, mergedDatesArr);
return(new CompactFinTimeSeries <DateOnly, uint, float, uint>(mergedDatesArr, values));
}
catch (Exception e)
{
Utils.Logger.Error(e, "Exception in ReloadHistoricalDataAndSetTimer()");
await HealthMonitorMessage.SendAsync($"Exception in SqCoreWebsite.C#.MemDb. Exception: '{ e.ToStringWithShortenedStackTrace(1600)}'", HealthMonitorMessageID.SqCoreWebCsError);
}
return(null);
}
public MemData(User[] newUsers, AssetsCache newAssetCache, CompactFinTimeSeries <DateOnly, uint, float, uint> newDailyHist)
{
Users = newUsers;
AssetsCache = newAssetCache;
DailyHist = newDailyHist;
}
private static async Task <Dictionary <AssetId32Bits, List <Split> > > GetPotentianMissingYfSplits(Db p_db, AssetsCache p_assetCache)
{
DateTime etNow = DateTime.UtcNow.FromUtcToEt();
Dictionary <string, List <Split> > missingYfSplitSqTickers = p_db.GetMissingYfSplits(); // first get it from Redis Db, then Add items from Nasdaq
Dictionary <AssetId32Bits, List <Split> > potentialMissingYfSplits = missingYfSplitSqTickers.ToDictionary(r => p_assetCache.GetAsset(r.Key).AssetId, r => r.Value);
// var url = $"https://api.nasdaq.com/api/calendar/splits?date=2021-01-19";
var url = $"https://api.nasdaq.com/api/calendar/splits?date={Utils.Date2hYYYYMMDD(etNow.AddDays(-7))}"; // go back 7 days before to include yesterdays splits
string?nasdaqSplitsJson = await Utils.DownloadStringWithRetryAsync(url, 3, TimeSpan.FromSeconds(5), true);
if (nasdaqSplitsJson != null)
{
var ndqSplitsJson = JsonSerializer.Deserialize <Dictionary <string, object> >(nasdaqSplitsJson);
using JsonDocument doc = JsonDocument.Parse(nasdaqSplitsJson);
var ndqSplits = doc.RootElement.GetProperty("data").GetProperty("rows").EnumerateArray()
.Select(row =>
{
string symbol = row.GetProperty("symbol").ToString() ?? string.Empty;
string sqTicker = Asset.BasicSqTicker(AssetType.Stock, symbol);
Asset?asset = p_assetCache.TryGetAsset(sqTicker);
return(new
{
Asset = asset,
Row = row
});
}).Where(r =>
{
return(r.Asset != null);
}).Select(r =>
{
string executionDateStr = r.Row.GetProperty("executionDate").ToString() ?? string.Empty; // "executionDate":"01/21/2021"
DateTime executionDate = Utils.FastParseMMDDYYYY(executionDateStr);
string ratioStr = r.Row.GetProperty("ratio").ToString() ?? string.Empty;
var splitArr = ratioStr.Split(':'); // "ratio":"2 : 1" or "ratio":"5.000%" while YF "2:1", which is the same order.
double beforeSplit = Double.MinValue, afterSplit = Double.MinValue;
if (splitArr.Length == 2) // "ratio":"2 : 1"
{
afterSplit = Utils.InvariantConvert <double>(splitArr[0]);
beforeSplit = Utils.InvariantConvert <double>(splitArr[1]);
}
else // "ratio":"5.000%", we can ignore it.
{
// {"symbol":"GNTY","name":"Guaranty Bancshares, Inc.","ratio":"10.000%","payableDate":"02/12/2021","executionDate":"02/04/2021","announcedDate":"01/20/2021"}
// "...has declared a 10% stock dividend for shareholders of record as of February 5, 2021. As an example, each shareholder will receive one additional share of stock for every ten shares owned on the effective date of February 12, 2021.
// however, YF doesn't handle at all these stock-dividends (not cash dividend)
// YF builds it into the quote, so the quote price is smooth, but this cannot be queried as YF split, neither as cash-dividend.
}
var split = new Split()
{
Date = executionDate, Before = beforeSplit, After = afterSplit
};
return(new
{
Asset = r.Asset !,
Split = split
});
}).Where(r =>
