private void ServerDiagnosticMemDb(StringBuilder p_sb, bool p_isHtml)
{
p_sb.Append($"#Users: {Users.Length}: <br>");
var hist = DailyHist.GetDataDirect();
int memUsedKb = hist.MemUsed() / 1024;
p_sb.Append($"#Assets: {AssetsCache.Assets.Count}, #HistoricalAssets: {hist.Data.Count}, Used RAM: {memUsedKb:N0}KB<br>"); // hist.Data.Count = Srv.LoadPrHist + DC Aggregated NAV
var lastDbReloadWithoutHist = m_lastDbReloadTs - m_lastHistoricalDataReloadTs;
p_sb.Append($"m_lastHistoricalDataReloadTimeUtc: '{m_lastHistoricalDataReload}', lastDbReloadWithoutHist: {lastDbReloadWithoutHist.TotalSeconds:0.000}sec, m_lastHistoricalDataReloadTs: {m_lastHistoricalDataReloadTs.TotalSeconds:0.000}sec.<br>");
var yfTickers = AssetsCache.Assets.Where(r => r.AssetId.AssetTypeID == AssetType.Stock).Select(r => ((Stock)r).YfTicker).ToArray();
p_sb.Append($"StockAssets (#{yfTickers.Length}): ");
p_sb.AppendLongListByLine(yfTickers, ",", 10, "<br>");
}
// ************** Helper methods for functions that are used frequently in the code-base
// p_dateExclLoc: not UTC, but ET (in USA stocks), CET (in EU stocks) or whatever is the local time at the exchange of the stock.
// The date of the query. If p_dateExclLoc = Monday, then LastCloseDate will be previous Friday.
// Imagine we want to find the LastClose price which was valid on day p_dateExclLoc
// Different assets can have different LastCloseDates. If there was an EU holiday on Friday, then the EU stock's LastClose is Thursday, while an USA stock LastClose is Friday
// keep the order and the length of p_assets list. So, it can be indexed. p_assets[i] is the same item as result[i]
// If an asset has no history, we return NaN as lastClose price.
// This requre more RAM than the other solution which only returns the filled rows, but it will save CPU time later, when the result is processed at the caller. He don't have to do double FORs to iterate.
public IEnumerable <AssetLastClose> GetSdaLastCloses(IEnumerable <Asset> p_assets, DateTime p_dateExclLoc /* usually given as current time today */)
{
DateOnly lookbackEnd = p_dateExclLoc.Date.AddDays(-1); // if (p_dateExclLoc is Monday), -1 days is Sunday, but we have to find Friday before
TsDateData <DateOnly, uint, float, uint> histData = DailyHist.GetDataDirect();
DateOnly[] dates = histData.Dates;
// At 16:00, or even intraday: YF gives even the today last-realtime price with a today-date. We have to find any date backwards, which is NOT today. That is the PreviousClose.
int iEndDay = 0;
for (int i = 0; i < dates.Length; i++)
{
if (dates[i] <= lookbackEnd)
{
iEndDay = i;
break;
}
}
Debug.WriteLine($"MemDb.GetSdaLastCloses().EndDate: {dates[iEndDay]}");
var lastCloses = p_assets.Select(r =>
{
if (!histData.Data.TryGetValue(r.AssetId, out Tuple <Dictionary <TickType, float[]>, Dictionary <TickType, uint[]> >?assetHist))
{
return(new AssetLastClose(r, DateTime.MinValue, float.NaN)); // the Asset might be in MemDb, but has no history at all.
}
float[] sdaCloses = assetHist.Item1[TickType.SplitDivAdjClose];
// If there was an EU holiday on Friday, then the EU stock's LastClose is Thursday, while an USA stock LastClose is Friday
int j = iEndDay;
do
{
float lastClose = sdaCloses[j];
if (!float.IsNaN(lastClose))
{
return(new AssetLastClose(r, dates[j], lastClose));
}
j++;
} while (j < dates.Length);
return(new AssetLastClose(r, DateTime.MinValue, float.NaN));
});
return(lastCloses);
}
// Backtests don't need the Statistical data (maxDD), just the prices. Dashboard.MarketHealth only needs the Statistical data, no historical prices
public IEnumerable <AssetHist> GetSdaHistCloses(IEnumerable <Asset> p_assets, DateTime p_startIncLoc, DateTime p_endExclLoc /* usually given as current time today */,
bool p_valuesNeeded, bool p_statNeeded)
{
DateOnly lookbackEnd = p_endExclLoc.Date.AddDays(-1); // if (p_dateExclLoc is Monday), -1 days is Sunday, but we have to find Friday before
TsDateData <DateOnly, uint, float, uint> histData = DailyHist.GetDataDirect();
DateOnly[] dates = histData.Dates;
// At 16:00, or even intraday: YF gives even the today last-realtime price with a today-date. We have to find any date backwards, which is NOT today. That is the PreviousClose.
int iEndDay = 0;
for (int i = 0; i < dates.Length; i++)
{
if (dates[i] <= lookbackEnd)
{
iEndDay = i;
break;
}
}
Debug.WriteLine($"MemDb.GetSdaHistCloses().EndDate: {dates[iEndDay]}");
int iStartDay = histData.IndexOfKeyOrAfter(new DateOnly(p_startIncLoc)); // the valid price at the weekend is the one on the previous Friday. After.
if (iStartDay == -1 || iStartDay >= dates.Length) // If not found then fix the startDate as the first available date of history.
{
iStartDay = dates.Length - 1;
}
Debug.WriteLine($"MemDb.GetSdaHistCloses().StartDate: {dates[iStartDay]}");
IEnumerable <AssetHist> assetHists = p_assets.Select(r =>
{
float[] sdaCloses = histData.Data[r.AssetId].Item1[TickType.SplitDivAdjClose];
// if startDate is not found, because e.g. we want to go back 3 years, while stock has only 2 years history
int iiStartDay = (iStartDay < sdaCloses.Length) ? iStartDay : sdaCloses.Length - 1;
if (Single.IsNaN(sdaCloses[iiStartDay]) && // if that date in the global MemDb was an USA stock market holiday (e.g. President days is on monday), price is NaN for stocks, but valid value for NAV
((iiStartDay + 1) <= sdaCloses.Length))
{
iiStartDay++; // that start 1 day earlier. It is better to give back more data, then less. Besides on that holiday day, the previous day price is valid.
}
List <AssetHistValue>?values = (p_valuesNeeded) ? new List <AssetHistValue>() : null;
// reverse marching from yesterday into past is not good, because we have to calculate running maxDD, maxDU.
float max = float.MinValue, min = float.MaxValue, maxDD = float.MaxValue, maxDU = float.MinValue;
int iStockEndDay = Int32.MinValue, iStockFirstDay = Int32.MinValue;
for (int i = iiStartDay; i >= iEndDay; i--) // iEndDay is index 0 or 1. Reverse marching from yesterday iEndDay to deeper into the past. Until startdate iStartDay or until history beginning reached
{
if (Single.IsNaN(sdaCloses[i]))
{
continue; // if that date in the global MemDb was an USA stock market holiday (e.g. President days is on monday), price is NaN for stocks, but valid value for NAV
}
if (iStockFirstDay == Int32.MinValue)
{
iStockFirstDay = i;
}
iStockEndDay = i;
float val = sdaCloses[i];
if (p_valuesNeeded && values != null)
{
values.Add(new AssetHistValue()
{
Date = dates[i], SdaValue = val
});
}
if (val > max)
{
max = val;
}
if (val < min)
{
min = val;
}
float dailyDD = val / max - 1; // -0.1 = -10%. daily Drawdown = how far from High = loss felt compared to Highest
if (dailyDD < maxDD) // dailyDD are a negative values, so we should do MIN-search to find the Maximum negative value
{
maxDD = dailyDD; // maxDD = maximum loss, pain felt over the period
}
float dailyDU = val / min - 1; // daily DrawUp = how far from Low = profit felt compared to Lowest
if (dailyDU > maxDU)
{
maxDU = dailyDU; // maxDU = maximum profit, happiness felt over the period
}
}
// it is possible that both iStockFirstDay, iStockEndDay are left as Int32.MinValue, because there is no valid value at all in that range. Fine.
AssetHistStat?stat = null;
if (p_statNeeded)
{
stat = new AssetHistStat()
{
PeriodStart = (iStockFirstDay >= 0) ? sdaCloses[iStockFirstDay] : Double.NaN,
PeriodEnd = (iStockEndDay >= 0) ? sdaCloses[iStockEndDay] : Double.NaN,
PeriodHigh = (max == float.MinValue) ? float.NaN : max,
PeriodLow = (min == float.MaxValue) ? float.NaN : min,
PeriodMaxDD = (maxDD == float.MaxValue) ? float.NaN : maxDD,
PeriodMaxDU = (maxDU == float.MinValue) ? float.NaN : maxDU
};
}
var periodStartDateInc = (iStockFirstDay >= 0) ? (DateTime)dates[iStockFirstDay] : DateTime.MaxValue; // it may be not the 'asked' start date if asset has less price history
var periodEndDateInc = (iStockEndDay >= 0) ? (DateTime)dates[iStockEndDay] : DateTime.MaxValue; // by default it is the date of yesterday, but the user can change it
AssetHist hist = new AssetHist(r, periodStartDateInc, periodEndDateInc, stat, values);
return(hist);
});
return(assetHists);
}
async Task ReloadDbDataIfChangedImpl() // if necessary it reloads Historical and Realtime data
{
Console.WriteLine("*MemDb is not yet ready! ReloadDbData is in progress...");
DateTime startTime = DateTime.UtcNow;
// GA.IM.NAV assets have user_id data, so User data has to be reloaded too before Assets
(bool isDbReloadNeeded, User[]? newUsers, List <Asset>?sqCoreAssets) = m_Db.GetDataIfReloadNeeded();
if (!isDbReloadNeeded)
{
return;
}
// to minimize the time memDb is not consintent we create everything into new pointers first, then update them quickly
var newAssetCache = AssetsCache.CreateAssetCache(sqCoreAssets !);
// var newPortfolios = GeneratePortfolios();
if (IsInitialized)
{
// if this is the periodic (not initial) reload of RedisDb, then we don't surprise clients by emptying HistPrices
// and not having HistPrices for 20minutes. So, we download HistPrices before swapping m_memData pointer
DateTime startTimeHist = DateTime.UtcNow;
var newDailyHist = await CreateDailyHist(m_Db, newUsers !, newAssetCache); // downloads historical prices from YF. Assume it takes 20min
if (newDailyHist == null)
{
newDailyHist = new CompactFinTimeSeries <DateOnly, uint, float, uint>();
}
m_lastHistoricalDataReload = DateTime.UtcNow;
m_lastHistoricalDataReloadTs = DateTime.UtcNow - startTimeHist;
var newMemData = new MemData(newUsers !, newAssetCache, newDailyHist);
m_memData = newMemData; // swap pointer in atomic operation
Console.WriteLine($"*MemDb is ready! (#Assets: {AssetsCache.Assets.Count}, #HistoricalAssets: {DailyHist.GetDataDirect().Data.Count}) in {m_lastHistoricalDataReloadTs.TotalSeconds:0.000}sec");
}
else
{
// if this is the first time to load DB from Redis, then we don't demand HistData. Assume HistData crawling takes 20min
// many clients can survive without historical data first. MarketDashboard. However, they need Asset and User data immediately.
// BrAccInfo is fine wihout historical. It will send NaN as a LastClose. Fine. Client will handle it.
// So, we don't need to wait for Historical to finish InitDb (that might take 20 minutes in the future).
// !!! Also, in development, we don't want to wait until All HistData arrives, but start Debugging code right away after starting the WebServer.
// Clients of MemDb should handle properly if HistData is not yet ready (NaN and later Refresh).
var newMemData = new MemData(newUsers !, newAssetCache, new CompactFinTimeSeries <DateOnly, uint, float, uint>());
m_memData = newMemData; // swap pointer in atomic operation
Console.WriteLine($"*MemDb is half-ready! (#Assets: {AssetsCache.Assets.Count}, #HistoricalAssets: 0)");
}
m_lastDbReload = DateTime.UtcNow;
m_lastDbReloadTs = DateTime.UtcNow - startTime;
foreach (var brAccount in BrAccounts)
{
UpdateBrAccPosAssetIds(brAccount.AccPoss);
}
OnReloadAssetData_ReloadRtDataAndSetTimer(); // downloads realtime prices from YF or IEX
OnReloadAssetData_ReloadRtNavDataAndSetTimer(); // downloads realtime NAVs from VBrokers
EvDbDataReloaded?.Invoke();
}