public static void RefreshNoDataDaysAndSlapCache(ThreadControl tc)
{
ZonedDateTime ie = SystemClock.Instance.GetCurrentInstant().InZone(UCDT.TimeZones.Eastern);
// set the current date to two days ago via eastern timezone
DateTime dt = new DateTime(ie.Year, ie.Month, ie.Day, 0, 0, 0, DateTimeKind.Unspecified).AddDays(-2);
// ZonedDateTime zdt = UCDT.ZonedDateTimetoZonedDateTime(currentDate, UCDT.TimeZones.Eastern);
List <FD> noDataDays = new List <FD>();
while (dt.Year > 2018 && tc.CheckNotStopped())
{
tc.Log.AddLine("Checking: " + dt.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture));
if (!StockAPI.Methods.DateHasData(dt.Year, dt.Month, dt.Day, false))
{
FD fd = new FD(dt);
tc.Log.AddLine("Found a day with no data: " + fd.ToStringLong());
noDataDays.Add(fd);
}
dt = dt.AddDays(-1);
}
if (tc.CheckNotStopped())
{
DBMethods.RefreshDaysTracker(noDataDays, "NoData");
noDataDays.Sort();
for (int n = 0; n < Global.State.DataTracker.NoDataDays.Count; n++)
{
// check if we didn't find one that was found before
if (noDataDays.BinarySearch(Global.State.DataTracker.NoDataDays[n]) < 0)
{
tc.Log.AddLine("FOUND A DAY IN THE OLD DATA THAT'S NOT IN THE NEW DATA: " + Global.State.DataTracker.NoDataDays[n].ToStringLong());
// This means that the scan before got a false nodataday.
// Could be API error or something.
// DELETE ANY CACHED DAY FOR THIS DAY BECAUSE WE MAY HAVE CACHED EMPTY NODES
}
}
App.InitDataTracker();
}
}
public static void VacuumAllDBs(ThreadControl tc)
{
tc.Log.AddLine("Starting Vacuuming all databases", Verbosity.Minimal);
List <string> keys = DBMethods.GetConnectionKeys();
long startSize = UC.GetFolderSize(Global.Constants.StockTicksDBPath);
for (int m = 0; m < keys.Count; m += 60)
{
Parallel.For(0, 60, new ParallelOptions {
MaxDegreeOfParallelism = 20
}, n =>
{
if (m + n < keys.Count)
{
tc.Log.AddLine("Vacuuming " + keys[m + n], Verbosity.Verbose);
DBMethods.Vacuum(keys[m + n]);
tc.Log.AddLine("Done Vacuuming " + keys[m + n], Verbosity.Verbose);
}
});
if (!tc.CheckNotStopped())
{
tc.Log.AddLine("Breaking VacuumAllDBs!");
break;
}
}
Thread.Sleep(500);
long endSize = UC.GetFolderSize(Global.Constants.StockTicksDBPath);
tc.Log.AddLine("Vacuuming Complete. Start size: " + startSize + ", End size: " + endSize, Verbosity.Minimal);
}
/// <summary>
/// When simulating, this method will load all of the needed nodes for simulating a live feed/stream.
/// </summary>
private void PrepareSimDayNodes()
{
if (AlgoTraderState.IsSim)
{
// load up today's nodes for StreamManager to pretend to add nodes by using today's nodes
int amount = 100;
Stopwatch sw = new Stopwatch();
sw.Start();
object lockObj = new object();
for (int m = 0; m < Global.State.AllSymbols.Count; m += amount)
{
Parallel.For(0, amount, new ParallelOptions {
MaxDegreeOfParallelism = 30
}, n =>
{
if (m + n < Global.State.AllSymbols.Count)
{
// NodesData nodes = null;
NodesData nodes = DataMethods.GetCachedDayNodesData(Global.State.AllSymbols[m + n], AlgoTraderState.CurrentDay, Interval.HalfSecond, justCache: false);
// NodesData nodesData = DataMethods.GetCachedDayNodesData(Global.State.AllSymbols[m + n], CurrentDay, Interval.HalfSecond, computeNodes: true, justCache: false);
// no need to ComputeGaps because it's 1 day... trusting well formatted data
lock (lockObj)
{
AlgoTraderShared.SimDayNodes.Add(Global.State.AllSymbols[m + n], nodes);
//nodes.Add(Global.State.AllSymbols[m + n], DataMethods.GetCachedDayNodesData(Global.State.AllSymbols[m + n], CurrentDay, Interval.HalfSecond, computeNodes: true, justCache: false));
}
TC.Log.AddLine("Loaded " + Global.State.AllSymbols[m + n] + " stock nodes for stream manager");
}
});
if (!TC.CheckNotStopped())
{
break;
}
}
sw.Stop();
TC.Log.AddLine("Finished loading sim nodes for stream manager in " + UC.MillisecondsToSeconds(sw.ElapsedMilliseconds, 3) + " sec(s). That's " + ((decimal)Global.State.AllSymbols.Count / UC.MillisecondsToSeconds(sw.ElapsedMilliseconds, 3)) + " per sec.");
}
}
public static void BuildMinuteDayNodes(ThreadControl tc)
{
// this should be called AFTER updated yesterday's ticks AND data tracker no data days updated
List <string> symbols = new List <string>(Global.State.AllSymbols);
symbols.Shuffle();
symbols.Remove("spy");
symbols.Remove("aapl");
symbols.Add("spy");
symbols.Add("aapl");
symbols.Reverse();
ZonedDateTime ie = SystemClock.Instance.GetCurrentInstant().InZone(UCDT.TimeZones.Eastern);
// set the current date to yesterday via eastern timezone just to be safe
DateTime dt = new DateTime(ie.Year, ie.Month, ie.Day, 0, 0, 0, DateTimeKind.Unspecified).AddDays(-20);
DateTime firstDataDay = Global.State.DataTracker.DataDays[0].DT;
while (dt > firstDataDay && tc.CheckNotStopped())
{
FD fd = new FD(dt);
Parallel.For(0, symbols.Count, new ParallelOptions {
MaxDegreeOfParallelism = 20
}, n =>
{
if (DataMethods.DayNodesDataCacheable(symbols[n], fd))
{
Stopwatch sw = new Stopwatch();
sw.Start();
NodesData nodes = DataMethods.GetCachedDayNodesData(symbols[n], fd, Interval.HalfSecond, computeNodes: false, justCache: true);
sw.Stop();
tc.Log.AddLine("[" + symbols[n] + "] " + fd.ToString() + " Done. Took " + UC.MillisecondsToSeconds(sw.ElapsedMilliseconds, 2) + " sec(s)");
}
else
{
tc.Log.AddLine("[" + symbols[n] + "] " + fd.ToString() + ". Not cacheable.");
}
});
dt = dt.AddDays(-1);
}
}
/*
* private bool ModeNeedsSystemClock()
* {
* return (
* AlgoTraderState.Mode == AlgoTraderModes.Live ||
* AlgoTraderState.Mode == AlgoTraderModes.Paper ||
* AlgoTraderState.Mode == AlgoTraderModes.FakeLive
* );
* }
*/
public static void WaitFor(int hour, int minute, ThreadControl tc)
{
int lastSecond = 0;
int currentSecond = 0;
while (!AlgoTraderState.CurrentTime.IsOnOrPassed(hour, minute) && tc.CheckNotStopped())
{
currentSecond = DateTime.UtcNow.Second;
if (currentSecond != lastSecond)
{
lastSecond = currentSecond;
tc.Log.AddLine("Waiting for " + hour.ToString("D2") + ":" + minute.ToString("D2") + ". It's currently " + AlgoTraderState.CurrentTime.GetTime().ToString(), Verbosity.Normal);
}
}
}
public static void UpdateHistoricalTicks(ThreadControl tc)
{
tc.Log.AddLine("Starting update historical ticks", Verbosity.Minimal);
ZonedDateTime ie = SystemClock.Instance.GetCurrentInstant().InZone(UCDT.TimeZones.Eastern);
// set the current date to yesterday via eastern timezone just to be safe
DateTime currentDate = new DateTime(ie.Year, ie.Month, ie.Day, 0, 0, 0, DateTimeKind.Utc).AddDays(-335);
while (tc.CheckNotStopped())
{
FD fd = new FD(currentDate);
UpdateHistoricalTicksByDay(Global.State.AllSymbols, fd, false, tc);
currentDate = currentDate.AddDays(-1);
}
}
public static void UpdateHistoricalTicksByDay(List <string> symbols, FD fd, bool force, ThreadControl tc)
{
tc.Log.AddLine("Starting update histocial ticks by day", Verbosity.Minimal);
ZonedDateTime ie = SystemClock.Instance.GetCurrentInstant().InZone(UCDT.TimeZones.Eastern);
// safety precaution
if (ie.Year == fd.DT.Year && ie.Month == fd.DT.Month && ie.Day == fd.DT.Day)
{
throw new Exception("What are you doing???");
}
int parallelAPICalls = 15;
if (!fd.IsOnWeekend())
{
bool hasDataForDay = StockAPI.Methods.DateHasData(fd.DT.Year, fd.DT.Month, fd.DT.Day, force);
if (hasDataForDay)
{
if (force)
{
tc.Log.AddLine("Forcing a historical update", Verbosity.Verbose);
}
else
{
tc.Log.AddLine("Date has data", Verbosity.Verbose);
}
tc.Log.AddLine("About to start API calls as fast as we can go", Verbosity.Minimal);
symbols.Shuffle();
for (int s = 0; s < symbols.Count; s += parallelAPICalls)
{
//string[] syms = new string[parallelAPICalls];
lock (Global.State.ThreadControlTreeLock)
{
for (int m = 0; m < parallelAPICalls; m++)
{
if (m + s < symbols.Count)
{
string sym = symbols[m + s];
ThreadControl singleQuoteTC = new ThreadControl(fd.ToString() + " : " + sym);
tc.Children.Add(singleQuoteTC);
Task.Factory.StartNew(() => Methods.ThreadRun("AT.SchedulerMethods, AT", "UpdateSingleDayAndSingleSymbolTicks", singleQuoteTC, null, new object[] { sym, fd }), CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default);
}
}
}
while (tc.Children.Exists(p => p.State != ThreadControlState.Complete && p.State != ThreadControlState.Done))
{
}
tc.ClearChildren();
tc.Log.AddLine("Done with small batch for " + fd.ToString() + ". About %" + Math.Round(((double)s / (double)symbols.Count) * (double)100, 2) + " done.");
if (!tc.CheckNotStopped())
{
tc.Log.AddLine("Breaking UpdateHistoricalTicksByDay!");
break;
}
}
}
else
{
tc.Log.AddLine("Not forcing and date does NOT have data for " + fd.ToString());
}
}
else
{
tc.Log.AddLine("Date is on weekend " + fd.ToString());
}
}
public void Run(ThreadControl tc)
{
PTC = tc;
BuildSchedule(ScheduleJSON);
NewDay();
ZonedDateTime dt = UCDT.GetCurrentEastCoastTime();
int currentSecond = -1;
int lastSecond = -1;
int currentDay = dt.Day;
int lastDay = dt.Day;
while (PTC.CheckNotStopped())
{
dt = UCDT.GetCurrentEastCoastTime();
if (dt.Day != lastDay)
{
// reset for new day
NewDay();
}
lastDay = dt.Day;
string currentKey = GetKey();
currentSecond = dt.Second;
if (currentSecond != lastSecond)
{
lastSecond = currentSecond;
// check any items that have finished and mark them complete
List <SchedulerItem> CompletedNotMarkedItems = SchedulerItems.FindAll(i => !i.MarkedComplete && i.TC != null && i.TC.State == ThreadControlState.Complete);
for (int n = 0; n < CompletedNotMarkedItems.Count; n++)
{
PTC.Log.AddLine("Marking " + CompletedNotMarkedItems[n].Id + " as completed");
CompletedNotMarkedItems[n].MarkedComplete = true;
DBMethods.MarkSchedulerItem(currentKey, CompletedNotMarkedItems[n].Id);
}
// stop any that are running and it's after their "endAt" date
List <SchedulerItem> ItemsNeedStopped = SchedulerItems.FindAll(i => !i.MarkedComplete && i.TC != null && i.TC.CanBeStopped() && i.EndAt != null && CurrentTimeAfterOrEqualSchedulerItemTime(dt, i.EndAt));
for (int n = 0; n < ItemsNeedStopped.Count; n++)
{
PTC.Log.AddLine("Stopping " + ItemsNeedStopped[n].Id + " because on or after EndAt. Will wait for stop before marking completed.");
PurposedlyStoppedIds.Add(ItemsNeedStopped[n].TC.Id);
ItemsNeedStopped[n].TC.AddSignalForChild(Signal.Stop);
}
// find all recently stopped items BY SCHEDULER
List <SchedulerItem> ItemsThatHaveStopped = SchedulerItems.FindAll(i => !i.MarkedComplete && i.TC != null && i.TC.State == ThreadControlState.Done && PurposedlyStoppedIds.Exists(p => p == i.TC.Id));
for (int n = 0; n < ItemsThatHaveStopped.Count; n++)
{
PTC.Log.AddLine("Found a Done. Marking it as complete.");
PurposedlyStoppedIds.Remove(ItemsThatHaveStopped[n].TC.Id);
ItemsThatHaveStopped[n].MarkedComplete = true;
DBMethods.MarkSchedulerItem(currentKey, ItemsThatHaveStopped[n].Id);
}
// start any that are within the current time and not running
List <SchedulerItem> ItemsThatNeedStarted = SchedulerItems.FindAll(i => !i.MarkedComplete && i.TC == null && ItemIsWithinCurrentTime(dt, i));
for (int n = 0; n < ItemsThatNeedStarted.Count; n++)
{
PTC.Log.AddLine("Starting " + ItemsThatNeedStarted[n].Id + " because current time is between [At, EndAt]");
string typeName = ItemsThatNeedStarted[n].Class + ", " + ItemsThatNeedStarted[n].Assembly;
string methodName = ItemsThatNeedStarted[n].Call;
ThreadControl childTc = new ThreadControl(ItemsThatNeedStarted[n].Id, ItemsThatNeedStarted[n].ObjectId);
PTC.Children.Add(childTc);
ItemsThatNeedStarted[n].TC = childTc;
Task.Factory.StartNew(() => Methods.ThreadRun(typeName, methodName, childTc, null, null), CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default);
}
}
else
{
Thread.Sleep(250);
}
}
}
public void Run(AlgoTraderModes mode, List <FD> days, ThreadControl tc)
{
TC = tc;
ResetData();
AlgoTraderState.Mode = mode;
SetIntervalBasedOnMode();
AlgoTraderState.IsSim =
AlgoTraderState.Mode == AlgoTraderModes.GatherStats ||
AlgoTraderState.Mode == AlgoTraderModes.Simulating ||
AlgoTraderState.Mode == AlgoTraderModes.Backtesting;
AlgoTraderState.UseSimOrdering = AlgoTraderState.IsSim || AlgoTraderState.Mode == AlgoTraderModes.FakeLive;
PrepDays(days);
bool allowToRun = true;
if (!AlgoTraderState.IsSim)
{
ZonedDateTime zdt = UCDT.GetCurrentEastCoastTime();
if (AlgoTraderState.Mode == AlgoTraderModes.Live && zdt.Hour >= 4)
{
allowToRun = false;
}
}
if (allowToRun)
{
// live or paper only has one day
for (int n = 0; n < Days.Count; n++)
{
ResetDayData();
AlgoTraderState.CurrentDay = Days[n];
// AlgoTraderUI.SelectedSymbol = Global.State.AllSymbols[UC.GetRandomInteger(0, Global.State.AllSymbols.Count - 1)];
// init the list where nodes, trend lines, etc are stored
// AlgoTraderShared.NodesData = new Dictionary<string, NodesData>();
for (int m = 0; m < Global.State.AllSymbols.Count; m++)
{
AlgoTraderShared.NodesData.Add(Global.State.AllSymbols[m], null);
}
CollectGarbage();
// this will take a while only when simulating
PrepareSimDayNodes();
// starts incrementing CurrentTime immediatly when this is called
PrepareTimer();
// waits for 3:55am
AlgoTraderMethods.WaitFor(AlgoTraderConfig.TicksStreamManagerAtHour, AlgoTraderConfig.TicksStreamManagerAtMinute, TC);
RunStocksDataUpdater();
RunOrdersDataUpdater();
RunTradingManager();
CollectGarbage();
// waits for trading manager to finish
WaitToContinue();
TC.StopAllChildrenAndWait();
if (!TC.CheckNotStopped())
{
break;
}
}
}
TC.Log.AddLine("AlgoTraderProcess Run() complete", Verbosity.Minimal);
// it will also set Ended = true
ResetData();
}