public static void RefreshNoDataDaysAndSlapCache(ThreadControl tc) { ZonedDateTime ie = SystemClock.Instance.GetCurrentInstant().InZone(UCDT.TimeZones.Eastern); // set the current date to two days ago via eastern timezone DateTime dt = new DateTime(ie.Year, ie.Month, ie.Day, 0, 0, 0, DateTimeKind.Unspecified).AddDays(-2); // ZonedDateTime zdt = UCDT.ZonedDateTimetoZonedDateTime(currentDate, UCDT.TimeZones.Eastern); List <FD> noDataDays = new List <FD>(); while (dt.Year > 2018 && tc.CheckNotStopped()) { tc.Log.AddLine("Checking: " + dt.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture)); if (!StockAPI.Methods.DateHasData(dt.Year, dt.Month, dt.Day, false)) { FD fd = new FD(dt); tc.Log.AddLine("Found a day with no data: " + fd.ToStringLong()); noDataDays.Add(fd); } dt = dt.AddDays(-1); } if (tc.CheckNotStopped()) { DBMethods.RefreshDaysTracker(noDataDays, "NoData"); noDataDays.Sort(); for (int n = 0; n < Global.State.DataTracker.NoDataDays.Count; n++) { // check if we didn't find one that was found before if (noDataDays.BinarySearch(Global.State.DataTracker.NoDataDays[n]) < 0) { tc.Log.AddLine("FOUND A DAY IN THE OLD DATA THAT'S NOT IN THE NEW DATA: " + Global.State.DataTracker.NoDataDays[n].ToStringLong()); // This means that the scan before got a false nodataday. // Could be API error or something. // DELETE ANY CACHED DAY FOR THIS DAY BECAUSE WE MAY HAVE CACHED EMPTY NODES } } App.InitDataTracker(); } }
public static void VacuumAllDBs(ThreadControl tc) { tc.Log.AddLine("Starting Vacuuming all databases", Verbosity.Minimal); List <string> keys = DBMethods.GetConnectionKeys(); long startSize = UC.GetFolderSize(Global.Constants.StockTicksDBPath); for (int m = 0; m < keys.Count; m += 60) { Parallel.For(0, 60, new ParallelOptions { MaxDegreeOfParallelism = 20 }, n => { if (m + n < keys.Count) { tc.Log.AddLine("Vacuuming " + keys[m + n], Verbosity.Verbose); DBMethods.Vacuum(keys[m + n]); tc.Log.AddLine("Done Vacuuming " + keys[m + n], Verbosity.Verbose); } }); if (!tc.CheckNotStopped()) { tc.Log.AddLine("Breaking VacuumAllDBs!"); break; } } Thread.Sleep(500); long endSize = UC.GetFolderSize(Global.Constants.StockTicksDBPath); tc.Log.AddLine("Vacuuming Complete. Start size: " + startSize + ", End size: " + endSize, Verbosity.Minimal); }
/// <summary> /// When simulating, this method will load all of the needed nodes for simulating a live feed/stream. /// </summary> private void PrepareSimDayNodes() { if (AlgoTraderState.IsSim) { // load up today's nodes for StreamManager to pretend to add nodes by using today's nodes int amount = 100; Stopwatch sw = new Stopwatch(); sw.Start(); object lockObj = new object(); for (int m = 0; m < Global.State.AllSymbols.Count; m += amount) { Parallel.For(0, amount, new ParallelOptions { MaxDegreeOfParallelism = 30 }, n => { if (m + n < Global.State.AllSymbols.Count) { // NodesData nodes = null; NodesData nodes = DataMethods.GetCachedDayNodesData(Global.State.AllSymbols[m + n], AlgoTraderState.CurrentDay, Interval.HalfSecond, justCache: false); // NodesData nodesData = DataMethods.GetCachedDayNodesData(Global.State.AllSymbols[m + n], CurrentDay, Interval.HalfSecond, computeNodes: true, justCache: false); // no need to ComputeGaps because it's 1 day... trusting well formatted data lock (lockObj) { AlgoTraderShared.SimDayNodes.Add(Global.State.AllSymbols[m + n], nodes); //nodes.Add(Global.State.AllSymbols[m + n], DataMethods.GetCachedDayNodesData(Global.State.AllSymbols[m + n], CurrentDay, Interval.HalfSecond, computeNodes: true, justCache: false)); } TC.Log.AddLine("Loaded " + Global.State.AllSymbols[m + n] + " stock nodes for stream manager"); } }); if (!TC.CheckNotStopped()) { break; } } sw.Stop(); TC.Log.AddLine("Finished loading sim nodes for stream manager in " + UC.MillisecondsToSeconds(sw.ElapsedMilliseconds, 3) + " sec(s). That's " + ((decimal)Global.State.AllSymbols.Count / UC.MillisecondsToSeconds(sw.ElapsedMilliseconds, 3)) + " per sec."); } }
public static void BuildMinuteDayNodes(ThreadControl tc) { // this should be called AFTER updated yesterday's ticks AND data tracker no data days updated List <string> symbols = new List <string>(Global.State.AllSymbols); symbols.Shuffle(); symbols.Remove("spy"); symbols.Remove("aapl"); symbols.Add("spy"); symbols.Add("aapl"); symbols.Reverse(); ZonedDateTime ie = SystemClock.Instance.GetCurrentInstant().InZone(UCDT.TimeZones.Eastern); // set the current date to yesterday via eastern timezone just to be safe DateTime dt = new DateTime(ie.Year, ie.Month, ie.Day, 0, 0, 0, DateTimeKind.Unspecified).AddDays(-20); DateTime firstDataDay = Global.State.DataTracker.DataDays[0].DT; while (dt > firstDataDay && tc.CheckNotStopped()) { FD fd = new FD(dt); Parallel.For(0, symbols.Count, new ParallelOptions { MaxDegreeOfParallelism = 20 }, n => { if (DataMethods.DayNodesDataCacheable(symbols[n], fd)) { Stopwatch sw = new Stopwatch(); sw.Start(); NodesData nodes = DataMethods.GetCachedDayNodesData(symbols[n], fd, Interval.HalfSecond, computeNodes: false, justCache: true); sw.Stop(); tc.Log.AddLine("[" + symbols[n] + "] " + fd.ToString() + " Done. Took " + UC.MillisecondsToSeconds(sw.ElapsedMilliseconds, 2) + " sec(s)"); } else { tc.Log.AddLine("[" + symbols[n] + "] " + fd.ToString() + ". Not cacheable."); } }); dt = dt.AddDays(-1); } }
/* * private bool ModeNeedsSystemClock() * { * return ( * AlgoTraderState.Mode == AlgoTraderModes.Live || * AlgoTraderState.Mode == AlgoTraderModes.Paper || * AlgoTraderState.Mode == AlgoTraderModes.FakeLive * ); * } */ public static void WaitFor(int hour, int minute, ThreadControl tc) { int lastSecond = 0; int currentSecond = 0; while (!AlgoTraderState.CurrentTime.IsOnOrPassed(hour, minute) && tc.CheckNotStopped()) { currentSecond = DateTime.UtcNow.Second; if (currentSecond != lastSecond) { lastSecond = currentSecond; tc.Log.AddLine("Waiting for " + hour.ToString("D2") + ":" + minute.ToString("D2") + ". It's currently " + AlgoTraderState.CurrentTime.GetTime().ToString(), Verbosity.Normal); } } }
public static void UpdateHistoricalTicks(ThreadControl tc) { tc.Log.AddLine("Starting update historical ticks", Verbosity.Minimal); ZonedDateTime ie = SystemClock.Instance.GetCurrentInstant().InZone(UCDT.TimeZones.Eastern); // set the current date to yesterday via eastern timezone just to be safe DateTime currentDate = new DateTime(ie.Year, ie.Month, ie.Day, 0, 0, 0, DateTimeKind.Utc).AddDays(-335); while (tc.CheckNotStopped()) { FD fd = new FD(currentDate); UpdateHistoricalTicksByDay(Global.State.AllSymbols, fd, false, tc); currentDate = currentDate.AddDays(-1); } }
public static void UpdateHistoricalTicksByDay(List <string> symbols, FD fd, bool force, ThreadControl tc) { tc.Log.AddLine("Starting update histocial ticks by day", Verbosity.Minimal); ZonedDateTime ie = SystemClock.Instance.GetCurrentInstant().InZone(UCDT.TimeZones.Eastern); // safety precaution if (ie.Year == fd.DT.Year && ie.Month == fd.DT.Month && ie.Day == fd.DT.Day) { throw new Exception("What are you doing???"); } int parallelAPICalls = 15; if (!fd.IsOnWeekend()) { bool hasDataForDay = StockAPI.Methods.DateHasData(fd.DT.Year, fd.DT.Month, fd.DT.Day, force); if (hasDataForDay) { if (force) { tc.Log.AddLine("Forcing a historical update", Verbosity.Verbose); } else { tc.Log.AddLine("Date has data", Verbosity.Verbose); } tc.Log.AddLine("About to start API calls as fast as we can go", Verbosity.Minimal); symbols.Shuffle(); for (int s = 0; s < symbols.Count; s += parallelAPICalls) { //string[] syms = new string[parallelAPICalls]; lock (Global.State.ThreadControlTreeLock) { for (int m = 0; m < parallelAPICalls; m++) { if (m + s < symbols.Count) { string sym = symbols[m + s]; ThreadControl singleQuoteTC = new ThreadControl(fd.ToString() + " : " + sym); tc.Children.Add(singleQuoteTC); Task.Factory.StartNew(() => Methods.ThreadRun("AT.SchedulerMethods, AT", "UpdateSingleDayAndSingleSymbolTicks", singleQuoteTC, null, new object[] { sym, fd }), CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default); } } } while (tc.Children.Exists(p => p.State != ThreadControlState.Complete && p.State != ThreadControlState.Done)) { } tc.ClearChildren(); tc.Log.AddLine("Done with small batch for " + fd.ToString() + ". About %" + Math.Round(((double)s / (double)symbols.Count) * (double)100, 2) + " done."); if (!tc.CheckNotStopped()) { tc.Log.AddLine("Breaking UpdateHistoricalTicksByDay!"); break; } } } else { tc.Log.AddLine("Not forcing and date does NOT have data for " + fd.ToString()); } } else { tc.Log.AddLine("Date is on weekend " + fd.ToString()); } }
public void Run(ThreadControl tc) { PTC = tc; BuildSchedule(ScheduleJSON); NewDay(); ZonedDateTime dt = UCDT.GetCurrentEastCoastTime(); int currentSecond = -1; int lastSecond = -1; int currentDay = dt.Day; int lastDay = dt.Day; while (PTC.CheckNotStopped()) { dt = UCDT.GetCurrentEastCoastTime(); if (dt.Day != lastDay) { // reset for new day NewDay(); } lastDay = dt.Day; string currentKey = GetKey(); currentSecond = dt.Second; if (currentSecond != lastSecond) { lastSecond = currentSecond; // check any items that have finished and mark them complete List <SchedulerItem> CompletedNotMarkedItems = SchedulerItems.FindAll(i => !i.MarkedComplete && i.TC != null && i.TC.State == ThreadControlState.Complete); for (int n = 0; n < CompletedNotMarkedItems.Count; n++) { PTC.Log.AddLine("Marking " + CompletedNotMarkedItems[n].Id + " as completed"); CompletedNotMarkedItems[n].MarkedComplete = true; DBMethods.MarkSchedulerItem(currentKey, CompletedNotMarkedItems[n].Id); } // stop any that are running and it's after their "endAt" date List <SchedulerItem> ItemsNeedStopped = SchedulerItems.FindAll(i => !i.MarkedComplete && i.TC != null && i.TC.CanBeStopped() && i.EndAt != null && CurrentTimeAfterOrEqualSchedulerItemTime(dt, i.EndAt)); for (int n = 0; n < ItemsNeedStopped.Count; n++) { PTC.Log.AddLine("Stopping " + ItemsNeedStopped[n].Id + " because on or after EndAt. Will wait for stop before marking completed."); PurposedlyStoppedIds.Add(ItemsNeedStopped[n].TC.Id); ItemsNeedStopped[n].TC.AddSignalForChild(Signal.Stop); } // find all recently stopped items BY SCHEDULER List <SchedulerItem> ItemsThatHaveStopped = SchedulerItems.FindAll(i => !i.MarkedComplete && i.TC != null && i.TC.State == ThreadControlState.Done && PurposedlyStoppedIds.Exists(p => p == i.TC.Id)); for (int n = 0; n < ItemsThatHaveStopped.Count; n++) { PTC.Log.AddLine("Found a Done. Marking it as complete."); PurposedlyStoppedIds.Remove(ItemsThatHaveStopped[n].TC.Id); ItemsThatHaveStopped[n].MarkedComplete = true; DBMethods.MarkSchedulerItem(currentKey, ItemsThatHaveStopped[n].Id); } // start any that are within the current time and not running List <SchedulerItem> ItemsThatNeedStarted = SchedulerItems.FindAll(i => !i.MarkedComplete && i.TC == null && ItemIsWithinCurrentTime(dt, i)); for (int n = 0; n < ItemsThatNeedStarted.Count; n++) { PTC.Log.AddLine("Starting " + ItemsThatNeedStarted[n].Id + " because current time is between [At, EndAt]"); string typeName = ItemsThatNeedStarted[n].Class + ", " + ItemsThatNeedStarted[n].Assembly; string methodName = ItemsThatNeedStarted[n].Call; ThreadControl childTc = new ThreadControl(ItemsThatNeedStarted[n].Id, ItemsThatNeedStarted[n].ObjectId); PTC.Children.Add(childTc); ItemsThatNeedStarted[n].TC = childTc; Task.Factory.StartNew(() => Methods.ThreadRun(typeName, methodName, childTc, null, null), CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default); } } else { Thread.Sleep(250); } } }
public void Run(AlgoTraderModes mode, List <FD> days, ThreadControl tc) { TC = tc; ResetData(); AlgoTraderState.Mode = mode; SetIntervalBasedOnMode(); AlgoTraderState.IsSim = AlgoTraderState.Mode == AlgoTraderModes.GatherStats || AlgoTraderState.Mode == AlgoTraderModes.Simulating || AlgoTraderState.Mode == AlgoTraderModes.Backtesting; AlgoTraderState.UseSimOrdering = AlgoTraderState.IsSim || AlgoTraderState.Mode == AlgoTraderModes.FakeLive; PrepDays(days); bool allowToRun = true; if (!AlgoTraderState.IsSim) { ZonedDateTime zdt = UCDT.GetCurrentEastCoastTime(); if (AlgoTraderState.Mode == AlgoTraderModes.Live && zdt.Hour >= 4) { allowToRun = false; } } if (allowToRun) { // live or paper only has one day for (int n = 0; n < Days.Count; n++) { ResetDayData(); AlgoTraderState.CurrentDay = Days[n]; // AlgoTraderUI.SelectedSymbol = Global.State.AllSymbols[UC.GetRandomInteger(0, Global.State.AllSymbols.Count - 1)]; // init the list where nodes, trend lines, etc are stored // AlgoTraderShared.NodesData = new Dictionary<string, NodesData>(); for (int m = 0; m < Global.State.AllSymbols.Count; m++) { AlgoTraderShared.NodesData.Add(Global.State.AllSymbols[m], null); } CollectGarbage(); // this will take a while only when simulating PrepareSimDayNodes(); // starts incrementing CurrentTime immediatly when this is called PrepareTimer(); // waits for 3:55am AlgoTraderMethods.WaitFor(AlgoTraderConfig.TicksStreamManagerAtHour, AlgoTraderConfig.TicksStreamManagerAtMinute, TC); RunStocksDataUpdater(); RunOrdersDataUpdater(); RunTradingManager(); CollectGarbage(); // waits for trading manager to finish WaitToContinue(); TC.StopAllChildrenAndWait(); if (!TC.CheckNotStopped()) { break; } } } TC.Log.AddLine("AlgoTraderProcess Run() complete", Verbosity.Minimal); // it will also set Ended = true ResetData(); }