public static void VacuumAllDBs(ThreadControl tc) { tc.Log.AddLine("Starting Vacuuming all databases", Verbosity.Minimal); List <string> keys = DBMethods.GetConnectionKeys(); long startSize = UC.GetFolderSize(Global.Constants.StockTicksDBPath); for (int m = 0; m < keys.Count; m += 60) { Parallel.For(0, 60, new ParallelOptions { MaxDegreeOfParallelism = 20 }, n => { if (m + n < keys.Count) { tc.Log.AddLine("Vacuuming " + keys[m + n], Verbosity.Verbose); DBMethods.Vacuum(keys[m + n]); tc.Log.AddLine("Done Vacuuming " + keys[m + n], Verbosity.Verbose); } }); if (!tc.CheckNotStopped()) { tc.Log.AddLine("Breaking VacuumAllDBs!"); break; } } Thread.Sleep(500); long endSize = UC.GetFolderSize(Global.Constants.StockTicksDBPath); tc.Log.AddLine("Vacuuming Complete. Start size: " + startSize + ", End size: " + endSize, Verbosity.Minimal); }
public static void InitDataTracker() { Dictionary <string, List <FD> > dataDaysBySymbol = DBMethods.GetDaysThatHaveDataEachSymbol(); List <FD> dataDays = new List <FD>(); foreach (List <FD> item in dataDaysBySymbol.Values) { // add them all, dedupe later dataDays.AddRange(item); } dataDays.Sort(); List <FD> newList = new List <FD>(dataDays.Distinct()); dataDays = dataDays.DistinctBy(p => new { p.StartNano, p.EndNano }).ToList(); List <FD> noDataDays = DBMethods.GetDaysFromTracker("NoData"); List <FD> holidays = DBMethods.GetDaysFromTracker("Holidays"); Global.State.DataTracker = new DataTracker(); Global.State.DataTracker.DataDays = dataDays; Global.State.DataTracker.DataDaysBySymbol = dataDaysBySymbol; Global.State.DataTracker.Holidays = holidays; Global.State.DataTracker.NoDataDays = noDataDays; }
public static void UpdateTestingDaysDB(ThreadControl tc) { List <FD> dataDays = Global.State.DataTracker.DataDays; tc.Log.AddLine("TOTAL DATA DAYS: " + dataDays.Count); List <string> testingList = new List <string>(); for (int n = 0; n < dataDays.Count; n++) { int year = dataDays[n].DT.Year; int month = dataDays[n].DT.Month; int day = dataDays[n].DT.Day; int total = year + month + day; if (total % 13 == 0 || total % 5 == 0 || total % 17 == 0) { testingList.Add(year + "-" + month.ToString("D2") + "-" + day.ToString("D2")); } } tc.Log.AddLine("TOTAL TESTING DAYS: " + testingList.Count); for (int n = 0; n < testingList.Count; n++) { int m = DBMethods.InsertTestingDay(testingList[n]); tc.Log.AddLine("(" + m + ") Inserted test day " + testingList[n]); } }
/* * public static void BuildMinuteDayNodes(ThreadControl tc) * { * * List<string> symbols = new List<string>(Global.State.AllSymbols); * * symbols.Remove("spy"); * symbols.Remove("aapl"); * symbols.Add("spy"); * symbols.Add("aapl"); * symbols.Reverse(); * * Dictionary<string, List<FD>> symbolsDataDays = new Dictionary<string, List<FD>>(); * * for(int n = 0; n < symbols.Count; n++) * { * symbolsDataDays.Add(symbols[n], DataMethods.ListOfDataDaysToListOfFD(DBMethods.GetDaysThatHaveDataBySymbol(symbols[n]))); * } * * for (int n = Global.State.DaysThatHaveData.Count - 1; n > 0; n--) * { * tc.Log.AddLine("Starting symbols for new day"); * * Parallel.For(0, symbols.Count, new ParallelOptions { MaxDegreeOfParallelism = 15 }, m => * { * // if the symbol has day data for this date * if (symbolsDataDays[symbols[m]].Exists(d => d.IsEqual(Global.State.DaysThatHaveData[n]))) * { * List<StockNode> nodes = DataMethods.GetCachedDayStockNodes(symbols[m], new List<FD>() { Global.State.DaysThatHaveData[n] }, Interval.OneMinute, false); * tc.Log.AddLine("[" + symbols[m] + "] " + Global.State.DaysThatHaveData[n].ToString() + " Done."); * } * else * { * tc.Log.AddLine("[" + symbols[m] + "] DOES NOT HAVE DAY DATA FOR THIS SYMBOL"); * } * }); * * if (!tc.CheckNotStopped()) * { * tc.Log.AddLine("Breaking!"); * break; * } * } * * * } */ public static void AddSymbols(ThreadControl tc) { List <string> symbols = new List <string>() { "adp", "azo", "avb", "avy", "bhge", "bll", "bac", "bk", "bax", "bbt", "bdx", "bby", "biib", "blk", "hrb", "ba", "bkng", "bwa", "bxp", "bsx", "bhf", "bmy", "avgo", "chrw", "ca", "cog", "cdns", "cpb", "cof", "cah", "kmx" }; int count = 0; for (int n = 0; n < symbols.Count; n++) { if (DBMethods.QuotesDBExists(symbols[n]) || DBMethods.TradesDBExists(symbols[n])) { tc.Log.AddLine("[" + symbols[n] + "] Already have this symbol"); } else { DBMethods.CreateQuotesDB(symbols[n]); DBMethods.CreateTradesDB(symbols[n]); DBMethods.InsertSymbol(symbols[n]); tc.Log.AddLine("[" + symbols[n] + "] Created Quotes and Trades tables and inserting into symbols.db"); count++; } } tc.Log.AddLine("All done. Added " + count + " new symbols."); }
/// <summary> /// This is executed at the start of the application. /// It getts all Strategies in AlgoTraderStrategies namespace. /// It will hash the code and store it in the DB and return the usable strategies that can be used in trading /// </summary> /// <returns></returns> public static Dictionary <string, string> RefreshStrategyNamesAndIds() { Dictionary <string, string> result = new Dictionary <string, string>(); Type[] types = UC.GetTypesInNamespace(Assembly.GetExecutingAssembly(), "AT.AlgoTraderStrategies"); string baseStrategyText = UC.NormalizeSourceCode(File.ReadAllText(Global.Constants.BaseStrategyPath)); string baseStrategyMD5 = UC.MD5Hash(baseStrategyText); string baseStrategyId = baseStrategyMD5.Substring(baseStrategyMD5.Length - 6); for (int n = 0; n < types.Length; n++) { string strategyText = UC.NormalizeSourceCode(File.ReadAllText(Global.Constants.StrategiesFolder + types[n].Name + ".cs")); string strategyMD5 = UC.MD5Hash(strategyText); string strategyId = strategyMD5.Substring(strategyMD5.Length - 6) + baseStrategyId; result.Add(strategyId, types[n].Name); DBMethods.InsertStrategyName(strategyId, types[n].Name, UCDT.GetCurrentNanoUnix()); } return(result); }
/// <summary> /// Checks and configures Days. /// Will remove any testing days if the mode can't have testing days. /// If not simulating, will limit the days to just today. /// </summary> /// <param name="days"></param> private void PrepDays(List <FD> days) { Days = new List <FD>(); // prep days (forces today when not sim, removes testing days if sim GatherStats) if (AlgoTraderState.IsSim) { if (CannotHaveTestingDays()) { List <FD> testingDays = DBMethods.GetTestingDays(); int daysCount = days.Count; // remove all testing days from days days.RemoveAll(d => testingDays.Exists(t => t.Equals(d))); TC.Log.AddLine("Removed " + (daysCount - days.Count) + " testing days because we cannot have testing days", Verbosity.Normal); } else { TC.Log.AddLine("Testing days are totally fine. Will add all " + days.Count + " day(s)", Verbosity.Normal); } Days.AddRange(days); // make sure it's ordered chronologically for tiddyness sake Days = Days.OrderBy(d => d.StartNano).ToList(); } else { ZonedDateTime zdt = UCDT.GetCurrentEastCoastTime(); FD fd = new FD(zdt.Year, zdt.Month, zdt.Day); TC.Log.AddLine("Added only today because it's not a simulation: " + fd.ToString(), Verbosity.Normal); Days.Add(fd); } }
public static void RefreshNoDataDaysAndSlapCache(ThreadControl tc) { ZonedDateTime ie = SystemClock.Instance.GetCurrentInstant().InZone(UCDT.TimeZones.Eastern); // set the current date to two days ago via eastern timezone DateTime dt = new DateTime(ie.Year, ie.Month, ie.Day, 0, 0, 0, DateTimeKind.Unspecified).AddDays(-2); // ZonedDateTime zdt = UCDT.ZonedDateTimetoZonedDateTime(currentDate, UCDT.TimeZones.Eastern); List <FD> noDataDays = new List <FD>(); while (dt.Year > 2018 && tc.CheckNotStopped()) { tc.Log.AddLine("Checking: " + dt.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture)); if (!StockAPI.Methods.DateHasData(dt.Year, dt.Month, dt.Day, false)) { FD fd = new FD(dt); tc.Log.AddLine("Found a day with no data: " + fd.ToStringLong()); noDataDays.Add(fd); } dt = dt.AddDays(-1); } if (tc.CheckNotStopped()) { DBMethods.RefreshDaysTracker(noDataDays, "NoData"); noDataDays.Sort(); for (int n = 0; n < Global.State.DataTracker.NoDataDays.Count; n++) { // check if we didn't find one that was found before if (noDataDays.BinarySearch(Global.State.DataTracker.NoDataDays[n]) < 0) { tc.Log.AddLine("FOUND A DAY IN THE OLD DATA THAT'S NOT IN THE NEW DATA: " + Global.State.DataTracker.NoDataDays[n].ToStringLong()); // This means that the scan before got a false nodataday. // Could be API error or something. // DELETE ANY CACHED DAY FOR THIS DAY BECAUSE WE MAY HAVE CACHED EMPTY NODES } } App.InitDataTracker(); } }
public static void CreateTradingStatsDBs(ThreadControl tc) { List <string> symbols = Global.State.AllSymbols; for (int n = 0; n < symbols.Count; n++) { if (DBMethods.TradingStatsDBExists(symbols[n])) { tc.Log.AddLine("[" + symbols[n] + "] Already has a trading stats DB"); } else { DBMethods.CreateTradingStatsDB(symbols[n]); tc.Log.AddLine("[" + symbols[n] + "] Created trading stats DB."); } } }
public static void UpdateSingleDayAndSingleSymbolTicks(string symbol, FD fd, ThreadControl tc) { if (DBMethods.TradesDBExists(symbol) && DBMethods.QuotesDBExists(symbol)) { if (!Global.State.DataTracker.SymbolHasDayData(symbol, fd)) { tc.Log.AddLine("[" + fd.ToString() + "]", Verbosity.Verbose); tc.Log.AddLine("[" + symbol + "] Making trades API calls", Verbosity.Verbose); List <Trade> trades = StockAPI.Methods.GetHistoricTradesFull(symbol, fd.DT.Year, fd.DT.Month, fd.DT.Day); tc.Log.AddLine("[" + symbol + "] Bulk inserting " + trades.Count + " trade(s)", Verbosity.Verbose); int tradesInserted = DBMethods.BulkInsertTrades(trades, symbol); tc.Log.AddLine("[" + symbol + "] Inserted " + tradesInserted + " trade(s)", Verbosity.Normal); tc.Log.AddLine("[" + symbol + "] Making quotes API calls", Verbosity.Verbose); List <Quote> quotes = StockAPI.Methods.GetHistoricQuotesFull(symbol, fd.DT.Year, fd.DT.Month, fd.DT.Day); tc.Log.AddLine("[" + symbol + "] Bulk inserting " + quotes.Count + " quote(s)", Verbosity.Verbose); int quotesInserted = DBMethods.BulkInsertQuotes(quotes, symbol); tc.Log.AddLine("[" + symbol + "] Inserted " + quotesInserted + " quote(s)", Verbosity.Normal); int p = DBMethods.MarkSymbolHasDayData(symbol, fd.DT.Month, fd.DT.Day, fd.DT.Year); tc.Log.AddLine("[" + symbol + "] Marked symbol has data (" + p + ")", Verbosity.Minimal); } else { tc.Log.AddLine("[" + symbol + "] Already has data for this date."); } } else { tc.Log.AddLine("[" + symbol + "] DBs not found"); } }
/// <summary> /// Executes once when the application starts /// </summary> public static void Init() { Global.State.ExecutablePath = AppDomain.CurrentDomain.BaseDirectory; string sourcePath = Global.State.ExecutablePath.Replace("\\compiled\\", ""); Global.Constants.StrategiesFolder = sourcePath + "\\AlgoTrader\\Strategies\\"; Global.Constants.BaseStrategyPath = sourcePath + "\\AlgoTrader\\AlgoTraderBaseStrategy.cs"; // D:\\Projects\\AlgoTrader - New\\Source\\AT\\AlgoTrader\\AlgoTraderBaseStrategy.cs Global.Constants.NodesPath = Global.State.ExecutablePath + "nodes\\"; Global.Constants.DBPath = Global.State.ExecutablePath + "dbs\\"; Global.Constants.StockTicksDBPath = Global.Constants.DBPath + Global.Constants.StockTicksDBFolder + "\\"; Global.Constants.TradingStatsDBPath = Global.Constants.DBPath + Global.Constants.TradingStatsDBFolder + "\\"; DBMethods.Init(); // ThreadPool.SetMinThreads(100, 100); string json = File.ReadAllText(Global.State.ExecutablePath + "schedules\\main.js"); Global.State.SchedulerTC = new ThreadControl("Scheduler"); Global.State.Scheduler = new Scheduler(json, "app"); Global.State.AllSymbols = DBMethods.GetAllSymbols(); //Global.State.AllSymbols.Shuffle(5); Global.State.AllSymbols.Clear(); Global.State.AllSymbols.Add("aapl"); Global.State.AllSymbols.Add("spy"); Global.State.AllSymbols.Add("qqq"); Global.State.AllSymbols.Add("msft"); Global.State.AllSymbols.Add("gld"); Global.State.AllSymbols.Add("amd"); /* * * Global.State.AllSymbols.Add("crc"); * Global.State.AllSymbols.Add("cnp"); * Global.State.AllSymbols.Add("meet"); * Global.State.AllSymbols.Add("eros"); * Global.State.AllSymbols.Add("bimi"); * */ //Global.State.AllSymbols.Add("tsla"); //Global.State.AllSymbols.Add("amd"); //Global.State.AllSymbols.Shuffle(); //Global.State.AllSymbols.RemoveRange(0, 380); /* * Global.State.AllSymbols.Add("spy"); * Global.State.AllSymbols.Add("amd"); * Global.State.AllSymbols.Add("tsla"); * Global.State.AllSymbols.Add("x"); * Global.State.AllSymbols.Add("aapl"); * Global.State.AllSymbols.Add("t"); * Global.State.AllSymbols.Add("xli"); * Global.State.AllSymbols.Add("tsg"); * Global.State.AllSymbols.Add("crc"); * Global.State.AllSymbols.Add("tza"); * Global.State.AllSymbols.Add("carg"); * Global.State.AllSymbols.Add("immu"); */ //Global.State.AllSymbols = new List<string>(); //Global.State.AllSymbols.Add("spy"); /* * if (Global.Constants.FastForTesting) * { * * } */ Global.State.ThreadControlTree = new List <ThreadControl>(); Global.State.ThreadControlTree.Add(Global.State.SchedulerTC); Global.State.Subscriptions = new Dictionary <string, Subscription>(); Global.State.AlgoTraderSubscriptions = new Dictionary <string, bool>(); //Global.State.ForceUpdate = new Dictionary<string, bool>(); // Global.State.SubscriptionStates = new Dictionary<string, object>(); Global.State.Subscriptions.Add("schedulerRunning", new Subscription() { State = false, Interval = 2 }); Global.State.Subscriptions.Add("threadControlTree", new Subscription() { State = long.MaxValue, Interval = 5 }); Global.State.Subscriptions.Add("threadControlState", new Subscription() { State = "md5hashstring", Interval = 7 }); Global.State.Subscriptions.Add("threadControlLog", new Subscription() { State = int.MaxValue, Interval = 3 }); Global.State.Subscriptions.Add("threadControlObject", new Subscription() { State = int.MaxValue, Interval = 2 }); Global.State.Subscriptions.Add("applicationStats", new Subscription() { State = "some string", Interval = 6 }); Global.State.AlgoTraderSubscriptions.Add("main", false); Global.State.AlgoTraderSubscriptions.Add("overview", false); Global.State.AlgoTraderSubscriptions.Add("positions", false); Global.State.AlgoTraderSubscriptions.Add("orders", false); Global.State.AlgoTraderSubscriptions.Add("actions", false); Global.State.AlgoTraderSubscriptions.Add("chart", false); Global.State.AlgoTraderSubscriptions.Add("log", false); InitDataTracker(); // this also stores new ids and names in DB Global.State.UsableStrategies = AlgoTraderMethods.RefreshStrategyNamesAndIds(); }