public static void VacuumAllDBs(ThreadControl tc)
{
tc.Log.AddLine("Starting Vacuuming all databases", Verbosity.Minimal);
List <string> keys = DBMethods.GetConnectionKeys();
long startSize = UC.GetFolderSize(Global.Constants.StockTicksDBPath);
for (int m = 0; m < keys.Count; m += 60)
{
Parallel.For(0, 60, new ParallelOptions {
MaxDegreeOfParallelism = 20
}, n =>
{
if (m + n < keys.Count)
{
tc.Log.AddLine("Vacuuming " + keys[m + n], Verbosity.Verbose);
DBMethods.Vacuum(keys[m + n]);
tc.Log.AddLine("Done Vacuuming " + keys[m + n], Verbosity.Verbose);
}
});
if (!tc.CheckNotStopped())
{
tc.Log.AddLine("Breaking VacuumAllDBs!");
break;
}
}
Thread.Sleep(500);
long endSize = UC.GetFolderSize(Global.Constants.StockTicksDBPath);
tc.Log.AddLine("Vacuuming Complete. Start size: " + startSize + ", End size: " + endSize, Verbosity.Minimal);
}
public static void InitDataTracker()
{
Dictionary <string, List <FD> > dataDaysBySymbol = DBMethods.GetDaysThatHaveDataEachSymbol();
List <FD> dataDays = new List <FD>();
foreach (List <FD> item in dataDaysBySymbol.Values)
{
// add them all, dedupe later
dataDays.AddRange(item);
}
dataDays.Sort();
List <FD> newList = new List <FD>(dataDays.Distinct());
dataDays = dataDays.DistinctBy(p => new { p.StartNano, p.EndNano }).ToList();
List <FD> noDataDays = DBMethods.GetDaysFromTracker("NoData");
List <FD> holidays = DBMethods.GetDaysFromTracker("Holidays");
Global.State.DataTracker = new DataTracker();
Global.State.DataTracker.DataDays = dataDays;
Global.State.DataTracker.DataDaysBySymbol = dataDaysBySymbol;
Global.State.DataTracker.Holidays = holidays;
Global.State.DataTracker.NoDataDays = noDataDays;
}
public static void UpdateTestingDaysDB(ThreadControl tc)
{
List <FD> dataDays = Global.State.DataTracker.DataDays;
tc.Log.AddLine("TOTAL DATA DAYS: " + dataDays.Count);
List <string> testingList = new List <string>();
for (int n = 0; n < dataDays.Count; n++)
{
int year = dataDays[n].DT.Year;
int month = dataDays[n].DT.Month;
int day = dataDays[n].DT.Day;
int total = year + month + day;
if (total % 13 == 0 || total % 5 == 0 || total % 17 == 0)
{
testingList.Add(year + "-" + month.ToString("D2") + "-" + day.ToString("D2"));
}
}
tc.Log.AddLine("TOTAL TESTING DAYS: " + testingList.Count);
for (int n = 0; n < testingList.Count; n++)
{
int m = DBMethods.InsertTestingDay(testingList[n]);
tc.Log.AddLine("(" + m + ") Inserted test day " + testingList[n]);
}
}
/*
* public static void BuildMinuteDayNodes(ThreadControl tc)
* {
*
* List<string> symbols = new List<string>(Global.State.AllSymbols);
*
* symbols.Remove("spy");
* symbols.Remove("aapl");
* symbols.Add("spy");
* symbols.Add("aapl");
* symbols.Reverse();
*
* Dictionary<string, List<FD>> symbolsDataDays = new Dictionary<string, List<FD>>();
*
* for(int n = 0; n < symbols.Count; n++)
* {
* symbolsDataDays.Add(symbols[n], DataMethods.ListOfDataDaysToListOfFD(DBMethods.GetDaysThatHaveDataBySymbol(symbols[n])));
* }
*
* for (int n = Global.State.DaysThatHaveData.Count - 1; n > 0; n--)
* {
* tc.Log.AddLine("Starting symbols for new day");
*
* Parallel.For(0, symbols.Count, new ParallelOptions { MaxDegreeOfParallelism = 15 }, m =>
* {
* // if the symbol has day data for this date
* if (symbolsDataDays[symbols[m]].Exists(d => d.IsEqual(Global.State.DaysThatHaveData[n])))
* {
* List<StockNode> nodes = DataMethods.GetCachedDayStockNodes(symbols[m], new List<FD>() { Global.State.DaysThatHaveData[n] }, Interval.OneMinute, false);
* tc.Log.AddLine("[" + symbols[m] + "] " + Global.State.DaysThatHaveData[n].ToString() + " Done.");
* }
* else
* {
* tc.Log.AddLine("[" + symbols[m] + "] DOES NOT HAVE DAY DATA FOR THIS SYMBOL");
* }
* });
*
* if (!tc.CheckNotStopped())
* {
* tc.Log.AddLine("Breaking!");
* break;
* }
* }
*
*
* }
*/
public static void AddSymbols(ThreadControl tc)
{
List <string> symbols = new List <string>()
{
"adp", "azo", "avb", "avy", "bhge", "bll", "bac", "bk", "bax", "bbt", "bdx", "bby", "biib", "blk", "hrb", "ba", "bkng", "bwa", "bxp", "bsx", "bhf", "bmy", "avgo", "chrw", "ca", "cog", "cdns", "cpb", "cof", "cah", "kmx"
};
int count = 0;
for (int n = 0; n < symbols.Count; n++)
{
if (DBMethods.QuotesDBExists(symbols[n]) || DBMethods.TradesDBExists(symbols[n]))
{
tc.Log.AddLine("[" + symbols[n] + "] Already have this symbol");
}
else
{
DBMethods.CreateQuotesDB(symbols[n]);
DBMethods.CreateTradesDB(symbols[n]);
DBMethods.InsertSymbol(symbols[n]);
tc.Log.AddLine("[" + symbols[n] + "] Created Quotes and Trades tables and inserting into symbols.db");
count++;
}
}
tc.Log.AddLine("All done. Added " + count + " new symbols.");
}
/// <summary>
/// This is executed at the start of the application.
/// It getts all Strategies in AlgoTraderStrategies namespace.
/// It will hash the code and store it in the DB and return the usable strategies that can be used in trading
/// </summary>
/// <returns></returns>
public static Dictionary <string, string> RefreshStrategyNamesAndIds()
{
Dictionary <string, string> result = new Dictionary <string, string>();
Type[] types = UC.GetTypesInNamespace(Assembly.GetExecutingAssembly(), "AT.AlgoTraderStrategies");
string baseStrategyText = UC.NormalizeSourceCode(File.ReadAllText(Global.Constants.BaseStrategyPath));
string baseStrategyMD5 = UC.MD5Hash(baseStrategyText);
string baseStrategyId = baseStrategyMD5.Substring(baseStrategyMD5.Length - 6);
for (int n = 0; n < types.Length; n++)
{
string strategyText = UC.NormalizeSourceCode(File.ReadAllText(Global.Constants.StrategiesFolder + types[n].Name + ".cs"));
string strategyMD5 = UC.MD5Hash(strategyText);
string strategyId = strategyMD5.Substring(strategyMD5.Length - 6) + baseStrategyId;
result.Add(strategyId, types[n].Name);
DBMethods.InsertStrategyName(strategyId, types[n].Name, UCDT.GetCurrentNanoUnix());
}
return(result);
}
/// <summary>
/// Checks and configures Days.
/// Will remove any testing days if the mode can't have testing days.
/// If not simulating, will limit the days to just today.
/// </summary>
/// <param name="days"></param>
private void PrepDays(List <FD> days)
{
Days = new List <FD>();
// prep days (forces today when not sim, removes testing days if sim GatherStats)
if (AlgoTraderState.IsSim)
{
if (CannotHaveTestingDays())
{
List <FD> testingDays = DBMethods.GetTestingDays();
int daysCount = days.Count;
// remove all testing days from days
days.RemoveAll(d => testingDays.Exists(t => t.Equals(d)));
TC.Log.AddLine("Removed " + (daysCount - days.Count) + " testing days because we cannot have testing days", Verbosity.Normal);
}
else
{
TC.Log.AddLine("Testing days are totally fine. Will add all " + days.Count + " day(s)", Verbosity.Normal);
}
Days.AddRange(days);
// make sure it's ordered chronologically for tiddyness sake
Days = Days.OrderBy(d => d.StartNano).ToList();
}
else
{
ZonedDateTime zdt = UCDT.GetCurrentEastCoastTime();
FD fd = new FD(zdt.Year, zdt.Month, zdt.Day);
TC.Log.AddLine("Added only today because it's not a simulation: " + fd.ToString(), Verbosity.Normal);
Days.Add(fd);
}
}
public static void RefreshNoDataDaysAndSlapCache(ThreadControl tc)
{
ZonedDateTime ie = SystemClock.Instance.GetCurrentInstant().InZone(UCDT.TimeZones.Eastern);
// set the current date to two days ago via eastern timezone
DateTime dt = new DateTime(ie.Year, ie.Month, ie.Day, 0, 0, 0, DateTimeKind.Unspecified).AddDays(-2);
// ZonedDateTime zdt = UCDT.ZonedDateTimetoZonedDateTime(currentDate, UCDT.TimeZones.Eastern);
List <FD> noDataDays = new List <FD>();
while (dt.Year > 2018 && tc.CheckNotStopped())
{
tc.Log.AddLine("Checking: " + dt.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture));
if (!StockAPI.Methods.DateHasData(dt.Year, dt.Month, dt.Day, false))
{
FD fd = new FD(dt);
tc.Log.AddLine("Found a day with no data: " + fd.ToStringLong());
noDataDays.Add(fd);
}
dt = dt.AddDays(-1);
}
if (tc.CheckNotStopped())
{
DBMethods.RefreshDaysTracker(noDataDays, "NoData");
noDataDays.Sort();
for (int n = 0; n < Global.State.DataTracker.NoDataDays.Count; n++)
{
// check if we didn't find one that was found before
if (noDataDays.BinarySearch(Global.State.DataTracker.NoDataDays[n]) < 0)
{
tc.Log.AddLine("FOUND A DAY IN THE OLD DATA THAT'S NOT IN THE NEW DATA: " + Global.State.DataTracker.NoDataDays[n].ToStringLong());
// This means that the scan before got a false nodataday.
// Could be API error or something.
// DELETE ANY CACHED DAY FOR THIS DAY BECAUSE WE MAY HAVE CACHED EMPTY NODES
}
}
App.InitDataTracker();
}
}
public static void CreateTradingStatsDBs(ThreadControl tc)
{
List <string> symbols = Global.State.AllSymbols;
for (int n = 0; n < symbols.Count; n++)
{
if (DBMethods.TradingStatsDBExists(symbols[n]))
{
tc.Log.AddLine("[" + symbols[n] + "] Already has a trading stats DB");
}
else
{
DBMethods.CreateTradingStatsDB(symbols[n]);
tc.Log.AddLine("[" + symbols[n] + "] Created trading stats DB.");
}
}
}
public static void UpdateSingleDayAndSingleSymbolTicks(string symbol, FD fd, ThreadControl tc)
{
if (DBMethods.TradesDBExists(symbol) && DBMethods.QuotesDBExists(symbol))
{
if (!Global.State.DataTracker.SymbolHasDayData(symbol, fd))
{
tc.Log.AddLine("[" + fd.ToString() + "]", Verbosity.Verbose);
tc.Log.AddLine("[" + symbol + "] Making trades API calls", Verbosity.Verbose);
List <Trade> trades = StockAPI.Methods.GetHistoricTradesFull(symbol, fd.DT.Year, fd.DT.Month, fd.DT.Day);
tc.Log.AddLine("[" + symbol + "] Bulk inserting " + trades.Count + " trade(s)", Verbosity.Verbose);
int tradesInserted = DBMethods.BulkInsertTrades(trades, symbol);
tc.Log.AddLine("[" + symbol + "] Inserted " + tradesInserted + " trade(s)", Verbosity.Normal);
tc.Log.AddLine("[" + symbol + "] Making quotes API calls", Verbosity.Verbose);
List <Quote> quotes = StockAPI.Methods.GetHistoricQuotesFull(symbol, fd.DT.Year, fd.DT.Month, fd.DT.Day);
tc.Log.AddLine("[" + symbol + "] Bulk inserting " + quotes.Count + " quote(s)", Verbosity.Verbose);
int quotesInserted = DBMethods.BulkInsertQuotes(quotes, symbol);
tc.Log.AddLine("[" + symbol + "] Inserted " + quotesInserted + " quote(s)", Verbosity.Normal);
int p = DBMethods.MarkSymbolHasDayData(symbol, fd.DT.Month, fd.DT.Day, fd.DT.Year);
tc.Log.AddLine("[" + symbol + "] Marked symbol has data (" + p + ")", Verbosity.Minimal);
}
else
{
tc.Log.AddLine("[" + symbol + "] Already has data for this date.");
}
}
else
{
tc.Log.AddLine("[" + symbol + "] DBs not found");
}
}
/// <summary>
/// Executes once when the application starts
/// </summary>
public static void Init()
{
Global.State.ExecutablePath = AppDomain.CurrentDomain.BaseDirectory;
string sourcePath = Global.State.ExecutablePath.Replace("\\compiled\\", "");
Global.Constants.StrategiesFolder = sourcePath + "\\AlgoTrader\\Strategies\\";
Global.Constants.BaseStrategyPath = sourcePath + "\\AlgoTrader\\AlgoTraderBaseStrategy.cs";
// D:\\Projects\\AlgoTrader - New\\Source\\AT\\AlgoTrader\\AlgoTraderBaseStrategy.cs
Global.Constants.NodesPath = Global.State.ExecutablePath + "nodes\\";
Global.Constants.DBPath = Global.State.ExecutablePath + "dbs\\";
Global.Constants.StockTicksDBPath = Global.Constants.DBPath + Global.Constants.StockTicksDBFolder + "\\";
Global.Constants.TradingStatsDBPath = Global.Constants.DBPath + Global.Constants.TradingStatsDBFolder + "\\";
DBMethods.Init();
// ThreadPool.SetMinThreads(100, 100);
string json = File.ReadAllText(Global.State.ExecutablePath + "schedules\\main.js");
Global.State.SchedulerTC = new ThreadControl("Scheduler");
Global.State.Scheduler = new Scheduler(json, "app");
Global.State.AllSymbols = DBMethods.GetAllSymbols();
//Global.State.AllSymbols.Shuffle(5);
Global.State.AllSymbols.Clear();
Global.State.AllSymbols.Add("aapl");
Global.State.AllSymbols.Add("spy");
Global.State.AllSymbols.Add("qqq");
Global.State.AllSymbols.Add("msft");
Global.State.AllSymbols.Add("gld");
Global.State.AllSymbols.Add("amd");
/*
*
* Global.State.AllSymbols.Add("crc");
* Global.State.AllSymbols.Add("cnp");
* Global.State.AllSymbols.Add("meet");
* Global.State.AllSymbols.Add("eros");
* Global.State.AllSymbols.Add("bimi");
*
*/
//Global.State.AllSymbols.Add("tsla");
//Global.State.AllSymbols.Add("amd");
//Global.State.AllSymbols.Shuffle();
//Global.State.AllSymbols.RemoveRange(0, 380);
/*
* Global.State.AllSymbols.Add("spy");
* Global.State.AllSymbols.Add("amd");
* Global.State.AllSymbols.Add("tsla");
* Global.State.AllSymbols.Add("x");
* Global.State.AllSymbols.Add("aapl");
* Global.State.AllSymbols.Add("t");
* Global.State.AllSymbols.Add("xli");
* Global.State.AllSymbols.Add("tsg");
* Global.State.AllSymbols.Add("crc");
* Global.State.AllSymbols.Add("tza");
* Global.State.AllSymbols.Add("carg");
* Global.State.AllSymbols.Add("immu");
*/
//Global.State.AllSymbols = new List<string>();
//Global.State.AllSymbols.Add("spy");
/*
* if (Global.Constants.FastForTesting)
* {
*
* }
*/
Global.State.ThreadControlTree = new List <ThreadControl>();
Global.State.ThreadControlTree.Add(Global.State.SchedulerTC);
Global.State.Subscriptions = new Dictionary <string, Subscription>();
Global.State.AlgoTraderSubscriptions = new Dictionary <string, bool>();
//Global.State.ForceUpdate = new Dictionary<string, bool>();
// Global.State.SubscriptionStates = new Dictionary<string, object>();
Global.State.Subscriptions.Add("schedulerRunning", new Subscription()
{
State = false, Interval = 2
});
Global.State.Subscriptions.Add("threadControlTree", new Subscription()
{
State = long.MaxValue, Interval = 5
});
Global.State.Subscriptions.Add("threadControlState", new Subscription()
{
State = "md5hashstring", Interval = 7
});
Global.State.Subscriptions.Add("threadControlLog", new Subscription()
{
State = int.MaxValue, Interval = 3
});
Global.State.Subscriptions.Add("threadControlObject", new Subscription()
{
State = int.MaxValue, Interval = 2
});
Global.State.Subscriptions.Add("applicationStats", new Subscription()
{
State = "some string", Interval = 6
});
Global.State.AlgoTraderSubscriptions.Add("main", false);
Global.State.AlgoTraderSubscriptions.Add("overview", false);
Global.State.AlgoTraderSubscriptions.Add("positions", false);
Global.State.AlgoTraderSubscriptions.Add("orders", false);
Global.State.AlgoTraderSubscriptions.Add("actions", false);
Global.State.AlgoTraderSubscriptions.Add("chart", false);
Global.State.AlgoTraderSubscriptions.Add("log", false);
InitDataTracker();
// this also stores new ids and names in DB
Global.State.UsableStrategies = AlgoTraderMethods.RefreshStrategyNamesAndIds();
}
