/// <summary>
/// Initializes the sim from the config object.
/// </summary>
/// <param name="config">Object with all the parameters that can be used to config how this sim runs</param>
public bool CreateFromConfig(SimulatorConfig config, TickerDataStore dataStore)
{
Config = config;
DataStore = dataStore;
DataOutput = new DataOutputter();
if (Config.UseTodaysDate)
{
Config.EndDate = DateTime.Now.Date;
}
// Load the config file with the instument list for all the symbols that we
// want to test.
SortedDictionary<string, TickerExchangePair> fileInstruments = new SortedDictionary<string, TickerExchangePair>();
string line;
try
{
StreamReader file = new StreamReader(Config.InstrumentListFile);
while ((line = file.ReadLine()) != null)
{
string[] pair = line.Split(',');
TickerExchangePair newTicker = new TickerExchangePair(pair[1], pair[0]);
string key = newTicker.ToString();
if (fileInstruments.ContainsKey(key) == false)
{
fileInstruments[key] = newTicker;
}
else
{
WriteMessage("Duplicate ticker in file: " + newTicker.ToString());
}
}
}
catch (Exception e)
{
WriteMessage("Error loading instrument file!\n" + e.Message);
return false;
}
WriteMessage("Initializing ticker data");
// TODO: there has to be a better place for this!
//if (Directory.Exists(Simulator.Config.OutputFolder + "\\higher"))
//{
// Directory.Delete(Simulator.Config.OutputFolder + "\\higher", true);
//}
// Add all the symbols as dependent strategies using the bestofsubstrategies
ConcurrentDictionary<string, BestOfRootStrategies> downloadedInstruments = new ConcurrentDictionary<string, BestOfRootStrategies>();
#if DEBUG
foreach (KeyValuePair<string, TickerExchangePair> item in fileInstruments)
#else
Parallel.ForEach(fileInstruments, item =>
#endif
{
// Get the data for the symbol and save it for later so we can output it.
TickerData tickerData = DataStore.GetTickerData(item.Value, config.StartDate, config.EndDate);
if (tickerData != null)
{
DataOutput.SaveTickerData(tickerData);
//RunnableFactory factory = new RunnableFactory(tickerData);
// This strategy will find the best strategy for this instrument everyday and save the value.
downloadedInstruments[item.Value.ToString()] = new BestOfRootStrategies(tickerData);
}
else
{
WriteMessage("No ticker data for " + item.Value.ToString());
}
#if DEBUG
}
#else
});
#endif
// Want to store the instrument data in a sorted way so that we always run things
// in the same order.
Instruments = new SortedDictionary<string, BestOfRootStrategies>(downloadedInstruments.ToDictionary(kvp => kvp.Key, kvp => kvp.Value));
NumberOfBars = DataStore.SimTickerDates.Count;
Broker = new Broker(Config.InitialAccountBalance, NumberOfBars);
return DataStore.SimTickerDates.Count > 0;
}
/// <summary>
/// Gets the symbol data from either memory, disk, or a server.
/// </summary>
/// <param name="ticker">Ticker to get data for</param>
/// <param name="start">Start date for the data</param>
/// <param name="end">End date for the data</param>
/// <returns>Data (price, volume, etc) for the ticker</returns>
public TickerData GetTickerData(TickerExchangePair ticker, DateTime start, DateTime end)
{
TickerData data = new TickerData(ticker);
// The symbol exists in memory already.
int key = ticker.GetHashCode();
if (_symbolsInMemory.ContainsKey(key))
{
TickerData inMemoryData = _symbolsInMemory[key];
// We don't have all the data in memory past the end, so we need to get that data and append it.
if (end > inMemoryData.End || start < inMemoryData.Start)
{
data = GetDataFromDiskOrServer(ticker, start, end);
// Update the data in memory so it has it next time it runs.
_symbolsInMemory[key] = data;
// Return only the dates requested.
data = data.SubSet(start, end);
}
// Not requesting everything that is in the memory. This is generally the case.
else if (start > inMemoryData.Start || end < inMemoryData.End)
{
data = inMemoryData.SubSet(start, end);
}
// We wanted everything that is memory.
else
{
data = inMemoryData;
}
}
// Symbol isn't in memory so we need to load from the disk or the server.
else
{
// Always start by loading everything we have our earliest date so that
// anytime we eventually will have all the data saved allowing us to
// test lots of different date ranges without having to hit the disk or internet.
data = GetDataFromDiskOrServer(ticker, start, end);
if (data != null)
{
// Save in memory for next time.
_symbolsInMemory[key] = data;
data = data.SubSet(start, end);
}
}
if (data != null)
{
// Save all the dates that this ticker has so that we have a list of dates that we can
// iterate through for trading periods. This is because each ticker can potentially have
// different trading dates but for the main sim we want to go through all dates and if
// the ticker has data for that time, we'll use it.
lock (_dateLock)
{
for (int i = 0; i < data.Dates.Count; i++)
{
if (SimTickerDates.ContainsKey(data.Dates[i]) == false)
{
SimTickerDates[data.Dates[i]] = true;
}
}
}
}
return(data);
}
/// <summary>
/// Appends the other data to the data already in this class. It doesn't overwrite data
/// for existing dates, so it can only prepend data to the start or append to the end.
/// </summary>
/// <param name="otherData">Data to append</param>
public void AppendData(TickerData otherData)
{
// Prepend
if (otherData.Start < Start)
{
// Find the index in the other data where this data starts.
int copyEndIndex;
for (copyEndIndex = 0; copyEndIndex < otherData.Dates.Count; copyEndIndex++)
{
if (otherData.Dates[copyEndIndex] >= Start)
{
break;
}
}
// Insert all the new data at the front of the existing data.
Dates.InsertRange(0, otherData.Dates.GetRange(0, copyEndIndex));
Open.InsertRange(0, otherData.Open.GetRange(0, copyEndIndex));
Close.InsertRange(0, otherData.Close.GetRange(0, copyEndIndex));
High.InsertRange(0, otherData.High.GetRange(0, copyEndIndex));
Low.InsertRange(0, otherData.Low.GetRange(0, copyEndIndex));
Volume.InsertRange(0, otherData.Volume.GetRange(0, copyEndIndex));
// Extras
Typical.InsertRange(0, otherData.Typical.GetRange(0, copyEndIndex));
Median.InsertRange(0, otherData.Median.GetRange(0, copyEndIndex));
HigherTimeframeTrend.InsertRange(0, otherData.HigherTimeframeTrend.GetRange(0, copyEndIndex));
for (int i = 0; i < HigherTimeframeValueStrings.Length; i++)
{
string key = HigherTimeframeValueStrings[i];
HigherTimeframeValues[key].InsertRange(0, otherData.HigherTimeframeValues[key].GetRange(0, copyEndIndex));
}
}
// Append
if (otherData.End > End)
{
// Find the index where the other data passes the end of the existing data.
int copyStartIndex;
for (copyStartIndex = 0; copyStartIndex < otherData.Dates.Count; copyStartIndex++)
{
if (otherData.Dates[copyStartIndex] > End)
{
break;
}
}
// Append the new data to the end of the existing data.
Dates.AddRange(otherData.Dates.GetRange(copyStartIndex, otherData.Dates.Count - copyStartIndex));
Open.AddRange(otherData.Open.GetRange(copyStartIndex, otherData.Open.Count - copyStartIndex));
Close.AddRange(otherData.Close.GetRange(copyStartIndex, otherData.Close.Count - copyStartIndex));
High.AddRange(otherData.High.GetRange(copyStartIndex, otherData.High.Count - copyStartIndex));
Low.AddRange(otherData.Low.GetRange(copyStartIndex, otherData.Low.Count - copyStartIndex));
Volume.AddRange(otherData.Volume.GetRange(copyStartIndex, otherData.Volume.Count - copyStartIndex));
// Extras
Typical.AddRange(otherData.Typical.GetRange(copyStartIndex, otherData.Typical.Count - copyStartIndex));
Median.AddRange(otherData.Median.GetRange(copyStartIndex, otherData.Median.Count - copyStartIndex));
HigherTimeframeTrend.AddRange(otherData.HigherTimeframeTrend.GetRange(copyStartIndex, otherData.HigherTimeframeTrend.Count - copyStartIndex));
for (int i = 0; i < HigherTimeframeValueStrings.Length; i++)
{
string key = HigherTimeframeValueStrings[i];
HigherTimeframeValues[key].AddRange(otherData.HigherTimeframeValues[key].GetRange(copyStartIndex, otherData.HigherTimeframeValues[key].Count - copyStartIndex));
}
}
Start = Dates[0];
End = Dates[Dates.Count - 1];
NumBars = Dates.Count;
SaveDates();
}
/// <summary>
/// Returns aggregated bars for the higher timeframe from the lower time frame data.
/// </summary>
/// <param name="ticker">Lower timeframe ticker data</param>
/// <returns>Higher timeframe ticker data</returns>
private TickerData GetHigherTimeframeBars(TickerData ticker)
{
double open = 0;
double high = 0;
double low = 0;
double close = 0;
long volume = 0;
int barCount = 0;
// Reset the states since we'll calculate them again.
TickerData higherData = new TickerData(ticker.TickerAndExchange);
int currentWeek = UtilityMethods.GetIso8601WeekOfYear(ticker.Dates.First());
// Aggregate all the data into the higher timeframe.
for (int i = 0; i < ticker.Dates.Count; i++)
{
// The first bar open we'll treat as the open price and set the high and low.
// Volume gets reset as it's cumulative through all the bars.
if (barCount == 0)
{
open = ticker.Open[i];
low = ticker.Low[i];
high = ticker.High[i];
volume = 0;
}
// If this low is lower than the saved low, we have a new low.
// Same for high but opposite of course.
if (ticker.Low[i] < low)
{
low = ticker.Low[i];
}
if (ticker.High[i] > high)
{
high = ticker.High[i];
}
// Move to the next bar to aggregate from.
++barCount;
volume += ticker.Volume[i];
// The last bar close is treated as the close. Now it's time to save all
// the aggregated data as one bar for the higher timeframe.
// We also want to do this if the for loop is just about to exit. We may not
// have the number of bars we wanted for the aggregate, but we want to at least have
// something for the last bar. Ex. We have 5 bars set for the higher timeframe length,
// but we've only got 3 bars of data and the for loop will end on the next iteration.
// In that case we want to use the 3 bars we have for the data.
if ((i + 1 == ticker.Dates.Count) ||
(Simulator.Config.DataType == "daily" && UtilityMethods.GetIso8601WeekOfYear(ticker.Dates[i + 1]) != currentWeek) ||
(Simulator.Config.DataType != "daily" && barCount == Simulator.Config.NumBarsHigherTimeframe))
{
close = ticker.Close[i];
higherData.Dates.Add(ticker.Dates[i]); // Use the ending aggregated date as the date for the higher timeframe.
higherData.Open.Add(open);
higherData.High.Add(high);
higherData.Low.Add(low);
higherData.Close.Add(close);
higherData.Volume.Add(volume);
higherData.NumBars = higherData.Dates.Count;
// Extras
higherData.Typical.Add((high + low + close) / 3);
higherData.Median.Add((high + low) / 2);
// Start aggregating a new set.
barCount = 0;
int nextWeekIndex = i + 1 == ticker.Dates.Count ? i : i + 1;
currentWeek = UtilityMethods.GetIso8601WeekOfYear(ticker.Dates[nextWeekIndex]);
}
}
return(higherData);
}
/// <summary>
/// Returns what type of orders are allowed for this bar based on the
/// higher time frame momentum analysis.
/// </summary>
/// <param name="ticker">Ticker data</param>
/// <param name="lastState">Last state of the higher timeframe</param>
/// <returns>Order type allowed for the last bar of the ticker data</returns>
private double GetHigherTimerframeExtras(TickerData ticker, double lastState)
{
// Get all the bars for the higher timeframe.
TickerData higherTickerData = GetHigherTimeframeBars(ticker);
Sma sma = new Sma(higherTickerData)
{
Period = 35
};
sma.Initialize();
sma.RunToBar(higherTickerData.NumBars - 1);
sma.Shutdown();
ticker.HigherTimeframeValues["Sma"].Add(sma.Avg.Last());
Atr atrInd = new Atr(higherTickerData)
{
Period = 14
};
atrInd.Initialize();
atrInd.RunToBar(higherTickerData.NumBars - 1);
atrInd.Shutdown();
ticker.HigherTimeframeValues["Atr"].Add(atrInd.Value.Last());
KeltnerChannel keltner = new KeltnerChannel(higherTickerData);
keltner.Initialize();
keltner.RunToBar(higherTickerData.NumBars - 1);
keltner.Shutdown();
ticker.HigherTimeframeValues["KeltnerUpper"].Add(keltner.Upper.Last());
ticker.HigherTimeframeValues["KeltnerMidline"].Add(keltner.Midline.Last());
ticker.HigherTimeframeValues["KeltnerLower"].Add(keltner.Lower.Last());
DtOscillator dtosc = new DtOscillator(higherTickerData);
dtosc.Initialize();
dtosc.RunToBar(higherTickerData.NumBars - 1);
dtosc.Shutdown();
ticker.HigherTimeframeValues["DtoscSK"].Add(dtosc.SK.Last());
ticker.HigherTimeframeValues["DtoscSD"].Add(dtosc.SK.Last());
ticker.HigherTimeframeValues["Close"].Add(higherTickerData.Close.Last());
// Return what kind orders are allowed.
double state = GetHigherTimeframeStateFromIndicator(dtosc, dtosc.Data.NumBars - 1, lastState);
////////////////// START HIGHER TIME FRAME DEBUGGING ////////////////////
if (Simulator.Config.OutputHigherTimeframeData)
{
DateTime outputDate = higherTickerData.Dates[higherTickerData.Dates.Count - 1];
List <double> states = new List <double>(ticker.HigherTimeframeTrend);
states.Add(state);
Simulator.DataOutput.OutputHigherTimeframeData(
outputDate,
new List <Indicator>()
{
dtosc, atrInd, keltner, sma
},
higherTickerData,
ticker,
states);
}
////////////////// END HIGHER TIME FRAME DEBUGGING ////////////////////
return(state);
}
/// <summary>
/// Creates an object of ticker data from the stream passed in.
/// </summary>
/// <param name="data">String of ticker data</param>
/// <param name="ticker">Ticker name for this data string</param>
/// <param name="isFromDisk">True if this string is loaded from our disk, saves a lot of calculation time</param>
/// <param name="start">Start date of the data</param>
/// <param name="end">End date of the data</param>
/// <returns>Returns an object created from the ticker data string</returns>
private TickerData CreateTickerDataFromString(string data, TickerExchangePair ticker, bool isFromDisk, DateTime start, DateTime end)
{
if (string.IsNullOrEmpty(data))
{
throw new Exception("No ticker data to parse.");
}
using (StringReader reader = new StringReader(data))
{
string line = string.Empty;
// Strip off the headers if the are present
if (reader.Peek() == 68) // "D"
{
reader.ReadLine();
}
TickerData tickerData = new TickerData(ticker);
// Value for an invalid date.
DateTime invalidDate = new DateTime(1970, 1, 1);
// Read each line of the string and convert it into numerical data and dates.
do
{
line = reader.ReadLine();
if (line != null)
{
string[] splitData = line.Split(new char[] { ',' });
DateTime lineDate = DateTime.MaxValue;
long lineDateDigit = 0;
if (long.TryParse(splitData[0], out lineDateDigit))
{
lineDate = UtilityMethods.ConvertFromUnixTimestamp(splitData[0]);
}
else
{
lineDate = DateTime.Parse(splitData[0]);
}
// Because of the way google returns data, we don't always get our exact dates.
// What we get is an interval of dates containing the ones we asked for, so
// we'll filter that data down to just the dates we want.
if (start != invalidDate && lineDate < start)
{
continue;
}
if (end != invalidDate && lineDate > end)
{
break;
}
// Sometimes google has random 0's or 0.01 values for things so we'll just skip those bars.
if (!IsDataFieldValid(splitData, 1) ||
!IsDataFieldValid(splitData, 2) ||
!IsDataFieldValid(splitData, 3) ||
!IsDataFieldValid(splitData, 4) ||
!IsDataFieldValid(splitData, 5))
{
continue;
}
// Add the data to our object.
double open = Convert.ToDouble(splitData[1]);
double high = Convert.ToDouble(splitData[2]);
double low = Convert.ToDouble(splitData[3]);
double close = Convert.ToDouble(splitData[4]);
long volume = Convert.ToInt64(splitData[5]);
tickerData.Dates.Add(lineDate);
tickerData.Open.Add(open);
tickerData.High.Add(high);
tickerData.Low.Add(low);
tickerData.Close.Add(close);
tickerData.Volume.Add(volume);
tickerData.NumBars = tickerData.Dates.Count;
// If this data is from the disk we don't need to calculate the extra fields since
// we've already saved them in the file.
if (isFromDisk)
{
tickerData.Typical.Add(Convert.ToDouble(splitData[(int)DataFields.Typical]));
tickerData.Median.Add(Convert.ToDouble(splitData[(int)DataFields.Median]));
tickerData.HigherTimeframeTrend.Add(Convert.ToDouble(splitData[(int)DataFields.HigherState]));
for (int i = 0; i < TickerData.HigherTimeframeValueStrings.Length; i++)
{
string key = TickerData.HigherTimeframeValueStrings[i];
List <double> higherValues = tickerData.HigherTimeframeValues[key];
higherValues.Add(Convert.ToDouble(splitData[(int)DataFields.HigherValuesStart + i]));
}
}
else
{
// Extra non-downloaded data.
high = tickerData.High[tickerData.NumBars - 1];
low = tickerData.Low[tickerData.NumBars - 1];
close = tickerData.Close[tickerData.NumBars - 1];
tickerData.Typical.Add((high + low + close) / 3);
tickerData.Median.Add((high + low) / 2);
// Calculate the higher momentum state for this bar. This is a pretty
// time consuming function since it has to loop back through all the
// bars before (and including) this one.
double lastValue = tickerData.HigherTimeframeTrend.Count > 0 ? tickerData.HigherTimeframeTrend[tickerData.HigherTimeframeTrend.Count - 1] : Order.OrderType.Long;
tickerData.HigherTimeframeTrend.Add(GetHigherTimerframeExtras(tickerData, lastValue));
}
}
} while (line != null);
tickerData.Start = tickerData.Dates[0];
tickerData.End = tickerData.Dates[tickerData.Dates.Count - 1];
tickerData.SaveDates();
return(tickerData);
}
}
/// <summary>
/// Constructor for the runnable.
/// </summary>
public Runnable(TickerData tickerData)
{
_dependents = new List <Runnable>();
Data = tickerData;
}
/// <summary>
/// Add this indicator to be saved for output later.
/// </summary>
/// <param name="tickerData">Ticker that the indicator is calculated with</param>
public Indicator(TickerData tickerData) : base(tickerData)
{
// Default to about 2 years of lookback data.
MaxSimulationBars = 500;
MaxPlotBars = 500;
}
