/// <summary>
/// Tries to get the data from the disk first. If all the data isn't on disk
/// then request it from the server.
/// </summary>
/// <param name="ticker">Ticker to get data for</param>
/// <param name="start">Start date for the data</param>
/// <param name="end">End date for the data</param>
/// <returns>Data (price, volume, etc) for the ticker</returns>
private TickerData GetDataFromDiskOrServer(TickerExchangePair ticker, DateTime start, DateTime end)
{
DateTime fileStartDate;
DateTime fileEndDate;
TickerData data = GetDataFromDisk(ticker, out fileStartDate, out fileEndDate);
bool diskDataNeedsUpdate = data == null || fileStartDate > start || fileEndDate < end;
// If the data is not on disk at all or there was a problem reading it, then
// we definitely get it from the server.
if (diskDataNeedsUpdate)
{
try
{
Simulator.WriteMessage("[" + ticker.ToString() + "] Downloading data");
TickerData serverData;
if (Simulator.Config.DataType == "daily")
{
serverData = GetDataFromGoogleServerAlt(ticker, start, end);
}
else
{
int interval = 60;
if (Simulator.Config.DataType == "fiveminute")
{
interval = 300;
}
else if (Simulator.Config.DataType == "threeminute")
{
interval = 180;
}
else if (Simulator.Config.DataType == "twominute")
{
interval = 120;
}
serverData = GetIntraDayDataFromGoogleServer(ticker, start, end, interval);
}
// Anytime we have to download data from the server always save the entire
// data set. This is because the data is split adjusted so the data from
// yesteray may not match the data from today.
data = serverData;
// Save the data so we can resuse it again without hitting the server.
SaveTickerData(ticker, data, start, end);
}
catch (Exception e)
{
Simulator.WriteMessage("[" + ticker.ToString() + "] Error downloading and parsing data-Exception: " + e.Message);
}
}
return(data);
}
/// <summary>
/// Calculates things like win/loss percent, gain, etc. for the ticker.
/// </summary>
/// <param name="tickerAndExchange">Ticker to calculate for</param>
/// <param name="currentBar">Current bar of the simulation</param>
/// <param name="maxBarsAgo">Maximum number of bars in the past to consider for calculating</param>
/// <returns>Class holding the statistics calculated</returns>
public StrategyStatistics GetTickerStatistics(TickerExchangePair tickerAndExchange, int currentBar, int maxBarsAgo)
{
// Orders that started less than this bar will not be considered.
int cutoffBar = currentBar - maxBarsAgo;
if (cutoffBar < 0)
{
cutoffBar = 0;
}
// Order type doesn't matter here since we are just using this class to
// output overall ticker info which could be from any order type. It will
// get ignored on the web output display.
StrategyStatistics stats = new StrategyStatistics(tickerAndExchange.ToString(), Order.OrderType.Long);
int tickerHash = tickerAndExchange.GetHashCode();
if (TickerDictionary.ContainsKey(tickerHash))
{
List <Order> tickerOrders = TickerDictionary[tickerHash];
for (int i = tickerOrders.Count - 1; i >= 0; i--)
{
Order order = tickerOrders[i];
if (order.BuyBar >= cutoffBar)
{
stats.AddOrder(order);
}
}
}
// Only count the statistics if we have a bit more data to deal with.
// We want to avoid having a strategy say it's 100% correct when it
// only has 1 winning trade.
if (stats.NumberOfOrders > Simulator.Config.MinRequiredOrders)
{
stats.CalculateStatistics();
}
else
{
// For the same reasons as earlier in this function, order type doesn't matter here.
stats = new StrategyStatistics(tickerAndExchange.ToString(), Order.OrderType.Long);
}
return(stats);
}
/// <summary>
/// Gets the data from the disk for the symbol dates requested. If it doesn't exist
/// on disk, then we'll have to get it from the internet and then save it on disk
/// for later use.
/// </summary>
/// <param name="ticker">Ticker to get data for</param>
/// <param name="fileStartDate">The date that was the start of the requested date range. This can differ from the actual date that the server returns as the first date</param>
/// <param name="fileEndDate">The date that was the end of the requested date range</param>
/// <returns>Data (price, volume, etc) for the ticker</returns>
private TickerData GetDataFromDisk(TickerExchangePair ticker, out DateTime fileStartDate, out DateTime fileEndDate)
{
string fileAndPath = GetTickerFilename(ticker);
// If the file doesn't exist then we for sure have to pull it from the internet later.
if (File.Exists(fileAndPath))
{
try
{
Simulator.WriteMessage("[" + ticker.ToString() + "] Loading from disk data");
StreamReader file = new StreamReader(fileAndPath);
string line;
StringBuilder sb = new StringBuilder();
// The first line should be the saved start and end dates. Don't include that in the string builder
// since it's only important for the callee of this function.
line = file.ReadLine();
string[] dates = line.Split(',');
fileStartDate = UtilityMethods.ConvertFromUnixTimestamp(dates[0]);
fileEndDate = UtilityMethods.ConvertFromUnixTimestamp(dates[1]);
while ((line = file.ReadLine()) != null)
{
sb.AppendLine(line);
}
file.Close();
return(CreateTickerDataFromString(sb.ToString(), ticker, true, new DateTime(1970, 1, 1), new DateTime(1970, 1, 1)));
}
catch (Exception e)
{
Simulator.WriteMessage("[" + ticker.ToString() + "] Error reading and parsing file-Exception: " + e.Message);
}
}
// If file != exist
fileStartDate = DateTime.Now;
fileEndDate = DateTime.Now;
return(null);
}
/// <summary>
/// Saves all the ticker data to a file so it can be resused without us downloading from the server.
/// </summary>
/// <param name="ticker">Ticker exchange name</param>
/// <param name="newData">Ticker data to save</param>
/// <param name="start">Start date requested</param>
/// <param name="end">End date requested</param>
private void SaveTickerData(TickerExchangePair ticker, TickerData newData, DateTime start, DateTime end)
{
string fileAndPath = GetTickerFilename(ticker);
string contents = UtilityMethods.UnixTicks(start) + "," + UtilityMethods.UnixTicks(end) + "," + Environment.NewLine;
contents += "Date,Open,High,Low,Close,Volume,Typical,Median,HigherState,";
for (int i = 0; i < TickerData.HigherTimeframeValueStrings.Length; i++)
{
contents += TickerData.HigherTimeframeValueStrings[i] + ",";
}
// Append all the data.
contents += newData.WriteToString();
try
{
File.WriteAllText(fileAndPath, contents);
}
catch (Exception e)
{
Simulator.WriteMessage("[" + ticker.ToString() + "] Save ticker exception: " + e.Message);
}
}
/// <summary>
/// Calculates things like win/loss percent, gain, etc. for the ticker.
/// </summary>
/// <param name="tickerAndExchange">Ticker to calculate for</param>
/// <param name="currentBar">Current bar of the simulation</param>
/// <param name="maxBarsAgo">Maximum number of bars in the past to consider for calculating</param>
/// <returns>Class holding the statistics calculated</returns>
public StrategyStatistics GetTickerStatistics(TickerExchangePair tickerAndExchange, int currentBar, int maxBarsAgo)
{
// TODO: add way to calculate this.
return(new StrategyStatistics(tickerAndExchange.ToString(), Order.OrderType.Long));
}
/// <summary>
/// Initializes the sim from the config object.
/// </summary>
/// <param name="config">Object with all the parameters that can be used to config how this sim runs</param>
public bool CreateFromConfig(SimulatorConfig config, TickerDataStore dataStore)
{
Config = config;
DataStore = dataStore;
DataOutput = new DataOutputter();
if (Config.UseTodaysDate)
{
Config.EndDate = DateTime.Now.Date;
}
// Load the config file with the instument list for all the symbols that we
// want to test.
SortedDictionary<string, TickerExchangePair> fileInstruments = new SortedDictionary<string, TickerExchangePair>();
string line;
try
{
StreamReader file = new StreamReader(Config.InstrumentListFile);
while ((line = file.ReadLine()) != null)
{
string[] pair = line.Split(',');
TickerExchangePair newTicker = new TickerExchangePair(pair[1], pair[0]);
string key = newTicker.ToString();
if (fileInstruments.ContainsKey(key) == false)
{
fileInstruments[key] = newTicker;
}
else
{
WriteMessage("Duplicate ticker in file: " + newTicker.ToString());
}
}
}
catch (Exception e)
{
WriteMessage("Error loading instrument file!\n" + e.Message);
return false;
}
WriteMessage("Initializing ticker data");
// TODO: there has to be a better place for this!
//if (Directory.Exists(Simulator.Config.OutputFolder + "\\higher"))
//{
// Directory.Delete(Simulator.Config.OutputFolder + "\\higher", true);
//}
// Add all the symbols as dependent strategies using the bestofsubstrategies
ConcurrentDictionary<string, BestOfRootStrategies> downloadedInstruments = new ConcurrentDictionary<string, BestOfRootStrategies>();
#if DEBUG
foreach (KeyValuePair<string, TickerExchangePair> item in fileInstruments)
#else
Parallel.ForEach(fileInstruments, item =>
#endif
{
// Get the data for the symbol and save it for later so we can output it.
TickerData tickerData = DataStore.GetTickerData(item.Value, config.StartDate, config.EndDate);
if (tickerData != null)
{
DataOutput.SaveTickerData(tickerData);
//RunnableFactory factory = new RunnableFactory(tickerData);
// This strategy will find the best strategy for this instrument everyday and save the value.
downloadedInstruments[item.Value.ToString()] = new BestOfRootStrategies(tickerData);
}
else
{
WriteMessage("No ticker data for " + item.Value.ToString());
}
#if DEBUG
}
#else
});
#endif
// Want to store the instrument data in a sorted way so that we always run things
// in the same order.
Instruments = new SortedDictionary<string, BestOfRootStrategies>(downloadedInstruments.ToDictionary(kvp => kvp.Key, kvp => kvp.Value));
NumberOfBars = DataStore.SimTickerDates.Count;
Broker = new Broker(Config.InitialAccountBalance, NumberOfBars);
return DataStore.SimTickerDates.Count > 0;
}
/// <summary>
/// Builds a cache file name for a ticker.
/// </summary>
/// <param name="ticker">Ticker exhange name</param>
/// <returns>Filename to for the ticker</returns>
private string GetTickerFilename(TickerExchangePair ticker)
{
return(_cacheFolder + @"\" + Simulator.Config.DataType + @"\" + ticker.ToString() + ".csv");
}