/// <summary>
/// Gets the data from the webserver and saves it onto disk for later usage.
/// </summary>
/// <param name="ticker">ticker to get data for</param>
/// <param name="start">Start date for the data</param>
/// <param name="end">End date for the data</param>
/// <param name="interval">Interval in seconds of the data to retreive</param>
/// <returns>Data (price, volume, etc) for the ticker</returns>
private TickerData GetIntraDayDataFromGoogleServer(TickerExchangePair ticker, DateTime start, DateTime end, int interval)
{
string downloadedData;
DownloadURIBuilder uriBuilder = new DownloadURIBuilder(ticker.Exchange, ticker.Ticker);
// Need to always get up till today from the server since google only supports a start date.
string uri = uriBuilder.getGetPricesUrlForIntraday(start, end, interval);
using (WebClient wClient = new WebClient())
{
downloadedData = wClient.DownloadString(uri);
}
using (MemoryStream ms = new MemoryStream(System.Text.Encoding.Default.GetBytes(downloadedData)))
{
DataProcessor dp = new DataProcessor();
string errorMessage;
string resultValue;
resultValue = dp.processIntradayStream(ms, out errorMessage);
if (!string.IsNullOrEmpty(errorMessage))
{
throw new Exception(errorMessage);
}
else
{
return(CreateTickerDataFromString(resultValue, ticker, false, start, end));
}
}
}
/// <summary>
/// Tries to get the data from the disk first. If all the data isn't on disk
/// then request it from the server.
/// </summary>
/// <param name="ticker">Ticker to get data for</param>
/// <param name="start">Start date for the data</param>
/// <param name="end">End date for the data</param>
/// <returns>Data (price, volume, etc) for the ticker</returns>
private TickerData GetDataFromDiskOrServer(TickerExchangePair ticker, DateTime start, DateTime end)
{
DateTime fileStartDate;
DateTime fileEndDate;
TickerData data = GetDataFromDisk(ticker, out fileStartDate, out fileEndDate);
bool diskDataNeedsUpdate = data == null || fileStartDate > start || fileEndDate < end;
// If the data is not on disk at all or there was a problem reading it, then
// we definitely get it from the server.
if (diskDataNeedsUpdate)
{
try
{
Simulator.WriteMessage("[" + ticker.ToString() + "] Downloading data");
TickerData serverData;
if (Simulator.Config.DataType == "daily")
{
serverData = GetDataFromGoogleServerAlt(ticker, start, end);
}
else
{
int interval = 60;
if (Simulator.Config.DataType == "fiveminute")
{
interval = 300;
}
else if (Simulator.Config.DataType == "threeminute")
{
interval = 180;
}
else if (Simulator.Config.DataType == "twominute")
{
interval = 120;
}
serverData = GetIntraDayDataFromGoogleServer(ticker, start, end, interval);
}
// Anytime we have to download data from the server always save the entire
// data set. This is because the data is split adjusted so the data from
// yesteray may not match the data from today.
data = serverData;
// Save the data so we can resuse it again without hitting the server.
SaveTickerData(ticker, data, start, end);
}
catch (Exception e)
{
Simulator.WriteMessage("[" + ticker.ToString() + "] Error downloading and parsing data-Exception: " + e.Message);
}
}
return(data);
}
/// <summary>
/// Check if the object is equal to another.
/// </summary>
/// <param name="obj">Object to compare</param>
/// <returns>True if the object passed in is equal to this object</returns>
public override bool Equals(object obj)
{
if (obj == null)
{
return(false);
}
TickerExchangePair ticker = obj as TickerExchangePair;
if ((object)ticker == null)
{
return(false);
}
return(ticker.Ticker == Ticker && ticker.Exchange == Exchange);
}
/// <summary>
/// Calculates things like win/loss percent, gain, etc. for the ticker.
/// </summary>
/// <param name="tickerAndExchange">Ticker to calculate for</param>
/// <param name="currentBar">Current bar of the simulation</param>
/// <param name="maxBarsAgo">Maximum number of bars in the past to consider for calculating</param>
/// <returns>Class holding the statistics calculated</returns>
public StrategyStatistics GetTickerStatistics(TickerExchangePair tickerAndExchange, int currentBar, int maxBarsAgo)
{
// Orders that started less than this bar will not be considered.
int cutoffBar = currentBar - maxBarsAgo;
if (cutoffBar < 0)
{
cutoffBar = 0;
}
// Order type doesn't matter here since we are just using this class to
// output overall ticker info which could be from any order type. It will
// get ignored on the web output display.
StrategyStatistics stats = new StrategyStatistics(tickerAndExchange.ToString(), Order.OrderType.Long);
int tickerHash = tickerAndExchange.GetHashCode();
if (TickerDictionary.ContainsKey(tickerHash))
{
List <Order> tickerOrders = TickerDictionary[tickerHash];
for (int i = tickerOrders.Count - 1; i >= 0; i--)
{
Order order = tickerOrders[i];
if (order.BuyBar >= cutoffBar)
{
stats.AddOrder(order);
}
}
}
// Only count the statistics if we have a bit more data to deal with.
// We want to avoid having a strategy say it's 100% correct when it
// only has 1 winning trade.
if (stats.NumberOfOrders > Simulator.Config.MinRequiredOrders)
{
stats.CalculateStatistics();
}
else
{
// For the same reasons as earlier in this function, order type doesn't matter here.
stats = new StrategyStatistics(tickerAndExchange.ToString(), Order.OrderType.Long);
}
return(stats);
}
/// <summary>
/// Gets the data from the disk for the symbol dates requested. If it doesn't exist
/// on disk, then we'll have to get it from the internet and then save it on disk
/// for later use.
/// </summary>
/// <param name="ticker">Ticker to get data for</param>
/// <param name="fileStartDate">The date that was the start of the requested date range. This can differ from the actual date that the server returns as the first date</param>
/// <param name="fileEndDate">The date that was the end of the requested date range</param>
/// <returns>Data (price, volume, etc) for the ticker</returns>
private TickerData GetDataFromDisk(TickerExchangePair ticker, out DateTime fileStartDate, out DateTime fileEndDate)
{
string fileAndPath = GetTickerFilename(ticker);
// If the file doesn't exist then we for sure have to pull it from the internet later.
if (File.Exists(fileAndPath))
{
try
{
Simulator.WriteMessage("[" + ticker.ToString() + "] Loading from disk data");
StreamReader file = new StreamReader(fileAndPath);
string line;
StringBuilder sb = new StringBuilder();
// The first line should be the saved start and end dates. Don't include that in the string builder
// since it's only important for the callee of this function.
line = file.ReadLine();
string[] dates = line.Split(',');
fileStartDate = UtilityMethods.ConvertFromUnixTimestamp(dates[0]);
fileEndDate = UtilityMethods.ConvertFromUnixTimestamp(dates[1]);
while ((line = file.ReadLine()) != null)
{
sb.AppendLine(line);
}
file.Close();
return(CreateTickerDataFromString(sb.ToString(), ticker, true, new DateTime(1970, 1, 1), new DateTime(1970, 1, 1)));
}
catch (Exception e)
{
Simulator.WriteMessage("[" + ticker.ToString() + "] Error reading and parsing file-Exception: " + e.Message);
}
}
// If file != exist
fileStartDate = DateTime.Now;
fileEndDate = DateTime.Now;
return(null);
}
/// <summary>
/// Saves all the ticker data to a file so it can be resused without us downloading from the server.
/// </summary>
/// <param name="ticker">Ticker exchange name</param>
/// <param name="newData">Ticker data to save</param>
/// <param name="start">Start date requested</param>
/// <param name="end">End date requested</param>
private void SaveTickerData(TickerExchangePair ticker, TickerData newData, DateTime start, DateTime end)
{
string fileAndPath = GetTickerFilename(ticker);
string contents = UtilityMethods.UnixTicks(start) + "," + UtilityMethods.UnixTicks(end) + "," + Environment.NewLine;
contents += "Date,Open,High,Low,Close,Volume,Typical,Median,HigherState,";
for (int i = 0; i < TickerData.HigherTimeframeValueStrings.Length; i++)
{
contents += TickerData.HigherTimeframeValueStrings[i] + ",";
}
// Append all the data.
contents += newData.WriteToString();
try
{
File.WriteAllText(fileAndPath, contents);
}
catch (Exception e)
{
Simulator.WriteMessage("[" + ticker.ToString() + "] Save ticker exception: " + e.Message);
}
}
/// <summary>
/// Gets the data from the webserver and saves it onto disk for later usage.
/// </summary>
/// <param name="ticker">ticker to get data for</param>
/// <param name="start">Start date for the data</param>
/// <param name="end">End date for the data</param>
/// <returns>Data (price, volume, etc) for the ticker</returns>
private TickerData GetDataFromGoogleServerAlt(TickerExchangePair ticker, DateTime start, DateTime end)
{
string downloadedData;
string baseUrl = "http://www.google.com/finance/historical?q={0}&startdate={1}&enddate={2}&ei=803jVKPmEoryrAGH34CgDA&output=csv";
string uri = string.Format(baseUrl,
ticker.Ticker,
start.ToString(@"MMM+d\%2C+yyyy"),
end.ToString(@"MMM+d\%2C+yyyy")
);
using (WebClient wClient = new WebClient())
{
downloadedData = wClient.DownloadString(uri);
}
using (MemoryStream ms = new MemoryStream(System.Text.Encoding.Default.GetBytes(downloadedData)))
{
StreamReader sr = new StreamReader(ms);
string line;
List <string> lines = new List <string>();
while ((line = sr.ReadLine()) != null)
{
lines.Add(line);
}
// Read all the lines from back to front and ignore the headers in the beginning of the file.
StringBuilder sb = new StringBuilder();
for (int i = lines.Count - 1; i > 0; i--)
{
sb.AppendLine(lines[i]);
}
string resultValue = sb.ToString();
return(CreateTickerDataFromString(resultValue, ticker, false, start, end));
}
}
/// <summary>
/// Creates a new object to hold all the data.
/// </summary>
/// <param name="start">Starting date of the data</param>
/// <param name="end">Ending date of the data</param>
public TickerData(TickerExchangePair tickerAndExchange)
{
TickerAndExchange = tickerAndExchange;
Start = DateTime.Now;
End = DateTime.Now;
Dates = new List <DateTime>();
Open = new List <double>();
Close = new List <double>();
High = new List <double>();
Low = new List <double>();
Volume = new List <long>();
// Extras.
Typical = new List <double>();
Median = new List <double>();
HigherTimeframeTrend = new List <double>();
HigherTimeframeValues = new Dictionary <string, List <double> >();
for (int i = 0; i < HigherTimeframeValueStrings.Length; i++)
{
HigherTimeframeValues[HigherTimeframeValueStrings[i]] = new List <double>();
}
}
/// <summary>
/// Calculates things like win/loss percent, gain, etc. for the ticker.
/// </summary>
/// <param name="tickerAndExchange">Ticker to calculate for</param>
/// <param name="currentBar">Current bar of the simulation</param>
/// <param name="maxBarsAgo">Maximum number of bars in the past to consider for calculating</param>
/// <returns>Class holding the statistics calculated</returns>
public StrategyStatistics GetTickerStatistics(TickerExchangePair tickerAndExchange, int currentBar, int maxBarsAgo)
{
// TODO: add way to calculate this.
return(new StrategyStatistics(tickerAndExchange.ToString(), Order.OrderType.Long));
}
/// <summary>
/// Calculates things like win/loss percent, gain, etc. for the strategy used on the ticker.
/// </summary>
/// <param name="strategyName">Name of the strategy the statistics are for</param>
/// <param name="orderType">Type of orders placed with this strategy (long or short)</param>
/// <param name="tickerAndExchange">Ticker the strategy used</param>
/// <param name="currentBar">Current bar of the simulation</param>
/// <param name="maxBarsAgo">Maximum number of bars in the past to consider for calculating</param>
/// <returns>Class holding the statistics calculated</returns>
public StrategyStatistics GetStrategyStatistics(string strategyName, double orderType, TickerExchangePair tickerAndExchange, int currentBar, int maxBarsAgo)
{
// Orders that started less than this bar will not be considered.
int cutoffBar = currentBar - maxBarsAgo;
if (cutoffBar < 0)
{
cutoffBar = 0;
}
// Get the list of orders to search.
StrategyStatistics stats = new StrategyStatistics(strategyName, orderType);
List <Order> orderList = null;
if (strategyName.Length > 0 && tickerAndExchange == null)
{
int strategyKey = strategyName.GetHashCode();
if (StrategyDictionary.ContainsKey(strategyKey))
{
orderList = StrategyDictionary[strategyKey].ToList();
}
}
else if (tickerAndExchange != null)
{
int tickerKey = tickerAndExchange.GetHashCode();
if (TickerStrategyOrders.ContainsKey(tickerKey))
{
Dictionary <int, List <Order> > tickerDictionary = TickerStrategyOrders[tickerKey];
int strategyKey = strategyName.GetHashCode();
if (tickerDictionary.ContainsKey(strategyKey))
{
orderList = tickerDictionary[strategyKey];
}
}
}
if (orderList != null)
{
for (int i = orderList.Count - 1; i >= 0; i--)
{
Order order = orderList[i];
// Add orders that are newer than the maximum lookback and only keep a set
// amount of orders.
if (order.IsFinished() &&
order.StrategyName == strategyName &&
order.BuyBar >= cutoffBar &&
order.Type == orderType &&
stats.NumberOfOrders < Simulator.Config.MaxLookBackOrders)
{
stats.AddOrder(order);
}
}
}
if (stats.NumberOfOrders > Simulator.Config.MinRequiredOrders)
{
stats.CalculateStatistics();
}
else
{
stats = new StrategyStatistics(strategyName, orderType);
}
return(stats);
}
/// <summary>
/// Frees the orders for a ticker when it finished.
/// </summary>
/// <param name="tickerAndExchange">Ticker to free</param>
public void PurgeTickerOrders(TickerExchangePair tickerAndExchange)
{
TickerStrategyOrders[tickerAndExchange.GetHashCode()] = null;
}
/// <summary>
/// Initializes the sim from the config object.
/// </summary>
/// <param name="config">Object with all the parameters that can be used to config how this sim runs</param>
public bool CreateFromConfig(SimulatorConfig config, TickerDataStore dataStore)
{
Config = config;
DataStore = dataStore;
DataOutput = new DataOutputter();
if (Config.UseTodaysDate)
{
Config.EndDate = DateTime.Now.Date;
}
// Load the config file with the instument list for all the symbols that we
// want to test.
SortedDictionary<string, TickerExchangePair> fileInstruments = new SortedDictionary<string, TickerExchangePair>();
string line;
try
{
StreamReader file = new StreamReader(Config.InstrumentListFile);
while ((line = file.ReadLine()) != null)
{
string[] pair = line.Split(',');
TickerExchangePair newTicker = new TickerExchangePair(pair[1], pair[0]);
string key = newTicker.ToString();
if (fileInstruments.ContainsKey(key) == false)
{
fileInstruments[key] = newTicker;
}
else
{
WriteMessage("Duplicate ticker in file: " + newTicker.ToString());
}
}
}
catch (Exception e)
{
WriteMessage("Error loading instrument file!\n" + e.Message);
return false;
}
WriteMessage("Initializing ticker data");
// TODO: there has to be a better place for this!
//if (Directory.Exists(Simulator.Config.OutputFolder + "\\higher"))
//{
// Directory.Delete(Simulator.Config.OutputFolder + "\\higher", true);
//}
// Add all the symbols as dependent strategies using the bestofsubstrategies
ConcurrentDictionary<string, BestOfRootStrategies> downloadedInstruments = new ConcurrentDictionary<string, BestOfRootStrategies>();
#if DEBUG
foreach (KeyValuePair<string, TickerExchangePair> item in fileInstruments)
#else
Parallel.ForEach(fileInstruments, item =>
#endif
{
// Get the data for the symbol and save it for later so we can output it.
TickerData tickerData = DataStore.GetTickerData(item.Value, config.StartDate, config.EndDate);
if (tickerData != null)
{
DataOutput.SaveTickerData(tickerData);
//RunnableFactory factory = new RunnableFactory(tickerData);
// This strategy will find the best strategy for this instrument everyday and save the value.
downloadedInstruments[item.Value.ToString()] = new BestOfRootStrategies(tickerData);
}
else
{
WriteMessage("No ticker data for " + item.Value.ToString());
}
#if DEBUG
}
#else
});
#endif
// Want to store the instrument data in a sorted way so that we always run things
// in the same order.
Instruments = new SortedDictionary<string, BestOfRootStrategies>(downloadedInstruments.ToDictionary(kvp => kvp.Key, kvp => kvp.Value));
NumberOfBars = DataStore.SimTickerDates.Count;
Broker = new Broker(Config.InitialAccountBalance, NumberOfBars);
return DataStore.SimTickerDates.Count > 0;
}
/// <summary>
/// Frees the orders for a ticker when it finished.
/// </summary>
/// <param name="tickerAndExchange">Ticker to free</param>
public void PurgeTickerOrders(TickerExchangePair tickerAndExchange)
{
}
/// <summary>
/// Gets the symbol data from either memory, disk, or a server.
/// </summary>
/// <param name="ticker">Ticker to get data for</param>
/// <param name="start">Start date for the data</param>
/// <param name="end">End date for the data</param>
/// <returns>Data (price, volume, etc) for the ticker</returns>
public TickerData GetTickerData(TickerExchangePair ticker, DateTime start, DateTime end)
{
TickerData data = new TickerData(ticker);
// The symbol exists in memory already.
int key = ticker.GetHashCode();
if (_symbolsInMemory.ContainsKey(key))
{
TickerData inMemoryData = _symbolsInMemory[key];
// We don't have all the data in memory past the end, so we need to get that data and append it.
if (end > inMemoryData.End || start < inMemoryData.Start)
{
data = GetDataFromDiskOrServer(ticker, start, end);
// Update the data in memory so it has it next time it runs.
_symbolsInMemory[key] = data;
// Return only the dates requested.
data = data.SubSet(start, end);
}
// Not requesting everything that is in the memory. This is generally the case.
else if (start > inMemoryData.Start || end < inMemoryData.End)
{
data = inMemoryData.SubSet(start, end);
}
// We wanted everything that is memory.
else
{
data = inMemoryData;
}
}
// Symbol isn't in memory so we need to load from the disk or the server.
else
{
// Always start by loading everything we have our earliest date so that
// anytime we eventually will have all the data saved allowing us to
// test lots of different date ranges without having to hit the disk or internet.
data = GetDataFromDiskOrServer(ticker, start, end);
if (data != null)
{
// Save in memory for next time.
_symbolsInMemory[key] = data;
data = data.SubSet(start, end);
}
}
if (data != null)
{
// Save all the dates that this ticker has so that we have a list of dates that we can
// iterate through for trading periods. This is because each ticker can potentially have
// different trading dates but for the main sim we want to go through all dates and if
// the ticker has data for that time, we'll use it.
lock (_dateLock)
{
for (int i = 0; i < data.Dates.Count; i++)
{
if (SimTickerDates.ContainsKey(data.Dates[i]) == false)
{
SimTickerDates[data.Dates[i]] = true;
}
}
}
}
return(data);
}
/// <summary>
/// Creates an object of ticker data from the stream passed in.
/// </summary>
/// <param name="data">String of ticker data</param>
/// <param name="ticker">Ticker name for this data string</param>
/// <param name="isFromDisk">True if this string is loaded from our disk, saves a lot of calculation time</param>
/// <param name="start">Start date of the data</param>
/// <param name="end">End date of the data</param>
/// <returns>Returns an object created from the ticker data string</returns>
private TickerData CreateTickerDataFromString(string data, TickerExchangePair ticker, bool isFromDisk, DateTime start, DateTime end)
{
if (string.IsNullOrEmpty(data))
{
throw new Exception("No ticker data to parse.");
}
using (StringReader reader = new StringReader(data))
{
string line = string.Empty;
// Strip off the headers if the are present
if (reader.Peek() == 68) // "D"
{
reader.ReadLine();
}
TickerData tickerData = new TickerData(ticker);
// Value for an invalid date.
DateTime invalidDate = new DateTime(1970, 1, 1);
// Read each line of the string and convert it into numerical data and dates.
do
{
line = reader.ReadLine();
if (line != null)
{
string[] splitData = line.Split(new char[] { ',' });
DateTime lineDate = DateTime.MaxValue;
long lineDateDigit = 0;
if (long.TryParse(splitData[0], out lineDateDigit))
{
lineDate = UtilityMethods.ConvertFromUnixTimestamp(splitData[0]);
}
else
{
lineDate = DateTime.Parse(splitData[0]);
}
// Because of the way google returns data, we don't always get our exact dates.
// What we get is an interval of dates containing the ones we asked for, so
// we'll filter that data down to just the dates we want.
if (start != invalidDate && lineDate < start)
{
continue;
}
if (end != invalidDate && lineDate > end)
{
break;
}
// Sometimes google has random 0's or 0.01 values for things so we'll just skip those bars.
if (!IsDataFieldValid(splitData, 1) ||
!IsDataFieldValid(splitData, 2) ||
!IsDataFieldValid(splitData, 3) ||
!IsDataFieldValid(splitData, 4) ||
!IsDataFieldValid(splitData, 5))
{
continue;
}
// Add the data to our object.
double open = Convert.ToDouble(splitData[1]);
double high = Convert.ToDouble(splitData[2]);
double low = Convert.ToDouble(splitData[3]);
double close = Convert.ToDouble(splitData[4]);
long volume = Convert.ToInt64(splitData[5]);
tickerData.Dates.Add(lineDate);
tickerData.Open.Add(open);
tickerData.High.Add(high);
tickerData.Low.Add(low);
tickerData.Close.Add(close);
tickerData.Volume.Add(volume);
tickerData.NumBars = tickerData.Dates.Count;
// If this data is from the disk we don't need to calculate the extra fields since
// we've already saved them in the file.
if (isFromDisk)
{
tickerData.Typical.Add(Convert.ToDouble(splitData[(int)DataFields.Typical]));
tickerData.Median.Add(Convert.ToDouble(splitData[(int)DataFields.Median]));
tickerData.HigherTimeframeTrend.Add(Convert.ToDouble(splitData[(int)DataFields.HigherState]));
for (int i = 0; i < TickerData.HigherTimeframeValueStrings.Length; i++)
{
string key = TickerData.HigherTimeframeValueStrings[i];
List <double> higherValues = tickerData.HigherTimeframeValues[key];
higherValues.Add(Convert.ToDouble(splitData[(int)DataFields.HigherValuesStart + i]));
}
}
else
{
// Extra non-downloaded data.
high = tickerData.High[tickerData.NumBars - 1];
low = tickerData.Low[tickerData.NumBars - 1];
close = tickerData.Close[tickerData.NumBars - 1];
tickerData.Typical.Add((high + low + close) / 3);
tickerData.Median.Add((high + low) / 2);
// Calculate the higher momentum state for this bar. This is a pretty
// time consuming function since it has to loop back through all the
// bars before (and including) this one.
double lastValue = tickerData.HigherTimeframeTrend.Count > 0 ? tickerData.HigherTimeframeTrend[tickerData.HigherTimeframeTrend.Count - 1] : Order.OrderType.Long;
tickerData.HigherTimeframeTrend.Add(GetHigherTimerframeExtras(tickerData, lastValue));
}
}
} while (line != null);
tickerData.Start = tickerData.Dates[0];
tickerData.End = tickerData.Dates[tickerData.Dates.Count - 1];
tickerData.SaveDates();
return(tickerData);
}
}
/// <summary>
/// Builds a cache file name for a ticker.
/// </summary>
/// <param name="ticker">Ticker exhange name</param>
/// <returns>Filename to for the ticker</returns>
private string GetTickerFilename(TickerExchangePair ticker)
{
return(_cacheFolder + @"\" + Simulator.Config.DataType + @"\" + ticker.ToString() + ".csv");
}