public StockDataSet(string symbol, DateTime start, StockDataFile file, long address = -1)
{
this.Symbol = symbol;
this.Start = start;
this.File = file;
this.StreamAddress = address;
}
public StockDataSetDerived(StockDataSet <U> source, StockDataFile file, StockDataCreator create, StockProcessingStateAccessor stateGetter)
{
this.SourceData = source;
this.File = file;
this.Start = source.Start;
this.Symbol = source.Symbol;
this.Create = create;
this.GetState = stateGetter;
}
/// <summary>
/// Creates a derived set of data from the given source
/// </summary>
/// <param name="source">The source data</param>
/// <returns>The derived data</returns>
public static Dictionary <string, List <StockDataSetDerived <T, U, V> > > Derive(Dictionary <string, List <StockDataSet <U> > > source, StockDataFile file, StockDataCreator create, StockProcessingStateAccessor stateGetter)
{
var derived = new Dictionary <string, List <StockDataSetDerived <T, U, V> > >();
foreach (KeyValuePair <string, List <StockDataSet <U> > > pair in source)
{
StockDataSetDerived <T, U, V> prevSet = null;
derived[pair.Key] = new List <StockDataSetDerived <T, U, V> >(pair.Value.Count);
foreach (StockDataSet <U> srcSet in pair.Value)
{
var newSet = new StockDataSetDerived <T, U, V>(srcSet, file, create, stateGetter);
newSet.Previous = prevSet;
prevSet = newSet;
derived[pair.Key].Add(newSet);
}
}
return(derived);
}
/// <summary>
/// Reads the specified data streams, and converts them into a basic stock data file
/// </summary>
/// <param name="sourceFiles">The legacy data files</param>
/// <returns>The new data file</returns>
public static StockDataFile Convert(List <string> sourceFiles, Stream destination)
{
StockDataFile newFile = new StockDataFile();
newFile.Interval = new TimeSpan(0, 1, 0);
Dictionary <string, List <StockDataSet <StockDataBase> > > segments = new Dictionary <string, List <StockDataSet <StockDataBase> > >();
foreach (string filename in sourceFiles)
{
// Parse the date from the filename
DateTime fileDate = GetDateFromFileName(filename);
DateTime fileStart = fileDate.AddHours(9.5);
DateTime fileEnd = fileDate.AddHours(16);
long delayedOffset = (filename.Contains("goog")) ? 0 : new TimeSpan(0, 15, 0).Ticks;
// Read the initial line of the file to learn which stocks are in the file
StreamReader s = new StreamReader(filename);
List <StockDataSet <StockDataBase> > fileData = new List <StockDataSet <StockDataBase> >();
string line = s.ReadLine();
string[] stockNamesStr = line.Split(new char[] { ',' });
for (int i = 1; i < stockNamesStr.Length; i++)
{
string symbol = stockNamesStr[i].ToUpper();
List <StockDataSet <StockDataBase> > sets;
// Check if a stock data set already exists for the symbol
if (!segments.TryGetValue(symbol, out sets))
{
sets = new List <StockDataSet <StockDataBase> >();
segments.Add(symbol, sets);
}
foreach (StockDataSet <StockDataBase> preExistingSet in sets)
{
// Add to an existing set if they are for the same day
if (preExistingSet.Start.Date == fileStart.Date)
{
fileData.Add(preExistingSet);
break;
}
}
if (fileData.Count <= (i - 1))
{
// No matching set was found, so create a new one
StockDataSet <StockDataBase> newSet = new StockDataSet <StockDataBase>(symbol, fileStart, newFile);
newSet.DataSet.Resize(391);
fileData.Add(newSet);
if (sets.Count > 0)
{
newSet.Previous = sets[sets.Count - 1];
}
sets.Add(newSet);
}
}
DateTime lineTime = fileStart;
while (!s.EndOfStream && (lineTime < fileEnd))
{
string[] stockPricesStr = s.ReadLine().Split(new char[] { ',' });
// Ensure the entry contains all of the stocks
if ((stockPricesStr.Length - 1) != fileData.Count)
{
continue;
}
// Get the time of the entry
DateTime newTime;
if (!DateTime.TryParse(stockPricesStr[0].Replace("\"", ""), out newTime))
{
continue;
}
newTime = fileStart.AddTicks((newTime.TimeOfDay.Ticks - fileStart.TimeOfDay.Ticks) - delayedOffset);
// Check if this new entry is valid
if ((newTime >= fileStart) && (newTime <= fileEnd))
{
// Update the prices of each of the stocks
for (int i = 0; i < (stockPricesStr.Length - 1); i++)
{
float price;
if (float.TryParse(stockPricesStr[i + 1], out price))
{
// Sometimes the first price is corrupted, so skip it
if ((newTime == fileStart)
//&& ((fileData[i].Previous != null) && (Math.Abs((price / fileData[i][-1].Price) - 1.0f) > 0.01f))
)
{
continue;
}
if ((price == 0.0f) || (newTime == fileEnd))
{
price = fileData[i][fileData[i].Count - 1].Price;
if (price == 0.0f)
{
continue;
}
}
while (fileData[i].End <= newTime)
{
fileData[i].DataSet.Add(new StockDataBase(price));
}
}
}
}
}
}
// Save the old file to disk
newFile.SetSegments(segments);
newFile.Save(destination, typeof(StockDataBase), segments);
return(newFile);
}
/// <summary>
/// Constructor for a file agregator
/// </summary>
/// <param name="files">The streams containing the source files</param>
public AggregatorFile(List <Stream> files)
{
// Open the source files
int count = 0;
List <FieldInfo> fields = new List <FieldInfo>();
List <FieldInfo[]> sourceFields = new List <FieldInfo[]>();
Interval = TimeSpan.MaxValue;
foreach (var s in files)
{
var f = StockDataFile.Open(s);
try
{
// Get the members in the source
string typeName = "Source" + count.ToString();
var scriptInstance = CSScript.LoadCode(f.GetSourceCode(typeName).Replace("StockDataSource", typeName), null);
Type t = scriptInstance.DefinedTypes.Where((scriptType) => { return(scriptType.Name.Equals(typeName)); }).First();
var fieldList = t.GetFields(BindingFlags.Public | BindingFlags.Instance);
sourceFields.Add(fieldList);
foreach (var field in fieldList)
{
if (!fields.Contains(field))
{
fields.Add(field);
}
}
// Check the date range included in the source
foreach (var pair in f.Segments)
{
List <Tuple <DateTime, long> > segList;
if (!this.Segments.TryGetValue(pair.Key, out segList))
{
segList = new List <Tuple <DateTime, long> >();
this.Segments.Add(pair.Key, segList);
}
foreach (var seg in pair.Value)
{
if (segList.Find((ele) => { return(ele.Item1.Equals(seg.Item1)); }) == null)
{
segList.Add(new Tuple <DateTime, long>(seg.Item1, long.MaxValue));
}
}
}
// Use the smallest interval from among the sources
this.Interval = ((f.Interval < this.Interval) ? f.Interval : this.Interval);
// Remember the source file
Sources.Add(f);
}
catch (Exception ex)
{
System.Windows.Forms.MessageBox.Show(ex.ToString());
}
}
// Generate the source code based on the member list
var code = GenSourceCode(fields, sourceFields).Replace("StockDataSink", "StockDataSource");
this.SourceCode = code.ToArray();
}
public static StockSession Start(List <string> sources, List <string> sinkScripts, string executeScript)
{
StockSession session = new StockSession();
session.Scripts.Clear();
Directory.CreateDirectory("tmp");
// Convert any legacy files before further processing
var legacyFiles = sources.Where((s) => { return(s.EndsWith(".csv")); }).ToList();
if (legacyFiles.Count() > 0)
{
System.Windows.Forms.SaveFileDialog saveDiag = new System.Windows.Forms.SaveFileDialog();
saveDiag.Title = "Save converted data file as...";
saveDiag.CheckFileExists = false;
if (saveDiag.ShowDialog() == System.Windows.Forms.DialogResult.OK)
{
List <string> convertedFileNames;
var convertedFiles = StockDataFile.ConvertByMonth(legacyFiles, Path.GetDirectoryName(saveDiag.FileName), out convertedFileNames);
foreach (var cf in convertedFileNames)
{
sources.Add(cf);
}
}
else
{
// Cancel running the script
return(null);
}
foreach (var l in legacyFiles)
{
sources.Remove(l);
}
}
session.SourceFile = StockDataFile.Open(sources.ConvertAll <Stream>((s) => { return(new FileStream(s, FileMode.Open)); }));
session.Scripts.Add("tmp/" + SOURCE_CLASS + ".cs");
using (var file = new StreamWriter(new FileStream(session.Scripts.Last(), FileMode.Create))) file.Write(session.SourceFile.GetSourceCode(SOURCE_CLASS));
// Put the data set reference script first
List <string> totalSinkScripts = sinkScripts.ToList();
totalSinkScripts.Insert(0, "Script\\Data\\DataSetReference.cs");
session.SinkFile = new StockDataFile(totalSinkScripts.ConvertAll <string>((f) => { return(Path.GetFileNameWithoutExtension(f)); }), totalSinkScripts.ConvertAll <string>((f) => { return(File.ReadAllText(f)); }));
session.SinkFile.Interval = session.SourceFile.Interval;
session.Scripts.Add("tmp/" + SINK_CLASS + ".cs");
using (var file = new StreamWriter(new FileStream(session.Scripts.Last(), FileMode.Create))) file.Write(session.SinkFile.GenStockDataSink());
session.Scripts.AddRange(totalSinkScripts);
// Create the evaluator file (needs to be compiled in the script since it references StockDataSource)
string[] embeddedFiles = new string[]
{
"RobinhoodDesktop.Script.StockEvaluator.cs",
"RobinhoodDesktop.Script.StockProcessor.cs"
};
foreach (var f in embeddedFiles)
{
session.Scripts.Add(string.Format("tmp/{0}.cs", f.Substring(24, f.Length - 27)));
StringBuilder analyzerCode = new StringBuilder();
analyzerCode.Append(new StreamReader(Assembly.GetExecutingAssembly().GetManifestResourceStream(f)).ReadToEnd());
using (var file = new StreamWriter(new FileStream(session.Scripts.Last(), FileMode.Create))) file.Write(StockDataFile.FormatSource(analyzerCode.ToString()));
}
// Add the user defined analyzers
foreach (string path in Directory.GetFiles(@"Script/Decision", "*.cs", SearchOption.AllDirectories))
{
session.Scripts.Add(path);
}
foreach (string path in Directory.GetFiles(@"Script/Action", "*.cs", SearchOption.AllDirectories))
{
session.Scripts.Add(path);
}
// Get the code that will actually run the session
if (!string.IsNullOrEmpty(executeScript))
{
session.Scripts.Add(executeScript);
}
// Build and run the session
session.LoadScripts(true);
StockSession.Instance = session;
return(session);
}
public static StockSession Start(List <string> sources, List <string> sinkScripts, string executeScript)
{
StockSession session = new StockSession();
List <string> script = new List <string>();
Directory.CreateDirectory("tmp");
// Convert any legacy files before further processing
var legacyFiles = sources.Where((s) => { return(s.EndsWith(".csv")); }).ToList();
if (legacyFiles.Count() > 0)
{
System.Windows.Forms.SaveFileDialog saveDiag = new System.Windows.Forms.SaveFileDialog();
saveDiag.Title = "Save converted data file as...";
if (saveDiag.ShowDialog() == System.Windows.Forms.DialogResult.OK)
{
var convertedFile = StockDataFile.Convert(legacyFiles, new FileStream(saveDiag.FileName, FileMode.Create));
convertedFile.Close();
sources.Add(saveDiag.FileName);
}
else
{
// Cancel running the script
return(null);
}
foreach (var l in legacyFiles)
{
sources.Remove(l);
}
}
session.SourceFile = StockDataFile.Open(sources.ConvertAll <Stream>((s) => { return(new FileStream(s, FileMode.Open)); }));
script.Add("tmp/" + SOURCE_CLASS + ".cs");
using (var file = new StreamWriter(new FileStream(script.Last(), FileMode.Create))) file.Write(session.SourceFile.GetSourceCode(SOURCE_CLASS));
session.SinkFile = new StockDataFile(sinkScripts.ConvertAll <string>((f) => { return(Path.GetFileNameWithoutExtension(f)); }), sinkScripts.ConvertAll <string>((f) => { return(File.ReadAllText(f)); }));
session.SinkFile.Interval = session.SourceFile.Interval;
script.Add("tmp/" + SINK_CLASS + ".cs");
using (var file = new StreamWriter(new FileStream(script.Last(), FileMode.Create))) file.Write(session.SinkFile.GetSourceCode(SINK_CLASS));
// Create the evaluator file (needs to be compiled in the script since it references StockDataSource)
string[] embeddedFiles = new string[]
{
"RobinhoodDesktop.Script.StockEvaluator.cs",
"RobinhoodDesktop.Script.StockProcessor.cs"
};
foreach (var f in embeddedFiles)
{
script.Add(string.Format("tmp/{0}.cs", f.Substring(24, f.Length - 27)));
StringBuilder analyzerCode = new StringBuilder();
analyzerCode.Append(new StreamReader(Assembly.GetExecutingAssembly().GetManifestResourceStream(f)).ReadToEnd());
using (var file = new StreamWriter(new FileStream(script.Last(), FileMode.Create))) file.Write(StockDataFile.FormatSource(analyzerCode.ToString()));
}
// Add the user defined analyzers
foreach (string path in Directory.GetFiles(@"Script/Decision", "*.cs", SearchOption.AllDirectories))
{
script.Add(path);
}
foreach (string path in Directory.GetFiles(@"Script/Action", "*.cs", SearchOption.AllDirectories))
{
script.Add(path);
}
// Get the code that will actually run the session
script.Add(executeScript);
// Build and run the session
#if DEBUG
var isDebug = true;
#else
var isDebug = false;
#endif
try
{
if (!string.IsNullOrEmpty(executeScript))
{
session.ScriptInstance = CSScript.LoadFiles(script.ToArray(), null, isDebug, "TensorFlow.NET.dll", "Google.Protobuf.dll", "NumSharp.Lite", "netstandard", "System.Memory", "System.Numerics");
var run = session.ScriptInstance.GetStaticMethod("*.Run", session);
run(session);
}
}
catch (Exception ex)
{
System.Windows.Forms.MessageBox.Show(ex.ToString());
session.SourceFile.Close();
session.SinkFile.Close();
}
return(session);
}
public void RunSession()
{
System.Windows.Forms.OpenFileDialog diag = new System.Windows.Forms.OpenFileDialog();
diag.Multiselect = false;
diag.Title = "Run Session Script...";
if (diag.ShowDialog() == System.Windows.Forms.DialogResult.OK)
{
List <string> script = new List <string>();
Directory.CreateDirectory("tmp");
SourceFile = new StockDataFile(new List <string>()
{
}, new List <string>()
{
});
script.Add("tmp/" + SOURCE_CLASS + ".cs");
using (var file = new StreamWriter(new FileStream(script.Last(), FileMode.Create))) file.Write(SourceFile.GetSourceCode(SOURCE_CLASS));
SinkFile = new StockDataFile(new List <string>()
{
"MovingAverage"
}, new List <string>()
{
File.ReadAllText(@"Script/Data/MovingAverage.cs")
});
script.Add("tmp/" + SINK_CLASS + ".cs");
using (var file = new StreamWriter(new FileStream(script.Last(), FileMode.Create))) file.Write(SinkFile.GetSourceCode(SINK_CLASS));
// Create the evaluator file (needs to be compiled in the script since it references StockDataSource)
string[] embeddedFiles = new string[]
{
"RobinhoodDesktop.Script.StockEvaluator.cs",
"RobinhoodDesktop.Script.StockProcessor.cs"
};
foreach (var f in embeddedFiles)
{
script.Add(string.Format("tmp/{0}.cs", f.Substring(24, f.Length - 27)));
StringBuilder analyzerCode = new StringBuilder();
analyzerCode.Append(new StreamReader(Assembly.GetExecutingAssembly().GetManifestResourceStream(f)).ReadToEnd());
using (var file = new StreamWriter(new FileStream(script.Last(), FileMode.Create))) file.Write(StockDataFile.FormatSource(analyzerCode.ToString()));
}
// Add the user defined analyzers
foreach (string path in Directory.GetFiles(@"Script/Decision", "*.cs", SearchOption.AllDirectories))
{
script.Add(path);
}
foreach (string path in Directory.GetFiles(@"Script/Action", "*.cs", SearchOption.AllDirectories))
{
script.Add(path);
}
// Get the code that will actually run the session
script.Add(diag.FileName);
// Build and run the session
#if DEBUG
var isDebug = true;
#else
var isDebug = false;
#endif
try
{
var scriptInstance = CSScript.LoadFiles(script.ToArray(), null, isDebug);
var run = scriptInstance.GetStaticMethod("*.Run", this);
run(this);
}
catch (Exception ex)
{
System.Windows.Forms.MessageBox.Show(ex.ToString());
}
}
}
public void Run()
{
System.Windows.Forms.OpenFileDialog diag = new System.Windows.Forms.OpenFileDialog();
diag.Multiselect = true;
diag.Title = "Open Stock Data File...";
if (diag.ShowDialog() == System.Windows.Forms.DialogResult.OK)
{
List <string> script = new List <string>();
Directory.CreateDirectory("tmp");
// Get the source file
if (diag.FileName.EndsWith(".csv"))
{
System.Windows.Forms.SaveFileDialog saveDiag = new System.Windows.Forms.SaveFileDialog();
if (saveDiag.ShowDialog() != System.Windows.Forms.DialogResult.OK)
{
return;
}
SourceFile = StockDataFile.Convert(diag.FileNames.ToList(), new FileStream(saveDiag.FileName, FileMode.Create));
}
else
{
SourceFile = StockDataFile.Open(new FileStream(diag.FileName, FileMode.Open));
}
script.Add("tmp/" + SOURCE_CLASS + ".cs");
using (var file = new StreamWriter(new FileStream(script.Last(), FileMode.Create))) file.Write(SourceFile.GetSourceCode(SOURCE_CLASS));
// Create the sink file
//SinkFile = new StockDataFile(new List<string>() { }, new List<string>() { });
SinkFile = new StockDataFile(new List <string>()
{
"MovingAverage"
}, new List <string>()
{
File.ReadAllText(@"Script/Data/MovingAverage.cs")
});
script.Add("tmp/" + SINK_CLASS + ".cs");
using (var file = new StreamWriter(new FileStream(script.Last(), FileMode.Create))) file.Write(SinkFile.GetSourceCode(SINK_CLASS));
// Create the analyzer file (needs to be compiled in the script since it references StockDataSource)
var analyzerFilename = "RobinhoodDesktop.Script.StockAnalyzer.cs";
script.Add("tmp/StockAnalyzer.cs");
StringBuilder analyzerCode = new StringBuilder();
analyzerCode.Append(new StreamReader(Assembly.GetExecutingAssembly().GetManifestResourceStream(analyzerFilename)).ReadToEnd());
using (var file = new StreamWriter(new FileStream(script.Last(), FileMode.Create))) file.Write(StockDataFile.FormatSource(analyzerCode.ToString()));
// Add the user defined analyzers
string[] analyzerPaths = Directory.GetFiles(@"Script/Decision", "*.cs", SearchOption.AllDirectories);
foreach (string path in analyzerPaths)
{
script.Add(path);
}
// Get the code that will actually run the session
script.Add("tmp/StockSessionScript.cs");
var sessionFilename = "RobinhoodDesktop.Script.StockSessionScript.cs";
using (var file = new StreamWriter(new FileStream(script.Last(), FileMode.Create))) file.Write(new StreamReader(Assembly.GetExecutingAssembly().GetManifestResourceStream(sessionFilename)).ReadToEnd());
// Build and run the session
#if DEBUG
var isDebug = true;
#else
var isDebug = false;
#endif
#if true
try
{
var scriptInstance = CSScript.LoadFiles(script.ToArray(), null, isDebug);
var run = scriptInstance.GetStaticMethod("RobinhoodDesktop.Script.StockSessionScript.Run", this);
run(this);
}
catch (Exception ex)
{
System.Windows.Forms.MessageBox.Show(ex.ToString());
}
#else
// Set up the derived data sink
var sourceData = SourceFile.GetSegments <StockDataBase>();
var sinkData = StockDataSetDerived <StockDataBase, StockDataBase> .Derive(sourceData, SinkFile, (data, idx) =>
{
var point = new StockDataBase();
point.Update(data, idx);
return(point);
});
SinkFile.SetSegments(sinkData);
// Load the first set of data
foreach (var pair in sinkData)
{
foreach (var set in pair.Value)
{
set.Load();
}
}
#endif
// Cleanup
SourceFile.Close();
}
}
