/// <summary>
/// Executes a run of processing the stock data
/// </summary>
/// <param name="session">The session configuration</param>
public static void Run(StockSession session)
{
// Create the stock processor
StockProcessor processor = new StockProcessor(session);
// Set the change percentages to monitor
StockDataSink.ChangePercentages = new float[] { -0.025f, -0.05f, -0.10f };
// Set the evaluator and resulting action
processor.Evaluator = new PriceEvaluator()
{
Percentage = 0.025f,
MonitorIncrease = true,
MonitorDecrease = false,
ReferencePrices = new Dictionary <string, float>()
{
{ "GNTX", 18.0f }
}
};
processor.Action = new StockAction();
processor.Action.Do += processor.Action.Notify;
// Add targets for the stocks to monitor
processor.Add(new StockProcessor.ProcessingTarget("GNTX"));
// Run the data through the processor
processor.Process(false, StockProcessor.MemoryScheme.MEM_KEEP_DERIVED);
}
/// <summary>
/// Loads the data from the source file
/// <param name="session">The session currently being processed</param>
/// </summary>
public virtual void Load(StockSession session)
{
if (!IsReady())
{
File.LoadSegment(this, session);
}
}
/// <summary>
/// Executes a run of processing the stock data
/// </summary>
/// <param name="session">The session configuration</param>
public static void Run(StockSession session)
{
// Set up the derived data sink
var sourceData = session.SourceFile.GetSegments <StockDataSource>();
var sinkData = StockDataSetDerived <StockDataSink, StockDataSource> .Derive(sourceData, session.SinkFile, (data, idx) =>
{
var point = new StockDataSink();
point.Update(data, idx);
return(point);
});
session.SinkFile.SetSegments(sinkData);
// Find when a stock increases by a certain amount
PriceMonitor priceMonitor = new PriceMonitor(0.025);
priceMonitor.MonitorDecrease = false;
sinkData["AA"][0].Load();
priceMonitor.Monitor("AA", sinkData["AA"][0][0].Price, (string stock, float startPrice, float endPrice, DateTime time) =>
{
System.Windows.Forms.MessageBox.Show(string.Format("{0} {1:C}->{2:C} @ {3}", stock, startPrice, endPrice, time));
return(true);
});
// Load the first set of data
foreach (var pair in sinkData)
{
foreach (var set in pair.Value)
{
set.Load();
priceMonitor.Process(set, 0);
}
}
}
/// <summary>
/// Loads the segment data from the source
/// </summary>
/// <typeparam name="T">The data point type</typeparam>
/// <param name="session">The session this is part of</param>
/// <param name="segment">The segment to populate</param>
public override void LoadSegment <T>(StockDataSet <T> segment, StockSession session = null)
{
if ((this.LoadMethod == null) && (session != null))
{
this.LoadMethod = session.ScriptInstance.GetStaticMethod("*.Load", this, "", DateTime.Now);
}
segment.DataSet.Initialize((T[])LoadMethod(this, segment.Symbol, segment.Start));
}
/// <summary>
/// Loads any additional static information from the file
/// <param name="session">The session this is part of</param>
/// </summary>
public virtual void LoadStaticData(StockSession session)
{
var loadMethod = session.ScriptInstance.GetStaticMethod("*.Load", File);
FileMutex.WaitOne();
File.Seek(LoadAddress, SeekOrigin.Begin);
loadMethod(File);
FileMutex.ReleaseMutex();
}
/// <summary>
/// Loads any additional static information from the file
/// <param name="session">The session this is being loaded as part of</param>
/// </summary>
public override void LoadStaticData(StockSession session)
{
for (int idx = 0; idx < Sources.Count; idx++)
{
var src = Sources[idx];
var loadMethod = session.ScriptInstance.GetStaticMethod("RobinhoodDesktop.Script.Source" + idx + ".Load", src.File);
src.FileMutex.WaitOne();
src.File.Seek(src.LoadAddress, SeekOrigin.Begin);
loadMethod(src.File);
src.FileMutex.ReleaseMutex();
}
}
public StockProcessor(StockSession session)
{
this.Session = session;
// Load the source file static information
Session.SourceFile.LoadStaticData(session);
// Create the derived data set
HistoricalData = StockDataSetDerived <StockDataSink, StockDataSource, StockProcessingState> .Derive(Session.SourceFile.GetSegments <StockDataSource>(), Session.SinkFile, CreateSink, GetProcessingState);
DerivedData = StockDataSetDerived <StockDataSink, StockDataSource, StockProcessingState> .CastToBase(HistoricalData);
Session.SinkFile.SetSegments <StockDataSink>(DerivedData);
}
/// <summary>
/// Loads the data from the source file
/// <param name="session">The session currently being processed</param>
/// </summary>
public override void Load(StockSession session)
{
if (!IsReady())
{
SourceData.Load(session);
int endIdx = (int)((SourceData.DataSet.Count * SourceData.Interval.Ticks) / Interval.Ticks);
DataSet.Resize(endIdx);
ProcessingState = GetState(this);
for (int idx = DataSet.Count; idx < endIdx; idx++)
{
Create(this, idx);
}
DataSet.Initialize(DataSet.InternalArray);
}
}
/// <summary>
/// Returns the stock processor instance, or creates one if there isn't one already
/// </summary>
/// <param name="session">The stock session to create the processor for</param>
/// <returns>The stock processor instance</returns>
public static StockProcessor GetInstance(StockSession session = null)
{
if (Instance == null)
{
if (session != null)
{
Instance = new StockProcessor(session);
}
else
{
throw new Exception("Must specify the session the first time the processor instance is accessed.");
}
}
return(Instance);
}
public StockProcessor(StockSession session)
{
this.Session = session;
// Load the source file static information
Session.SourceFile.LoadStaticData(session);
// Create the derived data set
HistoricalData = StockDataSetDerived <StockDataSink, StockDataSource, StockProcessingState> .Derive(Session.SourceFile.GetSegments <StockDataSource>(), Session.SinkFile, CreateSink, GetProcessingState);
DerivedData = StockDataSetDerived <StockDataSink, StockDataSource, StockProcessingState> .CastToInterface(HistoricalData);
Session.Data = DerivedData;
Session.SinkFile.SetSegments <StockDataSink>(StockDataSetDerived <StockDataSink, StockDataSource, StockProcessingState> .CastToBase(HistoricalData));
foreach (var pair in HistoricalData)
{
var state = new StockProcessingState();
state.DataSet = pair.Value;
ProcessingStates[pair.Key] = new Dictionary <TimeSpan, StockProcessingState>()
{
};
}
}
public static StockSession Start(List <string> sources, List <string> sinkScripts, string executeScript)
{
StockSession session = new StockSession();
session.Scripts.Clear();
Directory.CreateDirectory("tmp");
// Convert any legacy files before further processing
var legacyFiles = sources.Where((s) => { return(s.EndsWith(".csv")); }).ToList();
if (legacyFiles.Count() > 0)
{
System.Windows.Forms.SaveFileDialog saveDiag = new System.Windows.Forms.SaveFileDialog();
saveDiag.Title = "Save converted data file as...";
saveDiag.CheckFileExists = false;
if (saveDiag.ShowDialog() == System.Windows.Forms.DialogResult.OK)
{
List <string> convertedFileNames;
var convertedFiles = StockDataFile.ConvertByMonth(legacyFiles, Path.GetDirectoryName(saveDiag.FileName), out convertedFileNames);
foreach (var cf in convertedFileNames)
{
sources.Add(cf);
}
}
else
{
// Cancel running the script
return(null);
}
foreach (var l in legacyFiles)
{
sources.Remove(l);
}
}
session.SourceFile = StockDataFile.Open(sources.ConvertAll <Stream>((s) => { return(new FileStream(s, FileMode.Open)); }));
session.Scripts.Add("tmp/" + SOURCE_CLASS + ".cs");
using (var file = new StreamWriter(new FileStream(session.Scripts.Last(), FileMode.Create))) file.Write(session.SourceFile.GetSourceCode(SOURCE_CLASS));
// Put the data set reference script first
List <string> totalSinkScripts = sinkScripts.ToList();
totalSinkScripts.Insert(0, "Script\\Data\\DataSetReference.cs");
session.SinkFile = new StockDataFile(totalSinkScripts.ConvertAll <string>((f) => { return(Path.GetFileNameWithoutExtension(f)); }), totalSinkScripts.ConvertAll <string>((f) => { return(File.ReadAllText(f)); }));
session.SinkFile.Interval = session.SourceFile.Interval;
session.Scripts.Add("tmp/" + SINK_CLASS + ".cs");
using (var file = new StreamWriter(new FileStream(session.Scripts.Last(), FileMode.Create))) file.Write(session.SinkFile.GenStockDataSink());
session.Scripts.AddRange(totalSinkScripts);
// Create the evaluator file (needs to be compiled in the script since it references StockDataSource)
string[] embeddedFiles = new string[]
{
"RobinhoodDesktop.Script.StockEvaluator.cs",
"RobinhoodDesktop.Script.StockProcessor.cs"
};
foreach (var f in embeddedFiles)
{
session.Scripts.Add(string.Format("tmp/{0}.cs", f.Substring(24, f.Length - 27)));
StringBuilder analyzerCode = new StringBuilder();
analyzerCode.Append(new StreamReader(Assembly.GetExecutingAssembly().GetManifestResourceStream(f)).ReadToEnd());
using (var file = new StreamWriter(new FileStream(session.Scripts.Last(), FileMode.Create))) file.Write(StockDataFile.FormatSource(analyzerCode.ToString()));
}
// Add the user defined analyzers
foreach (string path in Directory.GetFiles(@"Script/Decision", "*.cs", SearchOption.AllDirectories))
{
session.Scripts.Add(path);
}
foreach (string path in Directory.GetFiles(@"Script/Action", "*.cs", SearchOption.AllDirectories))
{
session.Scripts.Add(path);
}
// Get the code that will actually run the session
if (!string.IsNullOrEmpty(executeScript))
{
session.Scripts.Add(executeScript);
}
// Build and run the session
session.LoadScripts(true);
StockSession.Instance = session;
return(session);
}
/// <summary>
/// Save this instance to the stream
/// </summary>
/// <param name="session">The session this is part of</param>
/// <param name="s">The stream to save this to</param>
public void Save <T>(Stream s, Type dataType, Dictionary <string, List <StockDataSet <T> > > segments, StockSession session = null) where T : struct, StockData
{
var headerSer = new Serializer(new List <Type>()
{
typeof(StockDataFile)
});
var dataSer = new Serializer(new List <Type>()
{
dataType.MakeArrayType()
});
this.File = s;
// Serialize the segments
s.Seek(0x10, SeekOrigin.End); // Leave some space at the beginning of the stream to store the offset to the serialized stock file
foreach (KeyValuePair <string, List <StockDataSet <T> > > pair in segments)
{
List <Tuple <DateTime, long> > allSegments = this.Segments[pair.Key];
int matchingIdx = 0;
for (int segIdx = 0; segIdx < pair.Value.Count; segIdx++)
{
StockDataSet <T> set = pair.Value[segIdx];
for (; matchingIdx < allSegments.Count; matchingIdx++)
{
if (set.Start == allSegments[matchingIdx].Item1)
{
allSegments[matchingIdx] = new Tuple <DateTime, long>(set.Start, s.Position);
set.StreamAddress = s.Position;
if (typeof(T) == dataType)
{
//dataSer.Serialize(s, set.DataSet.InternalArray);
Store(s, set.DataSet.InternalArray, set.DataSet.Count);
}
else
{
var data_points = Array.CreateInstance(dataType, set.DataSet.Count);
for (int pntIdx = 0; pntIdx < set.DataSet.Count; pntIdx++)
{
data_points.SetValue(set.DataSet.InternalArray[pntIdx], pntIdx);
}
dataSer.Serialize(s, data_points);
}
matchingIdx++;
break;
}
}
}
}
// Serialize the header with the updated segment addresses
long headerAddress = s.Position;
headerSer.Serialize(s, this);
// Save any script-specific data
if ((session != null) && (session.ScriptInstance != null))
{
var saveMethod = session.ScriptInstance.GetStaticMethod("*.Save", s);
saveMethod(s);
}
// Set the offset to the header
s.Seek(0, SeekOrigin.Begin);
headerSer.Serialize(s, headerAddress);
}
/// <summary>
/// Loads the segment data from the source
/// </summary>
/// <typeparam name="T">The data point type</typeparam>
/// <param name="segment">The segment to populate</param>
/// <param name="session">The session this is part of</param>
public virtual void LoadSegment <T>(StockDataSet <T> segment, StockSession session = null) where T : struct, StockData
{
FileMutex.WaitOne();
segment.DataSet.Initialize(LoadData <T>(segment.StreamAddress));
FileMutex.ReleaseMutex();
}
public static StockSession Start(List <string> sources, List <string> sinkScripts, string executeScript)
{
StockSession session = new StockSession();
List <string> script = new List <string>();
Directory.CreateDirectory("tmp");
// Convert any legacy files before further processing
var legacyFiles = sources.Where((s) => { return(s.EndsWith(".csv")); }).ToList();
if (legacyFiles.Count() > 0)
{
System.Windows.Forms.SaveFileDialog saveDiag = new System.Windows.Forms.SaveFileDialog();
saveDiag.Title = "Save converted data file as...";
if (saveDiag.ShowDialog() == System.Windows.Forms.DialogResult.OK)
{
var convertedFile = StockDataFile.Convert(legacyFiles, new FileStream(saveDiag.FileName, FileMode.Create));
convertedFile.Close();
sources.Add(saveDiag.FileName);
}
else
{
// Cancel running the script
return(null);
}
foreach (var l in legacyFiles)
{
sources.Remove(l);
}
}
session.SourceFile = StockDataFile.Open(sources.ConvertAll <Stream>((s) => { return(new FileStream(s, FileMode.Open)); }));
script.Add("tmp/" + SOURCE_CLASS + ".cs");
using (var file = new StreamWriter(new FileStream(script.Last(), FileMode.Create))) file.Write(session.SourceFile.GetSourceCode(SOURCE_CLASS));
session.SinkFile = new StockDataFile(sinkScripts.ConvertAll <string>((f) => { return(Path.GetFileNameWithoutExtension(f)); }), sinkScripts.ConvertAll <string>((f) => { return(File.ReadAllText(f)); }));
session.SinkFile.Interval = session.SourceFile.Interval;
script.Add("tmp/" + SINK_CLASS + ".cs");
using (var file = new StreamWriter(new FileStream(script.Last(), FileMode.Create))) file.Write(session.SinkFile.GetSourceCode(SINK_CLASS));
// Create the evaluator file (needs to be compiled in the script since it references StockDataSource)
string[] embeddedFiles = new string[]
{
"RobinhoodDesktop.Script.StockEvaluator.cs",
"RobinhoodDesktop.Script.StockProcessor.cs"
};
foreach (var f in embeddedFiles)
{
script.Add(string.Format("tmp/{0}.cs", f.Substring(24, f.Length - 27)));
StringBuilder analyzerCode = new StringBuilder();
analyzerCode.Append(new StreamReader(Assembly.GetExecutingAssembly().GetManifestResourceStream(f)).ReadToEnd());
using (var file = new StreamWriter(new FileStream(script.Last(), FileMode.Create))) file.Write(StockDataFile.FormatSource(analyzerCode.ToString()));
}
// Add the user defined analyzers
foreach (string path in Directory.GetFiles(@"Script/Decision", "*.cs", SearchOption.AllDirectories))
{
script.Add(path);
}
foreach (string path in Directory.GetFiles(@"Script/Action", "*.cs", SearchOption.AllDirectories))
{
script.Add(path);
}
// Get the code that will actually run the session
script.Add(executeScript);
// Build and run the session
#if DEBUG
var isDebug = true;
#else
var isDebug = false;
#endif
try
{
if (!string.IsNullOrEmpty(executeScript))
{
session.ScriptInstance = CSScript.LoadFiles(script.ToArray(), null, isDebug, "TensorFlow.NET.dll", "Google.Protobuf.dll", "NumSharp.Lite", "netstandard", "System.Memory", "System.Numerics");
var run = session.ScriptInstance.GetStaticMethod("*.Run", session);
run(session);
}
}
catch (Exception ex)
{
System.Windows.Forms.MessageBox.Show(ex.ToString());
session.SourceFile.Close();
session.SinkFile.Close();
}
return(session);
}
/// <summary>
/// Executes a run of processing the stock data
/// </summary>
/// <param name="session">The session configuration</param>
public static void Run(StockSession session)
{
// Create the stock processor
StockProcessor processor = new StockProcessor(session);
// Get the sizes of the features and labels
int dstIdx = 0;
int numDataPoints = 0;
int numFeatures, numLabels;
float[,] tmpDst = new float[1, 1024];
int[,] tmpLabels = new int[1, 1];
var tmpDataPoint = processor.DerivedData.First().Value[0];
tmpDataPoint.Load(session);
PopulateData(tmpDataPoint, tmpDst, tmpLabels, ref dstIdx, out numFeatures, out numLabels);
// Determine the total number of data points
List <string> symbols = new List <string>()
{
"GNTX"
};
foreach (var s in symbols)
{
List <StockDataSet <StockDataSink> > sources;
if (!processor.DerivedData.TryGetValue(s, out sources))
{
continue;
}
for (int i = 0; i < sources.Count; i++)
{
sources[i].Load(session);
numDataPoints += sources[i].Count;
}
}
// Allocate the feature and label arrays
float[,] features = new float[numDataPoints, numFeatures];
int[,] labels = new int[numDataPoints, numLabels];
// Load the data
dstIdx = 0;
foreach (var s in symbols)
{
List <StockDataSet <StockDataSink> > sources;
if (!processor.DerivedData.TryGetValue(s, out sources))
{
continue;
}
// Create a table of each data point in the specified range
for (int i = 0; i < sources.Count; i++)
{
sources[i].Load(session);
PopulateData(sources[i], features, labels, ref dstIdx, out numFeatures, out numLabels);
}
}
// Create the Machine Learning instance
MLInstance ml = new MLInstance();
ml.BuildFullyConnectedGraph(new int[] { numFeatures, numFeatures, numLabels });
ml.PrepareData(features, labels);
ml.Train();
}
/// <summary>
/// Executes a run of processing the stock data
/// </summary>
/// <param name="session">The session configuration</param>
public static void Run(StockSession session)
{
}
/// <summary>
/// Creates a stock data chart
/// </summary>
/// <param name="session">The session the chart is a part of</param>
/// <returns>The chart, as a generic control</returns>
public static Control CreateChart(StockSession session)
{
StockProcessor processor = new StockProcessor(session);
return((Control)(new DataChartGui <StockDataSink>(processor.DerivedData, session.SinkFile, session)).GuiPanel);
}
/// <summary>
/// Compiles a script to evaluate the specified expression
/// </summary>
/// <param name="expression">The expression to get a value from the dataset</param>
/// <returns>The delegate used to get the desired value from a dataset</returns>
public Func <StockDataInterface, int, object> GetExpressionEvaluator(string expression, StockSession session)
{
Func <StockDataSet <T>, int, object> accessor = null;
// Check for the special case of requesting the time
if (expression.Equals("Time"))
{
accessor = new Func <StockDataSet <T>, int, object>((data, index) =>
{
return(data.Time(index));
});
}
else
{
// Order the list based on the lame length
Type dataType = GetDataType();
var fields = new List <string>();
fields.AddRange(typeof(T).GetFields().ToList().ConvertAll((f) => { return(f.Name); }));
fields.AddRange(typeof(T).GetProperties().ToList().ConvertAll((f) => { return(f.Name); }));
fields.AddRange(typeof(T).GetMethods().ToList().ConvertAll((f) => { return(f.Name); }));
fields.Sort((f1, f2) => { return(f2.Length.CompareTo(f1.Length)); });
// First remove any string literals
string src = expression;
List <string> stringLiterals = new List <string>();
for (int i = src.IndexOf('"'); (i >= 0) && (i < src.Length); i = src.IndexOf('"'))
{
int end = src.IndexOf('"', i + 1);
if ((end >= 0) && (end < src.Length))
{
stringLiterals.Add(src.Substring(i, (end - i) + 1));
src = src.Replace(stringLiterals.Last(), string.Format("<=s{0}>", stringLiterals.Count - 1));
}
}
// First replace the fields with an index to prevent names within a name from getting messed up
for (int i = 0; i < fields.Count; i++)
{
src = System.Text.RegularExpressions.Regex.Replace(src, fields[i] + "([^a-zA-Z0-9_]|$)", string.Format("<={0}>$1", i));
}
// Next pre-pend the data set to the field names
for (int i = 0; i < fields.Count; i++)
{
src = src.Replace(string.Format("<={0}>", i), string.Format("data[updateIndex].{0}", fields[i]));
}
// Restore the string literals
for (int i = 0; i < stringLiterals.Count; i++)
{
src = src.Replace(string.Format("<=s{0}>", i), stringLiterals[i]);
}
// Build the expression into an accessor function
//src = "namespace RobinhoodDesktop.Script { public class ExpressionAccessor{ public static object GetValue(StockDataSet<" + typeof(T).Name + "> data, int updateIndex) { return " + src + ";} } }";
var compiler = CSScript.MonoEvaluator.ReferenceAssemblyOf <T>();
foreach (var s in session.Scripts.Values)
{
compiler = compiler.ReferenceAssembly(s.Location);
}
//var script = CSScript.LoadCode(src);
//CSScript.Evaluator.
//accessor = script.GetStaticMethod("*.*");
accessor = compiler.LoadDelegate <Func <StockDataSet <T>, int, object> >(@"object GetValue(RobinhoodDesktop.Script.StockDataSet<" + typeof(T).FullName + "> data, int updateIndex) { return " + src + ";}");
}
return(new Func <StockDataInterface, int, object>((data, index) => { return accessor((StockDataSet <T>)data, index); }));
}
/// <summary>
/// Loads the segment data from the source
/// </summary>
/// <typeparam name="T">The data point type</typeparam>
/// <param name="segment">The segment to populate</param>
/// <param name="session">The session this is part of</param>
public virtual void LoadSegment <T>(StockDataSet <T> segment, StockSession session = null) where T : struct, StockData
{
segment.DataSet.Initialize(LoadData <T>(segment.StreamAddress));
}