/// <summary> /// Executes a run of processing the stock data /// </summary> /// <param name="session">The session configuration</param> public static void Run(StockSession session) { // Create the stock processor StockProcessor processor = new StockProcessor(session); // Set the change percentages to monitor StockDataSink.ChangePercentages = new float[] { -0.025f, -0.05f, -0.10f }; // Set the evaluator and resulting action processor.Evaluator = new PriceEvaluator() { Percentage = 0.025f, MonitorIncrease = true, MonitorDecrease = false, ReferencePrices = new Dictionary <string, float>() { { "GNTX", 18.0f } } }; processor.Action = new StockAction(); processor.Action.Do += processor.Action.Notify; // Add targets for the stocks to monitor processor.Add(new StockProcessor.ProcessingTarget("GNTX")); // Run the data through the processor processor.Process(false, StockProcessor.MemoryScheme.MEM_KEEP_DERIVED); }
/// <summary> /// Loads the data from the source file /// <param name="session">The session currently being processed</param> /// </summary> public virtual void Load(StockSession session) { if (!IsReady()) { File.LoadSegment(this, session); } }
/// <summary> /// Executes a run of processing the stock data /// </summary> /// <param name="session">The session configuration</param> public static void Run(StockSession session) { // Set up the derived data sink var sourceData = session.SourceFile.GetSegments <StockDataSource>(); var sinkData = StockDataSetDerived <StockDataSink, StockDataSource> .Derive(sourceData, session.SinkFile, (data, idx) => { var point = new StockDataSink(); point.Update(data, idx); return(point); }); session.SinkFile.SetSegments(sinkData); // Find when a stock increases by a certain amount PriceMonitor priceMonitor = new PriceMonitor(0.025); priceMonitor.MonitorDecrease = false; sinkData["AA"][0].Load(); priceMonitor.Monitor("AA", sinkData["AA"][0][0].Price, (string stock, float startPrice, float endPrice, DateTime time) => { System.Windows.Forms.MessageBox.Show(string.Format("{0} {1:C}->{2:C} @ {3}", stock, startPrice, endPrice, time)); return(true); }); // Load the first set of data foreach (var pair in sinkData) { foreach (var set in pair.Value) { set.Load(); priceMonitor.Process(set, 0); } } }
/// <summary> /// Loads the segment data from the source /// </summary> /// <typeparam name="T">The data point type</typeparam> /// <param name="session">The session this is part of</param> /// <param name="segment">The segment to populate</param> public override void LoadSegment <T>(StockDataSet <T> segment, StockSession session = null) { if ((this.LoadMethod == null) && (session != null)) { this.LoadMethod = session.ScriptInstance.GetStaticMethod("*.Load", this, "", DateTime.Now); } segment.DataSet.Initialize((T[])LoadMethod(this, segment.Symbol, segment.Start)); }
/// <summary> /// Loads any additional static information from the file /// <param name="session">The session this is part of</param> /// </summary> public virtual void LoadStaticData(StockSession session) { var loadMethod = session.ScriptInstance.GetStaticMethod("*.Load", File); FileMutex.WaitOne(); File.Seek(LoadAddress, SeekOrigin.Begin); loadMethod(File); FileMutex.ReleaseMutex(); }
/// <summary> /// Loads any additional static information from the file /// <param name="session">The session this is being loaded as part of</param> /// </summary> public override void LoadStaticData(StockSession session) { for (int idx = 0; idx < Sources.Count; idx++) { var src = Sources[idx]; var loadMethod = session.ScriptInstance.GetStaticMethod("RobinhoodDesktop.Script.Source" + idx + ".Load", src.File); src.FileMutex.WaitOne(); src.File.Seek(src.LoadAddress, SeekOrigin.Begin); loadMethod(src.File); src.FileMutex.ReleaseMutex(); } }
public StockProcessor(StockSession session) { this.Session = session; // Load the source file static information Session.SourceFile.LoadStaticData(session); // Create the derived data set HistoricalData = StockDataSetDerived <StockDataSink, StockDataSource, StockProcessingState> .Derive(Session.SourceFile.GetSegments <StockDataSource>(), Session.SinkFile, CreateSink, GetProcessingState); DerivedData = StockDataSetDerived <StockDataSink, StockDataSource, StockProcessingState> .CastToBase(HistoricalData); Session.SinkFile.SetSegments <StockDataSink>(DerivedData); }
/// <summary> /// Loads the data from the source file /// <param name="session">The session currently being processed</param> /// </summary> public override void Load(StockSession session) { if (!IsReady()) { SourceData.Load(session); int endIdx = (int)((SourceData.DataSet.Count * SourceData.Interval.Ticks) / Interval.Ticks); DataSet.Resize(endIdx); ProcessingState = GetState(this); for (int idx = DataSet.Count; idx < endIdx; idx++) { Create(this, idx); } DataSet.Initialize(DataSet.InternalArray); } }
/// <summary> /// Returns the stock processor instance, or creates one if there isn't one already /// </summary> /// <param name="session">The stock session to create the processor for</param> /// <returns>The stock processor instance</returns> public static StockProcessor GetInstance(StockSession session = null) { if (Instance == null) { if (session != null) { Instance = new StockProcessor(session); } else { throw new Exception("Must specify the session the first time the processor instance is accessed."); } } return(Instance); }
public StockProcessor(StockSession session) { this.Session = session; // Load the source file static information Session.SourceFile.LoadStaticData(session); // Create the derived data set HistoricalData = StockDataSetDerived <StockDataSink, StockDataSource, StockProcessingState> .Derive(Session.SourceFile.GetSegments <StockDataSource>(), Session.SinkFile, CreateSink, GetProcessingState); DerivedData = StockDataSetDerived <StockDataSink, StockDataSource, StockProcessingState> .CastToInterface(HistoricalData); Session.Data = DerivedData; Session.SinkFile.SetSegments <StockDataSink>(StockDataSetDerived <StockDataSink, StockDataSource, StockProcessingState> .CastToBase(HistoricalData)); foreach (var pair in HistoricalData) { var state = new StockProcessingState(); state.DataSet = pair.Value; ProcessingStates[pair.Key] = new Dictionary <TimeSpan, StockProcessingState>() { }; } }
public static StockSession Start(List <string> sources, List <string> sinkScripts, string executeScript) { StockSession session = new StockSession(); session.Scripts.Clear(); Directory.CreateDirectory("tmp"); // Convert any legacy files before further processing var legacyFiles = sources.Where((s) => { return(s.EndsWith(".csv")); }).ToList(); if (legacyFiles.Count() > 0) { System.Windows.Forms.SaveFileDialog saveDiag = new System.Windows.Forms.SaveFileDialog(); saveDiag.Title = "Save converted data file as..."; saveDiag.CheckFileExists = false; if (saveDiag.ShowDialog() == System.Windows.Forms.DialogResult.OK) { List <string> convertedFileNames; var convertedFiles = StockDataFile.ConvertByMonth(legacyFiles, Path.GetDirectoryName(saveDiag.FileName), out convertedFileNames); foreach (var cf in convertedFileNames) { sources.Add(cf); } } else { // Cancel running the script return(null); } foreach (var l in legacyFiles) { sources.Remove(l); } } session.SourceFile = StockDataFile.Open(sources.ConvertAll <Stream>((s) => { return(new FileStream(s, FileMode.Open)); })); session.Scripts.Add("tmp/" + SOURCE_CLASS + ".cs"); using (var file = new StreamWriter(new FileStream(session.Scripts.Last(), FileMode.Create))) file.Write(session.SourceFile.GetSourceCode(SOURCE_CLASS)); // Put the data set reference script first List <string> totalSinkScripts = sinkScripts.ToList(); totalSinkScripts.Insert(0, "Script\\Data\\DataSetReference.cs"); session.SinkFile = new StockDataFile(totalSinkScripts.ConvertAll <string>((f) => { return(Path.GetFileNameWithoutExtension(f)); }), totalSinkScripts.ConvertAll <string>((f) => { return(File.ReadAllText(f)); })); session.SinkFile.Interval = session.SourceFile.Interval; session.Scripts.Add("tmp/" + SINK_CLASS + ".cs"); using (var file = new StreamWriter(new FileStream(session.Scripts.Last(), FileMode.Create))) file.Write(session.SinkFile.GenStockDataSink()); session.Scripts.AddRange(totalSinkScripts); // Create the evaluator file (needs to be compiled in the script since it references StockDataSource) string[] embeddedFiles = new string[] { "RobinhoodDesktop.Script.StockEvaluator.cs", "RobinhoodDesktop.Script.StockProcessor.cs" }; foreach (var f in embeddedFiles) { session.Scripts.Add(string.Format("tmp/{0}.cs", f.Substring(24, f.Length - 27))); StringBuilder analyzerCode = new StringBuilder(); analyzerCode.Append(new StreamReader(Assembly.GetExecutingAssembly().GetManifestResourceStream(f)).ReadToEnd()); using (var file = new StreamWriter(new FileStream(session.Scripts.Last(), FileMode.Create))) file.Write(StockDataFile.FormatSource(analyzerCode.ToString())); } // Add the user defined analyzers foreach (string path in Directory.GetFiles(@"Script/Decision", "*.cs", SearchOption.AllDirectories)) { session.Scripts.Add(path); } foreach (string path in Directory.GetFiles(@"Script/Action", "*.cs", SearchOption.AllDirectories)) { session.Scripts.Add(path); } // Get the code that will actually run the session if (!string.IsNullOrEmpty(executeScript)) { session.Scripts.Add(executeScript); } // Build and run the session session.LoadScripts(true); StockSession.Instance = session; return(session); }
/// <summary> /// Save this instance to the stream /// </summary> /// <param name="session">The session this is part of</param> /// <param name="s">The stream to save this to</param> public void Save <T>(Stream s, Type dataType, Dictionary <string, List <StockDataSet <T> > > segments, StockSession session = null) where T : struct, StockData { var headerSer = new Serializer(new List <Type>() { typeof(StockDataFile) }); var dataSer = new Serializer(new List <Type>() { dataType.MakeArrayType() }); this.File = s; // Serialize the segments s.Seek(0x10, SeekOrigin.End); // Leave some space at the beginning of the stream to store the offset to the serialized stock file foreach (KeyValuePair <string, List <StockDataSet <T> > > pair in segments) { List <Tuple <DateTime, long> > allSegments = this.Segments[pair.Key]; int matchingIdx = 0; for (int segIdx = 0; segIdx < pair.Value.Count; segIdx++) { StockDataSet <T> set = pair.Value[segIdx]; for (; matchingIdx < allSegments.Count; matchingIdx++) { if (set.Start == allSegments[matchingIdx].Item1) { allSegments[matchingIdx] = new Tuple <DateTime, long>(set.Start, s.Position); set.StreamAddress = s.Position; if (typeof(T) == dataType) { //dataSer.Serialize(s, set.DataSet.InternalArray); Store(s, set.DataSet.InternalArray, set.DataSet.Count); } else { var data_points = Array.CreateInstance(dataType, set.DataSet.Count); for (int pntIdx = 0; pntIdx < set.DataSet.Count; pntIdx++) { data_points.SetValue(set.DataSet.InternalArray[pntIdx], pntIdx); } dataSer.Serialize(s, data_points); } matchingIdx++; break; } } } } // Serialize the header with the updated segment addresses long headerAddress = s.Position; headerSer.Serialize(s, this); // Save any script-specific data if ((session != null) && (session.ScriptInstance != null)) { var saveMethod = session.ScriptInstance.GetStaticMethod("*.Save", s); saveMethod(s); } // Set the offset to the header s.Seek(0, SeekOrigin.Begin); headerSer.Serialize(s, headerAddress); }
/// <summary> /// Loads the segment data from the source /// </summary> /// <typeparam name="T">The data point type</typeparam> /// <param name="segment">The segment to populate</param> /// <param name="session">The session this is part of</param> public virtual void LoadSegment <T>(StockDataSet <T> segment, StockSession session = null) where T : struct, StockData { FileMutex.WaitOne(); segment.DataSet.Initialize(LoadData <T>(segment.StreamAddress)); FileMutex.ReleaseMutex(); }
public static StockSession Start(List <string> sources, List <string> sinkScripts, string executeScript) { StockSession session = new StockSession(); List <string> script = new List <string>(); Directory.CreateDirectory("tmp"); // Convert any legacy files before further processing var legacyFiles = sources.Where((s) => { return(s.EndsWith(".csv")); }).ToList(); if (legacyFiles.Count() > 0) { System.Windows.Forms.SaveFileDialog saveDiag = new System.Windows.Forms.SaveFileDialog(); saveDiag.Title = "Save converted data file as..."; if (saveDiag.ShowDialog() == System.Windows.Forms.DialogResult.OK) { var convertedFile = StockDataFile.Convert(legacyFiles, new FileStream(saveDiag.FileName, FileMode.Create)); convertedFile.Close(); sources.Add(saveDiag.FileName); } else { // Cancel running the script return(null); } foreach (var l in legacyFiles) { sources.Remove(l); } } session.SourceFile = StockDataFile.Open(sources.ConvertAll <Stream>((s) => { return(new FileStream(s, FileMode.Open)); })); script.Add("tmp/" + SOURCE_CLASS + ".cs"); using (var file = new StreamWriter(new FileStream(script.Last(), FileMode.Create))) file.Write(session.SourceFile.GetSourceCode(SOURCE_CLASS)); session.SinkFile = new StockDataFile(sinkScripts.ConvertAll <string>((f) => { return(Path.GetFileNameWithoutExtension(f)); }), sinkScripts.ConvertAll <string>((f) => { return(File.ReadAllText(f)); })); session.SinkFile.Interval = session.SourceFile.Interval; script.Add("tmp/" + SINK_CLASS + ".cs"); using (var file = new StreamWriter(new FileStream(script.Last(), FileMode.Create))) file.Write(session.SinkFile.GetSourceCode(SINK_CLASS)); // Create the evaluator file (needs to be compiled in the script since it references StockDataSource) string[] embeddedFiles = new string[] { "RobinhoodDesktop.Script.StockEvaluator.cs", "RobinhoodDesktop.Script.StockProcessor.cs" }; foreach (var f in embeddedFiles) { script.Add(string.Format("tmp/{0}.cs", f.Substring(24, f.Length - 27))); StringBuilder analyzerCode = new StringBuilder(); analyzerCode.Append(new StreamReader(Assembly.GetExecutingAssembly().GetManifestResourceStream(f)).ReadToEnd()); using (var file = new StreamWriter(new FileStream(script.Last(), FileMode.Create))) file.Write(StockDataFile.FormatSource(analyzerCode.ToString())); } // Add the user defined analyzers foreach (string path in Directory.GetFiles(@"Script/Decision", "*.cs", SearchOption.AllDirectories)) { script.Add(path); } foreach (string path in Directory.GetFiles(@"Script/Action", "*.cs", SearchOption.AllDirectories)) { script.Add(path); } // Get the code that will actually run the session script.Add(executeScript); // Build and run the session #if DEBUG var isDebug = true; #else var isDebug = false; #endif try { if (!string.IsNullOrEmpty(executeScript)) { session.ScriptInstance = CSScript.LoadFiles(script.ToArray(), null, isDebug, "TensorFlow.NET.dll", "Google.Protobuf.dll", "NumSharp.Lite", "netstandard", "System.Memory", "System.Numerics"); var run = session.ScriptInstance.GetStaticMethod("*.Run", session); run(session); } } catch (Exception ex) { System.Windows.Forms.MessageBox.Show(ex.ToString()); session.SourceFile.Close(); session.SinkFile.Close(); } return(session); }
/// <summary> /// Executes a run of processing the stock data /// </summary> /// <param name="session">The session configuration</param> public static void Run(StockSession session) { // Create the stock processor StockProcessor processor = new StockProcessor(session); // Get the sizes of the features and labels int dstIdx = 0; int numDataPoints = 0; int numFeatures, numLabels; float[,] tmpDst = new float[1, 1024]; int[,] tmpLabels = new int[1, 1]; var tmpDataPoint = processor.DerivedData.First().Value[0]; tmpDataPoint.Load(session); PopulateData(tmpDataPoint, tmpDst, tmpLabels, ref dstIdx, out numFeatures, out numLabels); // Determine the total number of data points List <string> symbols = new List <string>() { "GNTX" }; foreach (var s in symbols) { List <StockDataSet <StockDataSink> > sources; if (!processor.DerivedData.TryGetValue(s, out sources)) { continue; } for (int i = 0; i < sources.Count; i++) { sources[i].Load(session); numDataPoints += sources[i].Count; } } // Allocate the feature and label arrays float[,] features = new float[numDataPoints, numFeatures]; int[,] labels = new int[numDataPoints, numLabels]; // Load the data dstIdx = 0; foreach (var s in symbols) { List <StockDataSet <StockDataSink> > sources; if (!processor.DerivedData.TryGetValue(s, out sources)) { continue; } // Create a table of each data point in the specified range for (int i = 0; i < sources.Count; i++) { sources[i].Load(session); PopulateData(sources[i], features, labels, ref dstIdx, out numFeatures, out numLabels); } } // Create the Machine Learning instance MLInstance ml = new MLInstance(); ml.BuildFullyConnectedGraph(new int[] { numFeatures, numFeatures, numLabels }); ml.PrepareData(features, labels); ml.Train(); }
/// <summary> /// Executes a run of processing the stock data /// </summary> /// <param name="session">The session configuration</param> public static void Run(StockSession session) { }
/// <summary> /// Creates a stock data chart /// </summary> /// <param name="session">The session the chart is a part of</param> /// <returns>The chart, as a generic control</returns> public static Control CreateChart(StockSession session) { StockProcessor processor = new StockProcessor(session); return((Control)(new DataChartGui <StockDataSink>(processor.DerivedData, session.SinkFile, session)).GuiPanel); }
/// <summary> /// Compiles a script to evaluate the specified expression /// </summary> /// <param name="expression">The expression to get a value from the dataset</param> /// <returns>The delegate used to get the desired value from a dataset</returns> public Func <StockDataInterface, int, object> GetExpressionEvaluator(string expression, StockSession session) { Func <StockDataSet <T>, int, object> accessor = null; // Check for the special case of requesting the time if (expression.Equals("Time")) { accessor = new Func <StockDataSet <T>, int, object>((data, index) => { return(data.Time(index)); }); } else { // Order the list based on the lame length Type dataType = GetDataType(); var fields = new List <string>(); fields.AddRange(typeof(T).GetFields().ToList().ConvertAll((f) => { return(f.Name); })); fields.AddRange(typeof(T).GetProperties().ToList().ConvertAll((f) => { return(f.Name); })); fields.AddRange(typeof(T).GetMethods().ToList().ConvertAll((f) => { return(f.Name); })); fields.Sort((f1, f2) => { return(f2.Length.CompareTo(f1.Length)); }); // First remove any string literals string src = expression; List <string> stringLiterals = new List <string>(); for (int i = src.IndexOf('"'); (i >= 0) && (i < src.Length); i = src.IndexOf('"')) { int end = src.IndexOf('"', i + 1); if ((end >= 0) && (end < src.Length)) { stringLiterals.Add(src.Substring(i, (end - i) + 1)); src = src.Replace(stringLiterals.Last(), string.Format("<=s{0}>", stringLiterals.Count - 1)); } } // First replace the fields with an index to prevent names within a name from getting messed up for (int i = 0; i < fields.Count; i++) { src = System.Text.RegularExpressions.Regex.Replace(src, fields[i] + "([^a-zA-Z0-9_]|$)", string.Format("<={0}>$1", i)); } // Next pre-pend the data set to the field names for (int i = 0; i < fields.Count; i++) { src = src.Replace(string.Format("<={0}>", i), string.Format("data[updateIndex].{0}", fields[i])); } // Restore the string literals for (int i = 0; i < stringLiterals.Count; i++) { src = src.Replace(string.Format("<=s{0}>", i), stringLiterals[i]); } // Build the expression into an accessor function //src = "namespace RobinhoodDesktop.Script { public class ExpressionAccessor{ public static object GetValue(StockDataSet<" + typeof(T).Name + "> data, int updateIndex) { return " + src + ";} } }"; var compiler = CSScript.MonoEvaluator.ReferenceAssemblyOf <T>(); foreach (var s in session.Scripts.Values) { compiler = compiler.ReferenceAssembly(s.Location); } //var script = CSScript.LoadCode(src); //CSScript.Evaluator. //accessor = script.GetStaticMethod("*.*"); accessor = compiler.LoadDelegate <Func <StockDataSet <T>, int, object> >(@"object GetValue(RobinhoodDesktop.Script.StockDataSet<" + typeof(T).FullName + "> data, int updateIndex) { return " + src + ";}"); } return(new Func <StockDataInterface, int, object>((data, index) => { return accessor((StockDataSet <T>)data, index); })); }
/// <summary> /// Loads the segment data from the source /// </summary> /// <typeparam name="T">The data point type</typeparam> /// <param name="segment">The segment to populate</param> /// <param name="session">The session this is part of</param> public virtual void LoadSegment <T>(StockDataSet <T> segment, StockSession session = null) where T : struct, StockData { segment.DataSet.Initialize(LoadData <T>(segment.StreamAddress)); }