/// <summary>
        /// Loads a segment from the source stream into a usable object
        /// </summary>
        /// <typeparam name="T">The type of the data points in the segments</typeparam>
        /// <returns>The data segments</returns>
        public Dictionary <string, List <StockDataSet <T> > > GetSegments <T>() where T : struct, StockData
        {
            /*
             * var loader = CSScript.LoadCode(SourceCode).GetStaticMethod(".Load");
             * var data = (Dictionary<string, List<T>>)loader(s, Segments[segmentIndex].Item2);
             * var retVal = new Dictionary<string, StockDataSet<T>>();
             * var startTime = Segments[segmentIndex].Item1;
             * var endTime = startTime.AddTicks(Interval.Ticks * data.ElementAt(0).Value.Count);
             * foreach(KeyValuePair<string, List<T>> pair in data)
             * {
             *  var set = new StockDataSet<T>(pair.Key, startTime, endTime, Interval);
             *  set.DataSet = pair.Value;
             * }; */
            Dictionary <string, List <StockDataSet <T> > > dataSets = new Dictionary <string, List <StockDataSet <T> > >();

            foreach (KeyValuePair <string, List <Tuple <DateTime, long> > > pair in Segments)
            {
                StockDataSet <T> prevSet = null;
                dataSets[pair.Key] = new List <StockDataSet <T> >();
                foreach (Tuple <DateTime, long> set in pair.Value)
                {
                    var nextSet = new StockDataSet <T>(pair.Key, set.Item1, this, set.Item2);
                    nextSet.Previous = prevSet;
                    prevSet          = nextSet;
                    dataSets[pair.Key].Add(nextSet);
                }
            }
            return(dataSets);
        }
 /// <summary>
 /// Loads the segment data from the source
 /// </summary>
 /// <typeparam name="T">The data point type</typeparam>
 /// <param name="session">The session this is part of</param>
 /// <param name="segment">The segment to populate</param>
 public override void LoadSegment <T>(StockDataSet <T> segment, StockSession session = null)
 {
     if ((this.LoadMethod == null) && (session != null))
     {
         this.LoadMethod = session.ScriptInstance.GetStaticMethod("*.Load", this, "", DateTime.Now);
     }
     segment.DataSet.Initialize((T[])LoadMethod(this, segment.Symbol, segment.Start));
 }
 public StockDataSetDerived(StockDataSet <U> source, StockDataFile file, StockDataCreator create, StockProcessingStateAccessor stateGetter)
 {
     this.SourceData = source;
     this.File       = file;
     this.Start      = source.Start;
     this.Symbol     = source.Symbol;
     this.Create     = create;
     this.GetState   = stateGetter;
 }
예제 #4
0
        /// <summary>
        /// The main update function which sets all of the member variables based on other source data
        /// </summary>
        /// <param name="dataSource">The available source data</param>
        /// <param name="updateIndex">The index into the data that should be used as the source for this</param>
        public void Update <T>(StockDataSet <T> .StockDataArray dataSource, int updateIndex) where T : struct, StockData
        {
            var data = dataSource as StockDataSet <StockDataBase> .StockDataArray;

            if (data != null)
            {
                this.Price = data.InternalArray[updateIndex].Price;
            }
        }
예제 #5
0
        /// <summary>
        /// The main update function for the class
        /// </summary>
        /// <param name="data">The available source data</param>
        /// <param name="updateIndex">The index into the data that should be used as the source for this</param>
        partial void MovingAverage_Update(StockDataSet <StockDataSource> .StockDataArray data, int updateIndex)
        {
            this.Average10Min = 0.0f;
            int startIdx = Math.Max(0, updateIndex - 9);

            for (int idx = startIdx; idx <= updateIndex; idx++)
            {
                this.Average10Min += data.InternalArray[idx].Price;
            }
            this.Average10Min /= ((updateIndex + 1) - startIdx);
        }
            /// <summary>
            /// Returns the number of data points in the given segment
            /// </summary>
            /// <typeparam name="T">The data point type</typeparam>
            /// <param name="segment">The segment to check</param>
            /// <returns>The number of data points in the segment when it is loaded</returns>
            public override int GetSegmentSize <T>(StockDataSet <T> segment)
            {
                int count = 0;

                for (int s = 0; s < Sources.Count; s++)
                {
                    if ((Sources[s].Start <= segment.Start) && (Sources[s].End > segment.Start))
                    {
                        count = Sources[s].GetSegmentSize <T>(segment);
                        break;
                    }
                }
                return(count);
            }
        /// <summary>
        /// Evaluates
        /// </summary>
        /// <param name="data">The data set to work from</param>
        /// <param name="index">The index which should be evaluated</param>
        /// <param name="target">The reference point to use when evaluating the stock</param>
        /// <returns>True if the analyzer criteria is met at the given data point</returns>
        public override bool Evaluate(StockDataSet <StockDataSink> data, int index, StockProcessor.ProcessingTarget target)
        {
            float refPrice;

            if (!ReferencePrices.TryGetValue(target.Symbol, out refPrice))
            {
                refPrice = data[index].Price;
                ReferencePrices.Add(target.Symbol, refPrice);
            }
            var percentDiff = ((data[index].Price - refPrice) / refPrice);

            return(((percentDiff >= Percentage) && MonitorIncrease) ||
                   ((percentDiff <= -Percentage) && MonitorDecrease));
        }
        /// <summary>
        /// Returns the number of data points in the given segment
        /// </summary>
        /// <typeparam name="T">The data point type</typeparam>
        /// <param name="segment">The segment to check</param>
        /// <returns>The number of data points in the segment when it is loaded</returns>
        public virtual int GetSegmentSize <T>(StockDataSet <T> segment) where T : struct, StockData
        {
            int count = 0;

            foreach (Tuple <DateTime, long> t in this.Segments[segment.Symbol])
            {
                if (t.Item1 == segment.Start)
                {
                    File.Seek(t.Item2, SeekOrigin.Begin);
                    count = (File.ReadByte() << 8) | File.ReadByte();
                    break;
                }
            }
            return(count);
        }
예제 #9
0
        /// <summary>
        /// Pulls the processed stock data, and puts it into a format that can be fed into the ML model
        /// </summary>
        /// <param name="src">The processed stock data</param>
        /// <param name="dst">The array the features should be placed in</param>
        /// <param name="labels">The array the expected outputs should be placed in</param>
        /// <param name="dstIdx">Update the current postion within the arrays</param>
        /// <param name="numFeatures">Output the number of features</param>
        /// <param name="numLabels">Output the number of data labels</param>
        public static void PopulateData(StockDataSet <StockDataSink> src, float[,] dst, int[,] labels, ref int dstIdx, out int numFeatures, out int numLabels)
        {
            numFeatures = 0;
            numLabels   = 0;
            for (int srcIdx = 0; (srcIdx < src.Count) && (dstIdx < dst.GetLength(0)); srcIdx++, dstIdx++)
            {
                // Output features for the data point
                numFeatures = 0;

                StockDataSink s = src[srcIdx];
                dst[dstIdx, numFeatures++] = s.Average10Min;

                // Output the labels for the data point - the thing you are trying to predict
                numLabels = 0;

                labels[dstIdx, numLabels++] = ((s.Price > s.Average10Min) ? 1 : 0);
            }
        }
예제 #10
0
        /// <summary>
        /// Adds a new processing target
        /// </summary>
        /// <param name="target">The target point to add</param>
        public void Add(ProcessingTarget target)
        {
            Targets.Add(target);

            // If live, set up a subscription
            if (Live && !LiveData.ContainsKey(target.Symbol))
            {
                var sourceList = new StockDataSet <StockDataSource>(target.Symbol, DateTime.Now, Session.SourceFile);
                var data       = new StockDataSetDerived <StockDataSink, StockDataSource, StockProcessingState>(sourceList, Session.SinkFile, CreateSink, GetProcessingState);
                var sub        = DataAccessor.Subscribe(target.Symbol, LiveInterval);
                sub.Notify += (DataAccessor.Subscription s) =>
                {
                    LiveData[s.Symbol].Item1.Add(StockDataSource.CreateFromPrice((float)s.Price));
                    LiveProcessingQueue.Add(target);
                };

                LiveData[target.Symbol] = new Tuple <StockDataSetDerived <StockDataSink, StockDataSource, StockProcessingState>, DataAccessor.Subscription>(data, sub);
            }
        }
 /// <summary>
 /// Evaluates
 /// </summary>
 /// <param name="data">The data set to work from</param>
 /// <param name="index">The index which should be evaluated</param>
 /// <param name="target">The reference point to use when evaluating the stock</param>
 /// <returns>True if the analyzer criteria is met at the given data point</returns>
 public abstract bool Evaluate(StockDataSet <StockDataSink> data, int index, StockProcessor.ProcessingTarget target);
예제 #12
0
        /// <summary>
        /// Reads the specified data streams, and converts them into a basic stock data file
        /// </summary>
        /// <param name="sourceFiles">The legacy data files</param>
        /// <returns>The new data file</returns>
        public static StockDataFile Convert(List <string> sourceFiles, Stream destination)
        {
            StockDataFile newFile = new StockDataFile();

            newFile.Interval = new TimeSpan(0, 1, 0);
            Dictionary <string, List <StockDataSet <StockDataBase> > > segments = new Dictionary <string, List <StockDataSet <StockDataBase> > >();

            foreach (string filename in sourceFiles)
            {
                // Parse the date from the filename
                DateTime fileDate      = GetDateFromFileName(filename);
                DateTime fileStart     = fileDate.AddHours(9.5);
                DateTime fileEnd       = fileDate.AddHours(16);
                long     delayedOffset = (filename.Contains("goog")) ? 0 : new TimeSpan(0, 15, 0).Ticks;

                // Read the initial line of the file to learn which stocks are in the file
                StreamReader s = new StreamReader(filename);
                List <StockDataSet <StockDataBase> > fileData = new List <StockDataSet <StockDataBase> >();
                string   line          = s.ReadLine();
                string[] stockNamesStr = line.Split(new char[] { ',' });
                for (int i = 1; i < stockNamesStr.Length; i++)
                {
                    string symbol = stockNamesStr[i].ToUpper();
                    List <StockDataSet <StockDataBase> > sets;

                    // Check if a stock data set already exists for the symbol
                    if (!segments.TryGetValue(symbol, out sets))
                    {
                        sets = new List <StockDataSet <StockDataBase> >();
                        segments.Add(symbol, sets);
                    }
                    foreach (StockDataSet <StockDataBase> preExistingSet in sets)
                    {
                        // Add to an existing set if they are for the same day
                        if (preExistingSet.Start.Date == fileStart.Date)
                        {
                            fileData.Add(preExistingSet);
                            break;
                        }
                    }
                    if (fileData.Count <= (i - 1))
                    {
                        // No matching set was found, so create a new one
                        StockDataSet <StockDataBase> newSet = new StockDataSet <StockDataBase>(symbol, fileStart, newFile);
                        newSet.DataSet.Resize(391);
                        fileData.Add(newSet);
                        if (sets.Count > 0)
                        {
                            newSet.Previous = sets[sets.Count - 1];
                        }
                        sets.Add(newSet);
                    }
                }

                DateTime lineTime = fileStart;
                while (!s.EndOfStream && (lineTime < fileEnd))
                {
                    string[] stockPricesStr = s.ReadLine().Split(new char[] { ',' });

                    // Ensure the entry contains all of the stocks
                    if ((stockPricesStr.Length - 1) != fileData.Count)
                    {
                        continue;
                    }

                    // Get the time of the entry
                    DateTime newTime;
                    if (!DateTime.TryParse(stockPricesStr[0].Replace("\"", ""), out newTime))
                    {
                        continue;
                    }
                    newTime = fileStart.AddTicks((newTime.TimeOfDay.Ticks - fileStart.TimeOfDay.Ticks) - delayedOffset);

                    // Check if this new entry is valid
                    if ((newTime >= fileStart) && (newTime <= fileEnd))
                    {
                        // Update the prices of each of the stocks
                        for (int i = 0; i < (stockPricesStr.Length - 1); i++)
                        {
                            float price;
                            if (float.TryParse(stockPricesStr[i + 1], out price))
                            {
                                // Sometimes the first price is corrupted, so skip it
                                if ((newTime == fileStart)
                                    //&& ((fileData[i].Previous != null) && (Math.Abs((price / fileData[i][-1].Price) - 1.0f) > 0.01f))
                                    )
                                {
                                    continue;
                                }

                                if ((price == 0.0f) || (newTime == fileEnd))
                                {
                                    price = fileData[i][fileData[i].Count - 1].Price;
                                    if (price == 0.0f)
                                    {
                                        continue;
                                    }
                                }
                                while (fileData[i].End <= newTime)
                                {
                                    fileData[i].DataSet.Add(new StockDataBase(price));
                                }
                            }
                        }
                    }
                }
            }

            // Save the old file to disk
            newFile.SetSegments(segments);
            newFile.Save(destination, typeof(StockDataBase), segments);

            return(newFile);
        }
예제 #13
0
        /// <summary>
        /// Save this instance to the stream
        /// </summary>
        /// <param name="session">The session this is part of</param>
        /// <param name="s">The stream to save this to</param>
        public void Save <T>(Stream s, Type dataType, Dictionary <string, List <StockDataSet <T> > > segments, StockSession session = null) where T : struct, StockData
        {
            var headerSer = new Serializer(new List <Type>()
            {
                typeof(StockDataFile)
            });
            var dataSer = new Serializer(new List <Type>()
            {
                dataType.MakeArrayType()
            });

            this.File = s;


            // Serialize the segments
            s.Seek(0x10, SeekOrigin.End);   // Leave some space at the beginning of the stream to store the offset to the serialized stock file
            foreach (KeyValuePair <string, List <StockDataSet <T> > > pair in segments)
            {
                List <Tuple <DateTime, long> > allSegments = this.Segments[pair.Key];
                int matchingIdx = 0;
                for (int segIdx = 0; segIdx < pair.Value.Count; segIdx++)
                {
                    StockDataSet <T> set = pair.Value[segIdx];
                    for (; matchingIdx < allSegments.Count; matchingIdx++)
                    {
                        if (set.Start == allSegments[matchingIdx].Item1)
                        {
                            allSegments[matchingIdx] = new Tuple <DateTime, long>(set.Start, s.Position);
                            set.StreamAddress        = s.Position;
                            if (typeof(T) == dataType)
                            {
                                //dataSer.Serialize(s, set.DataSet.InternalArray);
                                Store(s, set.DataSet.InternalArray, set.DataSet.Count);
                            }
                            else
                            {
                                var data_points = Array.CreateInstance(dataType, set.DataSet.Count);
                                for (int pntIdx = 0; pntIdx < set.DataSet.Count; pntIdx++)
                                {
                                    data_points.SetValue(set.DataSet.InternalArray[pntIdx], pntIdx);
                                }
                                dataSer.Serialize(s, data_points);
                            }
                            matchingIdx++;
                            break;
                        }
                    }
                }
            }

            // Serialize the header with the updated segment addresses
            long headerAddress = s.Position;

            headerSer.Serialize(s, this);

            // Save any script-specific data
            if ((session != null) && (session.ScriptInstance != null))
            {
                var saveMethod = session.ScriptInstance.GetStaticMethod("*.Save", s);
                saveMethod(s);
            }

            // Set the offset to the header
            s.Seek(0, SeekOrigin.Begin);
            headerSer.Serialize(s, headerAddress);
        }
예제 #14
0
 /// <summary>
 /// Loads the segment data from the source
 /// </summary>
 /// <typeparam name="T">The data point type</typeparam>
 /// <param name="segment">The segment to populate</param>
 /// <param name="session">The session this is part of</param>
 public virtual void LoadSegment <T>(StockDataSet <T> segment, StockSession session = null) where T : struct, StockData
 {
     FileMutex.WaitOne();
     segment.DataSet.Initialize(LoadData <T>(segment.StreamAddress));
     FileMutex.ReleaseMutex();
 }
예제 #15
0
 /// <summary>
 /// The main update function which sets all of the member variables based on other source data
 /// </summary>
 /// <param name="data">The available source data</param>
 /// <param name="updateIndex">The index into the data that should be used as the source for this</param>
 public void Update(StockDataSet <StockDataBase> .StockDataArray data, int updateIndex)
 {
     this.Price = data.InternalArray[updateIndex].Price;
 }
예제 #16
0
 /// <summary>
 /// Loads the segment data from the source
 /// </summary>
 /// <typeparam name="T">The data point type</typeparam>
 /// <param name="segment">The segment to populate</param>
 /// <param name="session">The session this is part of</param>
 public virtual void LoadSegment <T>(StockDataSet <T> segment, StockSession session = null) where T : struct, StockData
 {
     segment.DataSet.Initialize(LoadData <T>(segment.StreamAddress));
 }