/// <summary>
/// Loads a segment from the source stream into a usable object
/// </summary>
/// <typeparam name="T">The type of the data points in the segments</typeparam>
/// <returns>The data segments</returns>
public Dictionary <string, List <StockDataSet <T> > > GetSegments <T>() where T : struct, StockData
{
/*
* var loader = CSScript.LoadCode(SourceCode).GetStaticMethod(".Load");
* var data = (Dictionary<string, List<T>>)loader(s, Segments[segmentIndex].Item2);
* var retVal = new Dictionary<string, StockDataSet<T>>();
* var startTime = Segments[segmentIndex].Item1;
* var endTime = startTime.AddTicks(Interval.Ticks * data.ElementAt(0).Value.Count);
* foreach(KeyValuePair<string, List<T>> pair in data)
* {
* var set = new StockDataSet<T>(pair.Key, startTime, endTime, Interval);
* set.DataSet = pair.Value;
* }; */
Dictionary <string, List <StockDataSet <T> > > dataSets = new Dictionary <string, List <StockDataSet <T> > >();
foreach (KeyValuePair <string, List <Tuple <DateTime, long> > > pair in Segments)
{
StockDataSet <T> prevSet = null;
dataSets[pair.Key] = new List <StockDataSet <T> >();
foreach (Tuple <DateTime, long> set in pair.Value)
{
var nextSet = new StockDataSet <T>(pair.Key, set.Item1, this, set.Item2);
nextSet.Previous = prevSet;
prevSet = nextSet;
dataSets[pair.Key].Add(nextSet);
}
}
return(dataSets);
}
/// <summary>
/// Loads the segment data from the source
/// </summary>
/// <typeparam name="T">The data point type</typeparam>
/// <param name="session">The session this is part of</param>
/// <param name="segment">The segment to populate</param>
public override void LoadSegment <T>(StockDataSet <T> segment, StockSession session = null)
{
if ((this.LoadMethod == null) && (session != null))
{
this.LoadMethod = session.ScriptInstance.GetStaticMethod("*.Load", this, "", DateTime.Now);
}
segment.DataSet.Initialize((T[])LoadMethod(this, segment.Symbol, segment.Start));
}
public StockDataSetDerived(StockDataSet <U> source, StockDataFile file, StockDataCreator create, StockProcessingStateAccessor stateGetter)
{
this.SourceData = source;
this.File = file;
this.Start = source.Start;
this.Symbol = source.Symbol;
this.Create = create;
this.GetState = stateGetter;
}
/// <summary>
/// The main update function which sets all of the member variables based on other source data
/// </summary>
/// <param name="dataSource">The available source data</param>
/// <param name="updateIndex">The index into the data that should be used as the source for this</param>
public void Update <T>(StockDataSet <T> .StockDataArray dataSource, int updateIndex) where T : struct, StockData
{
var data = dataSource as StockDataSet <StockDataBase> .StockDataArray;
if (data != null)
{
this.Price = data.InternalArray[updateIndex].Price;
}
}
/// <summary>
/// The main update function for the class
/// </summary>
/// <param name="data">The available source data</param>
/// <param name="updateIndex">The index into the data that should be used as the source for this</param>
partial void MovingAverage_Update(StockDataSet <StockDataSource> .StockDataArray data, int updateIndex)
{
this.Average10Min = 0.0f;
int startIdx = Math.Max(0, updateIndex - 9);
for (int idx = startIdx; idx <= updateIndex; idx++)
{
this.Average10Min += data.InternalArray[idx].Price;
}
this.Average10Min /= ((updateIndex + 1) - startIdx);
}
/// <summary>
/// Returns the number of data points in the given segment
/// </summary>
/// <typeparam name="T">The data point type</typeparam>
/// <param name="segment">The segment to check</param>
/// <returns>The number of data points in the segment when it is loaded</returns>
public override int GetSegmentSize <T>(StockDataSet <T> segment)
{
int count = 0;
for (int s = 0; s < Sources.Count; s++)
{
if ((Sources[s].Start <= segment.Start) && (Sources[s].End > segment.Start))
{
count = Sources[s].GetSegmentSize <T>(segment);
break;
}
}
return(count);
}
/// <summary>
/// Evaluates
/// </summary>
/// <param name="data">The data set to work from</param>
/// <param name="index">The index which should be evaluated</param>
/// <param name="target">The reference point to use when evaluating the stock</param>
/// <returns>True if the analyzer criteria is met at the given data point</returns>
public override bool Evaluate(StockDataSet <StockDataSink> data, int index, StockProcessor.ProcessingTarget target)
{
float refPrice;
if (!ReferencePrices.TryGetValue(target.Symbol, out refPrice))
{
refPrice = data[index].Price;
ReferencePrices.Add(target.Symbol, refPrice);
}
var percentDiff = ((data[index].Price - refPrice) / refPrice);
return(((percentDiff >= Percentage) && MonitorIncrease) ||
((percentDiff <= -Percentage) && MonitorDecrease));
}
/// <summary>
/// Returns the number of data points in the given segment
/// </summary>
/// <typeparam name="T">The data point type</typeparam>
/// <param name="segment">The segment to check</param>
/// <returns>The number of data points in the segment when it is loaded</returns>
public virtual int GetSegmentSize <T>(StockDataSet <T> segment) where T : struct, StockData
{
int count = 0;
foreach (Tuple <DateTime, long> t in this.Segments[segment.Symbol])
{
if (t.Item1 == segment.Start)
{
File.Seek(t.Item2, SeekOrigin.Begin);
count = (File.ReadByte() << 8) | File.ReadByte();
break;
}
}
return(count);
}
/// <summary>
/// Pulls the processed stock data, and puts it into a format that can be fed into the ML model
/// </summary>
/// <param name="src">The processed stock data</param>
/// <param name="dst">The array the features should be placed in</param>
/// <param name="labels">The array the expected outputs should be placed in</param>
/// <param name="dstIdx">Update the current postion within the arrays</param>
/// <param name="numFeatures">Output the number of features</param>
/// <param name="numLabels">Output the number of data labels</param>
public static void PopulateData(StockDataSet <StockDataSink> src, float[,] dst, int[,] labels, ref int dstIdx, out int numFeatures, out int numLabels)
{
numFeatures = 0;
numLabels = 0;
for (int srcIdx = 0; (srcIdx < src.Count) && (dstIdx < dst.GetLength(0)); srcIdx++, dstIdx++)
{
// Output features for the data point
numFeatures = 0;
StockDataSink s = src[srcIdx];
dst[dstIdx, numFeatures++] = s.Average10Min;
// Output the labels for the data point - the thing you are trying to predict
numLabels = 0;
labels[dstIdx, numLabels++] = ((s.Price > s.Average10Min) ? 1 : 0);
}
}
/// <summary>
/// Adds a new processing target
/// </summary>
/// <param name="target">The target point to add</param>
public void Add(ProcessingTarget target)
{
Targets.Add(target);
// If live, set up a subscription
if (Live && !LiveData.ContainsKey(target.Symbol))
{
var sourceList = new StockDataSet <StockDataSource>(target.Symbol, DateTime.Now, Session.SourceFile);
var data = new StockDataSetDerived <StockDataSink, StockDataSource, StockProcessingState>(sourceList, Session.SinkFile, CreateSink, GetProcessingState);
var sub = DataAccessor.Subscribe(target.Symbol, LiveInterval);
sub.Notify += (DataAccessor.Subscription s) =>
{
LiveData[s.Symbol].Item1.Add(StockDataSource.CreateFromPrice((float)s.Price));
LiveProcessingQueue.Add(target);
};
LiveData[target.Symbol] = new Tuple <StockDataSetDerived <StockDataSink, StockDataSource, StockProcessingState>, DataAccessor.Subscription>(data, sub);
}
}
/// <summary> /// Evaluates /// </summary> /// <param name="data">The data set to work from</param> /// <param name="index">The index which should be evaluated</param> /// <param name="target">The reference point to use when evaluating the stock</param> /// <returns>True if the analyzer criteria is met at the given data point</returns> public abstract bool Evaluate(StockDataSet <StockDataSink> data, int index, StockProcessor.ProcessingTarget target);
/// <summary>
/// Reads the specified data streams, and converts them into a basic stock data file
/// </summary>
/// <param name="sourceFiles">The legacy data files</param>
/// <returns>The new data file</returns>
public static StockDataFile Convert(List <string> sourceFiles, Stream destination)
{
StockDataFile newFile = new StockDataFile();
newFile.Interval = new TimeSpan(0, 1, 0);
Dictionary <string, List <StockDataSet <StockDataBase> > > segments = new Dictionary <string, List <StockDataSet <StockDataBase> > >();
foreach (string filename in sourceFiles)
{
// Parse the date from the filename
DateTime fileDate = GetDateFromFileName(filename);
DateTime fileStart = fileDate.AddHours(9.5);
DateTime fileEnd = fileDate.AddHours(16);
long delayedOffset = (filename.Contains("goog")) ? 0 : new TimeSpan(0, 15, 0).Ticks;
// Read the initial line of the file to learn which stocks are in the file
StreamReader s = new StreamReader(filename);
List <StockDataSet <StockDataBase> > fileData = new List <StockDataSet <StockDataBase> >();
string line = s.ReadLine();
string[] stockNamesStr = line.Split(new char[] { ',' });
for (int i = 1; i < stockNamesStr.Length; i++)
{
string symbol = stockNamesStr[i].ToUpper();
List <StockDataSet <StockDataBase> > sets;
// Check if a stock data set already exists for the symbol
if (!segments.TryGetValue(symbol, out sets))
{
sets = new List <StockDataSet <StockDataBase> >();
segments.Add(symbol, sets);
}
foreach (StockDataSet <StockDataBase> preExistingSet in sets)
{
// Add to an existing set if they are for the same day
if (preExistingSet.Start.Date == fileStart.Date)
{
fileData.Add(preExistingSet);
break;
}
}
if (fileData.Count <= (i - 1))
{
// No matching set was found, so create a new one
StockDataSet <StockDataBase> newSet = new StockDataSet <StockDataBase>(symbol, fileStart, newFile);
newSet.DataSet.Resize(391);
fileData.Add(newSet);
if (sets.Count > 0)
{
newSet.Previous = sets[sets.Count - 1];
}
sets.Add(newSet);
}
}
DateTime lineTime = fileStart;
while (!s.EndOfStream && (lineTime < fileEnd))
{
string[] stockPricesStr = s.ReadLine().Split(new char[] { ',' });
// Ensure the entry contains all of the stocks
if ((stockPricesStr.Length - 1) != fileData.Count)
{
continue;
}
// Get the time of the entry
DateTime newTime;
if (!DateTime.TryParse(stockPricesStr[0].Replace("\"", ""), out newTime))
{
continue;
}
newTime = fileStart.AddTicks((newTime.TimeOfDay.Ticks - fileStart.TimeOfDay.Ticks) - delayedOffset);
// Check if this new entry is valid
if ((newTime >= fileStart) && (newTime <= fileEnd))
{
// Update the prices of each of the stocks
for (int i = 0; i < (stockPricesStr.Length - 1); i++)
{
float price;
if (float.TryParse(stockPricesStr[i + 1], out price))
{
// Sometimes the first price is corrupted, so skip it
if ((newTime == fileStart)
//&& ((fileData[i].Previous != null) && (Math.Abs((price / fileData[i][-1].Price) - 1.0f) > 0.01f))
)
{
continue;
}
if ((price == 0.0f) || (newTime == fileEnd))
{
price = fileData[i][fileData[i].Count - 1].Price;
if (price == 0.0f)
{
continue;
}
}
while (fileData[i].End <= newTime)
{
fileData[i].DataSet.Add(new StockDataBase(price));
}
}
}
}
}
}
// Save the old file to disk
newFile.SetSegments(segments);
newFile.Save(destination, typeof(StockDataBase), segments);
return(newFile);
}
/// <summary>
/// Save this instance to the stream
/// </summary>
/// <param name="session">The session this is part of</param>
/// <param name="s">The stream to save this to</param>
public void Save <T>(Stream s, Type dataType, Dictionary <string, List <StockDataSet <T> > > segments, StockSession session = null) where T : struct, StockData
{
var headerSer = new Serializer(new List <Type>()
{
typeof(StockDataFile)
});
var dataSer = new Serializer(new List <Type>()
{
dataType.MakeArrayType()
});
this.File = s;
// Serialize the segments
s.Seek(0x10, SeekOrigin.End); // Leave some space at the beginning of the stream to store the offset to the serialized stock file
foreach (KeyValuePair <string, List <StockDataSet <T> > > pair in segments)
{
List <Tuple <DateTime, long> > allSegments = this.Segments[pair.Key];
int matchingIdx = 0;
for (int segIdx = 0; segIdx < pair.Value.Count; segIdx++)
{
StockDataSet <T> set = pair.Value[segIdx];
for (; matchingIdx < allSegments.Count; matchingIdx++)
{
if (set.Start == allSegments[matchingIdx].Item1)
{
allSegments[matchingIdx] = new Tuple <DateTime, long>(set.Start, s.Position);
set.StreamAddress = s.Position;
if (typeof(T) == dataType)
{
//dataSer.Serialize(s, set.DataSet.InternalArray);
Store(s, set.DataSet.InternalArray, set.DataSet.Count);
}
else
{
var data_points = Array.CreateInstance(dataType, set.DataSet.Count);
for (int pntIdx = 0; pntIdx < set.DataSet.Count; pntIdx++)
{
data_points.SetValue(set.DataSet.InternalArray[pntIdx], pntIdx);
}
dataSer.Serialize(s, data_points);
}
matchingIdx++;
break;
}
}
}
}
// Serialize the header with the updated segment addresses
long headerAddress = s.Position;
headerSer.Serialize(s, this);
// Save any script-specific data
if ((session != null) && (session.ScriptInstance != null))
{
var saveMethod = session.ScriptInstance.GetStaticMethod("*.Save", s);
saveMethod(s);
}
// Set the offset to the header
s.Seek(0, SeekOrigin.Begin);
headerSer.Serialize(s, headerAddress);
}
/// <summary>
/// Loads the segment data from the source
/// </summary>
/// <typeparam name="T">The data point type</typeparam>
/// <param name="segment">The segment to populate</param>
/// <param name="session">The session this is part of</param>
public virtual void LoadSegment <T>(StockDataSet <T> segment, StockSession session = null) where T : struct, StockData
{
FileMutex.WaitOne();
segment.DataSet.Initialize(LoadData <T>(segment.StreamAddress));
FileMutex.ReleaseMutex();
}
/// <summary>
/// The main update function which sets all of the member variables based on other source data
/// </summary>
/// <param name="data">The available source data</param>
/// <param name="updateIndex">The index into the data that should be used as the source for this</param>
public void Update(StockDataSet <StockDataBase> .StockDataArray data, int updateIndex)
{
this.Price = data.InternalArray[updateIndex].Price;
}
/// <summary>
/// Loads the segment data from the source
/// </summary>
/// <typeparam name="T">The data point type</typeparam>
/// <param name="segment">The segment to populate</param>
/// <param name="session">The session this is part of</param>
public virtual void LoadSegment <T>(StockDataSet <T> segment, StockSession session = null) where T : struct, StockData
{
segment.DataSet.Initialize(LoadData <T>(segment.StreamAddress));
}