/// <summary>
/// Processes the specified data using the evaluator and resulting action
/// <param name="multithreaded">Indicates if the processing should be done multithreaded</param>
/// <param name="keep">Indicates if the processed data should remain in memory,
/// or should be cleared once processing is complete.</param>
/// </summary>
public virtual void Process(bool multithreaded = false, MemoryScheme keep = MemoryScheme.MEM_KEEP_NONE)
{
System.Action <ProcessingTarget> processFunc = (ProcessingTarget target) =>
{
List <StockDataSetDerived <StockDataSink, StockDataSource, StockProcessingState> > sets = HistoricalData[target.Symbol];
var processingState = new StockProcessingState();
for (; target.ProcessedSetIdx < sets.Count; target.ProcessedSetIdx++)
{
// Process all of the data sets for the target
StockDataSetDerived <StockDataSink, StockDataSource, StockProcessingState> set = sets[target.ProcessedSetIdx];
set.Load(Session);
for (; (target.ProcessedDataIdx < set.Count); target.ProcessedDataIdx++)
{
if (Evaluator.Evaluate(set, target.ProcessedDataIdx, target))
{
Action.Do(this, target, set.Time(target.ProcessedDataIdx));
}
}
// Clean up the memory after processing has completed
if (keep == MemoryScheme.MEM_KEEP_NONE)
{
set.Clear();
}
else if (keep == MemoryScheme.MEM_KEEP_SOURCE)
{
set.ClearDerived();
}
}
};
// Process each of the targets
if (multithreaded)
{
System.Threading.Tasks.Parallel.ForEach(Targets, processFunc);
}
else
{
foreach (ProcessingTarget target in Targets)
{
processFunc(target);
}
}
// Indicate processing has completed
if (Complete != null)
{
Complete();
}
}
/// <summary>
/// Callback used to create a stock data instance
/// </summary>
public StockProcessingState GetProcessingState(StockDataInterface data)
{
Dictionary <TimeSpan, StockProcessingState> intervals;
StockProcessingState state;
string symbol;
DateTime start;
TimeSpan interval;
data.GetInfo(out symbol, out start, out interval);
if (!ProcessingStates.TryGetValue(symbol, out intervals) || (intervals == null))
{
state = new StockProcessingState();
state.DataSet = HistoricalData[symbol];
ProcessingStates[symbol] = new Dictionary <TimeSpan, StockProcessingState>()
{
{ interval, state }
};
}
else
{
if (!intervals.TryGetValue(interval, out state) || (state == null))
{
state = new StockProcessingState();
state.DataSet = HistoricalData[symbol];
intervals[interval] = state;
}
else
{
// Assume incrementing to the next data set
state.DataSetIndex++;
// Check if processing restarted
if (start < state.LastProcessedStartTime)
{
state = new StockProcessingState();
state.DataSet = HistoricalData[symbol];
intervals[interval] = state;
}
}
}
state.LastProcessedStartTime = start;
// Ensure the data set index is correct (should never evaluate to true)
if (data != HistoricalData[symbol][state.DataSetIndex])
{
throw new Exception("Data set index mismatch");
}
return(state);
}
/// <summary>
/// Callback used to create a stock data instance
/// </summary>
public StockProcessingState GetProcessingState(StockDataInterface data)
{
Dictionary <TimeSpan, StockProcessingState> intervals;
StockProcessingState state;
string symbol;
DateTime start;
TimeSpan interval;
data.GetInfo(out symbol, out start, out interval);
if (!ProcessingStates.TryGetValue(symbol, out intervals) || (intervals == null))
{
state = new StockProcessingState();
state.DataSet = HistoricalData[symbol];
ProcessingStates[symbol] = new Dictionary <TimeSpan, StockProcessingState>()
{
{ interval, state }
};
}
else
{
if (!intervals.TryGetValue(interval, out state) || (state == null))
{
state = new StockProcessingState();
state.DataSet = HistoricalData[symbol];
intervals[interval] = state;
}
else
{
// Assume incrementing to the next data set
for (; state.DataSetIndex < state.DataSet.Count(); state.DataSetIndex++)
{
if (state.DataSet[state.DataSetIndex] == data)
{
break;
}
}
// Check if processing restarted
if (start < state.LastProcessedStartTime)
{
state = new StockProcessingState();
state.DataSet = HistoricalData[symbol];
intervals[interval] = state;
}
}
}
state.LastProcessedStartTime = start;
return(state);
}
/// <summary>
/// Callback used to create a stock data instance
/// </summary>
public StockProcessingState GetProcessingState(StockDataSetInterface data)
{
Dictionary <TimeSpan, StockProcessingState> intervals;
StockProcessingState state;
string symbol;
DateTime start;
TimeSpan interval;
data.GetInfo(out symbol, out start, out interval);
if (!ProcessingStates.TryGetValue(symbol, out intervals) || (intervals == null))
{
state = new StockProcessingState();
ProcessingStates[symbol] = new Dictionary <TimeSpan, StockProcessingState>()
{
{ interval, state }
};
}
else
{
if (!intervals.TryGetValue(interval, out state) || (state == null))
{
state = new StockProcessingState();
intervals[interval] = state;
}
else
{
// Check if processing restarted
if (start < state.LastProcessedStartTime)
{
state = new StockProcessingState();
intervals[interval] = state;
}
}
}
state.LastProcessedStartTime = start;
return(state);
}
public StockProcessor(StockSession session)
{
this.Session = session;
// Load the source file static information
Session.SourceFile.LoadStaticData(session);
// Create the derived data set
HistoricalData = StockDataSetDerived <StockDataSink, StockDataSource, StockProcessingState> .Derive(Session.SourceFile.GetSegments <StockDataSource>(), Session.SinkFile, CreateSink, GetProcessingState);
DerivedData = StockDataSetDerived <StockDataSink, StockDataSource, StockProcessingState> .CastToInterface(HistoricalData);
Session.Data = DerivedData;
Session.SinkFile.SetSegments <StockDataSink>(StockDataSetDerived <StockDataSink, StockDataSource, StockProcessingState> .CastToBase(HistoricalData));
foreach (var pair in HistoricalData)
{
var state = new StockProcessingState();
state.DataSet = pair.Value;
ProcessingStates[pair.Key] = new Dictionary <TimeSpan, StockProcessingState>()
{
};
}
}
