/// <summary> /// Processes the specified data using the evaluator and resulting action /// <param name="multithreaded">Indicates if the processing should be done multithreaded</param> /// <param name="keep">Indicates if the processed data should remain in memory, /// or should be cleared once processing is complete.</param> /// </summary> public virtual void Process(bool multithreaded = false, MemoryScheme keep = MemoryScheme.MEM_KEEP_NONE) { System.Action <ProcessingTarget> processFunc = (ProcessingTarget target) => { List <StockDataSetDerived <StockDataSink, StockDataSource, StockProcessingState> > sets = HistoricalData[target.Symbol]; var processingState = new StockProcessingState(); for (; target.ProcessedSetIdx < sets.Count; target.ProcessedSetIdx++) { // Process all of the data sets for the target StockDataSetDerived <StockDataSink, StockDataSource, StockProcessingState> set = sets[target.ProcessedSetIdx]; set.Load(Session); for (; (target.ProcessedDataIdx < set.Count); target.ProcessedDataIdx++) { if (Evaluator.Evaluate(set, target.ProcessedDataIdx, target)) { Action.Do(this, target, set.Time(target.ProcessedDataIdx)); } } // Clean up the memory after processing has completed if (keep == MemoryScheme.MEM_KEEP_NONE) { set.Clear(); } else if (keep == MemoryScheme.MEM_KEEP_SOURCE) { set.ClearDerived(); } } }; // Process each of the targets if (multithreaded) { System.Threading.Tasks.Parallel.ForEach(Targets, processFunc); } else { foreach (ProcessingTarget target in Targets) { processFunc(target); } } // Indicate processing has completed if (Complete != null) { Complete(); } }
/// <summary> /// Callback used to create a stock data instance /// </summary> public StockProcessingState GetProcessingState(StockDataInterface data) { Dictionary <TimeSpan, StockProcessingState> intervals; StockProcessingState state; string symbol; DateTime start; TimeSpan interval; data.GetInfo(out symbol, out start, out interval); if (!ProcessingStates.TryGetValue(symbol, out intervals) || (intervals == null)) { state = new StockProcessingState(); state.DataSet = HistoricalData[symbol]; ProcessingStates[symbol] = new Dictionary <TimeSpan, StockProcessingState>() { { interval, state } }; } else { if (!intervals.TryGetValue(interval, out state) || (state == null)) { state = new StockProcessingState(); state.DataSet = HistoricalData[symbol]; intervals[interval] = state; } else { // Assume incrementing to the next data set state.DataSetIndex++; // Check if processing restarted if (start < state.LastProcessedStartTime) { state = new StockProcessingState(); state.DataSet = HistoricalData[symbol]; intervals[interval] = state; } } } state.LastProcessedStartTime = start; // Ensure the data set index is correct (should never evaluate to true) if (data != HistoricalData[symbol][state.DataSetIndex]) { throw new Exception("Data set index mismatch"); } return(state); }
/// <summary> /// Callback used to create a stock data instance /// </summary> public StockProcessingState GetProcessingState(StockDataInterface data) { Dictionary <TimeSpan, StockProcessingState> intervals; StockProcessingState state; string symbol; DateTime start; TimeSpan interval; data.GetInfo(out symbol, out start, out interval); if (!ProcessingStates.TryGetValue(symbol, out intervals) || (intervals == null)) { state = new StockProcessingState(); state.DataSet = HistoricalData[symbol]; ProcessingStates[symbol] = new Dictionary <TimeSpan, StockProcessingState>() { { interval, state } }; } else { if (!intervals.TryGetValue(interval, out state) || (state == null)) { state = new StockProcessingState(); state.DataSet = HistoricalData[symbol]; intervals[interval] = state; } else { // Assume incrementing to the next data set for (; state.DataSetIndex < state.DataSet.Count(); state.DataSetIndex++) { if (state.DataSet[state.DataSetIndex] == data) { break; } } // Check if processing restarted if (start < state.LastProcessedStartTime) { state = new StockProcessingState(); state.DataSet = HistoricalData[symbol]; intervals[interval] = state; } } } state.LastProcessedStartTime = start; return(state); }
/// <summary> /// Callback used to create a stock data instance /// </summary> public StockProcessingState GetProcessingState(StockDataSetInterface data) { Dictionary <TimeSpan, StockProcessingState> intervals; StockProcessingState state; string symbol; DateTime start; TimeSpan interval; data.GetInfo(out symbol, out start, out interval); if (!ProcessingStates.TryGetValue(symbol, out intervals) || (intervals == null)) { state = new StockProcessingState(); ProcessingStates[symbol] = new Dictionary <TimeSpan, StockProcessingState>() { { interval, state } }; } else { if (!intervals.TryGetValue(interval, out state) || (state == null)) { state = new StockProcessingState(); intervals[interval] = state; } else { // Check if processing restarted if (start < state.LastProcessedStartTime) { state = new StockProcessingState(); intervals[interval] = state; } } } state.LastProcessedStartTime = start; return(state); }
public StockProcessor(StockSession session) { this.Session = session; // Load the source file static information Session.SourceFile.LoadStaticData(session); // Create the derived data set HistoricalData = StockDataSetDerived <StockDataSink, StockDataSource, StockProcessingState> .Derive(Session.SourceFile.GetSegments <StockDataSource>(), Session.SinkFile, CreateSink, GetProcessingState); DerivedData = StockDataSetDerived <StockDataSink, StockDataSource, StockProcessingState> .CastToInterface(HistoricalData); Session.Data = DerivedData; Session.SinkFile.SetSegments <StockDataSink>(StockDataSetDerived <StockDataSink, StockDataSource, StockProcessingState> .CastToBase(HistoricalData)); foreach (var pair in HistoricalData) { var state = new StockProcessingState(); state.DataSet = pair.Value; ProcessingStates[pair.Key] = new Dictionary <TimeSpan, StockProcessingState>() { }; } }