/// <summary> /// Get historical data from data feed, if possible /// TODO implement /// </summary> /// <param name="subscriptionsRequest">Data subscriptions to request</param> /// <param name="start">Start date for history request</param> /// <param name="end">End date for history request</param> public override void GetDataHistory(DataSubscriptionRequest[] subscriptionsRequest, DateTime start, DateTime end) { var found = new ConcurrentBag <DataPoint>(); var tasks = new List <Task>(); foreach (var sub in subscriptionsRequest) { DateTime currentdate = start.Date; while (currentdate < end) { tasks.Add(Task.Run(() => { var data = LoadFromCacheAsync(sub, currentdate).Result; if (data != null) { GetDataFromStream(sub.Ticker.Name, data).ForEach(x => found.Add(x)); } })); currentdate = currentdate.AddDays(1); } } //Wait for all threads Task.WaitAll(tasks.ToArray()); //Set to backfilling //found.ForEach(x => x.IsBackfilling = true); //TODO: we need to be able to set to backfilling? //Return what we have found.OrderBy(x => x.OccuredUtc) .ThenBy(x => x.Ticker) .ForEach(x => CurrenDatapoints.Enqueue(x)); }
/// <summary> /// Start historical feed /// </summary> public void Start() { //Start running IsRunning = true; DateTime currentdate = StartDateTime; while (IsRunning && EndDateTime > currentdate) { //Get data for today List <Task> tasks = new List <Task>(); Dictionary <string, MemoryStream> streams = new Dictionary <string, MemoryStream>(); _subscriptions.ForEach(sub => { tasks.Add(Task.Run(async() => { var data = await LoadFromCacheAsync(sub, currentdate); if (data != null) { streams.Add(sub.Ticker.Name, data); } _log.Trace($"Loaded data: {currentdate.ToString("yyyyMMdd")} - {sub.Ticker} - {sub.DataType}"); })); }); Task.WaitAll(tasks.ToArray()); tasks.Clear(); //Set background tasks List <DataPoint> datapoints = new List <DataPoint>(); streams.Where(x => x.Value != null) .ForEach(x => tasks.Add(Task.Run(() => datapoints.AddRange(GetDataFromStream(x.Key, x.Value))))); Task.WaitAll(tasks.ToArray()); //Check order if (datapoints.DistinctBy(x => x.Ticker).Count() > 1) { datapoints = datapoints.OrderBy(x => x.OccuredUtc) .ThenBy(x => x.Ticker) .ToList(); } //Add to current data points datapoints.ForEach(x => CurrenDatapoints.Enqueue(x)); //Add date currentdate = currentdate.AddDays(1); //Check if we need to wait (max 1M in cache) while (CurrenDatapoints.Count > 1000000 && IsRunning) { Thread.Sleep(10); } } //Check end of simulation while (CurrenDatapoints.Count > 0) { Thread.Sleep(10); } //Stop IsRunning = false; }