/// <summary>
/// Get historical data from data feed, if possible
/// TODO implement
/// </summary>
/// <param name="subscriptionsRequest">Data subscriptions to request</param>
/// <param name="start">Start date for history request</param>
/// <param name="end">End date for history request</param>
public override void GetDataHistory(DataSubscriptionRequest[] subscriptionsRequest, DateTime start, DateTime end)
{
var found = new ConcurrentBag <DataPoint>();
var tasks = new List <Task>();
foreach (var sub in subscriptionsRequest)
{
DateTime currentdate = start.Date;
while (currentdate < end)
{
tasks.Add(Task.Run(() =>
{
var data = LoadFromCacheAsync(sub, currentdate).Result;
if (data != null)
{
GetDataFromStream(sub.Ticker.Name, data).ForEach(x => found.Add(x));
}
}));
currentdate = currentdate.AddDays(1);
}
}
//Wait for all threads
Task.WaitAll(tasks.ToArray());
//Set to backfilling
//found.ForEach(x => x.IsBackfilling = true); //TODO: we need to be able to set to backfilling?
//Return what we have
found.OrderBy(x => x.OccuredUtc)
.ThenBy(x => x.Ticker)
.ForEach(x => CurrenDatapoints.Enqueue(x));
}
/// <summary>
/// Start historical feed
/// </summary>
public void Start()
{
//Start running
IsRunning = true;
DateTime currentdate = StartDateTime;
while (IsRunning && EndDateTime > currentdate)
{
//Get data for today
List <Task> tasks = new List <Task>();
Dictionary <string, MemoryStream> streams = new Dictionary <string, MemoryStream>();
_subscriptions.ForEach(sub =>
{
tasks.Add(Task.Run(async() =>
{
var data = await LoadFromCacheAsync(sub, currentdate);
if (data != null)
{
streams.Add(sub.Ticker.Name, data);
}
_log.Trace($"Loaded data: {currentdate.ToString("yyyyMMdd")} - {sub.Ticker} - {sub.DataType}");
}));
});
Task.WaitAll(tasks.ToArray());
tasks.Clear();
//Set background tasks
List <DataPoint> datapoints = new List <DataPoint>();
streams.Where(x => x.Value != null)
.ForEach(x => tasks.Add(Task.Run(() => datapoints.AddRange(GetDataFromStream(x.Key, x.Value)))));
Task.WaitAll(tasks.ToArray());
//Check order
if (datapoints.DistinctBy(x => x.Ticker).Count() > 1)
{
datapoints = datapoints.OrderBy(x => x.OccuredUtc)
.ThenBy(x => x.Ticker)
.ToList();
}
//Add to current data points
datapoints.ForEach(x => CurrenDatapoints.Enqueue(x));
//Add date
currentdate = currentdate.AddDays(1);
//Check if we need to wait (max 1M in cache)
while (CurrenDatapoints.Count > 1000000 && IsRunning)
{
Thread.Sleep(10);
}
}
//Check end of simulation
while (CurrenDatapoints.Count > 0)
{
Thread.Sleep(10);
}
//Stop
IsRunning = false;
}