private IList <IngestionFetch> ListToFetches(SourceSeriesList sourceSeriesList)
{
var fetches = sourceSeriesList.Series.Select(BuildFetchesForSeries); //.Where(x => x.SourceName == "KGP.KJF.KEL01.0023.PV")
//Console.WriteLine("Built fetches: " + JsonConvert.SerializeObject(fetches));
return(fetches.MergeMany().ToList());
}
public void SetSourceSeriesList(SourceSeriesList sourceSeriesList)
{
lock (_subscriptionsLock)
{
var nextSourceNames = new HashSet <string>(sourceSeriesList.Series.Where(x => x.Realtime).Select(x => x.SourceName));
var newSourceNames = nextSourceNames.Where(nextSourceName => !_subscriptions.ContainsKey(nextSourceName));
var newSubscriptions = newSourceNames.Select(x => new NewSubscription {
CancellationTokenSource = new CancellationTokenSource(), SourceName = x
});
foreach (var newSubscription in newSubscriptions)
{
_subscriptions.TryAdd(newSubscription.SourceName, newSubscription.CancellationTokenSource);
_provider.SubscribeForRawData(newSubscription.SourceName, newSubscription.CancellationTokenSource.Token, dataRange => ReceiveRealtimeRaw(newSubscription.SourceName, dataRange));
_workers.GetOrAdd(newSubscription.SourceName, sourceName => new AggregationWorker(sourceName, _configuration.AggregationSeconds, _configuration.MaximumSourceDelaySeconds, _provider, dataRange => ReceiveRealtimeAggregates(sourceName, dataRange)));
}
var removeSourceNames = _subscriptions.Keys.Where(existingSourceName => !nextSourceNames.Contains(existingSourceName));
foreach (var sourceName in removeSourceNames)
{
_subscriptions.TryRemove(sourceName, out CancellationTokenSource cancellationTokenSource);
cancellationTokenSource.Cancel();
}
}
var nextQueue = PrepareNextQueue(sourceSeriesList);
SwapToNextQueue(nextQueue);
}
private QueueWrapper PrepareNextQueue(SourceSeriesList sourceSeriesList)
{
var fetches = ListToFetches(sourceSeriesList);
//fetches.Shuffle(); // Randomise the fetch order so we aren't always prioritising the same series
var queue = new QueueWrapper();
foreach (var ingestionFetch in fetches)
{
queue.Queue.Add(ingestionFetch);
}
return(queue);
}