Ejemplo n.º 1
0
        /// <summary>
        /// Get historical data from data feed, if possible
        /// TODO implement
        /// </summary>
        /// <param name="subscriptionsRequest">Data subscriptions to request</param>
        /// <param name="start">Start date for history request</param>
        /// <param name="end">End date for history request</param>
        public override void GetDataHistory(DataSubscriptionRequest[] subscriptionsRequest, DateTime start, DateTime end)
        {
            var found = new ConcurrentBag <DataPoint>();
            var tasks = new List <Task>();

            foreach (var sub in subscriptionsRequest)
            {
                DateTime currentdate = start.Date;
                while (currentdate < end)
                {
                    tasks.Add(Task.Run(() =>
                    {
                        var data = LoadFromCacheAsync(sub, currentdate).Result;
                        if (data != null)
                        {
                            GetDataFromStream(sub.Ticker.Name, data).ForEach(x => found.Add(x));
                        }
                    }));

                    currentdate = currentdate.AddDays(1);
                }
            }

            //Wait for all threads
            Task.WaitAll(tasks.ToArray());

            //Set to backfilling
            //found.ForEach(x => x.IsBackfilling = true); //TODO: we need to be able to set to backfilling?

            //Return what we have
            found.OrderBy(x => x.OccuredUtc)
            .ThenBy(x => x.Ticker)
            .ForEach(x => CurrenDatapoints.Enqueue(x));
        }
Ejemplo n.º 2
0
        /// <summary>
        /// Start historical feed
        /// </summary>
        public void Start()
        {
            //Start running
            IsRunning = true;
            DateTime currentdate = StartDateTime;

            while (IsRunning && EndDateTime > currentdate)
            {
                //Get data for today
                List <Task> tasks = new List <Task>();
                Dictionary <string, MemoryStream> streams = new Dictionary <string, MemoryStream>();
                _subscriptions.ForEach(sub =>
                {
                    tasks.Add(Task.Run(async() =>
                    {
                        var data = await LoadFromCacheAsync(sub, currentdate);
                        if (data != null)
                        {
                            streams.Add(sub.Ticker.Name, data);
                        }
                        _log.Trace($"Loaded data: {currentdate.ToString("yyyyMMdd")} - {sub.Ticker} - {sub.DataType}");
                    }));
                });

                Task.WaitAll(tasks.ToArray());
                tasks.Clear();

                //Set background tasks
                List <DataPoint> datapoints = new List <DataPoint>();
                streams.Where(x => x.Value != null)
                .ForEach(x => tasks.Add(Task.Run(() => datapoints.AddRange(GetDataFromStream(x.Key, x.Value)))));
                Task.WaitAll(tasks.ToArray());

                //Check order
                if (datapoints.DistinctBy(x => x.Ticker).Count() > 1)
                {
                    datapoints = datapoints.OrderBy(x => x.OccuredUtc)
                                 .ThenBy(x => x.Ticker)
                                 .ToList();
                }

                //Add to current data points
                datapoints.ForEach(x => CurrenDatapoints.Enqueue(x));

                //Add date
                currentdate = currentdate.AddDays(1);

                //Check if we need to wait (max 1M in cache)
                while (CurrenDatapoints.Count > 1000000 && IsRunning)
                {
                    Thread.Sleep(10);
                }
            }

            //Check end of simulation
            while (CurrenDatapoints.Count > 0)
            {
                Thread.Sleep(10);
            }

            //Stop
            IsRunning = false;
        }