Example #1
0
        /// <summary>
        /// Add a data retrieval subscription for the given security
        /// </summary>
        /// <param name="subscriptionRequest"></param>
        public void AddSubscription(DataSubscriptionRequest subscriptionRequest)
        {
            //Check for updates
            UpdateTickers();

            //Check for current subscription capabilities
            if (subscriptionRequest.DataSource != DataSource)
            {
                throw new Exception($"Cannot subscribe to data source {subscriptionRequest.DataSource} when using {DataSource}");
            }

            //Normalize ticker
            string ticker = GetFeedTicker(subscriptionRequest.Ticker);

            //Check for all tickers
            if (subscriptionRequest.Ticker == FireHoseTicker)
            {
                IsFirehose = true;
                _availableTickers.ForEach(x =>
                {
                    _hubProxy.Invoke("SubscribeToExchangeDeltas", x);
                    _connectedTickers.Add(x);
                });
                return;
            }

            //Check for subscription
            if (!_connectedTickers.Contains(ticker))
            {
                _log.Info("Subscribing to ticker " + ticker);
                _hubProxy.Invoke("SubscribeToExchangeDeltas", ticker);
                _connectedTickers.Add(ticker);
            }
        }
Example #2
0
 /// <summary>
 /// Add subscription to security
 /// </summary>
 /// <param name="subscriptionRequest"></param>
 public void AddSubscription(DataSubscriptionRequest subscriptionRequest)
 {
     if (!_subscriptions.Select(x => x.GetSubscriptionName()).Contains(subscriptionRequest.GetSubscriptionName()))
     {
         _subscriptions.Add(subscriptionRequest);
     }
 }
Example #3
0
        /// <summary>
        /// Stop retrieving data for the requested security
        /// </summary>
        /// <param name="subscriptionRequest"></param>
        public void RemoveSubscription(DataSubscriptionRequest subscriptionRequest)
        {
            var ticker = GetFeedTicker(subscriptionRequest.Ticker);

            _log.Info("Unsubscribing to ticker " + ticker);
            _hubProxy.Invoke("UmsubscribeToExchangeDeltas", ticker);
            _connectedTickers.Remove(ticker);
        }
Example #4
0
        /// <summary>
        /// Stop retrieving data for the requested security
        /// </summary>
        /// <param name="subscriptionRequest"></param>
        public void RemoveSubscription(DataSubscriptionRequest subscriptionRequest)
        {
            var ticker = subscriptionRequest.Ticker;

            if (CanSubscribe(ticker))
            {
                _socket.Emit("unsubscribe", GetFeedTicker(ticker));
                _subscribedSymbols.Remove(ticker.Name);
            }
        }
Example #5
0
        /// <summary>
        /// Add subscription to socket
        /// </summary>
        /// <param name="subscriptionRequest"></param>
        public void AddSubscription(DataSubscriptionRequest subscriptionRequest)
        {
            //Check for all subscription
            if (!_subscribed.Select(x => x.GetSubscriptionName()).Contains(subscriptionRequest.GetSubscriptionName()))
            {
                _subscribed.Add(subscriptionRequest);
            }

            //Logging
            _log.Info("Subscribed to subscription " + subscriptionRequest.GetSubscriptionName());
        }
Example #6
0
        /// <summary>
        /// Remove a subscription for a data feed
        /// </summary>
        /// <param name="subscriptionRequest"></param>
        public void RemoveSubscription(DataSubscriptionRequest subscriptionRequest)
        {
            string subscriptionname = subscriptionRequest.GetSubscriptionName();

            _log.Info("Unsubscribing to subscription " + subscriptionname);
            if (_subscribed.Contains(subscriptionname))
            {
                _subscribed.Remove(subscriptionname);
                _subscriberSocket.Unsubscribe(subscriptionname);
            }
        }
Example #7
0
        /// <summary>
        /// Try to load entry from archive
        /// </summary>
        /// <param name="sub"></param>
        /// <param name="date"></param>
        /// <returns></returns>
        private async Task <MemoryStream> LoadFromDiskAsync(DataSubscriptionRequest sub, DateTime date)
        {
            //Get correct date
            int qldate = Util.ToQLDate(date);

            //Get archive
            string archive = DataUtil.GetArchiveFileName(_fileFolder, _source, sub.DataType, sub.Aggregation.HasValue ? $"{sub.Aggregation.Value.TotalMinutes}m" : $"1t", qldate);

            //Get Filename
            string file = DataUtil.SafeFilename(sub.Ticker.Name, Util.ToQLDate(date));

            //Load from archive
            return(await Archive.ReadAsync(archive, file));
        }
Example #8
0
        /// <summary>
        /// Stop retrieving data for the requested security
        /// </summary>
        /// <param name="subscriptionRequest"></param>
        public void RemoveSubscription(DataSubscriptionRequest subscriptionRequest)
        {
            //Check if we are fire hose, than skip
            if (IsFirehose)
            {
                return;
            }

            //Check and request for removal
            string feedname = GetFeedTicker(subscriptionRequest.Ticker);

            if (_subscribedtickers.Contains(feedname))
            {
                UnsubscribeTicker(feedname);
            }
        }
Example #9
0
        /// <summary>
        /// Add a data retrieval subscription for the given security
        /// </summary>
        /// <param name="subscriptionRequest"></param>
        public void AddSubscription(DataSubscriptionRequest subscriptionRequest)
        {
            //Check for current subscription capabilities
            if (subscriptionRequest.DataSource != DataSource)
            {
                throw new Exception($"Cannot subscribe to data source {subscriptionRequest.DataSource} when using {DataSource}");
            }

            //Check ticker type and subscribe
            if (subscriptionRequest.Ticker == FireHoseTicker)
            {
                _socket.Emit("subscribe", "firehose");
                IsFirehose = true;
            }
            else if (CanSubscribe(subscriptionRequest.Ticker) && !IsFirehose && !_subscribedSymbols.Contains(subscriptionRequest.Ticker.Name))
            {
                _socket.Emit("subscribe", subscriptionRequest.Ticker);
                _subscribedSymbols.Add(subscriptionRequest.Ticker.Name);
            }
        }
Example #10
0
 /// <summary>
 /// Stop retrieving data for the requested security
 /// </summary>
 /// <param name="subscriptionRequest"></param>
 public void RemoveSubscription(DataSubscriptionRequest subscriptionRequest)
 {
     try
     {
         var ticker = GetFeedTicker(subscriptionRequest.Ticker);
         _log.Info("Unsubscribing to ticker " + ticker);
         _subscribedtickers.Remove(ticker);
         var socket = _connecteddepthsockets.FirstOrDefault(x => x.Key == ticker);
         socket.Value.Close();
         socket.Value.Dispose();
         _connecteddepthsockets.Remove(ticker);
         socket = _connectedtradesockets.FirstOrDefault(x => x.Key == ticker);
         socket.Value.Close();
         socket.Value.Dispose();
         _connectedtradesockets.Remove(ticker);
     }
     catch (Exception e)
     {
         _log.Error(e, $"Could not remove subscription {subscriptionRequest.GetSubscriptionName()} from binance feed!");
     }
 }
Example #11
0
        /// <summary>
        /// Add subscription to socket
        /// </summary>
        /// <param name="subscriptionRequest"></param>
        public void AddSubscription(DataSubscriptionRequest subscriptionRequest)
        {
            //Check for all subscription
            string subscriptionname = subscriptionRequest.GetSubscriptionName();

            if (subscriptionRequest.Ticker == FireHoseTicker)
            {
                _subscriberSocket.Subscribe("");     //Firehose
            }
            else if (!_subscribed.Contains(subscriptionname) && CanSubscribe(subscriptionRequest.Ticker))
            {
                //Add to known subscription
                _subscribed.Add(subscriptionname);

                //Add subscription
                _subscriberSocket.Subscribe(subscriptionname);
            }

            //Logging
            _log.Info("Subscribed to subscription " + subscriptionname);
        }
Example #12
0
        /// <summary>
        /// Add a data retrieval subscription for the given security
        /// </summary>
        /// <param name="subscriptionRequest"></param>
        public void AddSubscription(DataSubscriptionRequest subscriptionRequest)
        {
            //Check for current subscription capabilities
            if (subscriptionRequest.DataSource != DataSource)
            {
                throw new Exception($"Cannot subscribe to data source {subscriptionRequest.DataSource} when using {DataSource}");
            }

            //Not needed as data feed will retrieve all data already
            UpdateTickers();
            if (subscriptionRequest.Ticker != FireHoseTicker)
            {
                SubscribeTicker(GetFeedTicker(subscriptionRequest.Ticker));
            }
            //Check for all tickers
            else
            {
                IsFirehose = true;
                _availableTickers.ForEach(x => SubscribeTicker(x));
            }
        }
Example #13
0
        /// <summary>
        /// Add a data retrieval subscription for the given security
        /// </summary>
        /// <param name="subscriptionRequest"></param>
        /// <exception cref="Exception"></exception>
        public void AddSubscription(DataSubscriptionRequest subscriptionRequest)
        {
            //Get latest list of tickers
            UpdateTickers();

            //Check for current subscription capabilities
            if (subscriptionRequest.DataSource != DataSource)
            {
                throw new Exception($"Cannot subscribe to data source {subscriptionRequest.DataSource} when using {DataSource}");
            }

            //Normalize ticker
            string ticker = GetFeedTicker(subscriptionRequest.Ticker);

            //Helper function
            void AddTickerSocket(string tickersymbol)
            {
                _connecteddepthsockets.Add(tickersymbol,
                                           GetAndConnectSocket($"{_endpoint}{tickersymbol.ToLower()}@depth"));
                _connectedtradesockets.Add(tickersymbol,
                                           GetAndConnectSocket($"{_endpoint}{tickersymbol.ToLower()}@aggTrade"));
            }

            //Check for all tickers
            if (subscriptionRequest.Ticker == FireHoseTicker)
            {
                IsFirehose = true;
                _knowntickers.ForEach(AddTickerSocket);
                return;
            }

            //Check if this is a regular request and we are not subscribed already
            if (!_connecteddepthsockets.ContainsKey(ticker))
            {
                AddTickerSocket(ticker);
            }
        }
Example #14
0
        /// <summary>
        /// Try and load data for date from cache
        /// http://www.alphadevx.com/a/387-Changing-the-maximum-item-size-allowed-by-Memcache
        /// </summary>
        /// <param name="sub"></param>
        /// <param name="date"></param>
        /// <returns></returns>
        private async Task <MemoryStream> LoadFromCacheAsync(DataSubscriptionRequest sub, DateTime date)
        {
            string key = DataPointCached.GetKey(sub.Ticker.Name, sub.DataType, sub.Aggregation, date);

            byte[] found = null;

            //Check if we are allowed to use cache
            if (_useCache)
            {
                //Try and load from cache
                var cacheitem = _cachedFiles.Get(key);
                if (cacheitem != null)
                {
                    return(new MemoryStream(Compress.UncompressBytes(cacheitem.Data, key)));
                }
            }

            //Fall back to disk if not present
            if (found == null)
            {
                var disk = await LoadFromDiskAsync(sub, date);

                if (disk != null && _useCache)
                {
                    //Create cache item, we store the result compressed in cache
                    var cacheitem = DataPointCached.Create(sub.Ticker.Name, sub.DataType, sub.Aggregation, date, Compress.CompressBytes(disk.ToArray(), key));
                    _cachedFiles.Put(key, cacheitem);
                }

                //Return disk data
                return(disk);
            }

            //This should not happen
            return(null);
        }
Example #15
0
 public void RemoveSubscription(DataSubscriptionRequest subscriptionRequest)
 {
     throw new System.NotImplementedException();
 }
Example #16
0
 /// <summary>
 /// Remove a subscription (not supported for historical data)
 /// </summary>
 /// <param name="ticker"></param>
 public void RemoveSubscription(DataSubscriptionRequest ticker)
 {
 }
Example #17
0
        /// <summary>
        /// Main entry point
        /// </summary>
        /// <param name="args">The arguments.</param>
        /// <exception cref="Exception">Could not initialize instance of data feed " + options.Datafeed</exception>
        internal static void Main(string[] args)
        {
            //Check on action to perform
            try
            {
                //Get options in use
                var y = Parser.Default.ParseArguments <Options>(args);
                if (y.Tag != ParserResultType.Parsed)
                {
                    return;
                }

                Options options = ((Parsed <Options>)y).Value;

                WriteLine($"Initializing data capture instance.");

                //nlog is writer for data
                var config     = new LoggingConfiguration();
                var filetarget = new FileTarget();
                config.AddTarget("log", filetarget);
                filetarget.FileName                     = $"{options.Destination}/capture.txt";
                filetarget.ArchiveFileName              = $"{options.Destination}/archive/capture" + ".{#}.zip";
                filetarget.Layout                       = @"${message}";
                filetarget.MaxArchiveFiles              = 0;
                filetarget.ArchiveNumbering             = ArchiveNumberingMode.Date;
                filetarget.ArchiveEvery                 = FileArchivePeriod.Day;
                filetarget.KeepFileOpen                 = true;
                filetarget.OpenFileCacheTimeout         = 30;
                filetarget.EnableArchiveFileCompression = true;

                //Use async writer for better performance
                var asyncwrapper = new AsyncTargetWrapper("wrapper", filetarget);
                asyncwrapper.TimeToSleepBetweenBatches = 0;
                asyncwrapper.OverflowAction            = AsyncTargetWrapperOverflowAction.Grow;
                asyncwrapper.BatchSize = 50000;

                //Set new rules
                config.LoggingRules.Add(new LoggingRule("Quantler.Util.DataCapture.*", LogLevel.Trace, asyncwrapper));
                LogManager.Configuration = config;

                //Get instance
                WriteLine($"Initializing DataFeed instance {options.Datafeed}");
                if (!DynamicLoader.Instance.TryGetInstance(options.Datafeed, out DataFeed feed))
                {
                    throw new Exception("Could not initialize instance of data feed " + options.Datafeed);
                }
                feed.SetDependencies(null, null, new NoDataFilter());

                //Initialize feed
                ILogger Log = LogManager.GetCurrentClassLogger();
                feed.Initialize(new LiveTradingMessage());
                feed.SetDataCapture(data => Log.Info(DateTime.UtcNow.ToUnixTime(true).ToString() + "|" + Regex.Replace(data, "(\"(?:[^\"\\\\]|\\\\.)*\")|\\s+", "$1"))); //Regex is used to minify any json retrieved

                //Start feed
                WriteLine($"Starting feed");
                Thread datafeedthread = new Thread(feed.Start)
                {
                    IsBackground = true, Name = "DataFeed Thread"
                };
                datafeedthread.Start();

                //Wait for it to start and get the firehose
                WriteLine($"Waiting to be fully connected");
                Thread.Sleep(15000);
                WriteLine($"Adding firehose subscription");
                feed.AddSubscription(DataSubscriptionRequest.GetFireHoseSubscriptionRequest(feed.DataSource)); //Request firehose

                //Keep running
                while (feed.GetAvailableDataPackets().Any() || feed.IsRunning)
                {
                    //Release current available data points, so we are not filling up memory
                }

                //Closure
                WriteLine("Feed stopped");
            }
            catch (Exception exc)
            {
                WriteLine("Exception occured!");
                WriteLine(exc.Message);
            }
            finally
            {
                ReadKey();
            }
        }
Example #18
0
 /// <summary>
 /// Remove a subscription for a data feed
 /// </summary>
 /// <param name="subscriptionRequest"></param>
 public void RemoveSubscription(DataSubscriptionRequest subscriptionRequest)
 {
 }