/// <summary> /// Add a data retrieval subscription for the given security /// </summary> /// <param name="subscriptionRequest"></param> public void AddSubscription(DataSubscriptionRequest subscriptionRequest) { //Check for updates UpdateTickers(); //Check for current subscription capabilities if (subscriptionRequest.DataSource != DataSource) { throw new Exception($"Cannot subscribe to data source {subscriptionRequest.DataSource} when using {DataSource}"); } //Normalize ticker string ticker = GetFeedTicker(subscriptionRequest.Ticker); //Check for all tickers if (subscriptionRequest.Ticker == FireHoseTicker) { IsFirehose = true; _availableTickers.ForEach(x => { _hubProxy.Invoke("SubscribeToExchangeDeltas", x); _connectedTickers.Add(x); }); return; } //Check for subscription if (!_connectedTickers.Contains(ticker)) { _log.Info("Subscribing to ticker " + ticker); _hubProxy.Invoke("SubscribeToExchangeDeltas", ticker); _connectedTickers.Add(ticker); } }
/// <summary> /// Add subscription to security /// </summary> /// <param name="subscriptionRequest"></param> public void AddSubscription(DataSubscriptionRequest subscriptionRequest) { if (!_subscriptions.Select(x => x.GetSubscriptionName()).Contains(subscriptionRequest.GetSubscriptionName())) { _subscriptions.Add(subscriptionRequest); } }
/// <summary> /// Stop retrieving data for the requested security /// </summary> /// <param name="subscriptionRequest"></param> public void RemoveSubscription(DataSubscriptionRequest subscriptionRequest) { var ticker = GetFeedTicker(subscriptionRequest.Ticker); _log.Info("Unsubscribing to ticker " + ticker); _hubProxy.Invoke("UmsubscribeToExchangeDeltas", ticker); _connectedTickers.Remove(ticker); }
/// <summary> /// Stop retrieving data for the requested security /// </summary> /// <param name="subscriptionRequest"></param> public void RemoveSubscription(DataSubscriptionRequest subscriptionRequest) { var ticker = subscriptionRequest.Ticker; if (CanSubscribe(ticker)) { _socket.Emit("unsubscribe", GetFeedTicker(ticker)); _subscribedSymbols.Remove(ticker.Name); } }
/// <summary> /// Add subscription to socket /// </summary> /// <param name="subscriptionRequest"></param> public void AddSubscription(DataSubscriptionRequest subscriptionRequest) { //Check for all subscription if (!_subscribed.Select(x => x.GetSubscriptionName()).Contains(subscriptionRequest.GetSubscriptionName())) { _subscribed.Add(subscriptionRequest); } //Logging _log.Info("Subscribed to subscription " + subscriptionRequest.GetSubscriptionName()); }
/// <summary> /// Remove a subscription for a data feed /// </summary> /// <param name="subscriptionRequest"></param> public void RemoveSubscription(DataSubscriptionRequest subscriptionRequest) { string subscriptionname = subscriptionRequest.GetSubscriptionName(); _log.Info("Unsubscribing to subscription " + subscriptionname); if (_subscribed.Contains(subscriptionname)) { _subscribed.Remove(subscriptionname); _subscriberSocket.Unsubscribe(subscriptionname); } }
/// <summary> /// Try to load entry from archive /// </summary> /// <param name="sub"></param> /// <param name="date"></param> /// <returns></returns> private async Task <MemoryStream> LoadFromDiskAsync(DataSubscriptionRequest sub, DateTime date) { //Get correct date int qldate = Util.ToQLDate(date); //Get archive string archive = DataUtil.GetArchiveFileName(_fileFolder, _source, sub.DataType, sub.Aggregation.HasValue ? $"{sub.Aggregation.Value.TotalMinutes}m" : $"1t", qldate); //Get Filename string file = DataUtil.SafeFilename(sub.Ticker.Name, Util.ToQLDate(date)); //Load from archive return(await Archive.ReadAsync(archive, file)); }
/// <summary> /// Stop retrieving data for the requested security /// </summary> /// <param name="subscriptionRequest"></param> public void RemoveSubscription(DataSubscriptionRequest subscriptionRequest) { //Check if we are fire hose, than skip if (IsFirehose) { return; } //Check and request for removal string feedname = GetFeedTicker(subscriptionRequest.Ticker); if (_subscribedtickers.Contains(feedname)) { UnsubscribeTicker(feedname); } }
/// <summary> /// Add a data retrieval subscription for the given security /// </summary> /// <param name="subscriptionRequest"></param> public void AddSubscription(DataSubscriptionRequest subscriptionRequest) { //Check for current subscription capabilities if (subscriptionRequest.DataSource != DataSource) { throw new Exception($"Cannot subscribe to data source {subscriptionRequest.DataSource} when using {DataSource}"); } //Check ticker type and subscribe if (subscriptionRequest.Ticker == FireHoseTicker) { _socket.Emit("subscribe", "firehose"); IsFirehose = true; } else if (CanSubscribe(subscriptionRequest.Ticker) && !IsFirehose && !_subscribedSymbols.Contains(subscriptionRequest.Ticker.Name)) { _socket.Emit("subscribe", subscriptionRequest.Ticker); _subscribedSymbols.Add(subscriptionRequest.Ticker.Name); } }
/// <summary> /// Stop retrieving data for the requested security /// </summary> /// <param name="subscriptionRequest"></param> public void RemoveSubscription(DataSubscriptionRequest subscriptionRequest) { try { var ticker = GetFeedTicker(subscriptionRequest.Ticker); _log.Info("Unsubscribing to ticker " + ticker); _subscribedtickers.Remove(ticker); var socket = _connecteddepthsockets.FirstOrDefault(x => x.Key == ticker); socket.Value.Close(); socket.Value.Dispose(); _connecteddepthsockets.Remove(ticker); socket = _connectedtradesockets.FirstOrDefault(x => x.Key == ticker); socket.Value.Close(); socket.Value.Dispose(); _connectedtradesockets.Remove(ticker); } catch (Exception e) { _log.Error(e, $"Could not remove subscription {subscriptionRequest.GetSubscriptionName()} from binance feed!"); } }
/// <summary> /// Add subscription to socket /// </summary> /// <param name="subscriptionRequest"></param> public void AddSubscription(DataSubscriptionRequest subscriptionRequest) { //Check for all subscription string subscriptionname = subscriptionRequest.GetSubscriptionName(); if (subscriptionRequest.Ticker == FireHoseTicker) { _subscriberSocket.Subscribe(""); //Firehose } else if (!_subscribed.Contains(subscriptionname) && CanSubscribe(subscriptionRequest.Ticker)) { //Add to known subscription _subscribed.Add(subscriptionname); //Add subscription _subscriberSocket.Subscribe(subscriptionname); } //Logging _log.Info("Subscribed to subscription " + subscriptionname); }
/// <summary> /// Add a data retrieval subscription for the given security /// </summary> /// <param name="subscriptionRequest"></param> public void AddSubscription(DataSubscriptionRequest subscriptionRequest) { //Check for current subscription capabilities if (subscriptionRequest.DataSource != DataSource) { throw new Exception($"Cannot subscribe to data source {subscriptionRequest.DataSource} when using {DataSource}"); } //Not needed as data feed will retrieve all data already UpdateTickers(); if (subscriptionRequest.Ticker != FireHoseTicker) { SubscribeTicker(GetFeedTicker(subscriptionRequest.Ticker)); } //Check for all tickers else { IsFirehose = true; _availableTickers.ForEach(x => SubscribeTicker(x)); } }
/// <summary> /// Add a data retrieval subscription for the given security /// </summary> /// <param name="subscriptionRequest"></param> /// <exception cref="Exception"></exception> public void AddSubscription(DataSubscriptionRequest subscriptionRequest) { //Get latest list of tickers UpdateTickers(); //Check for current subscription capabilities if (subscriptionRequest.DataSource != DataSource) { throw new Exception($"Cannot subscribe to data source {subscriptionRequest.DataSource} when using {DataSource}"); } //Normalize ticker string ticker = GetFeedTicker(subscriptionRequest.Ticker); //Helper function void AddTickerSocket(string tickersymbol) { _connecteddepthsockets.Add(tickersymbol, GetAndConnectSocket($"{_endpoint}{tickersymbol.ToLower()}@depth")); _connectedtradesockets.Add(tickersymbol, GetAndConnectSocket($"{_endpoint}{tickersymbol.ToLower()}@aggTrade")); } //Check for all tickers if (subscriptionRequest.Ticker == FireHoseTicker) { IsFirehose = true; _knowntickers.ForEach(AddTickerSocket); return; } //Check if this is a regular request and we are not subscribed already if (!_connecteddepthsockets.ContainsKey(ticker)) { AddTickerSocket(ticker); } }
/// <summary> /// Try and load data for date from cache /// http://www.alphadevx.com/a/387-Changing-the-maximum-item-size-allowed-by-Memcache /// </summary> /// <param name="sub"></param> /// <param name="date"></param> /// <returns></returns> private async Task <MemoryStream> LoadFromCacheAsync(DataSubscriptionRequest sub, DateTime date) { string key = DataPointCached.GetKey(sub.Ticker.Name, sub.DataType, sub.Aggregation, date); byte[] found = null; //Check if we are allowed to use cache if (_useCache) { //Try and load from cache var cacheitem = _cachedFiles.Get(key); if (cacheitem != null) { return(new MemoryStream(Compress.UncompressBytes(cacheitem.Data, key))); } } //Fall back to disk if not present if (found == null) { var disk = await LoadFromDiskAsync(sub, date); if (disk != null && _useCache) { //Create cache item, we store the result compressed in cache var cacheitem = DataPointCached.Create(sub.Ticker.Name, sub.DataType, sub.Aggregation, date, Compress.CompressBytes(disk.ToArray(), key)); _cachedFiles.Put(key, cacheitem); } //Return disk data return(disk); } //This should not happen return(null); }
public void RemoveSubscription(DataSubscriptionRequest subscriptionRequest) { throw new System.NotImplementedException(); }
/// <summary> /// Remove a subscription (not supported for historical data) /// </summary> /// <param name="ticker"></param> public void RemoveSubscription(DataSubscriptionRequest ticker) { }
/// <summary> /// Main entry point /// </summary> /// <param name="args">The arguments.</param> /// <exception cref="Exception">Could not initialize instance of data feed " + options.Datafeed</exception> internal static void Main(string[] args) { //Check on action to perform try { //Get options in use var y = Parser.Default.ParseArguments <Options>(args); if (y.Tag != ParserResultType.Parsed) { return; } Options options = ((Parsed <Options>)y).Value; WriteLine($"Initializing data capture instance."); //nlog is writer for data var config = new LoggingConfiguration(); var filetarget = new FileTarget(); config.AddTarget("log", filetarget); filetarget.FileName = $"{options.Destination}/capture.txt"; filetarget.ArchiveFileName = $"{options.Destination}/archive/capture" + ".{#}.zip"; filetarget.Layout = @"${message}"; filetarget.MaxArchiveFiles = 0; filetarget.ArchiveNumbering = ArchiveNumberingMode.Date; filetarget.ArchiveEvery = FileArchivePeriod.Day; filetarget.KeepFileOpen = true; filetarget.OpenFileCacheTimeout = 30; filetarget.EnableArchiveFileCompression = true; //Use async writer for better performance var asyncwrapper = new AsyncTargetWrapper("wrapper", filetarget); asyncwrapper.TimeToSleepBetweenBatches = 0; asyncwrapper.OverflowAction = AsyncTargetWrapperOverflowAction.Grow; asyncwrapper.BatchSize = 50000; //Set new rules config.LoggingRules.Add(new LoggingRule("Quantler.Util.DataCapture.*", LogLevel.Trace, asyncwrapper)); LogManager.Configuration = config; //Get instance WriteLine($"Initializing DataFeed instance {options.Datafeed}"); if (!DynamicLoader.Instance.TryGetInstance(options.Datafeed, out DataFeed feed)) { throw new Exception("Could not initialize instance of data feed " + options.Datafeed); } feed.SetDependencies(null, null, new NoDataFilter()); //Initialize feed ILogger Log = LogManager.GetCurrentClassLogger(); feed.Initialize(new LiveTradingMessage()); feed.SetDataCapture(data => Log.Info(DateTime.UtcNow.ToUnixTime(true).ToString() + "|" + Regex.Replace(data, "(\"(?:[^\"\\\\]|\\\\.)*\")|\\s+", "$1"))); //Regex is used to minify any json retrieved //Start feed WriteLine($"Starting feed"); Thread datafeedthread = new Thread(feed.Start) { IsBackground = true, Name = "DataFeed Thread" }; datafeedthread.Start(); //Wait for it to start and get the firehose WriteLine($"Waiting to be fully connected"); Thread.Sleep(15000); WriteLine($"Adding firehose subscription"); feed.AddSubscription(DataSubscriptionRequest.GetFireHoseSubscriptionRequest(feed.DataSource)); //Request firehose //Keep running while (feed.GetAvailableDataPackets().Any() || feed.IsRunning) { //Release current available data points, so we are not filling up memory } //Closure WriteLine("Feed stopped"); } catch (Exception exc) { WriteLine("Exception occured!"); WriteLine(exc.Message); } finally { ReadKey(); } }
/// <summary> /// Remove a subscription for a data feed /// </summary> /// <param name="subscriptionRequest"></param> public void RemoveSubscription(DataSubscriptionRequest subscriptionRequest) { }