/// <summary>
/// Add a data retrieval subscription for the given security
/// </summary>
/// <param name="subscriptionRequest"></param>
public void AddSubscription(DataSubscriptionRequest subscriptionRequest)
{
//Check for updates
UpdateTickers();
//Check for current subscription capabilities
if (subscriptionRequest.DataSource != DataSource)
{
throw new Exception($"Cannot subscribe to data source {subscriptionRequest.DataSource} when using {DataSource}");
}
//Normalize ticker
string ticker = GetFeedTicker(subscriptionRequest.Ticker);
//Check for all tickers
if (subscriptionRequest.Ticker == FireHoseTicker)
{
IsFirehose = true;
_availableTickers.ForEach(x =>
{
_hubProxy.Invoke("SubscribeToExchangeDeltas", x);
_connectedTickers.Add(x);
});
return;
}
//Check for subscription
if (!_connectedTickers.Contains(ticker))
{
_log.Info("Subscribing to ticker " + ticker);
_hubProxy.Invoke("SubscribeToExchangeDeltas", ticker);
_connectedTickers.Add(ticker);
}
}
/// <summary>
/// Add subscription to security
/// </summary>
/// <param name="subscriptionRequest"></param>
public void AddSubscription(DataSubscriptionRequest subscriptionRequest)
{
if (!_subscriptions.Select(x => x.GetSubscriptionName()).Contains(subscriptionRequest.GetSubscriptionName()))
{
_subscriptions.Add(subscriptionRequest);
}
}
/// <summary>
/// Stop retrieving data for the requested security
/// </summary>
/// <param name="subscriptionRequest"></param>
public void RemoveSubscription(DataSubscriptionRequest subscriptionRequest)
{
var ticker = GetFeedTicker(subscriptionRequest.Ticker);
_log.Info("Unsubscribing to ticker " + ticker);
_hubProxy.Invoke("UmsubscribeToExchangeDeltas", ticker);
_connectedTickers.Remove(ticker);
}
/// <summary>
/// Stop retrieving data for the requested security
/// </summary>
/// <param name="subscriptionRequest"></param>
public void RemoveSubscription(DataSubscriptionRequest subscriptionRequest)
{
var ticker = subscriptionRequest.Ticker;
if (CanSubscribe(ticker))
{
_socket.Emit("unsubscribe", GetFeedTicker(ticker));
_subscribedSymbols.Remove(ticker.Name);
}
}
/// <summary>
/// Add subscription to socket
/// </summary>
/// <param name="subscriptionRequest"></param>
public void AddSubscription(DataSubscriptionRequest subscriptionRequest)
{
//Check for all subscription
if (!_subscribed.Select(x => x.GetSubscriptionName()).Contains(subscriptionRequest.GetSubscriptionName()))
{
_subscribed.Add(subscriptionRequest);
}
//Logging
_log.Info("Subscribed to subscription " + subscriptionRequest.GetSubscriptionName());
}
/// <summary>
/// Remove a subscription for a data feed
/// </summary>
/// <param name="subscriptionRequest"></param>
public void RemoveSubscription(DataSubscriptionRequest subscriptionRequest)
{
string subscriptionname = subscriptionRequest.GetSubscriptionName();
_log.Info("Unsubscribing to subscription " + subscriptionname);
if (_subscribed.Contains(subscriptionname))
{
_subscribed.Remove(subscriptionname);
_subscriberSocket.Unsubscribe(subscriptionname);
}
}
/// <summary>
/// Try to load entry from archive
/// </summary>
/// <param name="sub"></param>
/// <param name="date"></param>
/// <returns></returns>
private async Task <MemoryStream> LoadFromDiskAsync(DataSubscriptionRequest sub, DateTime date)
{
//Get correct date
int qldate = Util.ToQLDate(date);
//Get archive
string archive = DataUtil.GetArchiveFileName(_fileFolder, _source, sub.DataType, sub.Aggregation.HasValue ? $"{sub.Aggregation.Value.TotalMinutes}m" : $"1t", qldate);
//Get Filename
string file = DataUtil.SafeFilename(sub.Ticker.Name, Util.ToQLDate(date));
//Load from archive
return(await Archive.ReadAsync(archive, file));
}
/// <summary>
/// Stop retrieving data for the requested security
/// </summary>
/// <param name="subscriptionRequest"></param>
public void RemoveSubscription(DataSubscriptionRequest subscriptionRequest)
{
//Check if we are fire hose, than skip
if (IsFirehose)
{
return;
}
//Check and request for removal
string feedname = GetFeedTicker(subscriptionRequest.Ticker);
if (_subscribedtickers.Contains(feedname))
{
UnsubscribeTicker(feedname);
}
}
/// <summary>
/// Add a data retrieval subscription for the given security
/// </summary>
/// <param name="subscriptionRequest"></param>
public void AddSubscription(DataSubscriptionRequest subscriptionRequest)
{
//Check for current subscription capabilities
if (subscriptionRequest.DataSource != DataSource)
{
throw new Exception($"Cannot subscribe to data source {subscriptionRequest.DataSource} when using {DataSource}");
}
//Check ticker type and subscribe
if (subscriptionRequest.Ticker == FireHoseTicker)
{
_socket.Emit("subscribe", "firehose");
IsFirehose = true;
}
else if (CanSubscribe(subscriptionRequest.Ticker) && !IsFirehose && !_subscribedSymbols.Contains(subscriptionRequest.Ticker.Name))
{
_socket.Emit("subscribe", subscriptionRequest.Ticker);
_subscribedSymbols.Add(subscriptionRequest.Ticker.Name);
}
}
/// <summary>
/// Stop retrieving data for the requested security
/// </summary>
/// <param name="subscriptionRequest"></param>
public void RemoveSubscription(DataSubscriptionRequest subscriptionRequest)
{
try
{
var ticker = GetFeedTicker(subscriptionRequest.Ticker);
_log.Info("Unsubscribing to ticker " + ticker);
_subscribedtickers.Remove(ticker);
var socket = _connecteddepthsockets.FirstOrDefault(x => x.Key == ticker);
socket.Value.Close();
socket.Value.Dispose();
_connecteddepthsockets.Remove(ticker);
socket = _connectedtradesockets.FirstOrDefault(x => x.Key == ticker);
socket.Value.Close();
socket.Value.Dispose();
_connectedtradesockets.Remove(ticker);
}
catch (Exception e)
{
_log.Error(e, $"Could not remove subscription {subscriptionRequest.GetSubscriptionName()} from binance feed!");
}
}
/// <summary>
/// Add subscription to socket
/// </summary>
/// <param name="subscriptionRequest"></param>
public void AddSubscription(DataSubscriptionRequest subscriptionRequest)
{
//Check for all subscription
string subscriptionname = subscriptionRequest.GetSubscriptionName();
if (subscriptionRequest.Ticker == FireHoseTicker)
{
_subscriberSocket.Subscribe(""); //Firehose
}
else if (!_subscribed.Contains(subscriptionname) && CanSubscribe(subscriptionRequest.Ticker))
{
//Add to known subscription
_subscribed.Add(subscriptionname);
//Add subscription
_subscriberSocket.Subscribe(subscriptionname);
}
//Logging
_log.Info("Subscribed to subscription " + subscriptionname);
}
/// <summary>
/// Add a data retrieval subscription for the given security
/// </summary>
/// <param name="subscriptionRequest"></param>
public void AddSubscription(DataSubscriptionRequest subscriptionRequest)
{
//Check for current subscription capabilities
if (subscriptionRequest.DataSource != DataSource)
{
throw new Exception($"Cannot subscribe to data source {subscriptionRequest.DataSource} when using {DataSource}");
}
//Not needed as data feed will retrieve all data already
UpdateTickers();
if (subscriptionRequest.Ticker != FireHoseTicker)
{
SubscribeTicker(GetFeedTicker(subscriptionRequest.Ticker));
}
//Check for all tickers
else
{
IsFirehose = true;
_availableTickers.ForEach(x => SubscribeTicker(x));
}
}
/// <summary>
/// Add a data retrieval subscription for the given security
/// </summary>
/// <param name="subscriptionRequest"></param>
/// <exception cref="Exception"></exception>
public void AddSubscription(DataSubscriptionRequest subscriptionRequest)
{
//Get latest list of tickers
UpdateTickers();
//Check for current subscription capabilities
if (subscriptionRequest.DataSource != DataSource)
{
throw new Exception($"Cannot subscribe to data source {subscriptionRequest.DataSource} when using {DataSource}");
}
//Normalize ticker
string ticker = GetFeedTicker(subscriptionRequest.Ticker);
//Helper function
void AddTickerSocket(string tickersymbol)
{
_connecteddepthsockets.Add(tickersymbol,
GetAndConnectSocket($"{_endpoint}{tickersymbol.ToLower()}@depth"));
_connectedtradesockets.Add(tickersymbol,
GetAndConnectSocket($"{_endpoint}{tickersymbol.ToLower()}@aggTrade"));
}
//Check for all tickers
if (subscriptionRequest.Ticker == FireHoseTicker)
{
IsFirehose = true;
_knowntickers.ForEach(AddTickerSocket);
return;
}
//Check if this is a regular request and we are not subscribed already
if (!_connecteddepthsockets.ContainsKey(ticker))
{
AddTickerSocket(ticker);
}
}
/// <summary>
/// Try and load data for date from cache
/// http://www.alphadevx.com/a/387-Changing-the-maximum-item-size-allowed-by-Memcache
/// </summary>
/// <param name="sub"></param>
/// <param name="date"></param>
/// <returns></returns>
private async Task <MemoryStream> LoadFromCacheAsync(DataSubscriptionRequest sub, DateTime date)
{
string key = DataPointCached.GetKey(sub.Ticker.Name, sub.DataType, sub.Aggregation, date);
byte[] found = null;
//Check if we are allowed to use cache
if (_useCache)
{
//Try and load from cache
var cacheitem = _cachedFiles.Get(key);
if (cacheitem != null)
{
return(new MemoryStream(Compress.UncompressBytes(cacheitem.Data, key)));
}
}
//Fall back to disk if not present
if (found == null)
{
var disk = await LoadFromDiskAsync(sub, date);
if (disk != null && _useCache)
{
//Create cache item, we store the result compressed in cache
var cacheitem = DataPointCached.Create(sub.Ticker.Name, sub.DataType, sub.Aggregation, date, Compress.CompressBytes(disk.ToArray(), key));
_cachedFiles.Put(key, cacheitem);
}
//Return disk data
return(disk);
}
//This should not happen
return(null);
}
public void RemoveSubscription(DataSubscriptionRequest subscriptionRequest)
{
throw new System.NotImplementedException();
}
/// <summary>
/// Remove a subscription (not supported for historical data)
/// </summary>
/// <param name="ticker"></param>
public void RemoveSubscription(DataSubscriptionRequest ticker)
{
}
/// <summary>
/// Main entry point
/// </summary>
/// <param name="args">The arguments.</param>
/// <exception cref="Exception">Could not initialize instance of data feed " + options.Datafeed</exception>
internal static void Main(string[] args)
{
//Check on action to perform
try
{
//Get options in use
var y = Parser.Default.ParseArguments <Options>(args);
if (y.Tag != ParserResultType.Parsed)
{
return;
}
Options options = ((Parsed <Options>)y).Value;
WriteLine($"Initializing data capture instance.");
//nlog is writer for data
var config = new LoggingConfiguration();
var filetarget = new FileTarget();
config.AddTarget("log", filetarget);
filetarget.FileName = $"{options.Destination}/capture.txt";
filetarget.ArchiveFileName = $"{options.Destination}/archive/capture" + ".{#}.zip";
filetarget.Layout = @"${message}";
filetarget.MaxArchiveFiles = 0;
filetarget.ArchiveNumbering = ArchiveNumberingMode.Date;
filetarget.ArchiveEvery = FileArchivePeriod.Day;
filetarget.KeepFileOpen = true;
filetarget.OpenFileCacheTimeout = 30;
filetarget.EnableArchiveFileCompression = true;
//Use async writer for better performance
var asyncwrapper = new AsyncTargetWrapper("wrapper", filetarget);
asyncwrapper.TimeToSleepBetweenBatches = 0;
asyncwrapper.OverflowAction = AsyncTargetWrapperOverflowAction.Grow;
asyncwrapper.BatchSize = 50000;
//Set new rules
config.LoggingRules.Add(new LoggingRule("Quantler.Util.DataCapture.*", LogLevel.Trace, asyncwrapper));
LogManager.Configuration = config;
//Get instance
WriteLine($"Initializing DataFeed instance {options.Datafeed}");
if (!DynamicLoader.Instance.TryGetInstance(options.Datafeed, out DataFeed feed))
{
throw new Exception("Could not initialize instance of data feed " + options.Datafeed);
}
feed.SetDependencies(null, null, new NoDataFilter());
//Initialize feed
ILogger Log = LogManager.GetCurrentClassLogger();
feed.Initialize(new LiveTradingMessage());
feed.SetDataCapture(data => Log.Info(DateTime.UtcNow.ToUnixTime(true).ToString() + "|" + Regex.Replace(data, "(\"(?:[^\"\\\\]|\\\\.)*\")|\\s+", "$1"))); //Regex is used to minify any json retrieved
//Start feed
WriteLine($"Starting feed");
Thread datafeedthread = new Thread(feed.Start)
{
IsBackground = true, Name = "DataFeed Thread"
};
datafeedthread.Start();
//Wait for it to start and get the firehose
WriteLine($"Waiting to be fully connected");
Thread.Sleep(15000);
WriteLine($"Adding firehose subscription");
feed.AddSubscription(DataSubscriptionRequest.GetFireHoseSubscriptionRequest(feed.DataSource)); //Request firehose
//Keep running
while (feed.GetAvailableDataPackets().Any() || feed.IsRunning)
{
//Release current available data points, so we are not filling up memory
}
//Closure
WriteLine("Feed stopped");
}
catch (Exception exc)
{
WriteLine("Exception occured!");
WriteLine(exc.Message);
}
finally
{
ReadKey();
}
}
/// <summary>
/// Remove a subscription for a data feed
/// </summary>
/// <param name="subscriptionRequest"></param>
public void RemoveSubscription(DataSubscriptionRequest subscriptionRequest)
{
}