/********************************************************
* CLASS CONSTRUCTOR
*********************************************************/
/********************************************************
* CLASS METHODS
*********************************************************/
/// <summary>
/// Generic Reader Implementation for Quandl Data.
/// </summary>
/// <param name="config">Subscription configuration</param>
/// <param name="line">CSV line of data from the souce</param>
/// <param name="date">Date of the requested line</param>
/// <param name="datafeed">Datafeed type - live or backtest</param>
/// <returns></returns>
public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, DataFeedEndpoint datafeed)
{
var data = new Quandl();
data.Symbol = config.Symbol;
var csv = line.Split(',');
if (!_isInitialized)
{
_isInitialized = true;
foreach (var propertyName in csv)
{
var property = propertyName.TrimStart().TrimEnd();
// should we remove property names like Time?
// do we need to alias the Time??
data.SetProperty(property, 0m);
_propertyNames.Add(property);
}
return data;
}
data.Time = DateTime.ParseExact(csv[0], "yyyy-MM-dd", CultureInfo.InvariantCulture);
for (var i = 1; i < csv.Length; i++)
{
var value = csv[i].ToDecimal();
data.SetProperty(_propertyNames[i], value);
}
// we know that there is a close property, we want to set that to 'Value'
data.Value = (decimal) data.GetProperty("Close");
return data;
}
/********************************************************
* CLASS CONSTRUCTOR
*********************************************************/
/// <summary>
/// Live trading datafeed handler provides a base implementation of a live trading datafeed. Derived types
/// need only implement the GetNextTicks() function to return unprocessed ticks from a data source.
/// This creates a new data feed with a DataFeedEndpoint of LiveTrading.
/// </summary>
/// <param name="algorithm">Algorithm requesting data</param>
protected LiveTradingDataFeed(IAlgorithm algorithm)
{
//Subscription Count:
_subscriptions = algorithm.SubscriptionManager.Subscriptions;
//Set Properties:
_dataFeed = DataFeedEndpoint.LiveTrading;
_isActive = true;
_bridge = new ConcurrentQueue <List <BaseData> > [Subscriptions.Count];
_endOfBridge = new bool[Subscriptions.Count];
_subscriptionManagers = new SubscriptionDataReader[Subscriptions.Count];
_realtimePrices = new List <decimal>();
//Class Privates:
_algorithm = algorithm;
//Setup the arrays:
for (var i = 0; i < Subscriptions.Count; i++)
{
_endOfBridge[i] = false;
_bridge[i] = new ConcurrentQueue <List <BaseData> >();
//This is quantconnect data source, store here for speed/ease of access
_isDynamicallyLoadedData.Add(algorithm.Securities[_subscriptions[i].Symbol].IsDynamicallyLoadedData);
//Subscription managers for downloading user data:
_subscriptionManagers[i] = new SubscriptionDataReader(_subscriptions[i], algorithm.Securities[_subscriptions[i].Symbol], DataFeedEndpoint.LiveTrading, DateTime.MinValue, DateTime.MaxValue);
//Set up the source file for today:
_subscriptionManagers[i].RefreshSource(DateTime.Now.Date);
_realtimePrices.Add(0);
}
}
/********************************************************
* CLASS CONSTRUCTOR
*********************************************************/
/// <summary>
/// REST Streaming Reader. Each call will call to rest API again
/// </summary>
/// <param name="source">String location of the REST Endpoint</param>
/// <param name="datafeed">DataEndpoint</param>
public SubscriptionStreamReader(string source, DataFeedEndpoint datafeed)
{
//This is a physical file location, create a stream:
if (datafeed == DataFeedEndpoint.Backtesting || datafeed == DataFeedEndpoint.FileSystem)
{
_sr = new StreamReader(source);
}
_request = new RestRequest(source, Method.GET);
_dataFeed = datafeed;
}
public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, DataFeedEndpoint datafeed)
{
//New Bitcoin object
CashType cash = new CashType();
try
{
string[] data = line.Split(',');
cash.Time = DateTime.ParseExact(data[0], "yyyy-MM-dd", CultureInfo.InvariantCulture);
cash.Value = Convert.ToDecimal(data[1]);
}
catch { /* Do nothing, skip first title row */ }
return cash;
}
/// <summary>
/// 2. RETURN THE STRING URL SOURCE LOCATION FOR YOUR DATA:
/// This is a powerful and dynamic select source file method. If you have a large dataset, 10+mb we recommend you break it into smaller files. E.g. One zip per year.
/// We can accept raw text or ZIP files. We read the file extension to determine if it is a zip file.
/// </summary>
/// <param name="config">Subscription data, symbol name, data type</param>
/// <param name="date">Current date we're requesting. This allows you to break up the data source into daily files.</param>
/// <param name="datafeed">Datafeed type: Backtesting or the Live data broker who will provide live data. You can specify a different source for live trading! </param>
/// <returns>string URL end point.</returns>
public override string GetSource(SubscriptionDataConfig config, DateTime date, DataFeedEndpoint datafeed)
{
switch (datafeed)
{
//Backtesting Data Source: Example of a data source which varies by day (commented out)
default:
case DataFeedEndpoint.Backtesting:
//return "http://my-ftp-server.com/futures-data-" + date.ToString("Ymd") + ".zip";
// OR simply return a fixed small data file. Large files will slow down your backtest
return "http://www.quandl.com/api/v1/datasets/BITCOIN/BITSTAMPUSD.csv?sort_order=asc";
case DataFeedEndpoint.LiveTrading:
//Alternative live socket data source for live trading (soon)/
return "....";
}
}
/// <summary>
/// Live trading datafeed handler provides a base implementation of a live trading datafeed. Derived types
/// need only implement the GetNextTicks() function to return unprocessed ticks from a data source.
/// This creates a new data feed with a DataFeedEndpoint of LiveTrading.
/// </summary>
public LiveTradingDataFeed(IAlgorithm algorithm, LiveNodePacket job, IDataQueueHandler dataSource)
{
//Subscription Count:
_subscriptions = algorithm.SubscriptionManager.Subscriptions;
//Set Properties:
_isActive = true;
_dataFeed = DataFeedEndpoint.LiveTrading;
_bridge = new ConcurrentQueue <List <BaseData> > [Subscriptions.Count];
_endOfBridge = new bool[Subscriptions.Count];
_subscriptionManagers = new SubscriptionDataReader[Subscriptions.Count];
_realtimePrices = new List <decimal>();
//Set the source of the live data:
_dataQueue = dataSource;
//Class Privates:
_algorithm = algorithm;
_job = job;
//Setup the arrays:
for (var i = 0; i < Subscriptions.Count; i++)
{
_endOfBridge[i] = false;
_bridge[i] = new ConcurrentQueue <List <BaseData> >();
//This is quantconnect data source, store here for speed/ease of access
_isDynamicallyLoadedData.Add(algorithm.Securities[_subscriptions[i].Symbol].IsDynamicallyLoadedData);
//Subscription managers for downloading user data:
_subscriptionManagers[i] = new SubscriptionDataReader(_subscriptions[i], algorithm.Securities[_subscriptions[i].Symbol], DataFeedEndpoint.LiveTrading, DateTime.MinValue, DateTime.MaxValue);
//Set up the source file for today:
_subscriptionManagers[i].RefreshSource(DateTime.Now.Date);
_realtimePrices.Add(0);
}
// request for data from these symbols
_dataQueue.Subscribe(job, BuildTypeSymbolList(algorithm));
}
/// <summary>
/// Live trading datafeed handler provides a base implementation of a live trading datafeed. Derived types
/// need only implement the GetNextTicks() function to return unprocessed ticks from a data source.
/// This creates a new data feed with a DataFeedEndpoint of LiveTrading.
/// </summary>
public LiveTradingDataFeed(IAlgorithm algorithm, LiveNodePacket job, IDataQueueHandler dataSource)
{
//Subscription Count:
_subscriptions = algorithm.SubscriptionManager.Subscriptions;
//Set Properties:
_isActive = true;
_dataFeed = DataFeedEndpoint.LiveTrading;
_bridge = new ConcurrentQueue<List<BaseData>>[Subscriptions.Count];
_endOfBridge = new bool[Subscriptions.Count];
_subscriptionManagers = new SubscriptionDataReader[Subscriptions.Count];
_realtimePrices = new List<decimal>();
//Set the source of the live data:
_dataQueue = dataSource;
//Class Privates:
_algorithm = algorithm;
_job = job;
//Setup the arrays:
for (var i = 0; i < Subscriptions.Count; i++)
{
_endOfBridge[i] = false;
_bridge[i] = new ConcurrentQueue<List<BaseData>>();
//This is quantconnect data source, store here for speed/ease of access
_isDynamicallyLoadedData.Add(algorithm.Securities[_subscriptions[i].Symbol].IsDynamicallyLoadedData);
//Subscription managers for downloading user data:
_subscriptionManagers[i] = new SubscriptionDataReader(_subscriptions[i], algorithm.Securities[_subscriptions[i].Symbol], DataFeedEndpoint.LiveTrading, DateTime.MinValue, DateTime.MaxValue);
//Set up the source file for today:
_subscriptionManagers[i].RefreshSource(DateTime.Now.Date);
_realtimePrices.Add(0);
}
// request for data from these symbols
_dataQueue.Subscribe(job, BuildTypeSymbolList(algorithm));
}
/// <summary>
/// Source URL's of Backtesting and Live Streams:
/// </summary>
public override string GetSource(SubscriptionDataConfig config, DateTime date, DataFeedEndpoint datafeed)
{
var source = "";
switch (datafeed)
{
//Historical backtesting data:
case DataFeedEndpoint.Backtesting:
source = "http://www.quandl.com/api/v1/datasets/BITCOIN/BITSTAMPUSD.csv?sort_order=asc";
break;
//Live socket for bitcoin prices:
case DataFeedEndpoint.Tradier:
case DataFeedEndpoint.LiveTrading:
//Live refreshing endpoint.
source = "https://www.bitstamp.net/api/ticker/";
break;
}
return source;
}
/// <summary>
/// Parse a line from CSV data sources into our trade bars.
/// </summary>
/// <param name="config">Configuration class object for this data subscription</param>
/// <param name="baseDate">Base date of this tradebar line</param>
/// <param name="line">CSV line from source data file</param>
/// <param name="datafeed">Datafeed this csv line is sourced from (backtesting or live)</param>
public TradeBar(SubscriptionDataConfig config, string line, DateTime baseDate, DataFeedEndpoint datafeed = DataFeedEndpoint.Backtesting)
{
try
{
Period = config.Resolution.ToTimeSpan();
//Parse the data into a trade bar:
var csv = line.Split(',');
const decimal scaleFactor = 10000m;
Symbol = config.Symbol;
switch (config.SecurityType)
{
//Equity File Data Format:
case SecurityType.Equity:
Time = baseDate.Date.AddMilliseconds(Convert.ToInt32(csv[0]));
Open = config.GetNormalizedPrice(csv[1].ToDecimal() / scaleFactor); // Convert.ToDecimal(csv[1]) / scaleFactor;
High = config.GetNormalizedPrice(csv[2].ToDecimal() / scaleFactor); // Using custom "ToDecimal" conversion for speed.
Low = config.GetNormalizedPrice(csv[3].ToDecimal() / scaleFactor);
Close = config.GetNormalizedPrice(csv[4].ToDecimal() / scaleFactor);
Volume = Convert.ToInt64(csv[5]);
break;
//FOREX has a different data file format:
case SecurityType.Forex:
Time = DateTime.ParseExact(csv[0], "yyyyMMdd HH:mm:ss.ffff", CultureInfo.InvariantCulture);
Open = csv[1].ToDecimal();
High = csv[2].ToDecimal();
Low = csv[3].ToDecimal();
Close = csv[4].ToDecimal();
break;
}
//base.Value = Close;
}
catch (Exception err)
{
Log.Error("DataModels: TradeBar(): Error Initializing - " + config.SecurityType + " - " + err.Message + " - " + line);
}
}
/// <summary>
/// Parse a tick data line from quantconnect zip source files.
/// </summary>
/// <param name="line">CSV source line of the compressed source</param>
/// <param name="date">Base date for the tick (ticks date is stored as int milliseconds since midnight)</param>
/// <param name="config">Subscription configuration object</param>
/// <param name="datafeed">Datafeed for tick - live or backtesting.</param>
public Tick(SubscriptionDataConfig config, string line, DateTime date, DataFeedEndpoint datafeed)
{
try
{
var csv = line.Split(',');
// Which security type is this data feed:
switch (config.Security)
{
case SecurityType.Equity:
Symbol = config.Symbol;
Time = date.Date.AddMilliseconds(Convert.ToInt64(csv[0]));
Value = (csv[1].ToDecimal() / 10000m) * config.PriceScaleFactor;
DataType = MarketDataType.Tick;
TickType = TickType.Trade;
Quantity = Convert.ToInt32(csv[2]);
if (csv.Length > 3)
{
Exchange = csv[3];
SaleCondition = csv[4];
Suspicious = (csv[5] == "1");
}
break;
case SecurityType.Forex:
Symbol = config.Symbol;
TickType = TickType.Quote;
Time = DateTime.ParseExact(csv[0], "yyyyMMdd HH:mm:ss.ffff", CultureInfo.InvariantCulture);
BidPrice = csv[1].ToDecimal();
AskPrice = csv[2].ToDecimal();
Value = BidPrice + (AskPrice - BidPrice) / 2;
break;
}
}
catch (Exception err)
{
Log.Error("Error Generating Tick: " + err.Message);
}
}
/********************************************************
* CLASS CONSTRUCTOR
*********************************************************/
/// <summary>
/// Tradier datafeed handler for getting free data from the tradier brokerage api.
/// </summary>
/// <param name="algorithm">Algorithm requesting data</param>
/// <param name="job">Job packet requesting data</param>
/// <param name="brokerage">Brokerage instance to avoid access token duplication.</param>
public TradierDataFeed(IAlgorithm algorithm, IBrokerage brokerage, LiveNodePacket job)
{
//Subscription Count:
_subscriptions = algorithm.SubscriptionManager.Subscriptions;
_subscriptionCount = Subscriptions.Count;
//Set Properties:
_dataFeed = DataFeedEndpoint.Tradier;
_isActive = true;
_bridge = new ConcurrentQueue <List <BaseData> > [_subscriptionCount];
_endOfBridge = new bool[_subscriptionCount];
_subscriptionManagers = new SubscriptionDataReader[_subscriptionCount];
//Class Privates:
_job = job;
_algorithm = algorithm;
//Setup the arrays:
for (var i = 0; i < _subscriptionCount; i++)
{
_endOfBridge[i] = false;
_bridge[i] = new ConcurrentQueue <List <BaseData> >();
//This is quantconnect data source, store here for speed/ease of access
_isQuantConnectData.Add(algorithm.Securities[_subscriptions[i].Symbol].IsQuantConnectData);
//Subscription managers for downloading user data:
_subscriptionManagers[i] = new SubscriptionDataReader(_subscriptions[i], algorithm.Securities[_subscriptions[i].Symbol], DataFeedEndpoint.LiveTrading, new DateTime(), new DateTime(9999, 12, 12));
//Set up the source file for today:
_subscriptionManagers[i].RefreshSource(DateTime.Now.Date);
}
//Setup Brokerage Access:
_tradier = (TradierBrokerage)brokerage;
}
public override string GetSource(SubscriptionDataConfig config, DateTime date, DataFeedEndpoint datafeed)
{
return("https://www.dropbox.com/s/rsmg44jr6wexn2h/CNXNIFTY.csv?dl=1");
}
/********************************************************
* CLASS METHODS
*********************************************************/
/// <summary>
/// TradeBar Reader: Fetch the data from the QC storage and feed it line by line into the engine.
/// </summary>
/// <param name="datafeed">Destination for the this datafeed - live or backtesting</param>
/// <param name="config">Symbols, Resolution, DataType, </param>
/// <param name="line">Line from the data file requested</param>
/// <param name="date">Date of this reader request</param>
/// <returns>Enumerable iterator for returning each line of the required data.</returns>
public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, DataFeedEndpoint datafeed)
{
//Initialize:
var tradeBar = new TradeBar();
//Handle end of file:
if (line == null)
{
return(null);
}
//Select the URL source of the data depending on where the system is trading.
switch (datafeed)
{
//Amazon S3 Backtesting Data:
case DataFeedEndpoint.Backtesting:
//Create a new instance of our tradebar:
tradeBar = new TradeBar(config, line, date, datafeed);
break;
//Localhost Data Source
case DataFeedEndpoint.FileSystem:
//Create a new instance of our tradebar:
tradeBar = new TradeBar(config, line, date, datafeed);
break;
//QuantConnect Live Tick Stream:
case DataFeedEndpoint.LiveTrading:
break;
}
//Return initialized TradeBar:
return(tradeBar);
}
/// <summary>
/// Construct this from a normal stream reader
/// </summary>
/// <param name="sr"></param>
/// <param name="datafeed"></param>
public SubscriptionStreamReader(StreamReader sr, DataFeedEndpoint datafeed)
{
_sr = sr;
_dataFeed = datafeed;
}
/// <summary>
/// 2. RETURN THE STRING URL SOURCE LOCATION FOR YOUR DATA:
/// This is a powerful and dynamic select source file method. If you have a large dataset, 10+mb we recommend you break it into smaller files. E.g. One zip per year.
/// We can accept raw text or ZIP files. We read the file extension to determine if it is a zip file.
/// </summary>
/// <param name="config">Subscription data, symbol name, data type</param>
/// <param name="date">Current date we're requesting. This allows you to break up the data source into daily files.</param>
/// <param name="datafeed">Datafeed type: Backtesting or the Live data broker who will provide live data. You can specify a different source for live trading! </param>
/// <returns>string URL end point.</returns>
public override string GetSource(SubscriptionDataConfig config, DateTime date, DataFeedEndpoint datafeed)
{
switch (datafeed)
{
//Backtesting Data Source: Example of a data source which varies by day (commented out)
default:
case DataFeedEndpoint.Backtesting:
//return "http://my-ftp-server.com/futures-data-" + date.ToString("Ymd") + ".zip";
// OR simply return a fixed small data file. Large files will slow down your backtest
return("http://www.quandl.com/api/v1/datasets/BITCOIN/BITSTAMPUSD.csv?sort_order=asc");
case DataFeedEndpoint.LiveTrading:
//Alternative live socket data source for live trading (soon)/
return("....");
}
}
/// <summary>
/// Return the URL string source of the file. This will be converted to a stream
/// </summary>
public override string GetSource(SubscriptionDataConfig config, DateTime date, DataFeedEndpoint datafeed)
{
return "https://www.dropbox.com/s/m6ecmkg9aijwzy2/USDINR.csv?dl=1";
}
public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, DataFeedEndpoint datafeed)
{
if (_random.NextDouble() < 0.01)
{
// this is simulating that we don't have data for this time period for fill forward testing
return(null);
}
config.Security = _type;
var tradeBar = (TradeBar)base.Reader(config, line, date, datafeed);
return(CreateFromTradeBar(tradeBar));
}
/// <summary>
/// Parse a tick data line from Zip files.
/// </summary>
/// <param name="line">CSV Line</param>
/// <param name="date">Base date for the tick</param>
/// <param name="config">Subscription configuration object</param>
/// <param name="datafeed">Datafeed for </param>
public Tick(SubscriptionDataConfig config, string line, DateTime date, DataFeedEndpoint datafeed)
{
try
{
string[] csv = line.Split(',');
switch (config.Security)
{
case SecurityType.Equity:
base.Symbol = config.Symbol;
base.Time = date.Date.AddMilliseconds(Convert.ToInt64(csv[0]));
base.Value = (csv[1].ToDecimal() / 10000m) * config.PriceScaleFactor;
base.DataType = MarketDataType.Tick;
this.TickType = TickType.Trade;
this.Quantity = Convert.ToInt32(csv[2]);
if (csv.Length > 3)
{
this.Exchange = csv[3];
this.SaleCondition = csv[4];
this.Suspicious = (csv[5] == "1") ? true : false;
}
break;
case SecurityType.Forex:
base.Symbol = config.Symbol;
TickType = TickType.Quote;
Time = DateTime.ParseExact(csv[0], "yyyyMMdd HH:mm:ss.ffff", CultureInfo.InvariantCulture);
BidPrice = csv[1].ToDecimal();
AskPrice = csv[2].ToDecimal();
Value = BidPrice + (AskPrice - BidPrice) / 2;
break;
}
}
catch (Exception err)
{
Log.Error("Error Generating Tick: " + err.Message);
}
}
/// <summary>
/// Backtesting & Live Bitcoin Decoder:
/// </summary>
public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, DataFeedEndpoint datafeed)
{
Bitcoin coin = new Bitcoin();
switch (datafeed)
{
//Example Line Format:
//Date Open High Low Close Volume (BTC) Volume (Currency) Weighted Price
//2011-09-13 5.8 6.0 5.65 5.97 58.37138238, 346.0973893944 5.929230648356
case DataFeedEndpoint.Backtesting:
try
{
string[] data = line.Split(',');
coin.Time = DateTime.Parse(data[0]);
coin.Open = Convert.ToDecimal(data[1]);
coin.High = Convert.ToDecimal(data[2]);
coin.Low = Convert.ToDecimal(data[3]);
coin.Close = Convert.ToDecimal(data[4]);
coin.VolumeBTC = Convert.ToDecimal(data[5]);
coin.WeightedPrice = Convert.ToDecimal(data[7]);
coin.Symbol = "BTC";
coin.Value = coin.Close;
}
catch { /* Do nothing, skip first title row */ }
break;
//Example Line Format:
//{"high": "441.00", "last": "421.86", "timestamp": "1411606877", "bid": "421.96", "vwap": "428.58", "volume": "14120.40683975", "low": "418.83", "ask": "421.99"}
case DataFeedEndpoint.LiveTrading:
try
{
var liveBTC = JsonConvert.DeserializeObject<LiveBitcoin>(line);
coin.Time = DateTime.Now;
coin.Open = liveBTC.Last;
coin.High = liveBTC.High;
coin.Low = liveBTC.Low;
coin.Close = liveBTC.Last;
coin.VolumeBTC = liveBTC.Volume;
coin.WeightedPrice = liveBTC.VWAP;
coin.Symbol = "BTC";
coin.Value = coin.Close;
}
catch { /* Do nothing, possible error in json decoding */ }
break;
}
return coin;
}
public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, DataFeedEndpoint datafeed)
{
VIX fear = new VIX();
//try
//{
//Date Open High Low Close Volume Adjusted Close
//10/27/2014 17.24 17.87 16 16.04 0 16.04
string[] data = line.Split(',');
fear.Time = DateTime.ParseExact(data[0], "yyyy-MM-dd", CultureInfo.InvariantCulture);
fear.Open = Convert.ToDecimal(data[1]); fear.High = Convert.ToDecimal(data[2]);
fear.Low = Convert.ToDecimal(data[3]); fear.Close = Convert.ToDecimal(data[4]);
fear.Symbol = "VIX"; fear.Value = fear.Close;
//}
//catch
//{ }
return fear;
}
public override string GetSource(SubscriptionDataConfig config, DateTime date, DataFeedEndpoint datafeed)
{
return "https://www.quandl.com/api/v1/datasets/YAHOO/INDEX_VIX.csv?trim_start=2000-01-01&trim_end=2014-10-31&sort_order=asc&exclude_headers=true";
}
public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, DataFeedEndpoint datafeed)
{
throw new NotImplementedException();
}
public override string GetSource(SubscriptionDataConfig config, DateTime date, DataFeedEndpoint datafeed)
{
return("https://www.dropbox.com/s/m6ecmkg9aijwzy2/USDINR.csv?dl=1");
}
/********************************************************
* CLASS METHODS
*********************************************************/
/// <summary>
/// Tick Implementation of Reader Method: read a line and convert it to a tick.
/// </summary>
/// <param name="datafeed">Source of the datafeed</param>
/// <param name="config">Configuration object for algorith</param>
/// <param name="line">Line of the datafeed</param>
/// <param name="date">Date of this reader request</param>
/// <returns>New Initialized tick</returns>
public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, DataFeedEndpoint datafeed)
{
Tick _tick = new Tick();
//Select the URL source of the data depending on where the system is trading.
switch (datafeed)
{
//Local File System Storage and Backtesting QC Data Store Feed use same files:
case DataFeedEndpoint.FileSystem:
case DataFeedEndpoint.Backtesting:
//Create a new instance of our tradebar:
_tick = new Tick(config, line, date, datafeed);
break;
case DataFeedEndpoint.LiveTrading:
break;
case DataFeedEndpoint.Tradier:
break;
}
return _tick;
}
/// <summary>
/// Reader converts each line of the data source into BaseData objects. Each data type creates its own factory method, and returns a new instance of the object
/// each time it is called.
/// </summary>
/// <param name="config">Subscription data config setup object</param>
/// <param name="line">Line of the source document</param>
/// <param name="date">Date of the requested data</param>
/// <param name="datafeed">Type of datafeed we're requesting - a live or backtest feed.</param>
/// <returns>Instance of the T:BaseData object generated by this line of the CSV</returns>
public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, DataFeedEndpoint datafeed)
{
// this is implemented in the SubscriptionDataReader.RefreshSource
throw new NotImplementedException("This method is not supposed to be called on the Dividend type.");
}
/// <summary>
/// Return the URL string source of the file. This will be converted to a stream
/// </summary>
/// <param name="datafeed">Type of datafeed we're reqesting - backtest or live</param>
/// <param name="config">Configuration object</param>
/// <param name="date">Date of this source file</param>
/// <returns>String URL of source file.</returns>
public override string GetSource(SubscriptionDataConfig config, DateTime date, DataFeedEndpoint datafeed)
{
throw new NotSupportedException("RenkoBar does not support the GetSource function. This function should never be called on this type.");
}
/********************************************************
* CLASS METHODS
*********************************************************/
/// <summary>
/// Ticks Array Reader: Fetch the data from the QC storage and feed it line by line into the
/// system.
/// </summary>
/// <param name="datafeed">Who is requesting this data, backtest or live streamer</param>
/// <param name="config">Symbols, Resolution, DataType</param>
/// <param name="line">Line from the data file requested</param>
/// <param name="date">Date of the reader day:</param>
/// <returns>Enumerable iterator for returning each line of the required data.</returns>
public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, DataFeedEndpoint datafeed)
{
throw new NotImplementedException();
}
/// <summary>
/// Reader Method :: using set of arguements we specify read out type. Enumerate
/// until the end of the data stream or file. E.g. Read CSV file line by line and convert
/// into data types.
/// </summary>
/// <returns>BaseData type set by Subscription Method.</returns>
/// <param name="config">Config.</param>
/// <param name="line">Line.</param>
/// <param name="date">Date.</param>
/// <param name="datafeed">Datafeed.</param>
public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, DataFeedEndpoint datafeed)
{
throw new NotSupportedException("RenkoBar does not support the Reader function. This function should never be called on this type.");
}
public override string GetSource(SubscriptionDataConfig config, DateTime date, DataFeedEndpoint datafeed)
{
throw new NotImplementedException();
}
/******************************************************** * CLASS PROPERTIES *********************************************************/ /******************************************************** * CLASS METHODS *********************************************************/ /// <summary> /// Reader Method :: using set of arguements we specify read out type. Enumerate /// until the end of the data stream or file. E.g. Read CSV file line by line and convert /// into data types. /// </summary> /// <returns>BaseData type set by Subscription Method.</returns> public abstract BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, DataFeedEndpoint datafeed);
/// <summary>
/// 3. READER METHOD: Read 1 line from data source and convert it into Object.
/// Each line of the CSV File is presented in here. The backend downloads your file, loads it into memory and then line by line
/// feeds it into your algorithm
/// </summary>
/// <param name="line">string line from the data source file submitted above</param>
/// <param name="config">Subscription data, symbol name, data type</param>
/// <param name="date">Current date we're requesting. This allows you to break up the data source into daily files.</param>
/// <param name="datafeed">Datafeed type - Backtesting or LiveTrading</param>
/// <returns>New Weather Object which extends BaseData.</returns>
public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, DataFeedEndpoint datafeed)
{
//New weather object
Bitcoin coin = new Bitcoin();
try
{
//Example File Format:
//Date, Open High Low Close Volume (BTC) Volume (Currency) Weighted Price
//2011-09-13 5.8 6.0 5.65 5.97 58.37138238, 346.0973893944 5.929230648356
string[] data = line.Split(',');
coin.Time = DateTime.Parse(data[0]);
coin.Open = Convert.ToDecimal(data[1]);
coin.High = Convert.ToDecimal(data[2]);
coin.Low = Convert.ToDecimal(data[3]);
coin.Close = Convert.ToDecimal(data[4]);
coin.VolumeBTC = Convert.ToDecimal(data[5]);
coin.VolumeUSD = Convert.ToDecimal(data[6]);
coin.WeightedPrice = Convert.ToDecimal(data[7]);
coin.Symbol = "BTC";
coin.Value = coin.Close;
}
catch { /* Do nothing, skip first title row */ }
return(coin);
}
/// <summary> /// Return the URL string source of the file. This will be converted to a stream /// </summary> /// <param name="datafeed">Type of datafeed we're reqesting - backtest or live</param> /// <param name="config">Configuration object</param> /// <param name="date">Date of this source file</param> /// <returns>String URL of source file.</returns> public abstract string GetSource(SubscriptionDataConfig config, DateTime date, DataFeedEndpoint datafeed);
/********************************************************
* CLASS CONSTRUCTOR
*********************************************************/
/// <summary>
/// Subscription data reader takes a subscription request, loads the type, accepts the data source and enumerate on the results.
/// </summary>
/// <param name="config">Subscription configuration object</param>
/// <param name="security">Security asset</param>
/// <param name="feed">Feed type enum</param>
/// <param name="periodStart">Start date for the data request/backtest</param>
/// <param name="periodFinish">Finish date for the data request/backtest</param>
public SubscriptionDataReader(SubscriptionDataConfig config, Security security, DataFeedEndpoint feed, DateTime periodStart, DateTime periodFinish)
{
//Save configuration of data-subscription:
_config = config;
//Save access to fill foward flag:
_isFillForward = config.FillDataForward;
//Save Start and End Dates:
_periodStart = periodStart;
_periodFinish = periodFinish;
//Save access to securities
_security = security;
_isQCData = security.IsQuantConnectData;
_isQCEquity = (security.Type == SecurityType.Equity) && _isQCData;
//Set QC Type Flags:
IsQCTick = (config.Type.Name == "Tick" && _isQCData);
IsQCTradeBar = (config.Type.Name == "TradeBar" && _isQCData);
//Save the type of data we'll be getting from the source.
_feedEndpoint = feed;
//Create the dynamic type-activators:
_objectActivator = Loader.GetActivator(config.Type);
if (_objectActivator == null)
{
Engine.ResultHandler.ErrorMessage("Custom data type '" + config.Type.Name + "' missing parameterless constructor E.g. public " + config.Type.Name + "() { }");
_endOfStream = true;
return;
}
//Create an instance of the "Type":
var userObj = _objectActivator.Invoke(new object[] { });
_dataFactory = userObj as BaseData;
//Save Access to the "Reader" Method:
_readerMethod = _dataFactory.GetType().GetMethod("Reader", new[] { typeof(SubscriptionDataConfig), typeof(string), typeof(DateTime), typeof(DataFeedEndpoint) });
//Create a Delagate Accessor.
_readerMethodInvoker = _readerMethod.DelegateForCallMethod();
//Save access to the "GetSource" Method:
_getSourceMethod = _dataFactory.GetType().GetMethod("GetSource", new[] { typeof(SubscriptionDataConfig), typeof(DateTime), typeof(DataFeedEndpoint) });
//Load the entire factor and symbol mapping tables into memory
try
{
if (_isQCEquity)
{
_priceFactors = SubscriptionAdjustment.GetFactorTable(config.Symbol);
_symbolMap = SubscriptionAdjustment.GetMapTable(config.Symbol);
}
}
catch (Exception err)
{
Log.Error("SubscriptionDataReader(): Fetching Price/Map Factors: " + err.Message);
_priceFactors = new SortedDictionary <DateTime, decimal>();
_symbolMap = new SortedDictionary <DateTime, string>();
}
}
public override string GetSource(SubscriptionDataConfig config, DateTime date, DataFeedEndpoint datafeed)
{
// this is really a base security type, but we're pulling data from the equities/forex
config.Security = _type;
var file = base.GetSource(config, date, datafeed);
return file;
}
/// <summary>
/// Parse a line from CSV data sources into our trade bars.
/// </summary>
/// <param name="config">Configuration class object for this data subscription</param>
/// <param name="baseDate">Base date of this tradebar line</param>
/// <param name="line">CSV line from source data file</param>
/// <param name="datafeed">Datafeed this csv line is sourced from (backtesting or live)</param>
public TradeBar(SubscriptionDataConfig config, string line, DateTime baseDate, DataFeedEndpoint datafeed = DataFeedEndpoint.Backtesting)
{
try
{
//Parse the data into a trade bar:
var csv = line.Split(',');
const decimal scaleFactor = 10000m;
Symbol = config.Symbol;
switch (config.SecurityType)
{
//Equity File Data Format:
case SecurityType.Equity:
Time = baseDate.Date.AddMilliseconds(Convert.ToInt32(csv[0]));
Open = config.GetNormalizedPrice(csv[1].ToDecimal() / scaleFactor); // Convert.ToDecimal(csv[1]) / scaleFactor;
High = config.GetNormalizedPrice(csv[2].ToDecimal() / scaleFactor); // Using custom "ToDecimal" conversion for speed.
Low = config.GetNormalizedPrice(csv[3].ToDecimal() / scaleFactor);
Close = config.GetNormalizedPrice(csv[4].ToDecimal() / scaleFactor);
Volume = Convert.ToInt64(csv[5]);
break;
//FOREX has a different data file format:
case SecurityType.Forex:
Time = DateTime.ParseExact(csv[0], "yyyyMMdd HH:mm:ss.ffff", CultureInfo.InvariantCulture);
Open = csv[1].ToDecimal();
High = csv[2].ToDecimal();
Low = csv[3].ToDecimal();
Close = csv[4].ToDecimal();
break;
}
//base.Value = Close;
}
catch (Exception err)
{
Log.Error("DataModels: TradeBar(): Error Initializing - " + config.SecurityType + " - " + err.Message + " - " + line);
}
}
public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, DataFeedEndpoint datafeed)
{
if (_random.NextDouble() < 0.01)
{
// this is simulating that we don't have data for this time period for fill forward testing
return null;
}
config.Security = _type;
var tradeBar = (TradeBar)base.Reader(config, line, date, datafeed);
return CreateFromTradeBar(tradeBar);
}
/// <summary>
/// Get Source for Custom Data File
/// >> What source file location would you prefer for each type of usage:
/// </summary>
/// <param name="config">Configuration object</param>
/// <param name="date">Date of this source request if source spread across multiple files</param>
/// <param name="datafeed">Source of the datafeed</param>
/// <returns>String source location of the file</returns>
public override string GetSource(SubscriptionDataConfig config, DateTime date, DataFeedEndpoint datafeed)
{
var source = "";
var dataType = TickType.Trade;
switch (datafeed)
{
//Backtesting S3 Endpoint:
case DataFeedEndpoint.Backtesting:
case DataFeedEndpoint.FileSystem:
var dateFormat = "yyyyMMdd";
if (config.SecurityType == SecurityType.Forex)
{
dataType = TickType.Quote;
dateFormat = "yyMMdd";
}
var symbol = String.IsNullOrEmpty(config.MappedSymbol) ? config.Symbol : config.MappedSymbol;
source = Constants.DataFolder + config.SecurityType.ToString().ToLower();
source += @"/" + config.Resolution.ToString().ToLower() + @"/" + symbol.ToLower() + @"/";
source += date.ToString(dateFormat) + "_" + dataType.ToString().ToLower() + ".zip";
break;
//Live Trading Endpoint: Fake, not actually used but need for consistency with backtesting system. Set to "" so will not use subscription reader.
case DataFeedEndpoint.LiveTrading:
source = "";
break;
}
return(source);
}
/// <summary>
/// Live trading datafeed handler provides a base implementation of a live trading datafeed. Derived types
/// need only implement the GetNextTicks() function to return unprocessed ticks from a data source.
/// This creates a new data feed with a DataFeedEndpoint of LiveTrading.
/// </summary>
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler)
{
//Subscription Count:
_subscriptions = algorithm.SubscriptionManager.Subscriptions;
//Set Properties:
_isActive = true;
_dataFeed = DataFeedEndpoint.LiveTrading;
_bridge = new ConcurrentQueue<List<BaseData>>[Subscriptions.Count];
_endOfBridge = new bool[Subscriptions.Count];
_subscriptionManagers = new SubscriptionDataReader[Subscriptions.Count];
_realtimePrices = new List<decimal>();
//Set the source of the live data:
_dataQueue = Composer.Instance.GetExportedValueByTypeName<IDataQueueHandler>(Configuration.Config.Get("data-queue-handler", "LiveDataQueue"));
//Class Privates:
_algorithm = algorithm;
if (!(job is LiveNodePacket))
{
throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket.");
}
_job = (LiveNodePacket) job;
//Setup the arrays:
for (var i = 0; i < Subscriptions.Count; i++)
{
_endOfBridge[i] = false;
_bridge[i] = new ConcurrentQueue<List<BaseData>>();
//This is quantconnect data source, store here for speed/ease of access
_isDynamicallyLoadedData.Add(algorithm.Securities[_subscriptions[i].Symbol].IsDynamicallyLoadedData);
//Subscription managers for downloading user data:
_subscriptionManagers[i] = new SubscriptionDataReader(_subscriptions[i], algorithm.Securities[_subscriptions[i].Symbol], DataFeedEndpoint.LiveTrading, DateTime.MinValue, DateTime.MaxValue, resultHandler);
//Set up the source file for today:
_subscriptionManagers[i].RefreshSource(DateTime.Now.Date);
_realtimePrices.Add(0);
}
// request for data from these symbols
var symbols = BuildTypeSymbolList(algorithm);
if (symbols.Any())
{
// don't subscribe if there's nothing there, this allows custom data to
// work without an IDataQueueHandler implementation by specifying LiveDataQueue
// in the configuration, that implementation throws on every method, but we actually
// don't need it if we're only doing custom data
_dataQueue.Subscribe(_job, symbols);
}
}
public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, DataFeedEndpoint datafeed)
{
//New Nifty object
Nifty index = new Nifty();
try
{
//Example File Format:
//Date, Open High Low Close Volume Turnover
//2011-09-13 7792.9 7799.9 7722.65 7748.7 116534670 6107.78
string[] data = line.Split(',');
index.Time = DateTime.Parse(data[0]);
index.Open = Convert.ToDecimal(data[1], CultureInfo.InvariantCulture);
index.High = Convert.ToDecimal(data[2], CultureInfo.InvariantCulture);
index.Low = Convert.ToDecimal(data[3], CultureInfo.InvariantCulture);
index.Close = Convert.ToDecimal(data[4], CultureInfo.InvariantCulture);
index.Symbol = "NIFTY";
index.Value = index.Close;
}
catch
{
}
return(index);
}
/// <summary>
/// Return the URL string source of the file. This will be converted to a stream
/// </summary>
public override string GetSource(SubscriptionDataConfig config, DateTime date, DataFeedEndpoint datafeed)
{
return "https://www.dropbox.com/s/rsmg44jr6wexn2h/CNXNIFTY.csv?dl=1";
}
public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, DataFeedEndpoint datafeed)
{
//New USDINR object
DollarRupee currency = new DollarRupee();
try
{
string[] data = line.Split(',');
currency.Time = DateTime.Parse(data[0]);
currency.Close = Convert.ToDecimal(data[1], CultureInfo.InvariantCulture);
currency.Symbol = "USDINR";
currency.Value = currency.Close;
}
catch
{
}
return(currency);
}
public virtual BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, DataFeedEndpoint datafeed)
{
throw new InvalidOperationException(
$"Please implement Reader(SubscriptionDataConfig, string, DateTime, bool) on your custom data type: {GetType().Name}"
);
}
/// <summary>
/// Return the URL string source of the file. This will be converted to a stream
/// </summary>
/// <param name="datafeed">Type of datafeed we're reqesting - backtest or live</param>
/// <param name="config">Configuration object</param>
/// <param name="date">Date of this source file</param>
/// <returns>String URL of source file.</returns>
public override string GetSource(SubscriptionDataConfig config, DateTime date, DataFeedEndpoint datafeed)
{
return(Constants.DataFolder + @"/equity/factor_files/" + config.Symbol + ".csv");
}
/// <summary>
/// Get Source for Custom Data File
/// >> What source file location would you prefer for each type of usage:
/// </summary>
/// <param name="config">Configuration object</param>
/// <param name="date">Date of this source request if source spread across multiple files</param>
/// <param name="datafeed">Source of the datafeed</param>
/// <returns>String source location of the file</returns>
public override string GetSource(SubscriptionDataConfig config, DateTime date, DataFeedEndpoint datafeed)
{
string source = "";
switch (datafeed)
{
//Source location for backtesting. Commonly a dropbox or FTP link
case DataFeedEndpoint.Backtesting:
break;
//Source location for local testing: Not yet released :) Coming soon.
case DataFeedEndpoint.FileSystem:
break;
//Source location for live trading: do you have an endpoint for streaming data?
case DataFeedEndpoint.LiveTrading:
break;
}
return(source);
}
public override string GetSource(SubscriptionDataConfig config, DateTime date, DataFeedEndpoint datafeed)
{
// this is really a base security type, but we're pulling data from the equities/forex
config.Security = _type;
var file = base.GetSource(config, date, datafeed);
return(file);
}
/// <summary> /// Return the URL string source of the file. This will be converted to a stream /// </summary> /// <param name="datafeed">Type of datafeed we're reqesting - backtest or live</param> /// <param name="config">Configuration object</param> /// <param name="date">Date of this source file</param> /// <returns>String URL of source file.</returns> public abstract string GetSource(SubscriptionDataConfig config, DateTime date, DataFeedEndpoint datafeed);
/// <summary>
/// Get the source file for this tick subscription
/// </summary>
/// <param name="datafeed">Source of the datafeed / type of strings we'll be receiving</param>
/// <param name="config">Configuration for the subscription</param>
/// <param name="date">Date of the source file requested.</param>
/// <returns>String URL Source File</returns>
public override string GetSource(SubscriptionDataConfig config, DateTime date, DataFeedEndpoint datafeed)
{
throw new NotImplementedException();
}
/// <summary>
/// Source URL's of Backtesting and Live Streams:
/// </summary>
public override string GetSource(SubscriptionDataConfig config, DateTime date, DataFeedEndpoint datafeed)
{
var source = "";
switch (datafeed)
{
//Historical backtesting data:
case DataFeedEndpoint.Backtesting:
source = "http://www.quandl.com/api/v1/datasets/BITCOIN/BITSTAMPUSD.csv?sort_order=asc";
break;
//Live socket for bitcoin prices:
case DataFeedEndpoint.Tradier:
case DataFeedEndpoint.LiveTrading:
//Live refreshing endpoint.
source = "https://www.bitstamp.net/api/ticker/";
break;
}
return(source);
}
/// <summary>
/// Reader converts each line of the data source into BaseData objects. Each data type creates its own factory method, and returns a new instance of the object
/// each time it is called.
/// </summary>
public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, DataFeedEndpoint datafeed)
{
//New USDINR object
DollarRupee currency = new DollarRupee();
try
{
string[] data = line.Split(',');
currency.Time = DateTime.Parse(data[0]);
currency.Close = Convert.ToDecimal(data[1], CultureInfo.InvariantCulture);
currency.Symbol = "USDINR";
currency.Value = currency.Close;
}
catch
{
}
return currency;
}
/********************************************************
* CLASS CONSTRUCTOR
*********************************************************/
/// <summary>
/// REST Streaming Reader. Each call will call to rest API again
/// </summary>
/// <param name="source">String location of the REST Endpoint</param>
/// <param name="datafeed">DataEndpoint</param>
public SubscriptionStreamReader(string source, DataFeedEndpoint datafeed)
{
//This is a physical file location, create a stream:
if (datafeed == DataFeedEndpoint.Backtesting || datafeed == DataFeedEndpoint.FileSystem)
{
_sr = new StreamReader(source);
}
else
{
_client = new RestClient(source);
_request = new RestRequest(Method.GET);
}
_dataFeed = datafeed;
}
/// <summary>
/// Reader converts each line of the data source into BaseData objects. Each data type creates its own factory method, and returns a new instance of the object
/// each time it is called.
/// </summary>
public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, DataFeedEndpoint datafeed)
{
//New Nifty object
Nifty index = new Nifty();
try
{
//Example File Format:
//Date, Open High Low Close Volume Turnover
//2011-09-13 7792.9 7799.9 7722.65 7748.7 116534670 6107.78
string[] data = line.Split(',');
index.Time = DateTime.Parse(data[0]);
index.Open = Convert.ToDecimal(data[1], CultureInfo.InvariantCulture);
index.High = Convert.ToDecimal(data[2], CultureInfo.InvariantCulture);
index.Low = Convert.ToDecimal(data[3], CultureInfo.InvariantCulture);
index.Close = Convert.ToDecimal(data[4], CultureInfo.InvariantCulture);
index.Symbol = "NIFTY";
index.Value = index.Close;
}
catch
{
}
return index;
}
public virtual string GetSource(SubscriptionDataConfig config, DateTime date, DataFeedEndpoint datafeed)
{
throw new InvalidOperationException("Please implement GetSource(SubscriptionDataConfig, DateTime, bool) on your custom data type: " + GetType().Name);
}
/********************************************************
* CLASS METHODS
*********************************************************/
/// <summary>
/// Tick Implementation of Reader Method: read a line and convert it to a tick.
/// </summary>
/// <param name="datafeed">Source of the datafeed</param>
/// <param name="config">Configuration object for algorith</param>
/// <param name="line">Line of the datafeed</param>
/// <param name="date">Date of this reader request</param>
/// <returns>New Initialized tick</returns>
public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, DataFeedEndpoint datafeed)
{
Tick _tick = new Tick();
//Select the URL source of the data depending on where the system is trading.
switch (datafeed)
{
//Local File System Storage and Backtesting QC Data Store Feed use same files:
case DataFeedEndpoint.FileSystem:
case DataFeedEndpoint.Backtesting:
//Create a new instance of our tradebar:
_tick = new Tick(config, line, date, datafeed);
break;
case DataFeedEndpoint.LiveTrading:
break;
case DataFeedEndpoint.Tradier:
break;
}
return(_tick);
}
/// <summary>
/// Get Source for Custom Data File
/// >> What source file location would you prefer for each type of usage:
/// </summary>
/// <param name="config">Configuration object</param>
/// <param name="date">Date of this source request if source spread across multiple files</param>
/// <param name="datafeed">Source of the datafeed</param>
/// <returns>String source location of the file</returns>
public override string GetSource(SubscriptionDataConfig config, DateTime date, DataFeedEndpoint datafeed)
{
string source = "";
switch (datafeed)
{
//Source location for backtesting. Commonly a dropbox or FTP link
case DataFeedEndpoint.Backtesting:
//E.g.:
//source = @"https://www.dropbox.com/";
break;
//Source location for local testing: Not yet released :) Coming soon.
case DataFeedEndpoint.FileSystem:
break;
//Source location for live trading: do you have an endpoint for streaming data?
case DataFeedEndpoint.LiveTrading:
break;
}
return source;
}
/********************************************************
* CLASS PROPERTIES
*********************************************************/
/********************************************************
* CLASS METHODS
*********************************************************/
/// <summary>
/// Reader Method :: using set of arguements we specify read out type. Enumerate
/// until the end of the data stream or file. E.g. Read CSV file line by line and convert
/// into data types.
/// </summary>
/// <returns>BaseData type set by Subscription Method.</returns>
public abstract BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, DataFeedEndpoint datafeed);
/********************************************************
* CLASS CONSTRUCTOR
*********************************************************/
/// <summary>
/// Live trading datafeed handler provides a base implementation of a live trading datafeed. Derived types
/// need only implement the GetNextTicks() function to return unprocessed ticks from a data source.
/// This creates a new data feed with a DataFeedEndpoint of LiveTrading.
/// </summary>
/// <param name="algorithm">Algorithm requesting data</param>
protected LiveTradingDataFeed(IAlgorithm algorithm)
{
//Subscription Count:
_subscriptions = algorithm.SubscriptionManager.Subscriptions;
//Set Properties:
_dataFeed = DataFeedEndpoint.LiveTrading;
_isActive = true;
_bridge = new ConcurrentQueue<List<BaseData>>[Subscriptions.Count];
_endOfBridge = new bool[Subscriptions.Count];
_subscriptionManagers = new SubscriptionDataReader[Subscriptions.Count];
_realtimePrices = new List<decimal>();
//Class Privates:
_algorithm = algorithm;
//Setup the arrays:
for (var i = 0; i < Subscriptions.Count; i++)
{
_endOfBridge[i] = false;
_bridge[i] = new ConcurrentQueue<List<BaseData>>();
//This is quantconnect data source, store here for speed/ease of access
_isDynamicallyLoadedData.Add(algorithm.Securities[_subscriptions[i].Symbol].IsDynamicallyLoadedData);
//Subscription managers for downloading user data:
_subscriptionManagers[i] = new SubscriptionDataReader(_subscriptions[i], algorithm.Securities[_subscriptions[i].Symbol], DataFeedEndpoint.LiveTrading, DateTime.MinValue, DateTime.MaxValue);
//Set up the source file for today:
_subscriptionManagers[i].RefreshSource(DateTime.Now.Date);
_realtimePrices.Add(0);
}
}
/// <summary>
/// Backtesting & Live Bitcoin Decoder:
/// </summary>
public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, DataFeedEndpoint datafeed)
{
Bitcoin coin = new Bitcoin();
switch (datafeed)
{
//Example Line Format:
//Date Open High Low Close Volume (BTC) Volume (Currency) Weighted Price
//2011-09-13 5.8 6.0 5.65 5.97 58.37138238, 346.0973893944 5.929230648356
case DataFeedEndpoint.Backtesting:
try
{
string[] data = line.Split(',');
coin.Time = DateTime.Parse(data[0]);
coin.Open = Convert.ToDecimal(data[1]);
coin.High = Convert.ToDecimal(data[2]);
coin.Low = Convert.ToDecimal(data[3]);
coin.Close = Convert.ToDecimal(data[4]);
coin.VolumeBTC = Convert.ToDecimal(data[5]);
coin.WeightedPrice = Convert.ToDecimal(data[7]);
coin.Symbol = "BTC";
coin.Value = coin.Close;
}
catch { /* Do nothing, skip first title row */ }
break;
//Example Line Format:
//{"high": "441.00", "last": "421.86", "timestamp": "1411606877", "bid": "421.96", "vwap": "428.58", "volume": "14120.40683975", "low": "418.83", "ask": "421.99"}
case DataFeedEndpoint.Tradier:
case DataFeedEndpoint.LiveTrading:
try
{
var liveBTC = JsonConvert.DeserializeObject <LiveBitcoin>(line);
coin.Time = DateTime.Now;
coin.Open = liveBTC.Last;
coin.High = liveBTC.High;
coin.Low = liveBTC.Low;
coin.Close = liveBTC.Last;
coin.VolumeBTC = liveBTC.Volume;
coin.WeightedPrice = liveBTC.VWAP;
coin.Symbol = "BTC";
coin.Value = coin.Close;
}
catch { /* Do nothing, possible error in json decoding */ }
break;
}
return(coin);
}
/// <summary>
/// Get Source for Custom Data File
/// >> What source file location would you prefer for each type of usage:
/// </summary>
/// <param name="config">Configuration object</param>
/// <param name="date">Date of this source request if source spread across multiple files</param>
/// <param name="datafeed">Source of the datafeed</param>
/// <returns>String source location of the file</returns>
public override string GetSource(SubscriptionDataConfig config, DateTime date, DataFeedEndpoint datafeed)
{
var source = "";
var dataType = TickType.Trade;
switch (datafeed)
{
//Backtesting S3 Endpoint:
case DataFeedEndpoint.Backtesting:
case DataFeedEndpoint.FileSystem:
var dateFormat = "yyyyMMdd";
if (config.Security == SecurityType.Forex)
{
dataType = TickType.Quote;
dateFormat = "yyMMdd";
}
source = @"../../../Data/" + config.Security.ToString().ToLower();
source += @"/" + config.Resolution.ToString().ToLower() + @"/" + config.Symbol.ToLower() + @"/";
source += date.ToString(dateFormat) + "_" + dataType.ToString().ToLower() + ".zip";
break;
//Live Trading Endpoint: Fake, not actually used but need for consistency with backtesting system. Set to "" so will not use subscription reader.
case DataFeedEndpoint.LiveTrading:
source = "";
break;
}
return source;
}
public virtual string GetSource(SubscriptionDataConfig config, DateTime date, DataFeedEndpoint datafeed)
{
throw new InvalidOperationException("Please implement GetSource(SubscriptionDataConfig, DateTime, bool) on your custom data type: " + GetType().Name);
}
/********************************************************
* CLASS METHODS
*********************************************************/
/// <summary>
/// TradeBar Reader: Fetch the data from the QC storage and feed it line by line into the engine.
/// </summary>
/// <param name="datafeed">Destination for the this datafeed - live or backtesting</param>
/// <param name="config">Symbols, Resolution, DataType, </param>
/// <param name="line">Line from the data file requested</param>
/// <param name="date">Date of this reader request</param>
/// <returns>Enumerable iterator for returning each line of the required data.</returns>
public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, DataFeedEndpoint datafeed)
{
//Initialize:
var tradeBar = new TradeBar();
//Handle end of file:
if (line == null)
{
return null;
}
//Select the URL source of the data depending on where the system is trading.
switch (datafeed)
{
//Amazon S3 Backtesting Data:
case DataFeedEndpoint.Backtesting:
//Create a new instance of our tradebar:
tradeBar = new TradeBar(config, line, date, datafeed);
break;
//Localhost Data Source
case DataFeedEndpoint.FileSystem:
//Create a new instance of our tradebar:
tradeBar = new TradeBar(config, line, date, datafeed);
break;
//QuantConnect Live Tick Stream:
case DataFeedEndpoint.LiveTrading:
break;
}
//Return initialized TradeBar:
return tradeBar;
}
