/// <summary>
/// Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.
/// </summary>
public override void Initialize()
{
SetStartDate(2013, 10, 7); //Set Start Date
SetEndDate(2013, 10, 11); //Set End Date
SetCash(100000); //Set Strategy Cash
// Find more symbols here: http://quantconnect.com/data
AddSecurity(SecurityType.Equity, "SPY", Resolution.Second);
_consolidator = ResolveConsolidator("SPY", Resolution.Daily);
SubscriptionManager.AddConsolidator("SPY", _consolidator);
}
/// <summary>
/// Removes the specified consolidator for the symbol
/// </summary>
/// <param name="symbol">The symbol the consolidator is receiving data from</param>
/// <param name="consolidator">The consolidator instance to be removed</param>
public void RemoveConsolidator(Symbol symbol, IDataConsolidator consolidator)
{
// remove consolidator from each subscription
foreach (var subscription in _subscriptionManager.GetSubscriptionDataConfigs(symbol))
{
subscription.Consolidators.Remove(consolidator);
}
// dispose of the consolidator to remove any remaining event handlers
consolidator.DisposeSafely();
}
/// <summary>
/// Creates a new consolidator that will pump date through the first, and then the output
/// of the first into the second. This enables 'wrapping' or 'composing' of consolidators
/// </summary>
/// <param name="first">The first consolidator to receive data</param>
/// <param name="second">The consolidator to receive first's output</param>
public SequentialConsolidator(IDataConsolidator first, IDataConsolidator second, string name = "")
{
First = first;
Second = second;
Name = name;
// wire up the second one to get data from the first
first.Updated += (time, updated) => second.Update(time, updated);
// wire up the second one's events to also fire this consolidator's event so consumers
// can attach
second.Updated += (time, updated) => OnDataConsolidated(time, updated);
}
/// <summary>
/// Creates a new consolidator that will pump date through the first, and then the output
/// of the first into the second. This enables 'wrapping' or 'composing' of consolidators
/// </summary>
/// <param name="first">The first consolidator to receive data</param>
/// <param name="second">The consolidator to receive first's output</param>
public SequentialConsolidator(IDataConsolidator first, IDataConsolidator second)
{
if (first.OutputType != second.InputType)
{
throw new ArgumentException("first.OutputType must equal second.OutputType!");
}
First = first;
Second = second;
// wire up the second one to get data from the first
first.DataConsolidated += (sender, consolidated) => second.Update(consolidated);
}
public SymbolData(QCAlgorithm algorithm, Symbol symbol, int dailyLookback, int lookback, Resolution resolution)
{
Symbol = symbol;
_dailyReturn = new RateOfChangePercent($"{symbol}.DailyROCP(1)", 1);
_dailyConsolidator = algorithm.ResolveConsolidator(symbol, Resolution.Daily);
_dailyReturnHistory = new RollingWindow <IndicatorDataPoint>(dailyLookback);
_dailyReturn.Updated += (s, e) => _dailyReturnHistory.Add(e);
algorithm.RegisterIndicator(symbol, _dailyReturn, _dailyConsolidator);
Return = new RateOfChangePercent($"{symbol}.ROCP({lookback})", lookback);
_consolidator = algorithm.ResolveConsolidator(symbol, resolution);
algorithm.RegisterIndicator(symbol, Return, _consolidator);
}
/// <summary>
/// Registers the consolidator to receive automatic updates as well as configures the indicator to receive updates
/// from the consolidator.
/// </summary>
/// <param name="symbol">The symbol to register against</param>
/// <param name="indicator">The indicator to receive data from the consolidator</param>
/// <param name="consolidator">The consolidator to receive raw subscription data</param>
/// <param name="selector">Selects a value from the BaseData send into the indicator, if null defaults to a cast (x => (T)x)</param>
public void RegisterIndicator(Symbol symbol, PyObject indicator, IDataConsolidator consolidator, PyObject selector = null)
{
IndicatorBase <IndicatorDataPoint> indicatorDataPoint;
IndicatorBase <IBaseDataBar> indicatorDataBar;
IndicatorBase <TradeBar> indicatorTradeBar;
if (indicator.TryConvert(out indicatorDataPoint))
{
Func <IBaseData, decimal> func = null;
selector?.TryConvert(out func);
RegisterIndicator(symbol, indicatorDataPoint, consolidator, func);
return;
}
else if (indicator.TryConvert(out indicatorDataBar))
{
Func <IBaseData, IBaseDataBar> func = null;
selector?.TryConvert(out func);
RegisterIndicator(symbol, indicatorDataBar, consolidator, func);
return;
}
else if (indicator.TryConvert(out indicatorTradeBar))
{
Func <IBaseData, TradeBar> func = null;
selector?.TryConvert(out func);
RegisterIndicator(symbol, indicatorTradeBar, consolidator, func);
return;
}
using (Py.GIL())
{
if (!indicator.HasAttr("Update"))
{
throw new ArgumentException($"QCAlgorithm.RegisterIndicator(): Update method must be defined. Please checkout {indicator}");
}
}
// register the consolidator for automatic updates via SubscriptionManager
SubscriptionManager.AddConsolidator(symbol, consolidator);
// attach to the DataConsolidated event so it updates our indicator
consolidator.DataConsolidated += (sender, consolidated) =>
{
using (Py.GIL())
{
indicator.InvokeMethod("Update", new[] { consolidated.ToPython() });
}
};
}
/// <summary>
/// Creates a new consolidator that will pump date through the first, and then the output
/// of the first into the second. This enables 'wrapping' or 'composing' of consolidators
/// </summary>
/// <param name="first">The first consolidator to receive data</param>
/// <param name="second">The consolidator to receive first's output</param>
public SequentialConsolidator(IDataConsolidator first, IDataConsolidator second)
{
if (first.OutputType != second.InputType)
{
throw new ArgumentException("first.OutputType must equal second.OutputType!");
}
First = first;
Second = second;
// wire up the second one to get data from the first
first.DataConsolidated += (sender, consolidated) => second.Update(consolidated);
// wire up the second one's events to also fire this consolidator's event so consumers
// can attach
second.DataConsolidated += (sender, consolidated) => OnDataConsolidated(consolidated);
}
/// <summary>
/// Creates a new consolidator that will pump date through the first, and then the output
/// of the first into the second. This enables 'wrapping' or 'composing' of consolidators
/// </summary>
/// <param name="first">The first consolidator to receive data</param>
/// <param name="second">The consolidator to receive first's output</param>
public SequentialConsolidator(IDataConsolidator first, IDataConsolidator second)
{
if (!second.InputType.IsAssignableFrom(first.OutputType))
{
throw new ArgumentException("first.OutputType must equal second.OutputType!");
}
First = first;
Second = second;
// wire up the second one to get data from the first
first.DataConsolidated += (sender, consolidated) => second.Update(consolidated);
// wire up the second one's events to also fire this consolidator's event so consumers
// can attach
second.DataConsolidated += (sender, consolidated) => OnDataConsolidated(consolidated);
}
public SymbolData(QCAlgorithmFramework algorithm, Security security, int period, Resolution resolution)
{
Security = security;
Consolidator = algorithm.ResolveConsolidator(security.Symbol, resolution);
var smaName = algorithm.CreateIndicatorName(security.Symbol, "SMA" + period, resolution);
SMA = new SimpleMovingAverage(smaName, period);
var stdName = algorithm.CreateIndicatorName(security.Symbol, "STD" + period, resolution);
STD = new StandardDeviation(stdName, period);
algorithm.SubscriptionManager.AddConsolidator(security.Symbol, Consolidator);
Consolidator.DataConsolidated += (sender, consolidated) =>
{
SMA.Update(consolidated.EndTime, consolidated.Value);
STD.Update(consolidated.EndTime, consolidated.Value);
};
}
/// <summary>
/// Initializes a new instance of the <see cref="ScannableEnumerator{T}"/> class
/// </summary>
/// <param name="consolidator">Consolidator taking BaseData updates and firing events containing new 'consolidated' data</param>
/// <param name="timeZone">The time zone the raw data is time stamped in</param>
/// <param name="timeProvider">The time provider instance used to determine when bars are completed and can be emitted</param>
/// <param name="newDataAvailableHandler">The event handler for a new available data point</param>
/// <param name="isPeriodBased">The consolidator is period based, this will enable scanning on <see cref="MoveNext"/></param>
public ScannableEnumerator(IDataConsolidator consolidator, DateTimeZone timeZone, ITimeProvider timeProvider, EventHandler newDataAvailableHandler, bool isPeriodBased = true)
{
_timeZone = timeZone;
_timeProvider = timeProvider;
_consolidator = consolidator;
_isPeriodBase = isPeriodBased;
_queue = new ConcurrentQueue <T>();
_consolidatorInputType = consolidator.InputType;
_validateInputType = _consolidatorInputType != typeof(BaseData);
var newDataAvailableHandler1 = newDataAvailableHandler ?? ((s, e) => { });
_consolidator.DataConsolidated += (sender, data) =>
{
_consolidated = true;
Enqueue(data as T);
newDataAvailableHandler1(sender, EventArgs.Empty);
};
}
public void AddConsolidator(Symbol symbol, IDataConsolidator consolidator)
{
for (var i = 0; i < Subscriptions.Count; i++)
{
if (Subscriptions[i].Symbol == symbol)
{
if (!consolidator.InputType.IsAssignableFrom(Subscriptions[i].Type))
{
throw new ArgumentException(string.Format("Type mismatch found between consolidator and symbol. " +
"Symbol: {0} expects type {1} but tried to register consolidator with input type {2}",
symbol, Subscriptions[i].Type.Name, consolidator.InputType.Name)
);
}
Subscriptions[i].Consolidators.Add(consolidator);
return;
}
}
throw new ArgumentException("Please subscribe to this symbol before adding a consolidator for it. Symbol: " + symbol.ToString());
}
public SymbolData(QCAlgorithm algorithm, Security security, int period, Resolution resolution)
{
Security = security;
Consolidator = algorithm.ResolveConsolidator(security.Symbol, resolution);
var smaName = algorithm.CreateIndicatorName(security.Symbol, "SMA" + period, resolution);
SMA = new SimpleMovingAverage(smaName, period);
algorithm.RegisterIndicator(security.Symbol, SMA, Consolidator);
var stdName = algorithm.CreateIndicatorName(security.Symbol, "STD" + period, resolution);
STD = new StandardDeviation(stdName, period);
algorithm.RegisterIndicator(security.Symbol, STD, Consolidator);
// warmup our indicators by pushing history through the indicators
foreach (var bar in algorithm.History(Security.Symbol, period, resolution))
{
SMA.Update(bar.EndTime, bar.Value);
STD.Update(bar.EndTime, bar.Value);
}
}
/// <summary>
/// Add a consolidator for the symbol
/// </summary>
/// <param name="symbol">Symbol of the asset to consolidate</param>
/// <param name="consolidator">The consolidator</param>
public void AddConsolidator(Symbol symbol, IDataConsolidator consolidator)
{
//Find the right subscription and add the consolidator to it
for (var i = 0; i < Subscriptions.Count; i++)
{
if (Subscriptions[i].Symbol == symbol)
{
// we need to be able to pipe data directly from the data feed into the consolidator
if (!consolidator.InputType.IsAssignableFrom(Subscriptions[i].Type))
{
throw new ArgumentException(string.Format("Type mismatch found between consolidator and symbol. " +
"Symbol: {0} expects type {1} but tried to register consolidator with input type {2}",
symbol, Subscriptions[i].Type.Name, consolidator.InputType.Name)
);
}
Subscriptions[i].Consolidators.Add(consolidator);
return;
}
}
//If we made it here it is because we never found the symbol in the subscription list
throw new ArgumentException("Please subscribe to this symbol before adding a consolidator for it. Symbol: " + symbol.Value);
}
/// <summary>
/// Create a new AlgoSeekOptionsProcessor for enquing consolidated bars and flushing them to disk
/// </summary>
/// <param name="symbol">Symbol for the processor</param>
/// <param name="date">Reference date for the processor</param>
/// <param name="tickType">TradeBar or QuoteBar to generate</param>
/// <param name="resolution">Resolution to consolidate</param>
/// <param name="dataDirectory">Data directory for LEAN</param>
public AlgoSeekOptionsProcessor(Symbol symbol, DateTime date, TickType tickType, Resolution resolution, string dataDirectory)
{
_symbol = Safe(symbol);
_tickType = tickType;
_referenceDate = date;
_resolution = resolution;
_queue = new Queue <IBaseData>();
_dataDirectory = dataDirectory;
// Setup the consolidator for the requested resolution
if (resolution == Resolution.Tick)
{
throw new NotSupportedException();
}
switch (tickType)
{
case TickType.Trade:
_consolidator = new TickConsolidator(resolution.ToTimeSpan());
break;
case TickType.Quote:
_consolidator = new TickQuoteBarConsolidator(resolution.ToTimeSpan());
break;
case TickType.OpenInterest:
_consolidator = new OpenInterestConsolidator(resolution.ToTimeSpan());
break;
}
// On consolidating the bars put the bar into a queue in memory to be written to disk later.
_consolidator.DataConsolidated += (sender, consolidated) =>
{
_queue.Enqueue(consolidated);
};
}
/// <summary>
/// Add a consolidator for the symbol
/// </summary>
/// <param name="symbol">Symbol of the asset to consolidate</param>
/// <param name="consolidator">The consolidator</param>
public void AddConsolidator(string symbol, IDataConsolidator consolidator)
{
symbol = symbol.ToUpper();
//Find the right subscription and add the consolidator to it
for (var i = 0; i < Subscriptions.Count; i++)
{
if (Subscriptions[i].Symbol == symbol)
{
// we need to be able to pipe data directly from the data feed into the consolidator
if (!consolidator.InputType.IsAssignableFrom(Subscriptions[i].Type))
{
throw new ArgumentException(string.Format("Type mismatch found between consolidator and symbol. " +
"Symbol: {0} expects type {1} but tried to register consolidator with input type {2}",
symbol, Subscriptions[i].Type.Name, consolidator.InputType.Name)
);
}
Subscriptions[i].Consolidators.Add(consolidator);
return;
}
}
//If we made it here it is because we never found the symbol in the subscription list
throw new ArgumentException("Please subscribe to this symbol before adding a consolidator for it. Symbol: " + symbol);
}
/// <summary>
/// Registers the consolidator to receive automatic updates as well as configures the indicator to receive updates
/// from the consolidator.
/// </summary>
/// <param name="symbol">The symbol to register against</param>
/// <param name="indicator">The indicator to receive data from the consolidator</param>
/// <param name="consolidator">The consolidator to receive raw subscription data</param>
/// <param name="selector">Selects a value from the BaseData send into the indicator, if null defaults to a cast (x => (T)x)</param>
public void RegisterIndicator(Symbol symbol, IndicatorBase <TradeBar> indicator, IDataConsolidator consolidator, Func <IBaseData, TradeBar> selector)
{
RegisterIndicator <TradeBar>(symbol, indicator, consolidator, selector);
}
/// <summary>
/// Registers the consolidator to receive automatic updates as well as configures the indicator to receive updates
/// from the consolidator.
/// </summary>
/// <param name="symbol">The symbol to register against</param>
/// <param name="indicator">The indicator to receive data from the consolidator</param>
/// <param name="consolidator">The consolidator to receive raw subscription data</param>
/// <param name="selector">Selects a value from the BaseData send into the indicator, if null defaults to a cast (x => (T)x)</param>
public void RegisterIndicator <T>(string symbol, IndicatorBase <T> indicator, IDataConsolidator consolidator, Func <BaseData, T> selector = null)
where T : BaseData
{
// assign default using cast
selector = selector ?? (x => (T)x);
// register the consolidator for automatic updates via SubscriptionManager
SubscriptionManager.AddConsolidator(symbol, consolidator);
// check the output type of the consolidator and verify we can assign it to T
var type = typeof(T);
if (!type.IsAssignableFrom(consolidator.OutputType))
{
throw new ArgumentException(string.Format("Type mismatch found between consolidator and indicator for symbol: {0}." +
"Consolidator outputs type {1} but indicator expects input type {2}",
symbol, consolidator.OutputType.Name, type.Name)
);
}
// attach to the DataConsolidated event so it updates our indicator
consolidator.DataConsolidated += (sender, consolidated) =>
{
var value = selector(consolidated);
indicator.Update(value);
};
}
/// <summary>
/// Registers the consolidator to receive automatic updates as well as configures the indicator to receive updates
/// from the consolidator.
/// </summary>
/// <param name="symbol">The symbol to register against</param>
/// <param name="indicator">The indicator to receive data from the consolidator</param>
/// <param name="consolidator">The consolidator to receive raw subscription data</param>
/// <param name="selector">Selects a value from the BaseData send into the indicator, if null defaults to the Value property of BaseData (x => x.Value)</param>
public void RegisterIndicator(string symbol, IndicatorBase <IndicatorDataPoint> indicator, IDataConsolidator consolidator, Func <BaseData, decimal> selector = null)
{
// default our selector to the Value property on BaseData
selector = selector ?? (x => x.Value);
// register the consolidator for automatic updates via SubscriptionManager
SubscriptionManager.AddConsolidator(symbol, consolidator);
// attach to the DataConsolidated event so it updates our indicator
consolidator.DataConsolidated += (sender, consolidated) =>
{
var value = selector(consolidated);
indicator.Update(new IndicatorDataPoint(consolidated.Symbol, consolidated.EndTime, value));
};
}
/// <summary>
/// Registers the consolidator to receive automatic updates as well as configures the indicator to receive updates
/// from the consolidator.
/// </summary>
/// <param name="symbol">The symbol to register against</param>
/// <param name="indicator">The indicator to receive data from the consolidator</param>
/// <param name="consolidator">The consolidator to receive raw subscription data</param>
/// <param name="selector">Selects a value from the BaseData send into the indicator</param>
public void RegisterIndicator(string symbol, IndicatorBase <IndicatorDataPoint> indicator, IDataConsolidator consolidator, Func <BaseData, decimal> selector)
{
if (selector == null)
{
throw new ArgumentNullException("selector");
}
// register the consolidator for automatic updates via SubscriptionManager
SubscriptionManager.AddConsolidator(symbol, consolidator);
// attach to the DataConsolidated event so it updates our indicator
consolidator.DataConsolidated += (sender, consolidated) =>
{
var value = selector(consolidated);
indicator.Update(consolidated.Time, value);
};
}
/// <summary>
/// Simplified version of QCAlgorithm.RegisterIndicator
/// </summary>
/// <param name="indicator">The indicator to receive data from the consolidator</param>
/// <param name="consolidator">The consolidator to receive raw subscription data</param>
public void RegisterIndicator(IndicatorBase<IndicatorDataPoint> indicator, IDataConsolidator consolidator)
{
consolidator.DataConsolidated += (sender, consolidated) =>
{
indicator.Update(consolidated.EndTime, consolidated.Value);
};
}
/// <summary>
/// Checks if the subscription is valid for the consolidator
/// </summary>
/// <param name="subscription">The subscription configuration</param>
/// <param name="consolidator">The consolidator</param>
/// <returns>true if the subscription is valid for the consolidator</returns>
public static bool IsSubscriptionValidForConsolidator(SubscriptionDataConfig subscription, IDataConsolidator consolidator)
{
if (subscription.Type == typeof(Tick) &&
LeanData.IsCommonLeanDataType(consolidator.OutputType))
{
var tickType = LeanData.GetCommonTickTypeForCommonDataTypes(
consolidator.OutputType,
subscription.Symbol.SecurityType);
return(subscription.TickType == tickType);
}
return(consolidator.InputType.IsAssignableFrom(subscription.Type));
}
/// <summary>
/// Create a new AlgoSeekFuturesProcessor for enquing consolidated bars and flushing them to disk
/// </summary>
/// <param name="symbol">Symbol for the processor</param>
/// <param name="date">Reference date for the processor</param>
/// <param name="tickType">TradeBar or QuoteBar to generate</param>
/// <param name="resolution">Resolution to consolidate</param>
/// <param name="dataDirectory">Data directory for LEAN</param>
public AlgoSeekFuturesProcessor(Symbol symbol, DateTime date, TickType tickType, Resolution resolution, string dataDirectory)
{
_symbol = Safe(symbol);
_tickType = tickType;
_referenceDate = date;
_resolution = resolution;
_dataDirectory = dataDirectory;
// Setup the consolidator for the requested resolution
if (resolution == Resolution.Tick)
{
_consolidator = new IdentityDataConsolidator <Tick>();
}
else
{
switch (tickType)
{
case TickType.Trade:
_consolidator = new TickConsolidator(resolution.ToTimeSpan());
break;
case TickType.Quote:
_consolidator = new TickQuoteBarConsolidator(resolution.ToTimeSpan());
break;
case TickType.OpenInterest:
_consolidator = new OpenInterestConsolidator(resolution.ToTimeSpan());
break;
}
}
var path = ZipPath.Replace(".zip", string.Empty);
Directory.CreateDirectory(path);
var file = Path.Combine(path, EntryPath);
try
{
_streamWriter = new LazyStreamWriter(file);
}
catch (Exception err)
{
// we are unable to open new file - it is already opened due to bug in algoseek data
Log.Error("File: {0} Err: {1} Source: {2} Stack: {3}", file, err.Message, err.Source, err.StackTrace);
var newRandomizedName = (file + "-" + Math.Abs(file.GetHashCode()).ToString()).Replace(".csv", string.Empty) + ".csv";
// we store the information under different (randomized) name
Log.Trace("Changing name from {0} to {1}", file, newRandomizedName);
_streamWriter = new LazyStreamWriter(newRandomizedName);
}
// On consolidating the bars put the bar into a queue in memory to be written to disk later.
_consolidator.DataConsolidated += (sender, consolidated) =>
{
_streamWriter.WriteLine(LeanData.GenerateLine(consolidated, SecurityType.Future, Resolution));
};
Interlocked.Add(ref _curFileCount, 1);
if (_curFileCount % 1000 == 0)
{
Log.Trace("Opened more files: {0}", _curFileCount);
}
}
