/// <summary>
/// Adds the specified symbols to the subscription
/// </summary>
/// <param name="job">Job we're subscribing for:</param>
/// <param name="symbols">The symbols to be added keyed by SecurityType</param>
public void Subscribe(LiveNodePacket job, IEnumerable<Symbol> symbols)
{
var symbolsToSubscribe = (from symbol in symbols
where !_subscribedSymbols.Contains(symbol) && CanSubscribe(symbol)
select symbol).ToList();
if (symbolsToSubscribe.Count == 0)
return;
Log.Trace("FxcmBrokerage.Subscribe(): {0}", string.Join(",", symbolsToSubscribe));
var request = new MarketDataRequest();
foreach (var symbol in symbolsToSubscribe)
{
TradingSecurity fxcmSecurity;
if (_fxcmInstruments.TryGetValue(_symbolMapper.GetBrokerageSymbol(symbol), out fxcmSecurity))
{
request.addRelatedSymbol(fxcmSecurity);
}
}
request.setSubscriptionRequestType(SubscriptionRequestTypeFactory.SUBSCRIBE);
request.setMDEntryTypeSet(MarketDataRequest.MDENTRYTYPESET_ALL);
lock (_locker)
{
_gateway.sendMessage(request);
}
foreach (var symbol in symbolsToSubscribe)
{
_subscribedSymbols.Add(symbol);
}
}
/// <summary>
/// Adds the specified symbols to the subscription
/// </summary>
/// <param name="job">Job we're subscribing for:</param>
/// <param name="symbols">The symbols to be added keyed by SecurityType</param>
public void Subscribe(LiveNodePacket job, IDictionary<SecurityType, List<Symbol>> symbols)
{
var symbolsToSubscribe = (from secType in symbols
from symbol in secType.Value
where !_subscribedSymbols.Contains(symbol)
select symbol).ToList();
if (symbolsToSubscribe.Count == 0)
return;
Log.Trace("FxcmBrokerage.Subscribe(): {0}", string.Join(",", symbolsToSubscribe));
var request = new MarketDataRequest();
foreach (var symbol in symbolsToSubscribe)
{
request.addRelatedSymbol(_fxcmInstruments[ConvertSymbolToFxcmSymbol(symbol)]);
}
request.setSubscriptionRequestType(SubscriptionRequestTypeFactory.SUBSCRIBE);
request.setMDEntryTypeSet(MarketDataRequest.MDENTRYTYPESET_ALL);
lock (_locker)
{
_gateway.sendMessage(request);
}
foreach (var symbol in symbolsToSubscribe)
{
_subscribedSymbols.Add(symbol);
}
}
public void DoesNotSetRunTimeErrorWhenReconnectMessageComesThrough()
{
var algorithm = new AlgorithmStub(equities: new List<string> { "SPY" });
var referenceTime = DateTime.UtcNow;
algorithm.SetDateTime(referenceTime);
var localReferencTime = referenceTime.ConvertFromUtc(TimeZones.NewYork);
algorithm.Securities["SPY"].Exchange.SetMarketHours(localReferencTime.AddSeconds(1).TimeOfDay, TimeSpan.FromDays(1), localReferencTime.DayOfWeek);
var job = new LiveNodePacket();
var results = new TestResultHandler();//packet => Console.WriteLine(FieldsToString(packet)));
var api = new Api.Api();
var handler = new DefaultBrokerageMessageHandler(algorithm, job, results, api, TimeSpan.FromMinutes(15), TimeSpan.FromSeconds(.25));
Assert.IsNull(algorithm.RunTimeError);
handler.Handle(BrokerageMessageEvent.Disconnected("Disconnection!"));
Thread.Sleep(100);
handler.Handle(BrokerageMessageEvent.Reconnected("Reconnected!"));
Thread.Sleep(500);
Assert.IsNull(algorithm.RunTimeError);
results.Exit();
}
/// <summary>
/// Creates a new IBrokerage instance and set ups the environment for the brokerage
/// </summary>
/// <param name="job">The job packet to create the brokerage for</param>
/// <param name="algorithm">The algorithm instance</param>
/// <returns>A new brokerage instance</returns>
public override IBrokerage CreateBrokerage(LiveNodePacket job, IAlgorithm algorithm)
{
var errors = new List<string>();
// read values from the brokerage datas
var useTws = Config.GetBool("ib-use-tws");
var port = Config.GetInt("ib-port", 4001);
var host = Config.Get("ib-host", "127.0.0.1");
var twsDirectory = Config.Get("ib-tws-dir", "C:\\Jts");
var ibControllerDirectory = Config.Get("ib-controller-dir", "C:\\IBController");
var account = Read<string>(job.BrokerageData, "ib-account", errors);
var userID = Read<string>(job.BrokerageData, "ib-user-name", errors);
var password = Read<string>(job.BrokerageData, "ib-password", errors);
var agentDescription = Read<AgentDescription>(job.BrokerageData, "ib-agent-description", errors);
if (errors.Count != 0)
{
// if we had errors then we can't create the instance
throw new Exception(string.Join(Environment.NewLine, errors));
}
// launch the IB gateway
InteractiveBrokersGatewayRunner.Start(ibControllerDirectory, twsDirectory, userID, password, useTws);
var ib = new InteractiveBrokersBrokerage(algorithm.Transactions, algorithm.Portfolio, account, host, port, agentDescription);
Composer.Instance.AddPart<IDataQueueHandler>(ib);
return ib;
}
/// <summary>
/// Removes the specified symbols to the subscription
/// </summary>
/// <param name="job">Job we're processing.</param>
/// <param name="symbols">The symbols to be removed keyed by SecurityType</param>
public void Unsubscribe(LiveNodePacket job, IEnumerable<Symbol> symbols)
{
foreach (var symbol in symbols)
{
lock (_lock) _subscriptions.Remove(symbol);
}
}
/// <summary>
/// Creates a test live trading data feed with the specified fast forward factor
/// </summary>
/// <param name="algorithm">The algorithm under analysis</param>
/// <param name="job">The job for the algorithm</param>
public TestLiveTradingDataFeed(IAlgorithm algorithm, LiveNodePacket job)
: base(algorithm, job)
{
_start = DateTime.Now;
_current = DateTime.Now;
_tickResolution = TimeSpan.FromSeconds(1);
}
/// <summary>
/// Adds the specified symbols to the subscription: new IQLevel1WatchItem("IBM", true)
/// </summary>
/// <param name="job">Job we're subscribing for:</param>
/// <param name="symbols">The symbols to be added keyed by SecurityType</param>
public void Subscribe(LiveNodePacket job, IEnumerable<Symbol> symbols)
{
try
{
foreach (var symbol in symbols)
{
if (CanSubscribe(symbol))
{
lock (_sync)
{
Log.Trace("IQFeed.Subscribe(): Subscribe Request: " + symbol.ToString());
var type = symbol.ID.SecurityType;
if (_symbols.Add(symbol))
{
var ticker = symbol.Value;
if (type == SecurityType.Forex) ticker += ".FXCM";
_level1Port.Subscribe(ticker);
Log.Trace("IQFeed.Subscribe(): Subscribe Processed: " + symbol.ToString());
}
}
}
}
}
catch (Exception err)
{
Log.Error("IQFeed.Subscribe(): " + err.Message);
}
}
/// <summary>
/// Adds the specified symbols to the subscription
/// </summary>
/// <param name="job">Job we're subscribing for:</param>
/// <param name="symbols">The symbols to be added keyed by SecurityType</param>
public void Subscribe(LiveNodePacket job, IDictionary<SecurityType, List<Symbol>> symbols)
{
foreach (var kvp in symbols)
{
foreach (var symbol in kvp.Value)
{
lock (_lock) _subscriptions.Add(symbol);
}
}
}
/// <summary>
/// Creates a new IBrokerage instance
/// </summary>
/// <param name="job">The job packet to create the brokerage for</param>
/// <param name="algorithm">The algorithm instance</param>
/// <returns>A new brokerage instance</returns>
public IBrokerage CreateBrokerage(LiveNodePacket job, IAlgorithm algorithm)
{
//Try and use the live job packet cash if exists, otherwise resort to the user algo cash:
if (job.BrokerageData.ContainsKey("project-paper-equity"))
{
var consistentCash = Convert.ToDecimal(job.BrokerageData["project-paper-equity"]);
algorithm.SetCash(consistentCash);
}
return new PaperBrokerage(algorithm);
}
public void InitializesInstanceFromComposer()
{
var composer = Composer.Instance;
using (var factory = composer.Single<IBrokerageFactory>(instance => instance.BrokerageType == typeof (InteractiveBrokersBrokerage)))
{
Assert.IsNotNull(factory);
var job = new LiveNodePacket {BrokerageData = factory.BrokerageData};
var brokerage = factory.CreateBrokerage(job, AlgorithmDependency);
Assert.IsNotNull(brokerage);
Assert.IsInstanceOf<InteractiveBrokersBrokerage>(brokerage);
brokerage.Connect();
Assert.IsTrue(brokerage.IsConnected);
}
}
public void EmitsData()
{
var algorithm = new AlgorithmStub(forex: new List<string> {"EURUSD"});
// job is used to send into DataQueueHandler
var job = new LiveNodePacket();
// result handler is used due to dependency in SubscriptionDataReader
var resultHandler = new BacktestingResultHandler();
var dataFileProvider = new DefaultDataFileProvider();
var lastTime = DateTime.MinValue;
var timeProvider = new RealTimeProvider();
var dataQueueHandler = new FuncDataQueueHandler(fdqh =>
{
var time = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.EasternStandard);
if (time == lastTime) return Enumerable.Empty<BaseData>();
lastTime = time;
return Enumerable.Range(0, 9).Select(x => new Tick(time.AddMilliseconds(x*100), Symbols.EURUSD, 1.3m, 1.2m, 1.3m));
});
var feed = new TestableLiveTradingDataFeed(dataQueueHandler, timeProvider);
var mapFileProvider = new LocalDiskMapFileProvider();
feed.Initialize(algorithm, job, resultHandler, mapFileProvider, new LocalDiskFactorFileProvider(mapFileProvider), dataFileProvider);
var feedThreadStarted = new ManualResetEvent(false);
Task.Factory.StartNew(() =>
{
feedThreadStarted.Set();
feed.Run();
});
// wait for feed.Run to actually begin
feedThreadStarted.WaitOne();
var emittedData = false;
ConsumeBridge(feed, TimeSpan.FromSeconds(10), true, ts =>
{
if (ts.Slice.Count != 0)
{
emittedData = true;
Console.WriteLine("HasData: " + ts.Slice.Bars[Symbols.EURUSD].EndTime);
Console.WriteLine();
}
});
Assert.IsTrue(emittedData);
}
/// <summary>
/// Desktop/Local Get Next Task - Get task from the Algorithm folder of VS Solution.
/// </summary>
/// <returns></returns>
public AlgorithmNodePacket NextJob(out string location)
{
location = AlgorithmLocation;
Log.Trace("JobQueue.NextJob(): Selected " + location);
//If this isn't a backtesting mode/request, attempt a live job.
if (_liveMode)
{
var liveJob = new LiveNodePacket
{
Type = PacketType.LiveNode,
Algorithm = File.ReadAllBytes(AlgorithmLocation),
Brokerage = Config.Get("live-mode-brokerage", PaperBrokerageTypeName),
Channel = Config.Get("job-channel"),
UserId = Config.GetInt("job-user-id"),
Version = Constants.Version,
DeployId = Config.Get("algorithm-type-name"),
RamAllocation = int.MaxValue
};
try
{
// import the brokerage data for the configured brokerage
var brokerageFactory = Composer.Instance.Single<IBrokerageFactory>(factory => factory.BrokerageType.MatchesTypeName(liveJob.Brokerage));
liveJob.BrokerageData = brokerageFactory.BrokerageData;
}
catch (Exception err)
{
Log.Error(string.Format("JobQueue.NextJob(): Error resoliving BrokerageData for live job for brokerage {0}. {1}", liveJob.Brokerage, err.Message));
}
return liveJob;
}
//Default run a backtesting job.
var backtestJob = new BacktestNodePacket(0, 0, "", new byte[] {}, 10000, "local")
{
Type = PacketType.BacktestNode,
Algorithm = File.ReadAllBytes(AlgorithmLocation),
Version = Constants.Version,
BacktestId = Config.Get("algorithm-type-name"),
RamAllocation = int.MaxValue,
Language = (Language)Enum.Parse(typeof(Language), Config.Get("algorithm-language"))
};
return backtestJob;
}
/// <summary>
/// Compose Live Result Data Packet - With tradable dates
/// </summary>
/// <param name="job">Job that started this request</param>
/// <param name="results">Results class for the Backtest job</param>
public LiveResultPacket(LiveNodePacket job, LiveResult results)
: base(PacketType.LiveResult)
{
try
{
SessionId = job.SessionId;
CompileId = job.CompileId;
DeployId = job.DeployId;
Results = results;
UserId = job.UserId;
ProjectId = job.ProjectId;
SessionId = job.SessionId;
Channel = job.Channel;
}
catch (Exception err) {
Log.Error(err);
}
}
/// <summary>
/// Removes the specified symbols from the subscription
/// </summary>
/// <param name="job">Job we're processing.</param>
/// <param name="symbols">The symbols to be removed keyed by SecurityType</param>
public void Unsubscribe(LiveNodePacket job, IEnumerable<Symbol> symbols)
{
var symbolsToUnsubscribe = (from symbol in symbols
where _subscribedSymbols.Contains(symbol)
select symbol).ToList();
if (symbolsToUnsubscribe.Count == 0)
return;
Log.Trace("OandaBrokerage.Unsubscribe(): {0}", string.Join(",", symbolsToUnsubscribe.Select(x => x.Value)));
// Oanda does not allow more than a few rate streaming sessions,
// so we only use a single session for all currently subscribed symbols
var symbolsToSubscribe = _subscribedSymbols.ToList().Where(x => !symbolsToUnsubscribe.Contains(x)).ToList();
SubscribeSymbols(symbolsToSubscribe);
_subscribedSymbols = symbolsToSubscribe.ToHashSet();
}
public void DoesNotSetAlgorithmRunTimeErrorOnDisconnectIfAllSecuritiesClosed()
{
var referenceTime = DateTime.UtcNow;
var algorithm = new AlgorithmStub(equities: new List<string> { "SPY" });
algorithm.SetDateTime(referenceTime);
algorithm.Securities["SPY"].Exchange.SetMarketHours(TimeSpan.Zero, TimeSpan.Zero, referenceTime.ConvertFromUtc(TimeZones.NewYork).DayOfWeek);
var job = new LiveNodePacket();
var results = new TestResultHandler();//packet => Console.WriteLine(FieldsToString(packet)));
var api = new Api.Api();
var handler = new DefaultBrokerageMessageHandler(algorithm, job, results, api, TimeSpan.FromMinutes(15));
Assert.IsNull(algorithm.RunTimeError);
handler.Handle(BrokerageMessageEvent.Disconnected("Disconnection!"));
Assert.IsNull(algorithm.RunTimeError);
results.Exit();
}
public void SetsAlgorithmRunTimeErrorOnDisconnectIfNonCustomSecurityIsOpen()
{
var algorithm = new AlgorithmStub(equities: new List<string> { "SPY" });
algorithm.Securities[Symbols.SPY].Exchange = new SecurityExchange(SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork));
var job = new LiveNodePacket();
var results = new TestResultHandler();//packet => Console.WriteLine(FieldsToString(packet)));
var api = new Api.Api();
var handler = new DefaultBrokerageMessageHandler(algorithm, job, results, api, TimeSpan.Zero, TimeSpan.Zero);
Assert.IsNull(algorithm.RunTimeError);
handler.Handle(BrokerageMessageEvent.Disconnected("Disconnection!"));
Thread.Sleep(100);
Assert.IsNotNull(algorithm.RunTimeError);
results.Exit();
}
/// <summary>
/// Creates a new <see cref="IBrokerage"/> instance
/// </summary>
/// <param name="job">The job packet to create the brokerage for</param>
/// <param name="algorithm">The algorithm instance</param>
/// <returns>A new brokerage instance</returns>
public override IBrokerage CreateBrokerage(LiveNodePacket job, IAlgorithm algorithm)
{
var errors = new List<string>();
// read values from the brokerage data
var environment = Read<Environment>(job.BrokerageData, "oanda-environment", errors);
var accessToken = Read<string>(job.BrokerageData, "oanda-access-token", errors);
var accountId = Read<int>(job.BrokerageData, "oanda-account-id", errors);
if (errors.Count != 0)
{
// if we had errors then we can't create the instance
throw new Exception(string.Join(System.Environment.NewLine, errors));
}
var brokerage = new OandaBrokerage(algorithm.Transactions, algorithm.Portfolio, environment, accessToken, accountId);
Composer.Instance.AddPart<IDataQueueHandler>(brokerage);
return brokerage;
}
/// <summary>
/// Live trading datafeed handler provides a base implementation of a live trading datafeed. Derived types
/// need only implement the GetNextTicks() function to return unprocessed ticks from a data source.
/// This creates a new data feed with a DataFeedEndpoint of LiveTrading.
/// </summary>
public LiveTradingDataFeed(IAlgorithm algorithm, LiveNodePacket job, IDataQueueHandler dataSource)
{
//Subscription Count:
_subscriptions = algorithm.SubscriptionManager.Subscriptions;
//Set Properties:
_isActive = true;
_dataFeed = DataFeedEndpoint.LiveTrading;
_bridge = new ConcurrentQueue<List<BaseData>>[Subscriptions.Count];
_endOfBridge = new bool[Subscriptions.Count];
_subscriptionManagers = new SubscriptionDataReader[Subscriptions.Count];
_realtimePrices = new List<decimal>();
//Set the source of the live data:
_dataQueue = dataSource;
//Class Privates:
_algorithm = algorithm;
_job = job;
//Setup the arrays:
for (var i = 0; i < Subscriptions.Count; i++)
{
_endOfBridge[i] = false;
_bridge[i] = new ConcurrentQueue<List<BaseData>>();
//This is quantconnect data source, store here for speed/ease of access
_isDynamicallyLoadedData.Add(algorithm.Securities[_subscriptions[i].Symbol].IsDynamicallyLoadedData);
//Subscription managers for downloading user data:
_subscriptionManagers[i] = new SubscriptionDataReader(_subscriptions[i], algorithm.Securities[_subscriptions[i].Symbol], DataFeedEndpoint.LiveTrading, DateTime.MinValue, DateTime.MaxValue);
//Set up the source file for today:
_subscriptionManagers[i].RefreshSource(DateTime.Now.Date);
_realtimePrices.Add(0);
}
// request for data from these symbols
_dataQueue.Subscribe(job, BuildTypeSymbolList(algorithm));
}
/// <summary>
/// Creates a new PaperTradingDataFeed for the algorithm/job
/// </summary>
/// <param name="algorithm">The algorithm to receive the data, used for a complete listing of active securities</param>
/// <param name="job">The job being run</param>
public PaperTradingDataFeed(IAlgorithm algorithm, LiveNodePacket job)
: base(algorithm)
{
_job = job;
// create a lookup keyed by SecurityType
var symbols = new Dictionary<SecurityType, List<string>>();
// Only subscribe equities and forex symbols
foreach (var security in algorithm.Securities.Values)
{
if (security.Type == SecurityType.Equity || security.Type == SecurityType.Forex)
{
if (!symbols.ContainsKey(security.Type)) symbols.Add(security.Type, new List<string>());
symbols[security.Type].Add(security.Symbol);
}
}
// request for data from these symbols
Engine.Queue.Subscribe(symbols);
}
/// <summary>
/// Adds the specified symbols to the subscription
/// </summary>
/// <param name="job">Job we're subscribing for:</param>
/// <param name="symbols">The symbols to be added keyed by SecurityType</param>
public void Subscribe(LiveNodePacket job, IEnumerable<Symbol> symbols)
{
lock (_lockerSubscriptions)
{
var symbolsToSubscribe = (from symbol in symbols
where !_subscribedSymbols.Contains(symbol) && CanSubscribe(symbol)
select symbol).ToList();
if (symbolsToSubscribe.Count == 0)
return;
Log.Trace("OandaBrokerage.Subscribe(): {0}", string.Join(",", symbolsToSubscribe.Select(x => x.Value)));
// Oanda does not allow more than a few rate streaming sessions,
// so we only use a single session for all currently subscribed symbols
symbolsToSubscribe = symbolsToSubscribe.Union(_subscribedSymbols.ToList()).ToList();
_subscribedSymbols = symbolsToSubscribe.ToHashSet();
ProcessSubscriptionRequest();
}
}
/// <summary>
/// Creates a new <see cref="IBrokerage"/> instance
/// </summary>
/// <param name="job">The job packet to create the brokerage for</param>
/// <param name="algorithm">The algorithm instance</param>
/// <returns>A new brokerage instance</returns>
public override IBrokerage CreateBrokerage(LiveNodePacket job, IAlgorithm algorithm)
{
var errors = new List<string>();
// read values from the brokerage data
var server = Read<string>(job.BrokerageData, "fxcm-server", errors);
var terminal = Read<string>(job.BrokerageData, "fxcm-terminal", errors);
var userName = Read<string>(job.BrokerageData, "fxcm-user-name", errors);
var password = Read<string>(job.BrokerageData, "fxcm-password", errors);
var accountId = Read<string>(job.BrokerageData, "fxcm-account-id", errors);
if (errors.Count != 0)
{
// if we had errors then we can't create the instance
throw new Exception(string.Join(Environment.NewLine, errors));
}
var brokerage = new FxcmBrokerage(algorithm.Transactions, algorithm.Portfolio, server, terminal, userName, password, accountId);
Composer.Instance.AddPart<IDataQueueHandler>(brokerage);
Composer.Instance.AddPart<IHistoryProvider>(brokerage);
return brokerage;
}
/// <summary>
/// Initializes the data feed for the specified job and algorithm
/// </summary>
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider)
{
if (!(job is LiveNodePacket))
{
throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket.");
}
_cancellationTokenSource = new CancellationTokenSource();
_algorithm = algorithm;
_job = (LiveNodePacket) job;
_resultHandler = resultHandler;
_timeProvider = GetTimeProvider();
_dataQueueHandler = GetDataQueueHandler();
_frontierTimeProvider = new ManualTimeProvider(_timeProvider.GetUtcNow());
_customExchange = new BaseDataExchange("CustomDataExchange") {SleepInterval = 10};
// sleep is controlled on this exchange via the GetNextTicksEnumerator
_exchange = new BaseDataExchange("DataQueueExchange"){SleepInterval = 0};
_exchange.AddEnumerator(DataQueueHandlerSymbol, GetNextTicksEnumerator());
_subscriptions = new SubscriptionCollection();
_bridge = new BusyBlockingCollection<TimeSlice>();
_universeSelection = new UniverseSelection(this, algorithm, job.Controls);
// run the exchanges
Task.Run(() => _exchange.Start(_cancellationTokenSource.Token));
Task.Run(() => _customExchange.Start(_cancellationTokenSource.Token));
// this value will be modified via calls to AddSubscription/RemoveSubscription
var ffres = Time.OneMinute;
_fillForwardResolution = Ref.Create(() => ffres, v => ffres = v);
// wire ourselves up to receive notifications when universes are added/removed
var start = _timeProvider.GetUtcNow();
algorithm.UniverseManager.CollectionChanged += (sender, args) =>
{
switch (args.Action)
{
case NotifyCollectionChangedAction.Add:
foreach (var universe in args.NewItems.OfType<Universe>())
{
var config = universe.Configuration;
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var exchangeHours = marketHoursDatabase.GetExchangeHours(config);
Security security;
if (!_algorithm.Securities.TryGetValue(config.Symbol, out security))
{
// create a canonical security object
security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency));
}
AddSubscription(new SubscriptionRequest(true, universe, security, config, start, Time.EndOfTime));
// Not sure if this is needed but left here because of this:
// https://github.com/QuantConnect/Lean/commit/029d70bde6ca83a1eb0c667bb5cc4444bea05678
UpdateFillForwardResolution();
}
break;
case NotifyCollectionChangedAction.Remove:
foreach (var universe in args.OldItems.OfType<Universe>())
{
RemoveSubscription(universe.Configuration);
}
break;
default:
throw new NotImplementedException("The specified action is not implemented: " + args.Action);
}
};
}
/// <summary>
/// Desktop/Local Get Next Task - Get task from the Algorithm folder of VS Solution.
/// </summary>
/// <returns></returns>
public AlgorithmNodePacket NextJob(out string location)
{
location = AlgorithmLocation;
Log.Trace("JobQueue.NextJob(): Selected " + location);
// check for parameters in the config
var parameters = new Dictionary<string, string>();
var parametersConfigString = Config.Get("parameters");
if (parametersConfigString != string.Empty)
{
parameters = JsonConvert.DeserializeObject<Dictionary<string, string>>(parametersConfigString);
}
//If this isn't a backtesting mode/request, attempt a live job.
if (_liveMode)
{
var liveJob = new LiveNodePacket
{
Type = PacketType.LiveNode,
Algorithm = File.ReadAllBytes(AlgorithmLocation),
Brokerage = Config.Get("live-mode-brokerage", PaperBrokerageTypeName),
Channel = AccessToken,
UserId = UserId,
ProjectId = ProjectId,
Version = Globals.Version,
DeployId = AlgorithmTypeName,
RamAllocation = int.MaxValue,
Parameters = parameters,
Language = Language,
};
try
{
// import the brokerage data for the configured brokerage
var brokerageFactory = Composer.Instance.Single<IBrokerageFactory>(factory => factory.BrokerageType.MatchesTypeName(liveJob.Brokerage));
liveJob.BrokerageData = brokerageFactory.BrokerageData;
}
catch (Exception err)
{
Log.Error(err, string.Format("Error resolving BrokerageData for live job for brokerage {0}:", liveJob.Brokerage));
}
return liveJob;
}
//Default run a backtesting job.
var backtestJob = new BacktestNodePacket(0, 0, "", new byte[] {}, 10000, "local")
{
Type = PacketType.BacktestNode,
Algorithm = File.ReadAllBytes(AlgorithmLocation),
Channel = AccessToken,
UserId = UserId,
ProjectId = ProjectId,
Version = Globals.Version,
BacktestId = AlgorithmTypeName,
RamAllocation = int.MaxValue,
Language = Language,
Parameters = parameters
};
return backtestJob;
}
/// <summary>
/// Removes the specified symbols to the subscription
/// </summary>
/// <param name="job">Job we're processing.</param>
/// <param name="symbols">The symbols to be removed keyed by SecurityType</param>
public void Unsubscribe(LiveNodePacket job, IEnumerable<Symbol> symbols)
{
foreach (var symbol in symbols)
{
var res = default(int);
if (_subscribedSymbols.TryRemove(symbol, out res))
{
Client.CancelMarketData(res);
var secRes = default(Symbol);
_subscribedTickets.TryRemove(res, out secRes);
}
}
}
/// <summary>
/// Adds the specified symbols to the subscription
/// </summary>
/// <param name="job">Job we're subscribing for:</param>
/// <param name="symbols">The symbols to be added keyed by SecurityType</param>
public void Subscribe(LiveNodePacket job, IEnumerable<Symbol> symbols)
{
foreach (var symbol in symbols.Where(CanSubscribe))
{
var id = GetNextRequestID();
var contract = CreateContract(symbol);
Client.RequestMarketData(id, contract, null, false, false);
_subscribedSymbols[symbol] = id;
_subscribedTickets[id] = symbol;
}
}
public LiveConsoleStatusHandler(LiveNodePacket _job)
{
this._job = _job;
}
private IDataFeed RunDataFeed(IAlgorithm algorithm, out FuncDataQueueHandler dataQueueHandler, ITimeProvider timeProvider = null, Func<FuncDataQueueHandler, IEnumerable<BaseData>> getNextTicksFunction = null)
{
getNextTicksFunction = getNextTicksFunction ?? (fdqh => fdqh.Subscriptions.Select(symbol => new Tick(DateTime.Now, symbol, 1, 2){Quantity = 1}));
// job is used to send into DataQueueHandler
var job = new LiveNodePacket();
// result handler is used due to dependency in SubscriptionDataReader
var resultHandler = new ConsoleResultHandler(); // new ResultHandlerStub();
dataQueueHandler = new FuncDataQueueHandler(getNextTicksFunction);
var feed = new TestableLiveTradingDataFeed(dataQueueHandler, timeProvider);
var mapFileProvider = new LocalDiskMapFileProvider();
feed.Initialize(algorithm, job, resultHandler, mapFileProvider, new LocalDiskFactorFileProvider(mapFileProvider));
var feedThreadStarted = new ManualResetEvent(false);
Task.Factory.StartNew(() =>
{
feedThreadStarted.Set();
feed.Run();
});
// wait for feed.Run to actually begin
feedThreadStarted.WaitOne();
return feed;
}
/// <summary>
/// Creates a new PaperBrokerage
/// </summary>
/// <param name="algorithm">The algorithm under analysis</param>
/// <param name="job">The job packet</param>
public PaperBrokerage(IAlgorithm algorithm, LiveNodePacket job)
: base(algorithm, "Paper Brokerage")
{
_job = job;
}
/// <summary>
/// Initializes the data feed for the specified job and algorithm
/// </summary>
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler)
{
if (!(job is LiveNodePacket))
{
throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket.");
}
_cancellationTokenSource = new CancellationTokenSource();
_algorithm = algorithm;
_job = (LiveNodePacket) job;
_resultHandler = resultHandler;
_timeProvider = GetTimeProvider();
_dataQueueHandler = GetDataQueueHandler();
_frontierTimeProvider = new ManualTimeProvider(_timeProvider.GetUtcNow());
_customExchange = new BaseDataExchange("CustomDataExchange") {SleepInterval = 10};
// sleep is controlled on this exchange via the GetNextTicksEnumerator
_exchange = new BaseDataExchange("DataQueueExchange", GetNextTicksEnumerator()){SleepInterval = 0};
_subscriptions = new ConcurrentDictionary<SymbolSecurityType, Subscription>();
Bridge = new BusyBlockingCollection<TimeSlice>();
// run the exchanges
_exchange.Start();
_customExchange.Start();
// find the minimum resolution, ignoring ticks
_fillForwardResolution = algorithm.SubscriptionManager.Subscriptions
.Where(x => x.Resolution != Resolution.Tick)
.Select(x => x.Resolution)
.Union(algorithm.Universes.Select(x => x.SubscriptionSettings.Resolution))
.DefaultIfEmpty(algorithm.UniverseSettings.Resolution)
.Min();
// add user defined subscriptions
var start = _timeProvider.GetUtcNow();
foreach (var kvp in _algorithm.Securities.OrderBy(x => x.Key.ToString()))
{
var security = kvp.Value;
AddSubscription(security, start, Time.EndOfTime, true);
}
// add universe subscriptions
foreach (var universe in _algorithm.Universes)
{
var subscription = CreateUniverseSubscription(universe, start, Time.EndOfTime);
_subscriptions[new SymbolSecurityType(subscription)] = subscription;
}
}
/// <summary>
/// Initialize the result handler with this result packet.
/// </summary>
/// <param name="job">Algorithm job packet for this result handler</param>
/// <param name="messagingHandler"></param>
/// <param name="api"></param>
/// <param name="dataFeed"></param>
/// <param name="setupHandler"></param>
/// <param name="transactionHandler"></param>
public void Initialize(AlgorithmNodePacket job, IMessagingHandler messagingHandler, IApi api, IDataFeed dataFeed, ISetupHandler setupHandler, ITransactionHandler transactionHandler)
{
_api = api;
_dataFeed = dataFeed;
_messagingHandler = messagingHandler;
_setupHandler = setupHandler;
_transactionHandler = transactionHandler;
_job = (LiveNodePacket)job;
if (_job == null) throw new Exception("LiveResultHandler.Constructor(): Submitted Job type invalid.");
_deployId = _job.DeployId;
_compileId = _job.CompileId;
}
/// <summary>
/// Removes the specified symbols to the subscription
/// </summary>
/// <param name="job">Job we're processing.</param>
/// <param name="symbols">The symbols to be removed keyed by SecurityType</param>
public void Unsubscribe(LiveNodePacket job, IEnumerable<Symbol> symbols)
{
try
{
foreach (var symbol in symbols)
{
lock (_sync)
{
Log.Trace("IQFeed.Unsubscribe(): " + symbol.ToString());
var type = symbol.ID.SecurityType;
_symbols.Remove(symbol);
var ticker = symbol.Value;
if (type == SecurityType.Forex) ticker += ".FXCM";
if (_level1Port.Contains(ticker))
{
_level1Port.Unsubscribe(ticker);
}
}
}
}
catch (Exception err)
{
Log.Error("IQFeed.Unsubscribe(): " + err.Message);
}
}
