/// <summary>
/// Creates a new instance
/// </summary>
/// <param name="timeProvider">The time provider to use</param>
public RealTimeScheduleEventService(ITimeProvider timeProvider)
{
_nextUtcScheduledEvent = Ref.Create(new ReferenceWrapper <DateTime>(DateTime.MinValue));
_timer = new Timer(
async state =>
{
var nextUtcScheduledEvent = ((Ref <ReferenceWrapper <DateTime> >)state).Value.Value;
var diff = nextUtcScheduledEvent - timeProvider.GetUtcNow();
// we need to guarantee we trigger the event after the requested due time
// has past, if we got called earlier lets wait until time is right
while (diff.Ticks > 0)
{
if (diff.Milliseconds >= 1)
{
Thread.Sleep(diff);
}
else
{
Thread.SpinWait(1000);
}
// testing has shown that it sometimes requires more than one loop
diff = nextUtcScheduledEvent - timeProvider.GetUtcNow();
}
NewEvent?.Invoke(this, EventArgs.Empty);
},
_nextUtcScheduledEvent,
// Due time is never, has to be scheduled
Timeout.InfiniteTimeSpan,
// Do not trigger periodically
Timeout.InfiniteTimeSpan);
}
/// <summary>
/// Initializes a new instance of the <see cref="DataFeedPacket"/> class
/// </summary>
/// <param name="security">The security whose data is held in this packet</param>
/// <param name="configuration">The subscription configuration that produced this data</param>
/// <param name="data">The data to add to this packet. The list reference is reused
/// internally and NOT copied.</param>
/// <param name="isSubscriptionRemoved">Reference to whether or not the subscription has since been removed, defaults to false</param>
public DataFeedPacket(ISecurityPrice security, SubscriptionDataConfig configuration, List <BaseData> data, IReadOnlyRef <bool> isSubscriptionRemoved = null)
{
Security = security;
Configuration = configuration;
Data = data;
_isRemoved = isSubscriptionRemoved ?? Ref.Create(false);
}
static void ColorCommand(IEnumerable <string> args)
{
var help = Ref.Create(false);
var globDir = Ref.Create((DirectoryInfo)null);
var showLineEndings = false;
var options = new OptionSet(CreateStrictOptionSetArgumentParser())
{
Options.Help(help),
Options.Verbose(Verbose),
Options.Debug,
Options.Glob(globDir),
{ "eol", "Show line endings", _ => showLineEndings = true }
};
var tail = options.Parse(args);
if (help)
{
Help("color", options);
return;
}
var defaultColor = Color.Console;
var magenta = new Color(ConsoleColor.Magenta, defaultColor.Background);
var black = new Color(ConsoleColor.Black, ConsoleColor.DarkGray);
var green = new Color(ConsoleColor.Green, defaultColor.Background);
var yellow = new Color(ConsoleColor.Yellow, defaultColor.Background);
var cyan = new Color(ConsoleColor.Cyan, defaultColor.Background);
var blue = new Color(ConsoleColor.Blue, defaultColor.Background);
var gray = new Color(ConsoleColor.Gray, defaultColor.Background);
var defaultPalette = new(TokenKind TokenKind, Color)[]
/// <summary>
/// Initializes the data feed for the specified job and algorithm
/// </summary>
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, IDataProvider dataProvider)
{
_algorithm = algorithm;
_resultHandler = resultHandler;
_mapFileProvider = mapFileProvider;
_factorFileProvider = factorFileProvider;
_dataProvider = dataProvider;
_subscriptions = new SubscriptionCollection();
_universeSelection = new UniverseSelection(this, algorithm);
_cancellationTokenSource = new CancellationTokenSource();
_subscriptionfactory = new SubscriptionDataReaderSubscriptionEnumeratorFactory(_resultHandler, _mapFileProvider, _factorFileProvider, _dataProvider, false, true);
IsActive = true;
var threadCount = Math.Max(1, Math.Min(4, Environment.ProcessorCount - 3));
_controller = new ParallelRunnerController(threadCount);
_controller.Start(_cancellationTokenSource.Token);
var ffres = Time.OneMinute;
_fillForwardResolution = Ref.Create(() => ffres, res => ffres = res);
// wire ourselves up to receive notifications when universes are added/removed
algorithm.UniverseManager.CollectionChanged += (sender, args) =>
{
switch (args.Action)
{
case NotifyCollectionChangedAction.Add:
foreach (var universe in args.NewItems.OfType <Universe>())
{
var config = universe.Configuration;
var start = _algorithm.UtcTime;
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var exchangeHours = marketHoursDatabase.GetExchangeHours(config);
Security security;
if (!_algorithm.Securities.TryGetValue(config.Symbol, out security))
{
// create a canonical security object if it doesn't exist
security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency));
}
var end = _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone);
AddSubscription(new SubscriptionRequest(true, universe, security, config, start, end));
}
break;
case NotifyCollectionChangedAction.Remove:
foreach (var universe in args.OldItems.OfType <Universe>())
{
RemoveSubscription(universe.Configuration);
}
break;
default:
throw new NotImplementedException("The specified action is not implemented: " + args.Action);
}
};
}
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler)
{
if (algorithm.SubscriptionManager.Subscriptions.Count == 0 && algorithm.Universes.IsNullOrEmpty())
{
throw new Exception("No subscriptions registered and no universe defined.");
}
_algorithm = algorithm;
_resultHandler = resultHandler;
_subscriptions = new ConcurrentDictionary <SymbolSecurityType, Subscription>();
_cancellationTokenSource = new CancellationTokenSource();
IsActive = true;
Bridge = new BusyBlockingCollection <TimeSlice>(100);
var ffres = Time.OneSecond;
_fillForwardResolution = Ref.Create(() => ffres, res => ffres = res);
// find the minimum resolution, ignoring ticks
ffres = ResolveFillForwardResolution(algorithm);
// add each universe selection subscription to the feed
foreach (var universe in _algorithm.Universes)
{
var startTimeUtc = _algorithm.StartDate.ConvertToUtc(_algorithm.TimeZone);
var endTimeUtc = _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone);
AddUniverseSubscription(universe, startTimeUtc, endTimeUtc);
}
}
static void GlobCommand(IEnumerable <string> args)
{
var help = Ref.Create(false);
var globDir = Ref.Create((DirectoryInfo)null);
var options = new OptionSet(CreateStrictOptionSetArgumentParser())
{
Options.Help(help),
Options.Verbose(Verbose),
Options.Debug,
Options.Glob(globDir)
};
var tail = options.Parse(args);
if (help)
{
Help("glob", options);
return;
}
var dir = globDir.Value != null
? globDir
: new DirectoryInfo(Environment.CurrentDirectory);
foreach (var(p, _) in ReadSources(tail, dir, () => (string)null, _ => null))
{
Console.WriteLine(p);
}
}
/// <summary>
/// Creates an enumerator to read the specified request.
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider)
{
var config = request.Configuration;
// frontier value used to prevent emitting duplicate time stamps between refreshed enumerators
// also provides some immediate fast-forward to handle spooling through remote files quickly
var frontier = Ref.Create(DateTime.MinValue);
var lastSourceRefreshTime = DateTime.MinValue;
var sourceFactory = (BaseData)ObjectActivator.GetActivator(config.Type).Invoke(new object[] { config.Type });
// this is refreshing the enumerator stack for each new source
var refresher = new RefreshEnumerator <BaseData>(() =>
{
// rate limit the refresh of this enumerator stack
var utcNow = _timeProvider.GetUtcNow();
var minimumTimeBetweenCalls = GetMinimumTimeBetweenCalls(config.Increment);
if (utcNow - lastSourceRefreshTime < minimumTimeBetweenCalls)
{
return(Enumerable.Empty <BaseData>().GetEnumerator());
}
lastSourceRefreshTime = utcNow;
var localDate = utcNow.ConvertFromUtc(config.ExchangeTimeZone).Date;
var source = sourceFactory.GetSource(config, localDate, true);
// fetch the new source and enumerate the data source reader
var enumerator = EnumerateDataSourceReader(config, dataProvider, frontier, source, localDate);
if (SourceRequiresFastForward(source))
{
// apply fast forward logic for file transport mediums
var maximumDataAge = GetMaximumDataAge(config.Increment);
enumerator = new FastForwardEnumerator(enumerator, _timeProvider, config.ExchangeTimeZone, maximumDataAge);
}
else
{
// rate limit calls to this enumerator stack
enumerator = new RateLimitEnumerator <BaseData>(enumerator, _timeProvider, minimumTimeBetweenCalls);
}
if (source.Format == FileFormat.Collection)
{
// unroll collections into individual data points after fast forward/rate limiting applied
enumerator = enumerator.SelectMany(data =>
{
var collection = data as BaseDataCollection;
return(collection?.Data.GetEnumerator() ?? new List <BaseData> {
data
}.GetEnumerator());
});
}
return(enumerator);
});
// prevent calls to the enumerator stack if current is in the future
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
return(new FrontierAwareEnumerator(refresher, _timeProvider, timeZoneOffsetProvider));
}
/// <summary>
/// Initializes a new instance of the <see cref="SubscriptionCollection"/> class
/// </summary>
public SubscriptionCollection()
{
_subscriptions = new ConcurrentDictionary <SubscriptionDataConfig, Subscription>();
_subscriptionsByTickType = new List <Subscription>();
var ffres = Time.OneMinute;
_fillForwardResolution = Ref.Create(() => ffres, res => ffres = res);
}
/// <summary>
/// Initializes the data feed for the specified job and algorithm
/// </summary>
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler)
{
if (!(job is LiveNodePacket))
{
throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket.");
}
if (algorithm.SubscriptionManager.Subscriptions.Count == 0 && algorithm.Universes.IsNullOrEmpty())
{
throw new Exception("No subscriptions registered and no universe defined.");
}
_cancellationTokenSource = new CancellationTokenSource();
_algorithm = algorithm;
_job = (LiveNodePacket)job;
_resultHandler = resultHandler;
_timeProvider = GetTimeProvider();
_dataQueueHandler = GetDataQueueHandler();
_frontierTimeProvider = new ManualTimeProvider(_timeProvider.GetUtcNow());
_customExchange = new BaseDataExchange("CustomDataExchange")
{
SleepInterval = 10
};
// sleep is controlled on this exchange via the GetNextTicksEnumerator
_exchange = new BaseDataExchange("DataQueueExchange", GetNextTicksEnumerator())
{
SleepInterval = 0
};
_subscriptions = new ConcurrentDictionary <SymbolSecurityType, Subscription>();
Bridge = new BusyBlockingCollection <TimeSlice>();
// run the exchanges
_exchange.Start();
_customExchange.Start();
// this value will be modified via calls to AddSubscription/RemoveSubscription
var ffres = Time.OneSecond;
_fillForwardResolution = Ref.Create(() => ffres, v => ffres = v);
ffres = ResolveFillForwardResolution(algorithm);
// add subscriptions
var start = _timeProvider.GetUtcNow();
foreach (var universe in _algorithm.Universes)
{
var subscription = CreateUniverseSubscription(universe, start, Time.EndOfTime);
_subscriptions[new SymbolSecurityType(subscription)] = subscription;
}
}
/// <summary>
/// Initializes the data feed for the specified job and algorithm
/// </summary>
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider)
{
_algorithm = algorithm;
_resultHandler = resultHandler;
_mapFileProvider = mapFileProvider;
_factorFileProvider = factorFileProvider;
_subscriptions = new ConcurrentDictionary <Symbol, Subscription>();
_universeSelection = new UniverseSelection(this, algorithm, job.Controls);
_cancellationTokenSource = new CancellationTokenSource();
IsActive = true;
var threadCount = Math.Max(1, Math.Min(4, Environment.ProcessorCount - 3));
_controller = new ParallelRunnerController(threadCount);
_controller.Start(_cancellationTokenSource.Token);
var ffres = Time.OneSecond;
_fillForwardResolution = Ref.Create(() => ffres, res => ffres = res);
// find the minimum resolution, ignoring ticks
ffres = ResolveFillForwardResolution(algorithm);
// wire ourselves up to receive notifications when universes are added/removed
algorithm.UniverseManager.CollectionChanged += (sender, args) =>
{
switch (args.Action)
{
case NotifyCollectionChangedAction.Add:
foreach (var universe in args.NewItems.OfType <Universe>())
{
var start = _frontierUtc != DateTime.MinValue ? _frontierUtc : _algorithm.StartDate.ConvertToUtc(_algorithm.TimeZone);
AddUniverseSubscription(universe, start, _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone));
}
break;
case NotifyCollectionChangedAction.Remove:
foreach (var universe in args.OldItems.OfType <Universe>())
{
Subscription subscription;
if (_subscriptions.TryGetValue(universe.Configuration.Symbol, out subscription))
{
RemoveSubscription(subscription);
}
}
break;
default:
throw new NotImplementedException("The specified action is not implemented: " + args.Action);
}
};
}
public void HandlesDaylightSavingTimeChange()
{
var reference = new DateTime(2018, 3, 10);
var period = Time.OneDay;
var underlying = new List <TradeBar>
{
new TradeBar(reference, Symbols.SPY, 10, 20, 5, 15, 123456, period),
// Daylight Saving Time change -> add 1 hour
new TradeBar(reference.AddDays(1).AddHours(1), Symbols.SPY, 100, 200, 50, 150, 1234560, period)
};
var timeProvider = new ManualTimeProvider(TimeZones.NewYork);
timeProvider.SetCurrentTime(reference);
var exchange = new SecurityExchange(SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork));
var fillForward = new LiveFillForwardEnumerator(
timeProvider,
underlying.GetEnumerator(),
exchange,
Ref.Create(Time.OneDay),
false,
Time.EndOfTime,
Time.OneDay,
exchange.TimeZone,
Time.BeginningOfTime);
// first point is always emitted
Assert.IsTrue(fillForward.MoveNext());
Assert.IsFalse(fillForward.Current.IsFillForward);
Assert.AreEqual(underlying[0], fillForward.Current);
//Assert.AreEqual(underlying[0].EndTime, fillForward.Current.EndTime);
Assert.AreEqual(123456, ((TradeBar)fillForward.Current).Volume);
// Daylight Saving Time change -> add 1 hour
timeProvider.SetCurrentTime(reference.AddDays(1).AddHours(1));
// second data point emitted
Assert.IsTrue(fillForward.MoveNext());
Assert.IsFalse(fillForward.Current.IsFillForward);
Assert.AreEqual(underlying[1], fillForward.Current);
//Assert.AreEqual(underlying[1].EndTime, fillForward.Current.EndTime);
Assert.AreEqual(1234560, ((TradeBar)fillForward.Current).Volume);
Assert.IsTrue(fillForward.MoveNext());
Assert.IsTrue(fillForward.Current.IsFillForward);
Assert.AreEqual(underlying[1].EndTime, fillForward.Current.Time);
Assert.AreEqual(underlying[1].Value, fillForward.Current.Value);
Assert.AreEqual(0, ((TradeBar)fillForward.Current).Volume);
fillForward.Dispose();
}
/// <summary>
/// Gets an <see cref="IConfigProvider"/> using the specified list of preferred config types.
/// </summary>
/// <param name="configName">the name of the mod for this config</param>
/// <param name="extensions">the preferred config types to try to get</param>
/// <returns>an <see cref="IConfigProvider"/> of the requested type, or of type JSON.</returns>
public static IConfigProvider GetProviderFor(string configName, params string[] extensions)
{
var chosenExt = extensions.FirstOrDefault(s => registeredProviders.ContainsKey(s)) ?? "json";
var type = registeredProviders[chosenExt];
var provider = Activator.CreateInstance(type) as IConfigProvider;
if (provider != null)
{
provider.Filename = Path.Combine(BeatSaber.UserDataPath, configName);
configProviders.Add(Tuple.Create(Ref.Create(provider.LastModified), provider));
}
return(provider);
}
public async Task Fact()
{
var cancelled = Ref <bool> .Create(false);
Task.Run(async() =>
{
await Task.Delay(500);
cancelled.Value = true;
});
var task = Method(cancelled);
await Task.Delay(1000);
task.Status.Should().Be(TaskStatus.RanToCompletion);
}
static void GrepCommand(IEnumerable <string> args)
{
var help = Ref.Create(false);
var globDir = Ref.Create((DirectoryInfo)null);
var options = new OptionSet(CreateStrictOptionSetArgumentParser())
{
Options.Help(help),
Options.Verbose(Verbose),
Options.Debug,
Options.Glob(globDir),
};
var tail = new Queue <string>(options.Parse(args));
if (help)
{
Help("grep", options);
return;
}
if (!tail.Any())
{
throw new Exception("Missing search regular expression.");
}
var pattern = tail.Dequeue();
foreach (var(path, source) in ReadSources(tail, globDir))
{
foreach (var t in from e in Scanner.ParseStrings(source, (t, _, s) => (Token: t, Value: s))
where Regex.IsMatch(e.Value, pattern)
select e.Token)
{
Console.WriteLine($"{path}({t.Start.Line},{t.Start.Column}): {JsonString.Encode(source, t.Start.Offset, t.Length)}");
}
}
}
static int HashCommand(IEnumerable <string> args)
{
var help = Ref.Create(false);
var globDir = Ref.Create((DirectoryInfo)null);
var comparand = (byte[])null;
var algoName = HashAlgorithmName.SHA256;
var format = HashOutputFormat.Hexadecimal;
var commentFilterPattern = Ref.Create((string)null);
var keepLeadComment = Ref.Create(false);
var keepImportantComment = Ref.Create(false);
var options = new OptionSet(CreateStrictOptionSetArgumentParser())
{
Options.Help(help),
Options.Verbose(Verbose),
Options.Debug,
Options.Glob(globDir),
Options.CommentFilterPattern(commentFilterPattern),
Options.KeepLeadComment(keepLeadComment),
Options.KeepImportantComments(keepImportantComment),
{ "c|compare=", "set non-zero exit code if {HASH} (in hexadecimal) is different",
v => comparand = TryParseHexadecimalString(v, out var hc) ? hc
: throw new Exception("Hash comparand is not a valid hexadecimal string.") },
/// <summary>
/// Setups a new <see cref="Subscription"/> which will consume a blocking <see cref="EnqueueableEnumerator{T}"/>
/// that will be feed by a worker task
/// </summary>
/// <param name="request">The subscription data request</param>
/// <param name="enumerator">The data enumerator stack</param>
/// <param name="firstLoopLimit">The first loop data point count for which the worker will stop</param>
/// <returns>A new subscription instance ready to consume</returns>
public static Subscription CreateAndScheduleWorker(
SubscriptionRequest request,
IEnumerator <BaseData> enumerator,
int firstLoopLimit = 50)
{
var upperThreshold = GetUpperThreshold(request.Configuration.Resolution);
var lowerThreshold = GetLowerThreshold(request.Configuration.Resolution);
if (request.Configuration.Type == typeof(CoarseFundamental))
{
// the lower threshold will be when we start the worker again, if he is stopped
lowerThreshold = 200;
// the upper threshold will stop the worker from loading more data. This is roughly 1 GB
upperThreshold = 500;
}
var exchangeHours = request.Security.Exchange.Hours;
var enqueueable = new EnqueueableEnumerator <SubscriptionData>(true);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
var subscription = new Subscription(request, enqueueable, timeZoneOffsetProvider);
// The first loop of a backtest can load hundreds of subscription feeds, resulting in long delays while the thresholds
// for the buffer are reached. For the first loop start up with just 50 items in the buffer.
var firstLoop = Ref.Create(true);
Action produce = () =>
{
try
{
var count = 0;
while (enumerator.MoveNext())
{
// subscription has been removed, no need to continue enumerating
if (enqueueable.HasFinished)
{
enumerator.DisposeSafely();
return;
}
var subscriptionData = SubscriptionData.Create(subscription.Configuration, exchangeHours,
subscription.OffsetProvider, enumerator.Current);
// drop the data into the back of the enqueueable
enqueueable.Enqueue(subscriptionData);
count++;
// stop executing if we have more data than the upper threshold in the enqueueable, we don't want to fill the ram
if (count > upperThreshold || count > firstLoopLimit && firstLoop.Value)
{
// we use local count for the outside if, for performance, and adjust here
count = enqueueable.Count;
if (count > upperThreshold || firstLoop.Value)
{
firstLoop.Value = false;
// we will be re scheduled to run by the consumer, see EnqueueableEnumerator
// if the consumer is already waiting for us wake him up, he will rescheduled us if required
enqueueable.CancellationTokenSource.Cancel();
return;
}
}
}
}
catch (Exception exception)
{
Log.Error(exception, $"Subscription worker task exception {request.Configuration}.");
}
// we made it here because MoveNext returned false or we exploded, stop the enqueueable
enqueueable.Stop();
// we have to dispose of the enumerator
enumerator.DisposeSafely();
};
enqueueable.SetProducer(produce, lowerThreshold);
return(subscription);
}
public void EmitsDailyQuandlFutureDataOverWeekends()
{
RemoteFileSubscriptionStreamReader.SetDownloadProvider(new Api.Api());
var tickers = new[] { "CHRIS/CME_ES1", "CHRIS/CME_ES2" };
var startDate = new DateTime(2018, 4, 1);
var endDate = new DateTime(2018, 4, 20);
// delete temp files
foreach (var ticker in tickers)
{
var fileName = TestableQuandlFuture.GetLocalFileName(ticker, "test");
File.Delete(fileName);
}
var algorithm = new QCAlgorithm();
CreateDataFeed();
var dataManager = new DataManagerStub(algorithm, _feed);
algorithm.SubscriptionManager.SetDataManager(dataManager);
var symbols = tickers.Select(ticker => algorithm.AddData <TestableQuandlFuture>(ticker, Resolution.Daily).Symbol).ToList();
var timeProvider = new ManualTimeProvider(TimeZones.NewYork);
timeProvider.SetCurrentTime(startDate);
var dataPointsEmitted = 0;
RunLiveDataFeed(algorithm, startDate, symbols, timeProvider, dataManager);
var cancellationTokenSource = new CancellationTokenSource();
var lastFileWriteDate = DateTime.MinValue;
// create a timer to advance time much faster than realtime and to simulate live Quandl data file updates
var timerInterval = TimeSpan.FromMilliseconds(20);
var timer = Ref.Create <Timer>(null);
timer.Value = new Timer(state =>
{
try
{
var currentTime = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork);
if (currentTime.Date > endDate.Date)
{
Log.Trace($"Total data points emitted: {dataPointsEmitted.ToStringInvariant()}");
_feed.Exit();
cancellationTokenSource.Cancel();
return;
}
if (currentTime.Date > lastFileWriteDate.Date)
{
foreach (var ticker in tickers)
{
var source = TestableQuandlFuture.GetLocalFileName(ticker, "csv");
// write new local file including only rows up to current date
var outputFileName = TestableQuandlFuture.GetLocalFileName(ticker, "test");
var sb = new StringBuilder();
{
using (var reader = new StreamReader(source))
{
var firstLine = true;
string line;
while ((line = reader.ReadLine()) != null)
{
if (firstLine)
{
sb.AppendLine(line);
firstLine = false;
continue;
}
var csv = line.Split(',');
var time = Parse.DateTimeExact(csv[0], "yyyy-MM-dd");
if (time.Date >= currentTime.Date)
{
break;
}
sb.AppendLine(line);
}
}
}
if (currentTime.Date.DayOfWeek != DayOfWeek.Saturday && currentTime.Date.DayOfWeek != DayOfWeek.Sunday)
{
var fileContent = sb.ToString();
try
{
File.WriteAllText(outputFileName, fileContent);
}
catch (IOException)
{
Log.Error("IOException: will sleep 200ms and retry once more");
// lets sleep 200ms and retry once more, consumer could be reading the file
// this exception happens in travis intermittently, GH issue 3273
Thread.Sleep(200);
File.WriteAllText(outputFileName, fileContent);
}
Log.Trace($"Time:{currentTime} - Ticker:{ticker} - Files written:{++_countFilesWritten}");
}
}
lastFileWriteDate = currentTime;
}
// 30 minutes is the check interval for daily remote files, so we choose a smaller one to advance time
timeProvider.Advance(TimeSpan.FromMinutes(20));
//Log.Trace($"Time advanced to: {timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork)}");
// restart the timer
timer.Value.Change(timerInterval.Milliseconds, Timeout.Infinite);
}
catch (Exception exception)
{
Log.Error(exception);
_feed.Exit();
cancellationTokenSource.Cancel();
}
}, null, timerInterval.Milliseconds, Timeout.Infinite);
try
{
foreach (var timeSlice in _synchronizer.StreamData(cancellationTokenSource.Token))
{
foreach (var dataPoint in timeSlice.Slice.Values)
{
Log.Trace($"Data point emitted at {timeSlice.Slice.Time.ToStringInvariant()}: " +
$"{dataPoint.Symbol.Value} {dataPoint.Value.ToStringInvariant()} " +
$"{dataPoint.EndTime.ToStringInvariant()}"
);
dataPointsEmitted++;
}
}
}
catch (Exception exception)
{
Log.Trace($"Error: {exception}");
}
timer.Value.Dispose();
dataManager.RemoveAllSubscriptions();
Assert.AreEqual(14 * tickers.Length, dataPointsEmitted);
}
public void RemoteDataDoesNotIncreaseNumberOfSlices()
{
Config.Set("quandl-auth-token", "QUANDL-TOKEN");
var startDate = new DateTime(2018, 4, 2);
var endDate = new DateTime(2018, 4, 19);
var algorithm = new QCAlgorithm();
var timeProvider = new ManualTimeProvider(TimeZones.NewYork);
timeProvider.SetCurrentTime(startDate);
var dataQueueHandler = new FuncDataQueueHandler(fdqh =>
{
var time = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork);
var tick = new Tick(time, Symbols.SPY, 1.3m, 1.2m, 1.3m)
{
TickType = TickType.Trade
};
var tick2 = new Tick(time, Symbols.AAPL, 1.3m, 1.2m, 1.3m)
{
TickType = TickType.Trade
};
return(new[] { tick, tick2 });
}, timeProvider);
CreateDataFeed(dataQueueHandler);
var dataManager = new DataManagerStub(algorithm, _feed);
algorithm.SubscriptionManager.SetDataManager(dataManager);
var symbols = new List <Symbol>
{
algorithm.AddData <Quandl>("CBOE/VXV", Resolution.Daily).Symbol,
algorithm.AddData <QuandlVix>("CBOE/VIX", Resolution.Daily).Symbol,
algorithm.AddEquity("SPY", Resolution.Daily).Symbol,
algorithm.AddEquity("AAPL", Resolution.Daily).Symbol
};
algorithm.PostInitialize();
var cancellationTokenSource = new CancellationTokenSource();
var dataPointsEmitted = 0;
var slicesEmitted = 0;
RunLiveDataFeed(algorithm, startDate, symbols, timeProvider, dataManager);
Thread.Sleep(5000); // Give remote sources a handicap, so the data is available in time
// create a timer to advance time much faster than realtime and to simulate live Quandl data file updates
var timerInterval = TimeSpan.FromMilliseconds(100);
var timer = Ref.Create <Timer>(null);
timer.Value = new Timer(state =>
{
// stop the timer to prevent reentrancy
timer.Value.Change(Timeout.Infinite, Timeout.Infinite);
var currentTime = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork);
if (currentTime.Date > endDate.Date)
{
_feed.Exit();
cancellationTokenSource.Cancel();
return;
}
timeProvider.Advance(TimeSpan.FromHours(3));
// restart the timer
timer.Value.Change(timerInterval, timerInterval);
}, null, TimeSpan.FromSeconds(2), timerInterval);
try
{
foreach (var timeSlice in _synchronizer.StreamData(cancellationTokenSource.Token))
{
if (timeSlice.Slice.HasData)
{
slicesEmitted++;
dataPointsEmitted += timeSlice.Slice.Values.Count;
Assert.IsTrue(timeSlice.Slice.Values.Any(x => x.Symbol == symbols[0]), $"Slice doesn't contain {symbols[0]}");
Assert.IsTrue(timeSlice.Slice.Values.Any(x => x.Symbol == symbols[1]), $"Slice doesn't contain {symbols[1]}");
Assert.IsTrue(timeSlice.Slice.Values.Any(x => x.Symbol == symbols[2]), $"Slice doesn't contain {symbols[2]}");
Assert.IsTrue(timeSlice.Slice.Values.Any(x => x.Symbol == symbols[3]), $"Slice doesn't contain {symbols[3]}");
}
}
}
catch (Exception exception)
{
Log.Trace($"Error: {exception}");
}
timer.Value.Dispose();
dataManager.RemoveAllSubscriptions();
dataQueueHandler.DisposeSafely();
Assert.AreEqual(14, slicesEmitted);
Assert.AreEqual(14 * symbols.Count, dataPointsEmitted);
}
public void FillsForwardHourlyOnMinutesBeginningOfDay()
{
var dataResolution = Time.OneHour;
var reference = new DateTime(2015, 6, 25);
var data = new BaseData[]
{
// thurs 6/25
new TradeBar {
Value = 0, Time = reference, Period = dataResolution, Volume = 100
},
// fri 6/26
new TradeBar {
Value = 1, Time = reference.Date.AddHours(9), Period = dataResolution, Volume = 200
},
}.ToList();
var enumerator = data.GetEnumerator();
var exchange = new EquityExchange();
bool isExtendedMarketHours = false;
var ffResolution = TimeSpan.FromMinutes(15);
var fillForwardEnumerator = new FillForwardEnumerator(enumerator, exchange, Ref.Create(ffResolution), isExtendedMarketHours, data.Last().EndTime, dataResolution, exchange.TimeZone, data.First().EndTime);
// 12:00
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddHours(1), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(100, ((TradeBar)fillForwardEnumerator.Current).Volume);
// 9:45 (ff)
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddHours(9.75), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(0, ((TradeBar)fillForwardEnumerator.Current).Volume);
// 10:00
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddHours(10), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(1, fillForwardEnumerator.Current.Value);
Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(200, ((TradeBar)fillForwardEnumerator.Current).Volume);
Assert.IsFalse(fillForwardEnumerator.MoveNext());
fillForwardEnumerator.Dispose();
}
/// <summary>
/// Initializes the data feed for the specified job and algorithm
/// </summary>
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider)
{
if (!(job is LiveNodePacket))
{
throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket.");
}
_cancellationTokenSource = new CancellationTokenSource();
_algorithm = algorithm;
_job = (LiveNodePacket)job;
_resultHandler = resultHandler;
_timeProvider = GetTimeProvider();
_dataQueueHandler = GetDataQueueHandler();
_frontierTimeProvider = new ManualTimeProvider(_timeProvider.GetUtcNow());
_customExchange = new BaseDataExchange("CustomDataExchange")
{
SleepInterval = 10
};
// sleep is controlled on this exchange via the GetNextTicksEnumerator
_exchange = new BaseDataExchange("DataQueueExchange")
{
SleepInterval = 0
};
_exchange.AddEnumerator(DataQueueHandlerSymbol, GetNextTicksEnumerator());
_subscriptions = new SubscriptionCollection();
_bridge = new BusyBlockingCollection <TimeSlice>();
_universeSelection = new UniverseSelection(this, algorithm, job.Controls);
// run the exchanges
Task.Run(() => _exchange.Start(_cancellationTokenSource.Token));
Task.Run(() => _customExchange.Start(_cancellationTokenSource.Token));
// this value will be modified via calls to AddSubscription/RemoveSubscription
var ffres = Time.OneMinute;
_fillForwardResolution = Ref.Create(() => ffres, v => ffres = v);
// wire ourselves up to receive notifications when universes are added/removed
var start = _timeProvider.GetUtcNow();
algorithm.UniverseManager.CollectionChanged += (sender, args) =>
{
switch (args.Action)
{
case NotifyCollectionChangedAction.Add:
foreach (var universe in args.NewItems.OfType <Universe>())
{
if (!_subscriptions.Contains(universe.Configuration))
{
_subscriptions.TryAdd(CreateUniverseSubscription(universe, start, Time.EndOfTime));
}
// Not sure if this is needed but left here because of this:
// https://github.com/VigiothCapital.QuantTrader.commit/029d70bde6ca83a1eb0c667bb5cc4444bea05678
UpdateFillForwardResolution();
}
break;
case NotifyCollectionChangedAction.Remove:
foreach (var universe in args.OldItems.OfType <Universe>())
{
RemoveSubscription(universe.Configuration);
}
break;
default:
throw new NotImplementedException("The specified action is not implemented: " + args.Action);
}
};
}
static int CacheCommand(IEnumerable <string> args)
{
var help = Ref.Create(false);
var verbose = Ref.Create(false);
var options = new OptionSet(CreateStrictOptionSetArgumentParser())
{
Options.Help(help),
Options.Verbose(verbose),
Options.Debug,
};
var tail = options.Parse(args);
if (tail.FirstOrNone() is (SomeT, var arg))
{
throw new Exception("Invalid argument: " + arg);
}
if (verbose)
{
Trace.Listeners.Add(new TextWriterTraceListener(Console.Error));
}
if (help)
{
Help(options);
return(0);
}
var baseDir = new DirectoryInfo(GetCacheDirPath(GetSearchPaths(new DirectoryInfo(Environment.CurrentDirectory))));
var binDir = new DirectoryInfo(Path.Join(baseDir.FullName, "bin"));
if (!binDir.Exists)
{
return(0);
}
foreach (var dir in
from dir in binDir.EnumerateDirectories("*")
where 0 == (dir.Attributes & (FileAttributes.Hidden | FileAttributes.System)) &&
dir.Name.Length == 40 &&
dir.Name[0] != '.' &&
Regex.IsMatch(dir.Name, @"^[a-zA-Z0-9]{40}$")
select dir)
{
var runLogPath = Path.Join(dir.FullName, "runs.log");
var log
= File.Exists(runLogPath)
? File.ReadLines(runLogPath)
: Enumerable.Empty <string>();
var(count, lastRunTime) =
ParseRunLog(log, (lrt, _) => lrt)
.Aggregate(0, (a, _) => a + 1,
DateTimeOffset.MinValue, (a, lrt) => lrt > a ? lrt : a,
ValueTuple.Create);
var output = count > 0
? $"{dir.Name} (runs = {count}; last = {lastRunTime:yyyy'-'MM'-'ddTHH':'mm':'sszzz})"
: dir.Name;
Console.WriteLine(output);
}
return(0);
}
public void FillsForwardFromPreMarket()
{
var dataResolution = Time.OneMinute;
var reference = new DateTime(2015, 6, 25, 9, 28, 0);
var data = new []
{
new TradeBar
{
Time = reference,
Value = 0,
Period = dataResolution,
Volume = 100
},
new TradeBar
{
Time = reference.AddMinutes(4),
Value = 1,
Period = dataResolution,
Volume = 200
}
}.ToList();
var enumerator = data.GetEnumerator();
var exchange = new EquityExchange();
var isExtendedMarketHours = false;
var fillForwardEnumerator = new FillForwardEnumerator(enumerator, exchange, Ref.Create(TimeSpan.FromMinutes(1)), isExtendedMarketHours, data.Last().EndTime, dataResolution, exchange.TimeZone, data.First().EndTime);
// 9:29
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddMinutes(1), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
Assert.AreEqual(100, ((TradeBar)fillForwardEnumerator.Current).Volume);
// 9:31 (ff)
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddMinutes(3), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
Assert.AreEqual(0, ((TradeBar)fillForwardEnumerator.Current).Volume);
// 9:32 (ff)
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddMinutes(4), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
Assert.AreEqual(0, ((TradeBar)fillForwardEnumerator.Current).Volume);
// 9:33
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddMinutes(5), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(1, fillForwardEnumerator.Current.Value);
Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(dataResolution, fillForwardEnumerator.Current.EndTime - fillForwardEnumerator.Current.Time);
Assert.AreEqual(200, ((TradeBar)fillForwardEnumerator.Current).Volume);
Assert.IsFalse(fillForwardEnumerator.MoveNext());
fillForwardEnumerator.Dispose();
}
public void OandaFillsForwardDailyForexOnWeekends()
{
var dailyBarsEmitted = 0;
var fillForwardBars = new List <BaseData>();
// 3 QuoteBars as they would be read from the EURUSD oanda daily file by QuoteBar.Reader()
// The conversion from dataTimeZone to exchangeTimeZone has been done by hand
// dataTimeZone == UTC
/*
* 20120719 00:00,1.22769,1.2324,1.22286,1.22759,0,1.22781,1.23253,1.22298,1.22771,0
* 20120720 00:00,1.22757,1.22823,1.21435,1.21542,0,1.22769,1.22835,1.21449,1.21592,0
* 20120722 00:00,1.21542,1.21542,1.21037,1.21271,0,1.21592,1.21592,1.21087,1.21283,0
* 20120723 00:00,1.21273,1.21444,1.20669,1.21238,0,1.21285,1.21454,1.20685,1.21249,0
*/
var data = new BaseData[]
{
// fri 7/20
new QuoteBar {
Value = 0, Time = new DateTime(2012, 7, 19, 20, 0, 0), Period = Time.OneDay
},
// sunday 7/22
new QuoteBar {
Value = 1, Time = new DateTime(2012, 7, 21, 20, 0, 0), Period = Time.OneDay
},
// monday 7/23
new QuoteBar {
Value = 2, Time = new DateTime(2012, 7, 22, 20, 0, 0), Period = Time.OneDay
},
}.ToList();
var enumerator = data.GetEnumerator();
var market = Market.Oanda;
var symbol = Symbol.Create("EURUSD", SecurityType.Forex, market);
var marketHours = MarketHoursDatabase.FromDataFolder();
var exchange = new ForexExchange(marketHours.GetExchangeHours(market, symbol, SecurityType.Forex));
var fillForwardEnumerator = new FillForwardEnumerator(enumerator, exchange, Ref.Create(TimeSpan.FromDays(1)), false, data.Last().EndTime, Time.OneDay, TimeZones.Utc, data.First().EndTime);
while (fillForwardEnumerator.MoveNext())
{
fillForwardBars.Add(fillForwardEnumerator.Current);
Console.WriteLine(fillForwardEnumerator.Current.Time.DayOfWeek + " " + fillForwardEnumerator.Current.Time + " - " + fillForwardEnumerator.Current.EndTime.DayOfWeek + " " + fillForwardEnumerator.Current.EndTime);
dailyBarsEmitted++;
}
Assert.AreEqual(3, dailyBarsEmitted);
Assert.AreEqual(new DateTime(2012, 7, 19, 20, 0, 0), fillForwardBars[0].Time);
Assert.AreEqual(new DateTime(2012, 7, 21, 20, 0, 0), fillForwardBars[1].Time);
Assert.AreEqual(new DateTime(2012, 7, 22, 20, 0, 0), fillForwardBars[2].Time);
fillForwardEnumerator.Dispose();
}
public void FillsForwardMissingDaysOnFillForwardResolutionOfAnHour()
{
var dataResolution = Time.OneDay;
var reference = new DateTime(2015, 6, 23);
var data = new BaseData[]
{
// tues 6/23
new TradeBar {
Value = 0, Time = reference, Period = dataResolution, Volume = 100
},
// wed 7/1
new TradeBar {
Value = 1, Time = reference.AddDays(8), Period = dataResolution, Volume = 200
},
}.ToList();
var enumerator = data.GetEnumerator();
var exchange = new EquityExchange();
bool isExtendedMarketHours = false;
var ffResolution = TimeSpan.FromHours(1);
var fillForwardEnumerator = new FillForwardEnumerator(enumerator, exchange, Ref.Create(ffResolution), isExtendedMarketHours, data.Last().EndTime, dataResolution, exchange.TimeZone, data.First().EndTime);
int dailyBars = 0;
int hourlyBars = 0;
while (fillForwardEnumerator.MoveNext())
{
Console.WriteLine(fillForwardEnumerator.Current.EndTime);
if (fillForwardEnumerator.Current.Time.TimeOfDay == TimeSpan.Zero)
{
dailyBars++;
}
else
{
hourlyBars++;
Assert.AreEqual(0, ((TradeBar)fillForwardEnumerator.Current).Volume);
}
}
// we expect 7 daily bars here, beginning tues, wed, thurs, fri, mon, tues, wed
Assert.AreEqual(7, dailyBars);
// we expect 6 days worth of ff hourly bars at 7 bars a day
Assert.AreEqual(42, hourlyBars);
fillForwardEnumerator.Dispose();
}
/// <summary>
/// Creates an enumerator to read the specified request.
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider)
{
var config = request.Configuration;
// frontier value used to prevent emitting duplicate time stamps between refreshed enumerators
// also provides some immediate fast-forward to handle spooling through remote files quickly
var frontier = Ref.Create(request.StartTimeLocal);
var lastSourceRefreshTime = DateTime.MinValue;
var sourceFactory = (BaseData)ObjectActivator.GetActivator(config.Type).Invoke(new object[] { config.Type });
// this is refreshing the enumerator stack for each new source
var refresher = new RefreshEnumerator <BaseData>(() =>
{
// rate limit the refresh of this enumerator stack
var utcNow = _timeProvider.GetUtcNow();
var minimumTimeBetweenCalls = GetMinimumTimeBetweenCalls(config.Increment);
if (utcNow - lastSourceRefreshTime < minimumTimeBetweenCalls)
{
return(Enumerable.Empty <BaseData>().GetEnumerator());
}
lastSourceRefreshTime = utcNow;
var localDate = utcNow.ConvertFromUtc(config.ExchangeTimeZone).Date;
var source = sourceFactory.GetSource(config, localDate, true);
// fetch the new source and enumerate the data source reader
var enumerator = EnumerateDataSourceReader(config, dataProvider, frontier, source, localDate);
if (SourceRequiresFastForward(source))
{
// The FastForwardEnumerator implements these two features:
// (1) make sure we never emit past data
// (2) data filtering based on a maximum data age
// For custom data we don't want feature (2) because we would reject data points emitted later
// (e.g. Quandl daily data after a weekend), so we disable it using a huge maximum data age.
// apply fast forward logic for file transport mediums
var maximumDataAge = GetMaximumDataAge(Time.MaxTimeSpan);
enumerator = new FastForwardEnumerator(enumerator, _timeProvider, config.ExchangeTimeZone, maximumDataAge);
}
else
{
// rate limit calls to this enumerator stack
enumerator = new RateLimitEnumerator <BaseData>(enumerator, _timeProvider, minimumTimeBetweenCalls);
}
if (source.Format == FileFormat.Collection)
{
// unroll collections into individual data points after fast forward/rate limiting applied
enumerator = enumerator.SelectMany(data =>
{
var collection = data as BaseDataCollection;
return(collection?.Data.GetEnumerator() ?? new List <BaseData> {
data
}.GetEnumerator());
});
}
return(enumerator);
});
return(refresher);
}
public void CoarseUniverseRotatesActiveSecurity()
{
var startDate = new DateTime(2014, 3, 24);
var endDate = new DateTime(2014, 3, 29);
var timeProvider = new ManualTimeProvider(TimeZones.NewYork);
timeProvider.SetCurrentTime(startDate);
var coarseTimes = new List <DateTime>
{
new DateTime(2014, 3, 25, 5, 0, 0, 0),
new DateTime(2014, 3, 26, 5, 0, 0, 0),
new DateTime(2014, 3, 27, 5, 0, 0, 0),
new DateTime(2014, 3, 28, 5, 0, 0, 0),
new DateTime(2014, 3, 29, 5, 0, 0, 0)
}.ToHashSet();
var coarseSymbols = new List <Symbol> {
Symbols.SPY, Symbols.AAPL, Symbols.MSFT
};
var emitted = new AutoResetEvent(false);
var dataQueueHandler = new FuncDataQueueHandler(fdqh => Enumerable.Empty <BaseData>(), timeProvider);
var feed = new TestableLiveTradingDataFeed(dataQueueHandler);
var algorithm = new AlgorithmStub(feed);
algorithm.SetLiveMode(true);
var mock = new Mock <ITransactionHandler>();
mock.Setup(m => m.GetOpenOrders(It.IsAny <Func <Order, bool> >())).Returns(new List <Order>());
algorithm.Transactions.SetOrderProcessor(mock.Object);
var synchronizer = new TestableLiveSynchronizer(timeProvider);
synchronizer.Initialize(algorithm, algorithm.DataManager);
var mapFileProvider = new LocalDiskMapFileProvider();
feed.Initialize(algorithm, new LiveNodePacket(), new BacktestingResultHandler(),
mapFileProvider, new LocalDiskFactorFileProvider(mapFileProvider), new DefaultDataProvider(), algorithm.DataManager, synchronizer, new DataChannelProvider());
var symbolIndex = 0;
var coarseUniverseSelectionCount = 0;
algorithm.AddUniverse(
coarse =>
{
Log.Trace($"Emitted at {algorithm.Time}. Coarse {coarse.First().Time} to {coarse.First().EndTime}");
Interlocked.Increment(ref coarseUniverseSelectionCount);
emitted.Set();
// rotate single symbol in universe
if (symbolIndex == coarseSymbols.Count)
{
symbolIndex = 0;
}
return(new[] { coarseSymbols[symbolIndex++] });
});
algorithm.PostInitialize();
var cancellationTokenSource = new CancellationTokenSource();
Exception exceptionThrown = null;
// create a timer to advance time much faster than realtime
var timerInterval = TimeSpan.FromMilliseconds(5);
var timer = Ref.Create <Timer>(null);
timer.Value = new Timer(state =>
{
try
{
var currentTime = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork);
if (currentTime.Date > endDate.Date)
{
feed.Exit();
cancellationTokenSource.Cancel();
return;
}
timeProvider.Advance(TimeSpan.FromHours(1));
var time = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork);
algorithm.SetDateTime(timeProvider.GetUtcNow());
if (coarseTimes.Contains(time))
{
// lets wait for coarse to emit
if (!emitted.WaitOne(TimeSpan.FromMilliseconds(15000)))
{
throw new TimeoutException($"Timeout waiting for coarse to emit at {time}");
}
}
var activeSecuritiesCount = algorithm.ActiveSecurities.Count;
Assert.That(activeSecuritiesCount <= 1);
// restart the timer
timer.Value.Change(timerInterval, Timeout.InfiniteTimeSpan);
}
catch (Exception exception)
{
Log.Error(exception);
exceptionThrown = exception;
feed.Exit();
cancellationTokenSource.Cancel();
}
}, null, timerInterval, Timeout.InfiniteTimeSpan);
foreach (var _ in synchronizer.StreamData(cancellationTokenSource.Token))
{
}
timer.Value.DisposeSafely();
algorithm.DataManager.RemoveAllSubscriptions();
dataQueueHandler.DisposeSafely();
synchronizer.DisposeSafely();
emitted.DisposeSafely();
if (exceptionThrown != null)
{
throw new Exception("Exception in timer: ", exceptionThrown);
}
Assert.AreEqual(coarseTimes.Count, coarseUniverseSelectionCount, message: "coarseUniverseSelectionCount");
}
public void HandlesDaylightSavingTimeChange()
{
var dailyBarsEmitted = 0;
var fillForwardBars = new List <BaseData>();
// 3 QuoteBars as they would be read from the EURUSD oanda daily file by QuoteBar.Reader()
// The conversion from dataTimeZone to exchangeTimeZone has been done by hand
// dataTimeZone == UTC
/*
* 20180311 00:00,1.2308,1.2308,1.2308,1.2308,0,1.23096,1.23096,1.23096,1.23096,0
* 20180312 00:00,1.23082,1.23449,1.22898,1.23382,0,1.23097,1.23463,1.22911,1.23396,0
*/
var data = new BaseData[]
{
// Sunday 3/11
new QuoteBar {
Value = 0, Time = new DateTime(2018, 3, 10, 19, 0, 0), Period = Time.OneDay
},
// Monday 3/12
new QuoteBar {
Value = 1, Time = new DateTime(2018, 3, 11, 20, 0, 0), Period = Time.OneDay
},
}.ToList();
var enumerator = data.GetEnumerator();
var market = Market.Oanda;
var symbol = Symbol.Create("EURUSD", SecurityType.Forex, market);
var marketHours = MarketHoursDatabase.FromDataFolder();
var exchange = new ForexExchange(marketHours.GetExchangeHours(market, symbol, SecurityType.Forex));
var fillForwardEnumerator = new FillForwardEnumerator(enumerator, exchange, Ref.Create(TimeSpan.FromDays(1)), false, data.Last().EndTime, Time.OneDay, TimeZones.Utc, data.First().EndTime);
while (fillForwardEnumerator.MoveNext())
{
fillForwardBars.Add(fillForwardEnumerator.Current);
Console.WriteLine(fillForwardEnumerator.Current.Time.DayOfWeek + " " + fillForwardEnumerator.Current.Time + " - " + fillForwardEnumerator.Current.EndTime.DayOfWeek + " " + fillForwardEnumerator.Current.EndTime);
dailyBarsEmitted++;
}
Assert.AreEqual(2, dailyBarsEmitted);
Assert.AreEqual(new DateTime(2018, 3, 10, 19, 0, 0), fillForwardBars[0].Time);
Assert.AreEqual(new DateTime(2018, 3, 11, 20, 0, 0), fillForwardBars[1].Time);
fillForwardEnumerator.Dispose();
}
public void FillsForwardDailyMissingDays()
{
var dataResolution = Time.OneDay;
var reference = new DateTime(2015, 6, 25);
var data = new BaseData[]
{
// thurs 6/25
new TradeBar {
Value = 0, Time = reference, Period = Time.OneDay, Volume = 100
},
// fri 6/26
new TradeBar {
Value = 1, Time = reference.AddDays(5), Period = Time.OneDay, Volume = 200
},
}.ToList();
var enumerator = data.GetEnumerator();
var exchange = new EquityExchange();
bool isExtendedMarketHours = false;
var fillForwardEnumerator = new FillForwardEnumerator(enumerator, exchange, Ref.Create(TimeSpan.FromDays(1)), isExtendedMarketHours, data.Last().EndTime.AddDays(1), dataResolution, exchange.TimeZone, data.First().EndTime);
// 6/25
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddDays(1), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
Assert.IsTrue(((TradeBar)fillForwardEnumerator.Current).Period == dataResolution);
Assert.AreEqual(100, ((TradeBar)fillForwardEnumerator.Current).Volume);
// 6/26
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddDays(2), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
Assert.IsTrue(((TradeBar)fillForwardEnumerator.Current).Period == dataResolution);
Assert.AreEqual(0, ((TradeBar)fillForwardEnumerator.Current).Volume);
// 6/29
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddDays(5), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
Assert.IsTrue(((TradeBar)fillForwardEnumerator.Current).Period == dataResolution);
Assert.AreEqual(0, ((TradeBar)fillForwardEnumerator.Current).Volume);
// 6/30
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddDays(6), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(1, fillForwardEnumerator.Current.Value);
Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
Assert.IsTrue(((TradeBar)fillForwardEnumerator.Current).Period == dataResolution);
Assert.AreEqual(200, ((TradeBar)fillForwardEnumerator.Current).Volume);
// 7/1
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.AreEqual(reference.AddDays(7), fillForwardEnumerator.Current.EndTime);
Assert.AreEqual(1, fillForwardEnumerator.Current.Value);
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
Assert.IsTrue(((TradeBar)fillForwardEnumerator.Current).Period == dataResolution);
Assert.AreEqual(0, ((TradeBar)fillForwardEnumerator.Current).Volume);
Assert.IsFalse(fillForwardEnumerator.MoveNext());
fillForwardEnumerator.Dispose();
}
/// <summary>
/// Initializes the data feed for the specified job and algorithm
/// </summary>
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, IDataFileProvider dataFileProvider)
{
if (!(job is LiveNodePacket))
{
throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket.");
}
_cancellationTokenSource = new CancellationTokenSource();
_algorithm = algorithm;
_job = (LiveNodePacket)job;
_resultHandler = resultHandler;
_timeProvider = GetTimeProvider();
_dataQueueHandler = GetDataQueueHandler();
_dataFileProvider = dataFileProvider;
_frontierTimeProvider = new ManualTimeProvider(_timeProvider.GetUtcNow());
_customExchange = new BaseDataExchange("CustomDataExchange")
{
SleepInterval = 10
};
// sleep is controlled on this exchange via the GetNextTicksEnumerator
_exchange = new BaseDataExchange("DataQueueExchange")
{
SleepInterval = 0
};
_exchange.AddEnumerator(DataQueueHandlerSymbol, GetNextTicksEnumerator());
_subscriptions = new SubscriptionCollection();
_bridge = new BusyBlockingCollection <TimeSlice>();
_universeSelection = new UniverseSelection(this, algorithm, job.Controls);
// run the exchanges
Task.Run(() => _exchange.Start(_cancellationTokenSource.Token));
Task.Run(() => _customExchange.Start(_cancellationTokenSource.Token));
// this value will be modified via calls to AddSubscription/RemoveSubscription
var ffres = Time.OneMinute;
_fillForwardResolution = Ref.Create(() => ffres, v => ffres = v);
// wire ourselves up to receive notifications when universes are added/removed
var start = _timeProvider.GetUtcNow();
algorithm.UniverseManager.CollectionChanged += (sender, args) =>
{
switch (args.Action)
{
case NotifyCollectionChangedAction.Add:
foreach (var universe in args.NewItems.OfType <Universe>())
{
var config = universe.Configuration;
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var exchangeHours = marketHoursDatabase.GetExchangeHours(config);
Security security;
if (!_algorithm.Securities.TryGetValue(config.Symbol, out security))
{
// create a canonical security object
security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency));
}
AddSubscription(new SubscriptionRequest(true, universe, security, config, start, Time.EndOfTime));
// Not sure if this is needed but left here because of this:
// https://github.com/QuantConnect/Lean/commit/029d70bde6ca83a1eb0c667bb5cc4444bea05678
UpdateFillForwardResolution();
}
break;
case NotifyCollectionChangedAction.Remove:
foreach (var universe in args.OldItems.OfType <Universe>())
{
RemoveSubscription(universe.Configuration);
}
break;
default:
throw new NotImplementedException("The specified action is not implemented: " + args.Action);
}
};
}
public void HandlesDaylightSavingTimeChange_InifinteLoop()
{
var dailyBarsEmitted = 0;
var fillForwardBars = new List <BaseData>();
var data = new BaseData[]
{
new QuoteBar {
Value = 0, Time = new DateTime(2019, 10, 4, 10, 0, 0), Period = Time.OneDay
},
new QuoteBar {
Value = 1, Time = new DateTime(2019, 10, 8, 11, 0, 0), Period = Time.OneDay
}
}.ToList();
var enumerator = data.GetEnumerator();
var algo = new AlgorithmStub();
var market = Market.Oanda;
var security = algo.AddCfd("AU200AUD", Resolution.Daily, market);
var fillForwardEnumerator = new FillForwardEnumerator(enumerator, security.Exchange, Ref.Create(TimeSpan.FromDays(1)), false, data.Last().EndTime, Time.OneDay, TimeZones.Utc, data.First().EndTime);
while (fillForwardEnumerator.MoveNext())
{
fillForwardBars.Add(fillForwardEnumerator.Current);
Console.WriteLine(fillForwardEnumerator.Current.Time.DayOfWeek + " " + fillForwardEnumerator.Current.Time + " - " + fillForwardEnumerator.Current.EndTime.DayOfWeek + " " + fillForwardEnumerator.Current.EndTime + " " + fillForwardEnumerator.Current.IsFillForward);
dailyBarsEmitted++;
}
Assert.AreEqual(4, dailyBarsEmitted);
Assert.AreEqual(new DateTime(2019, 10, 4, 10, 0, 0), fillForwardBars[0].Time);
Assert.AreEqual(new DateTime(2019, 10, 6, 11, 0, 0), fillForwardBars[1].Time);
Assert.AreEqual(new DateTime(2019, 10, 7, 11, 0, 0), fillForwardBars[2].Time);
Assert.AreEqual(new DateTime(2019, 10, 8, 11, 0, 0), fillForwardBars[3].Time);
fillForwardEnumerator.Dispose();
}