public void TestGetFivePmYesterday()
{
ITimeProvider timeProvider = new ManualTimeProvider();
TimeZoneInfo easternTz = TimeZoneInfo.FindSystemTimeZoneById("Eastern Standard Time"); // used to run check on the time
DateTimeOffset dt = DateTimeUtils.GetFivePmYesterday(timeProvider);
Assert.IsNotNull(dt);
// Test for edge case: between 12am and 5am local time
timeProvider.SetCurrentTime(DateTime.Now.Year, DateTime.Now.Month, DateTime.Now.Day, 1, TimeSpan.FromHours(8), 0, 0);
dt = DateTimeUtils.GetFivePmYesterday(timeProvider);
Assert.IsTrue(dt.Date == timeProvider.Today().AddDays(-1).Date, "Testing GetFivePmYesterday with current time set to 1am: expect 5pm date to be on the previous HKT date");
if (easternTz.IsDaylightSavingTime(timeProvider.Now()))
{
Assert.IsTrue(dt.Hour == 5, "Currently EDT. Expect 5pm EDT = 5am HKT");
}
else
{
Assert.IsTrue(dt.Hour == 6, "Currently EDT. Expect 5pm EDT = 5am HKT");
}
// Test for between 5am to 11:59pm local time
timeProvider.SetCurrentTime(DateTime.Now.Year, DateTime.Now.Month, DateTime.Now.Day, 13, TimeSpan.FromHours(8), 0, 0);
dt = DateTimeUtils.GetFivePmYesterday(timeProvider);
Assert.IsTrue(dt.Date == timeProvider.Today().Date, "Testing GetFivePmYesterday with current time set to 1pm: expect 5pm date to be on the current HKT date");
if (easternTz.IsDaylightSavingTime(timeProvider.Now()))
{
Assert.IsTrue(dt.Hour == 5, "Currently EDT. Expect 5pm EDT = 5am HKT");
}
else
{
Assert.IsTrue(dt.Hour == 6, "Currently EDT. Expect 5pm EDT = 5am HKT");
}
// Test in winter (Dec 15th)
timeProvider.SetCurrentTime(DateTime.Now.Year, 12, 15, 13, TimeSpan.FromHours(8), 0, 0);
dt = DateTimeUtils.GetFivePmYesterday(timeProvider);
Assert.IsTrue(dt.Hour == 6, "Currently EDT. Expect 5pm EDT = 5am HKT");
// Test in summer (June 15th)
timeProvider.SetCurrentTime(DateTime.Now.Year, 6, 15, 13, TimeSpan.FromHours(8), 0, 0);
dt = DateTimeUtils.GetFivePmYesterday(timeProvider);
Assert.IsTrue(dt.Hour == 5, "Currently EDT. Expect 5pm EDT = 5am HKT");
}
public void HandlesDaylightSavingTimeChange()
{
var reference = new DateTime(2018, 3, 10);
var period = Time.OneDay;
var underlying = new List <TradeBar>
{
new TradeBar(reference, Symbols.SPY, 10, 20, 5, 15, 123456, period),
// Daylight Saving Time change -> add 1 hour
new TradeBar(reference.AddDays(1).AddHours(1), Symbols.SPY, 100, 200, 50, 150, 1234560, period)
};
var timeProvider = new ManualTimeProvider(TimeZones.NewYork);
timeProvider.SetCurrentTime(reference);
var exchange = new SecurityExchange(SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork));
var fillForward = new LiveFillForwardEnumerator(
timeProvider,
underlying.GetEnumerator(),
exchange,
Ref.Create(Time.OneDay),
false,
Time.EndOfTime,
Time.OneDay,
exchange.TimeZone,
Time.BeginningOfTime);
// first point is always emitted
Assert.IsTrue(fillForward.MoveNext());
Assert.IsFalse(fillForward.Current.IsFillForward);
Assert.AreEqual(underlying[0], fillForward.Current);
//Assert.AreEqual(underlying[0].EndTime, fillForward.Current.EndTime);
Assert.AreEqual(123456, ((TradeBar)fillForward.Current).Volume);
// Daylight Saving Time change -> add 1 hour
timeProvider.SetCurrentTime(reference.AddDays(1).AddHours(1));
// second data point emitted
Assert.IsTrue(fillForward.MoveNext());
Assert.IsFalse(fillForward.Current.IsFillForward);
Assert.AreEqual(underlying[1], fillForward.Current);
//Assert.AreEqual(underlying[1].EndTime, fillForward.Current.EndTime);
Assert.AreEqual(1234560, ((TradeBar)fillForward.Current).Volume);
Assert.IsTrue(fillForward.MoveNext());
Assert.IsTrue(fillForward.Current.IsFillForward);
Assert.AreEqual(underlying[1].EndTime, fillForward.Current.Time);
Assert.AreEqual(underlying[1].Value, fillForward.Current.Value);
Assert.AreEqual(0, ((TradeBar)fillForward.Current).Volume);
fillForward.Dispose();
}
public void AggregatesHourlyTicksIntoHourlyBars()
{
var timeProvider = new ManualTimeProvider(TimeZones.NewYork);
var enumerator = new TradeBarBuilderEnumerator(Time.OneHour, TimeZones.NewYork, timeProvider, false);
// noon new york time
var currentTime = new DateTime(2015, 10, 08, 12, 0, 0);
timeProvider.SetCurrentTime(currentTime);
var price = 0m;
for (var i = 0; i < 5; i++)
{
// advance one hour
currentTime = currentTime.AddHours(1);
timeProvider.SetCurrentTime(currentTime);
// we advanced time, a new bar should be generated
Assert.IsTrue(enumerator.MoveNext());
// the first loop no trade bar was generated yet, end time not reached
if (i > 0)
{
Assert.IsNotNull(enumerator.Current);
var bar = (TradeBar)enumerator.Current;
Assert.AreEqual(currentTime.AddHours(-1), bar.Time);
Assert.AreEqual(currentTime, bar.EndTime);
Assert.AreEqual(price, bar.Open);
Assert.AreEqual(price, bar.High);
Assert.AreEqual(price, bar.Low);
Assert.AreEqual(price, bar.Close);
}
// add a tick, will generate a new bar
price++;
enumerator.ProcessData(new Tick(currentTime, Symbols.SPY, price, price, price));
}
enumerator.Dispose();
}
public void EmitsDailyQuandlFutureDataOverWeekends()
{
RemoteFileSubscriptionStreamReader.SetDownloadProvider(new Api.Api());
var tickers = new[] { "CHRIS/CME_ES1", "CHRIS/CME_ES2" };
var startDate = new DateTime(2018, 4, 1);
var endDate = new DateTime(2018, 4, 20);
// delete temp files
foreach (var ticker in tickers)
{
var fileName = TestableQuandlFuture.GetLocalFileName(ticker, "test");
File.Delete(fileName);
}
var algorithm = new QCAlgorithm();
CreateDataFeed();
var dataManager = new DataManagerStub(algorithm, _feed);
algorithm.SubscriptionManager.SetDataManager(dataManager);
var symbols = tickers.Select(ticker => algorithm.AddData <TestableQuandlFuture>(ticker, Resolution.Daily).Symbol).ToList();
var timeProvider = new ManualTimeProvider(TimeZones.NewYork);
timeProvider.SetCurrentTime(startDate);
var dataPointsEmitted = 0;
RunLiveDataFeed(algorithm, startDate, symbols, timeProvider, dataManager);
var cancellationTokenSource = new CancellationTokenSource();
var lastFileWriteDate = DateTime.MinValue;
// create a timer to advance time much faster than realtime and to simulate live Quandl data file updates
var timerInterval = TimeSpan.FromMilliseconds(20);
var timer = Ref.Create <Timer>(null);
timer.Value = new Timer(state =>
{
try
{
var currentTime = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork);
if (currentTime.Date > endDate.Date)
{
Log.Trace($"Total data points emitted: {dataPointsEmitted.ToStringInvariant()}");
_feed.Exit();
cancellationTokenSource.Cancel();
return;
}
if (currentTime.Date > lastFileWriteDate.Date)
{
foreach (var ticker in tickers)
{
var source = TestableQuandlFuture.GetLocalFileName(ticker, "csv");
// write new local file including only rows up to current date
var outputFileName = TestableQuandlFuture.GetLocalFileName(ticker, "test");
var sb = new StringBuilder();
{
using (var reader = new StreamReader(source))
{
var firstLine = true;
string line;
while ((line = reader.ReadLine()) != null)
{
if (firstLine)
{
sb.AppendLine(line);
firstLine = false;
continue;
}
var csv = line.Split(',');
var time = Parse.DateTimeExact(csv[0], "yyyy-MM-dd");
if (time.Date >= currentTime.Date)
{
break;
}
sb.AppendLine(line);
}
}
}
if (currentTime.Date.DayOfWeek != DayOfWeek.Saturday && currentTime.Date.DayOfWeek != DayOfWeek.Sunday)
{
var fileContent = sb.ToString();
try
{
File.WriteAllText(outputFileName, fileContent);
}
catch (IOException)
{
Log.Error("IOException: will sleep 200ms and retry once more");
// lets sleep 200ms and retry once more, consumer could be reading the file
// this exception happens in travis intermittently, GH issue 3273
Thread.Sleep(200);
File.WriteAllText(outputFileName, fileContent);
}
Log.Trace($"Time:{currentTime} - Ticker:{ticker} - Files written:{++_countFilesWritten}");
}
}
lastFileWriteDate = currentTime;
}
// 30 minutes is the check interval for daily remote files, so we choose a smaller one to advance time
timeProvider.Advance(TimeSpan.FromMinutes(20));
//Log.Trace($"Time advanced to: {timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork)}");
// restart the timer
timer.Value.Change(timerInterval.Milliseconds, Timeout.Infinite);
}
catch (Exception exception)
{
Log.Error(exception);
_feed.Exit();
cancellationTokenSource.Cancel();
}
}, null, timerInterval.Milliseconds, Timeout.Infinite);
try
{
foreach (var timeSlice in _synchronizer.StreamData(cancellationTokenSource.Token))
{
foreach (var dataPoint in timeSlice.Slice.Values)
{
Log.Trace($"Data point emitted at {timeSlice.Slice.Time.ToStringInvariant()}: " +
$"{dataPoint.Symbol.Value} {dataPoint.Value.ToStringInvariant()} " +
$"{dataPoint.EndTime.ToStringInvariant()}"
);
dataPointsEmitted++;
}
}
}
catch (Exception exception)
{
Log.Trace($"Error: {exception}");
}
timer.Value.Dispose();
dataManager.RemoveAllSubscriptions();
Assert.AreEqual(14 * tickers.Length, dataPointsEmitted);
}
public void RemoteDataDoesNotIncreaseNumberOfSlices()
{
Config.Set("quandl-auth-token", "QUANDL-TOKEN");
var startDate = new DateTime(2018, 4, 2);
var endDate = new DateTime(2018, 4, 19);
var algorithm = new QCAlgorithm();
var timeProvider = new ManualTimeProvider(TimeZones.NewYork);
timeProvider.SetCurrentTime(startDate);
var dataQueueHandler = new FuncDataQueueHandler(fdqh =>
{
var time = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork);
var tick = new Tick(time, Symbols.SPY, 1.3m, 1.2m, 1.3m)
{
TickType = TickType.Trade
};
var tick2 = new Tick(time, Symbols.AAPL, 1.3m, 1.2m, 1.3m)
{
TickType = TickType.Trade
};
return(new[] { tick, tick2 });
}, timeProvider);
CreateDataFeed(dataQueueHandler);
var dataManager = new DataManagerStub(algorithm, _feed);
algorithm.SubscriptionManager.SetDataManager(dataManager);
var symbols = new List <Symbol>
{
algorithm.AddData <Quandl>("CBOE/VXV", Resolution.Daily).Symbol,
algorithm.AddData <QuandlVix>("CBOE/VIX", Resolution.Daily).Symbol,
algorithm.AddEquity("SPY", Resolution.Daily).Symbol,
algorithm.AddEquity("AAPL", Resolution.Daily).Symbol
};
algorithm.PostInitialize();
var cancellationTokenSource = new CancellationTokenSource();
var dataPointsEmitted = 0;
var slicesEmitted = 0;
RunLiveDataFeed(algorithm, startDate, symbols, timeProvider, dataManager);
Thread.Sleep(5000); // Give remote sources a handicap, so the data is available in time
// create a timer to advance time much faster than realtime and to simulate live Quandl data file updates
var timerInterval = TimeSpan.FromMilliseconds(100);
var timer = Ref.Create <Timer>(null);
timer.Value = new Timer(state =>
{
// stop the timer to prevent reentrancy
timer.Value.Change(Timeout.Infinite, Timeout.Infinite);
var currentTime = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork);
if (currentTime.Date > endDate.Date)
{
_feed.Exit();
cancellationTokenSource.Cancel();
return;
}
timeProvider.Advance(TimeSpan.FromHours(3));
// restart the timer
timer.Value.Change(timerInterval, timerInterval);
}, null, TimeSpan.FromSeconds(2), timerInterval);
try
{
foreach (var timeSlice in _synchronizer.StreamData(cancellationTokenSource.Token))
{
if (timeSlice.Slice.HasData)
{
slicesEmitted++;
dataPointsEmitted += timeSlice.Slice.Values.Count;
Assert.IsTrue(timeSlice.Slice.Values.Any(x => x.Symbol == symbols[0]), $"Slice doesn't contain {symbols[0]}");
Assert.IsTrue(timeSlice.Slice.Values.Any(x => x.Symbol == symbols[1]), $"Slice doesn't contain {symbols[1]}");
Assert.IsTrue(timeSlice.Slice.Values.Any(x => x.Symbol == symbols[2]), $"Slice doesn't contain {symbols[2]}");
Assert.IsTrue(timeSlice.Slice.Values.Any(x => x.Symbol == symbols[3]), $"Slice doesn't contain {symbols[3]}");
}
}
}
catch (Exception exception)
{
Log.Trace($"Error: {exception}");
}
timer.Value.Dispose();
dataManager.RemoveAllSubscriptions();
dataQueueHandler.DisposeSafely();
Assert.AreEqual(14, slicesEmitted);
Assert.AreEqual(14 * symbols.Count, dataPointsEmitted);
}
public void CoarseUniverseRotatesActiveSecurity()
{
var startDate = new DateTime(2014, 3, 24);
var endDate = new DateTime(2014, 3, 29);
var timeProvider = new ManualTimeProvider(TimeZones.NewYork);
timeProvider.SetCurrentTime(startDate);
var coarseTimes = new List <DateTime>
{
new DateTime(2014, 3, 25, 5, 0, 0, 0),
new DateTime(2014, 3, 26, 5, 0, 0, 0),
new DateTime(2014, 3, 27, 5, 0, 0, 0),
new DateTime(2014, 3, 28, 5, 0, 0, 0),
new DateTime(2014, 3, 29, 5, 0, 0, 0)
}.ToHashSet();
var coarseSymbols = new List <Symbol> {
Symbols.SPY, Symbols.AAPL, Symbols.MSFT
};
var emitted = new AutoResetEvent(false);
var dataQueueHandler = new FuncDataQueueHandler(fdqh => Enumerable.Empty <BaseData>(), timeProvider);
var feed = new TestableLiveTradingDataFeed(dataQueueHandler);
var algorithm = new AlgorithmStub(feed);
algorithm.SetLiveMode(true);
var mock = new Mock <ITransactionHandler>();
mock.Setup(m => m.GetOpenOrders(It.IsAny <Func <Order, bool> >())).Returns(new List <Order>());
algorithm.Transactions.SetOrderProcessor(mock.Object);
var synchronizer = new TestableLiveSynchronizer(timeProvider);
synchronizer.Initialize(algorithm, algorithm.DataManager);
var mapFileProvider = new LocalDiskMapFileProvider();
feed.Initialize(algorithm, new LiveNodePacket(), new BacktestingResultHandler(),
mapFileProvider, new LocalDiskFactorFileProvider(mapFileProvider), new DefaultDataProvider(), algorithm.DataManager, synchronizer, new DataChannelProvider());
var symbolIndex = 0;
var coarseUniverseSelectionCount = 0;
algorithm.AddUniverse(
coarse =>
{
Log.Trace($"Emitted at {algorithm.Time}. Coarse {coarse.First().Time} to {coarse.First().EndTime}");
Interlocked.Increment(ref coarseUniverseSelectionCount);
emitted.Set();
// rotate single symbol in universe
if (symbolIndex == coarseSymbols.Count)
{
symbolIndex = 0;
}
return(new[] { coarseSymbols[symbolIndex++] });
});
algorithm.PostInitialize();
var cancellationTokenSource = new CancellationTokenSource();
Exception exceptionThrown = null;
// create a timer to advance time much faster than realtime
var timerInterval = TimeSpan.FromMilliseconds(5);
var timer = Ref.Create <Timer>(null);
timer.Value = new Timer(state =>
{
try
{
var currentTime = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork);
if (currentTime.Date > endDate.Date)
{
feed.Exit();
cancellationTokenSource.Cancel();
return;
}
timeProvider.Advance(TimeSpan.FromHours(1));
var time = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork);
algorithm.SetDateTime(timeProvider.GetUtcNow());
if (coarseTimes.Contains(time))
{
// lets wait for coarse to emit
if (!emitted.WaitOne(TimeSpan.FromMilliseconds(15000)))
{
throw new TimeoutException($"Timeout waiting for coarse to emit at {time}");
}
}
var activeSecuritiesCount = algorithm.ActiveSecurities.Count;
Assert.That(activeSecuritiesCount <= 1);
// restart the timer
timer.Value.Change(timerInterval, Timeout.InfiniteTimeSpan);
}
catch (Exception exception)
{
Log.Error(exception);
exceptionThrown = exception;
feed.Exit();
cancellationTokenSource.Cancel();
}
}, null, timerInterval, Timeout.InfiniteTimeSpan);
foreach (var _ in synchronizer.StreamData(cancellationTokenSource.Token))
{
}
timer.Value.DisposeSafely();
algorithm.DataManager.RemoveAllSubscriptions();
dataQueueHandler.DisposeSafely();
synchronizer.DisposeSafely();
emitted.DisposeSafely();
if (exceptionThrown != null)
{
throw new Exception("Exception in timer: ", exceptionThrown);
}
Assert.AreEqual(coarseTimes.Count, coarseUniverseSelectionCount, message: "coarseUniverseSelectionCount");
}
public void CoarseUniverseRotatesActiveSecurity()
{
var startDate = new DateTime(2014, 3, 24);
var endDate = new DateTime(2014, 3, 28);
var timeProvider = new ManualTimeProvider(TimeZones.NewYork);
timeProvider.SetCurrentTime(startDate);
var coarseTimes = new List <DateTime>
{
new DateTime(2014, 3, 25),
new DateTime(2014, 3, 25, 23, 0, 0),
new DateTime(2014, 3, 27, 1, 0, 0)
}.ToHashSet();
var coarseSymbols = new List <Symbol> {
Symbols.SPY, Symbols.AAPL, Symbols.MSFT
};
var coarseUsaSymbol = CoarseFundamental.CreateUniverseSymbol(Market.USA, false);
var coarseDataEmittedCount = 0;
var lastTime = DateTime.MinValue;
var dataQueueHandler = new FuncDataQueueHandler(fdqh =>
{
var time = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork);
if (time != lastTime)
{
lastTime = time;
if (coarseTimes.Contains(time))
{
// emit coarse data at selected times
var coarseData = new BaseDataCollection {
Symbol = coarseUsaSymbol
};
foreach (var symbol in coarseSymbols)
{
coarseData.Data.Add(
new CoarseFundamental
{
Symbol = symbol,
Time = time,
Market = Market.USA,
Value = 100
});
}
coarseDataEmittedCount++;
return(new List <BaseData> {
coarseData
});
}
}
return(Enumerable.Empty <BaseData>());
});
var feed = new TestableLiveTradingDataFeed(dataQueueHandler);
var algorithm = new AlgorithmStub(feed);
algorithm.SetLiveMode(true);
var mock = new Mock <ITransactionHandler>();
mock.Setup(m => m.GetOpenOrders(It.IsAny <Func <Order, bool> >())).Returns(new List <Order>());
algorithm.Transactions.SetOrderProcessor(mock.Object);
var synchronizer = new TestableLiveSynchronizer(timeProvider);
synchronizer.Initialize(algorithm, algorithm.DataManager);
var mapFileProvider = new LocalDiskMapFileProvider();
feed.Initialize(algorithm, new LiveNodePacket(), new BacktestingResultHandler(),
mapFileProvider, new LocalDiskFactorFileProvider(mapFileProvider), new DefaultDataProvider(), algorithm.DataManager, synchronizer);
var symbolIndex = 0;
var coarseUniverseSelectionCount = 0;
algorithm.AddUniverse(
coarse =>
{
coarseUniverseSelectionCount++;
// rotate single symbol in universe
if (symbolIndex == coarseSymbols.Count)
{
symbolIndex = 0;
}
return(new[] { coarseSymbols[symbolIndex++] });
});
algorithm.PostInitialize();
var cancellationTokenSource = new CancellationTokenSource();
Exception exceptionThrown = null;
// create a timer to advance time much faster than realtime
var timerInterval = TimeSpan.FromMilliseconds(50);
var timer = Ref.Create <Timer>(null);
timer.Value = new Timer(state =>
{
try
{
// stop the timer to prevent reentrancy
timer.Value.Change(Timeout.Infinite, Timeout.Infinite);
var currentTime = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork);
if (currentTime.Date > endDate.Date)
{
feed.Exit();
cancellationTokenSource.Cancel();
return;
}
timeProvider.Advance(TimeSpan.FromHours(1));
var activeSecuritiesCount = algorithm.ActiveSecurities.Count;
Assert.That(activeSecuritiesCount <= 1);
// restart the timer
timer.Value.Change(timerInterval, timerInterval);
}
catch (Exception exception)
{
Log.Error(exception);
exceptionThrown = exception;
feed.Exit();
cancellationTokenSource.Cancel();
}
}, null, TimeSpan.FromSeconds(1), timerInterval);
foreach (var _ in synchronizer.StreamData(cancellationTokenSource.Token))
{
}
timer.Value.Dispose();
if (exceptionThrown != null)
{
throw new Exception("Exception in timer: ", exceptionThrown);
}
Assert.AreEqual(coarseTimes.Count, coarseDataEmittedCount);
Assert.AreEqual(coarseTimes.Count, coarseUniverseSelectionCount);
}
public void AggregatesTicksIntoSecondBars()
{
var timeProvider = new ManualTimeProvider(TimeZones.NewYork);
var enumerator = new TradeBarBuilderEnumerator(Time.OneSecond, TimeZones.NewYork, timeProvider);
// noon new york time
var currentTime = new DateTime(2015, 10, 08, 12, 0, 0);
timeProvider.SetCurrentTime(currentTime);
// add some ticks
var ticks = new List <Tick>
{
new Tick(currentTime, Symbols.SPY, 199.55m, 199, 200)
{
Quantity = 10
},
new Tick(currentTime, Symbols.SPY, 199.56m, 199.21m, 200.02m)
{
Quantity = 5
},
new Tick(currentTime, Symbols.SPY, 199.53m, 198.77m, 199.75m)
{
Quantity = 20
},
new Tick(currentTime, Symbols.SPY, 198.77m, 199.75m)
{
Quantity = 0
},
new Tick(currentTime, Symbols.SPY, 199.73m, 198.77m, 199.75m)
{
Quantity = 20
},
new Tick(currentTime, Symbols.SPY, 198.77m, 199.75m)
{
Quantity = 0
},
};
foreach (var tick in ticks)
{
enumerator.ProcessData(tick);
}
// even though no data is here, it will still return true
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNull(enumerator.Current);
// advance a second
currentTime = currentTime.AddSeconds(1);
timeProvider.SetCurrentTime(currentTime);
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNotNull(enumerator.Current);
// in the spirit of not duplicating the above code 5 times (OHLCV, we'll assert these ere as well)
var bar = (TradeBar)enumerator.Current;
Assert.AreEqual(currentTime.AddSeconds(-1), bar.Time);
Assert.AreEqual(currentTime, bar.EndTime);
Assert.AreEqual(Symbols.SPY, bar.Symbol);
Assert.AreEqual(ticks.First().LastPrice, bar.Open);
Assert.AreEqual(ticks.Max(x => x.LastPrice), bar.High);
Assert.AreEqual(ticks.Min(x => x.LastPrice), bar.Low);
Assert.AreEqual(ticks.Last().LastPrice, bar.Close);
Assert.AreEqual(ticks.Sum(x => x.Quantity), bar.Volume);
}
public void EmitsDailyQuandlFutureDataOverWeekends()
{
var tickers = new[] { "CHRIS/CME_ES1", "CHRIS/CME_ES2" };
var startDate = new DateTime(2018, 4, 1);
var endDate = new DateTime(2018, 4, 20);
// delete temp files
foreach (var ticker in tickers)
{
var fileName = TestableQuandlFuture.GetLocalFileName(ticker, "test");
File.Delete(fileName);
}
var algorithm = new QCAlgorithm();
var dataManager = new DataManagerStub(algorithm);
algorithm.SubscriptionManager.SetDataManager(dataManager);
var symbols = tickers.Select(ticker => algorithm.AddData <TestableQuandlFuture>(ticker, Resolution.Daily).Symbol).ToList();
algorithm.PostInitialize();
var timeProvider = new ManualTimeProvider(TimeZones.NewYork);
timeProvider.SetCurrentTime(startDate);
var dataPointsEmitted = 0;
var feed = RunLiveDataFeed(algorithm, startDate, symbols, timeProvider, dataManager);
var lastFileWriteDate = DateTime.MinValue;
// create a timer to advance time much faster than realtime and to simulate live Quandl data file updates
var timerInterval = TimeSpan.FromMilliseconds(100);
var timer = Ref.Create <Timer>(null);
timer.Value = new Timer(state =>
{
// stop the timer to prevent reentrancy
timer.Value.Change(Timeout.Infinite, Timeout.Infinite);
var currentTime = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork);
if (currentTime.Date > endDate.Date)
{
Log.Trace($"Total data points emitted: {dataPointsEmitted}");
feed.Exit();
return;
}
if (currentTime.Date > lastFileWriteDate.Date)
{
foreach (var ticker in tickers)
{
var source = TestableQuandlFuture.GetLocalFileName(ticker, "csv");
// write new local file including only rows up to current date
var outputFileName = TestableQuandlFuture.GetLocalFileName(ticker, "test");
var sb = new StringBuilder();
{
using (var reader = new StreamReader(source))
{
var firstLine = true;
string line;
while ((line = reader.ReadLine()) != null)
{
if (firstLine)
{
sb.AppendLine(line);
firstLine = false;
continue;
}
var csv = line.Split(',');
var time = DateTime.ParseExact(csv[0], "yyyy-MM-dd", CultureInfo.InvariantCulture);
if (time.Date >= currentTime.Date)
{
break;
}
sb.AppendLine(line);
}
}
}
if (currentTime.Date.DayOfWeek != DayOfWeek.Saturday && currentTime.Date.DayOfWeek != DayOfWeek.Sunday)
{
File.WriteAllText(outputFileName, sb.ToString());
Log.Trace($"Time:{currentTime} - Ticker:{ticker} - Files written:{++_countFilesWritten}");
}
}
lastFileWriteDate = currentTime;
}
// 30 minutes is the check interval for daily remote files, so we choose a smaller one to advance time
timeProvider.Advance(TimeSpan.FromMinutes(15));
//Log.Trace($"Time advanced to: {timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork)}");
// restart the timer
timer.Value.Change(timerInterval, timerInterval);
}, null, TimeSpan.FromSeconds(2), timerInterval);
try
{
foreach (var timeSlice in feed)
{
foreach (var dataPoint in timeSlice.Slice.Values)
{
Log.Trace($"Data point emitted at {timeSlice.Slice.Time}: {dataPoint.Symbol.Value} {dataPoint.Value} {dataPoint.EndTime}");
dataPointsEmitted++;
}
}
}
catch (Exception exception)
{
Log.Trace($"Error: {exception}");
}
timer.Value.Dispose();
Assert.AreEqual(14 * tickers.Length, dataPointsEmitted);
}
public void FillsForwardOnNulls()
{
var reference = new DateTime(2015, 10, 08);
var period = Time.OneSecond;
var underlying = new List <BaseData>
{
// 0 seconds
new TradeBar(reference, Symbols.SPY, 10, 20, 5, 15, 123456, period),
// 1 seconds
null,
// 3 seconds
new TradeBar(reference.AddSeconds(2), Symbols.SPY, 100, 200, 50, 150, 1234560, period),
null,
null,
null,
null
};
var timeProvider = new ManualTimeProvider(TimeZones.NewYork);
timeProvider.SetCurrentTime(reference);
var exchange = new SecurityExchange(SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork));
var fillForward = new LiveFillForwardEnumerator(timeProvider, underlying.GetEnumerator(), exchange, Ref.Create(Time.OneSecond), false, Time.EndOfTime, Time.OneSecond, exchange.TimeZone, Time.BeginningOfTime);
// first point is always emitted
Assert.IsTrue(fillForward.MoveNext());
Assert.AreEqual(underlying[0], fillForward.Current);
Assert.AreEqual(123456, ((TradeBar)fillForward.Current).Volume);
// stepping again without advancing time does nothing, but we'll still
// return true as per IEnumerator contract
Assert.IsTrue(fillForward.MoveNext());
Assert.IsNull(fillForward.Current);
timeProvider.SetCurrentTime(reference.AddSeconds(1));
// non-null next will fill forward in between
Assert.IsTrue(fillForward.MoveNext());
Assert.AreEqual(underlying[0].EndTime, fillForward.Current.Time);
Assert.AreEqual(underlying[0].Value, fillForward.Current.Value);
Assert.IsTrue(fillForward.Current.IsFillForward);
Assert.AreEqual(0, ((TradeBar)fillForward.Current).Volume);
// even without stepping the time this will advance since non-null data is ready
Assert.IsTrue(fillForward.MoveNext());
Assert.AreEqual(underlying[2], fillForward.Current);
Assert.AreEqual(1234560, ((TradeBar)fillForward.Current).Volume);
// step ahead into null data territory
timeProvider.SetCurrentTime(reference.AddSeconds(4));
Assert.IsTrue(fillForward.MoveNext());
Assert.AreEqual(underlying[2].Value, fillForward.Current.Value);
Assert.AreEqual(timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork), fillForward.Current.EndTime);
Assert.IsTrue(fillForward.Current.IsFillForward);
Assert.AreEqual(0, ((TradeBar)fillForward.Current).Volume);
Assert.IsTrue(fillForward.MoveNext());
Assert.IsNull(fillForward.Current);
timeProvider.SetCurrentTime(reference.AddSeconds(5));
Assert.IsTrue(fillForward.MoveNext());
Assert.AreEqual(underlying[2].Value, fillForward.Current.Value);
Assert.AreEqual(timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork), fillForward.Current.EndTime);
Assert.IsTrue(fillForward.Current.IsFillForward);
Assert.AreEqual(0, ((TradeBar)fillForward.Current).Volume);
timeProvider.SetCurrentTime(reference.AddSeconds(6));
Assert.IsTrue(fillForward.MoveNext());
Assert.AreEqual(underlying[2].Value, fillForward.Current.Value);
Assert.AreEqual(timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork), fillForward.Current.EndTime);
Assert.IsTrue(fillForward.Current.IsFillForward);
Assert.AreEqual(0, ((TradeBar)fillForward.Current).Volume);
fillForward.Dispose();
}
public void LiveFillForwardEnumeratorDoesNotStall()
{
var now = DateTime.UtcNow;
var timeProvider = new ManualTimeProvider(new DateTime(2020, 5, 21, 9, 40, 0, 100), TimeZones.NewYork);
var enqueueableEnumerator = new EnqueueableEnumerator <BaseData>();
var fillForwardEnumerator = new LiveFillForwardEnumerator(
timeProvider,
enqueueableEnumerator,
new SecurityExchange(MarketHoursDatabase.FromDataFolder()
.ExchangeHoursListing
.First(kvp => kvp.Key.Market == Market.USA && kvp.Key.SecurityType == SecurityType.Equity)
.Value
.ExchangeHours),
Ref.CreateReadOnly(() => Resolution.Minute.ToTimeSpan()),
false,
now,
Resolution.Minute.ToTimeSpan(),
TimeZones.NewYork,
new DateTime(2020, 5, 21)
);
var openingBar = new TradeBar
{
Open = 0.01m,
High = 0.01m,
Low = 0.01m,
Close = 0.01m,
Volume = 1,
EndTime = new DateTime(2020, 5, 21, 9, 40, 0),
Symbol = Symbols.AAPL
};
var secondBar = new TradeBar
{
Open = 1m,
High = 2m,
Low = 1m,
Close = 2m,
Volume = 100,
EndTime = new DateTime(2020, 5, 21, 9, 42, 0),
Symbol = Symbols.AAPL
};
// Enqueue the first point, which will be emitted ASAP.
enqueueableEnumerator.Enqueue(openingBar);
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.NotNull(fillForwardEnumerator.Current);
Assert.AreEqual(openingBar.Open, ((TradeBar)fillForwardEnumerator.Current).Open);
Assert.AreEqual(openingBar.EndTime, fillForwardEnumerator.Current.EndTime);
// Advance the time, we expect a fill-forward bar.
timeProvider.SetCurrentTime(new DateTime(2020, 5, 21, 9, 41, 0, 100));
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(openingBar.Open, ((TradeBar)fillForwardEnumerator.Current).Open);
Assert.AreEqual(openingBar.EndTime.AddMinutes(1), fillForwardEnumerator.Current.EndTime);
// Now we expect data. The secondBar should be fill-forwarded from here on out after the MoveNext
timeProvider.SetCurrentTime(new DateTime(2020, 5, 21, 9, 42, 0, 100));
enqueueableEnumerator.Enqueue(secondBar);
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
}
public void MoveNextScanQuoteBar()
{
var offset = Offset.FromHours(-5);
var timeZone = DateTimeZone.ForOffset(offset);
var utc = new DateTimeOffset(DateTime.Today);
var reference = utc.ToOffset(offset.ToTimeSpan());
var timeProvider = new ManualTimeProvider(reference.DateTime, timeZone);
var enumerator = new ScannableEnumerator <Data.BaseData>(
new TickQuoteBarConsolidator(TimeSpan.FromMinutes(1)),
timeZone,
timeProvider,
(s, e) => { }
);
var tick1 = new Tick
{
Symbol = Symbols.SPY,
Time = reference.DateTime,
BidPrice = 10,
BidSize = 20,
TickType = TickType.Quote
};
enumerator.Update(tick1);
var tick2 = new Tick
{
Symbol = Symbols.SPY,
Time = reference.DateTime.AddSeconds(1),
AskPrice = 20,
AskSize = 10,
TickType = TickType.Quote
};
enumerator.Update(tick2);
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNull(enumerator.Current);
var tick3 = new Tick
{
Symbol = Symbols.SPY,
Time = reference.DateTime.AddSeconds(2),
BidPrice = 12,
BidSize = 50,
TickType = TickType.Quote
};
enumerator.Update(tick3);
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNull(enumerator.Current);
var tick4 = new Tick
{
Symbol = Symbols.SPY,
Time = reference.DateTime.AddSeconds(3),
AskPrice = 17,
AskSize = 15,
TickType = TickType.Quote
};
enumerator.Update(tick4);
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNull(enumerator.Current);
timeProvider.SetCurrentTime(reference.DateTime.AddMinutes(2));
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNotNull(enumerator.Current);
QuoteBar quoteBar = enumerator.Current as QuoteBar;
Assert.IsNotNull(quoteBar);
Assert.AreEqual(Symbols.SPY, quoteBar.Symbol);
Assert.AreEqual(tick1.Time, quoteBar.Time);
Assert.AreNotEqual(tick4.EndTime, quoteBar.EndTime);
Assert.AreEqual(tick1.BidPrice, quoteBar.Bid.Open);
Assert.AreEqual(tick1.BidPrice, quoteBar.Bid.Low);
Assert.AreEqual(tick3.BidPrice, quoteBar.Bid.High);
Assert.AreEqual(tick3.BidPrice, quoteBar.Bid.Close);
Assert.AreEqual(tick3.BidSize, quoteBar.LastBidSize);
Assert.AreEqual(tick2.AskPrice, quoteBar.Ask.Open);
Assert.AreEqual(tick4.AskPrice, quoteBar.Ask.Low);
Assert.AreEqual(tick2.AskPrice, quoteBar.Ask.High);
Assert.AreEqual(tick4.AskPrice, quoteBar.Ask.Close);
Assert.AreEqual(tick4.AskSize, quoteBar.LastAskSize);
}
public void LastCloseAndCurrentOpenPriceShouldBeSameConsolidated()
{
var timeProvider = new ManualTimeProvider(TimeZones.NewYork);
var enumerator = new QuoteBarBuilderEnumerator(Time.OneSecond, TimeZones.NewYork, timeProvider, false);
// noon new york time
var currentTime = new DateTime(2015, 10, 08, 12, 0, 0);
timeProvider.SetCurrentTime(currentTime);
var reference = DateTime.Today;
var tick1 = new Tick
{
Symbol = Symbols.SPY,
Time = reference,
TickType = TickType.Quote,
AskPrice = 0,
BidPrice = 24,
};
enumerator.ProcessData(tick1);
var tick2 = new Tick
{
Symbol = Symbols.SPY,
Time = reference.AddSeconds(1),
TickType = TickType.Quote,
AskPrice = 25,
BidPrice = 0,
};
enumerator.ProcessData(tick2);
currentTime = currentTime.AddSeconds(1);
timeProvider.SetCurrentTime(currentTime);
Assert.IsTrue(enumerator.MoveNext());
var quoteBar = enumerator.Current as QuoteBar;
// bar 1 emitted
Assert.AreEqual(tick2.AskPrice, quoteBar.Ask.Open);
Assert.AreEqual(tick1.BidPrice, quoteBar.Bid.Open);
Assert.AreEqual(tick2.AskPrice, quoteBar.Ask.Close);
Assert.AreEqual(tick1.BidPrice, quoteBar.Bid.Close);
var tick3 = new Tick
{
Symbol = Symbols.SPY,
Time = reference.AddSeconds(1),
TickType = TickType.Quote,
AskPrice = 36,
BidPrice = 35,
};
enumerator.ProcessData(tick3);
currentTime = currentTime.AddSeconds(1);
timeProvider.SetCurrentTime(currentTime);
Assert.IsTrue(enumerator.MoveNext());
quoteBar = enumerator.Current as QuoteBar;
// bar 2 emitted
Assert.AreEqual(tick2.AskPrice, quoteBar.Ask.Open, "Ask Open not equal to Previous Close");
Assert.AreEqual(tick1.BidPrice, quoteBar.Bid.Open, "Bid Open not equal to Previous Close");
Assert.AreEqual(tick3.AskPrice, quoteBar.Ask.Close, "Ask Close incorrect");
Assert.AreEqual(tick3.BidPrice, quoteBar.Bid.Close, "Bid Close incorrect");
enumerator.Dispose();
}
public void SubscribeMultipleDataTypes()
{
var reference = DateTime.Today;
var timeProvider = new ManualTimeProvider(reference);
var aggregator = GetDataAggregator(timeProvider);
var symbol = Symbols.AAPL;
var configs = new[] {
GetSubscriptionDataConfig <TradeBar>(symbol, Resolution.Minute),
GetSubscriptionDataConfig <QuoteBar>(symbol, Resolution.Minute),
GetSubscriptionDataConfig <Tick>(symbol, Resolution.Tick, TickType.Trade),
GetSubscriptionDataConfig <Tick>(symbol, Resolution.Tick, TickType.Quote),
GetSubscriptionDataConfig <Dividend>(symbol, Resolution.Tick),
GetSubscriptionDataConfig <Split>(symbol, Resolution.Tick)
};
var enumerators = new Queue <IEnumerator <BaseData> >();
Array.ForEach(configs, (c) => enumerators.Enqueue(aggregator.Add(c, (s, e) => { })));
var expectedBars = new[] { 2, 2, 100, 100, 1, 1 };
for (int i = 1; i <= 100; i++)
{
aggregator.Update(new Tick(reference.AddSeconds(i), symbol, 20 + i, 20 + i)
{
TickType = TickType.Trade
});
aggregator.Update(new Tick(reference.AddSeconds(i), symbol, 20 + i, 20 + i)
{
TickType = TickType.Quote
});
}
aggregator.Update(new Dividend(symbol, reference.AddSeconds(1), 0.47m, 108.60m));
aggregator.Update(new Split(symbol, reference.AddSeconds(1), 645.57m, 0.142857m, SplitType.SplitOccurred));
timeProvider.SetCurrentTime(reference.AddMinutes(2));
var dividendCount = 0;
var splitCount = 0;
var j = 0;
foreach (var enumerator in enumerators)
{
for (int i = 0; i < expectedBars[j]; i++)
{
enumerator.MoveNext();
Assert.IsNotNull(enumerator.Current);
if (enumerator.Current is Dividend)
{
dividendCount++;
}
if (enumerator.Current is Split)
{
splitCount++;
}
}
enumerator.MoveNext();
Assert.IsNull(enumerator.Current);
j++;
}
Assert.AreEqual(1, dividendCount);
Assert.AreEqual(1, splitCount);
}