public void AddDataOptionsTickerHasNoChainedUnderlyingSymbols(string ticker, Type customDataType)
{
SymbolCache.Clear();
var qcAlgorithm = new QCAlgorithm();
qcAlgorithm.SubscriptionManager.SetDataManager(new DataManagerStub(qcAlgorithm));
var asset = qcAlgorithm.AddOption(ticker);
// Dummy here is meant to try to corrupt the SymbolCache. Ideally, SymbolCache should return non-custom data types with higher priority
// in case we want to add two custom data types, but still have them associated with the equity from the cache if we're using it.
// This covers the case where two idential data subscriptions are created.
var dummy = qcAlgorithm.AddData(customDataType, ticker, Resolution.Daily, qcAlgorithm.SubscriptionManager.Subscriptions.Where(x => x.SecurityType == SecurityType.Option).Single().DataTimeZone);
var customData = qcAlgorithm.AddData(customDataType, ticker, Resolution.Daily, qcAlgorithm.SubscriptionManager.Subscriptions.Where(x => x.SecurityType == SecurityType.Option).Single().DataTimeZone);
// Check to see if we have an underlying symbol when we shouldn't
Assert.IsFalse(customData.Symbol.HasUnderlying, $"{customDataType.Name} has an underlying Symbol");
var assetSubscription = qcAlgorithm.SubscriptionManager.Subscriptions.Where(x => x.SecurityType == SecurityType.Option).Single();
var customDataSubscription = qcAlgorithm.SubscriptionManager.Subscriptions.Where(x => x.SecurityType == SecurityType.Base).Single();
Assert.IsTrue(assetSubscription.TickerShouldBeMapped());
Assert.IsFalse(customDataSubscription.TickerShouldBeMapped());
//Assert.AreNotEqual(assetSubscription.MappedSymbol, customDataSubscription.MappedSymbol);
}
public EquityHighLowAlphaModel(
QCAlgorithm algorithm,
int insightPeriod = 1,
Resolution resolution = Resolution.Daily
)
{
_resolution = resolution;
// Add Quandl data for the Federal Interest Rate
_highs = algorithm.AddData <Quandl52WeekHigh>("URC/NASDAQ_52W_HI", _resolution).Symbol;
_lows = algorithm.AddData <Quandl52WeekLow>("URC/NYSE_52W_LO", _resolution).Symbol;
_insightDuration = resolution.ToTimeSpan().Multiply(insightPeriod);
}
public CostOfLivingAlphaModel(
QCAlgorithm algorithm,
Resolution resolution = Resolution.Daily,
int insightPeriod = 90
)
{
_resolution = resolution;
// Add quarterly compensation change data
_compensationDelta = algorithm.AddData <QuandlCompensationData>("BLSP/PRS85006062", _resolution).Symbol;
// Add quarterly cost of living data (Consumer Price Index)
_costOfLivingDelta = algorithm.AddData <QuandlCostOfLivingData>("RATEINF/CPI_USA", _resolution).Symbol;
_insightPeriod = resolution.ToTimeSpan().Multiply(insightPeriod);
}
public void CustomDataTypes_AreAddedToSubscriptions_Successfully()
{
var qcAlgorithm = new QCAlgorithm();
// Add a bitcoin subscription
qcAlgorithm.AddData <Bitcoin>("BTC");
var bitcoinSubscription = qcAlgorithm.SubscriptionManager.Subscriptions.FirstOrDefault(x => x.Type == typeof(Bitcoin));
Assert.AreEqual(bitcoinSubscription.Type, typeof(Bitcoin));
// Add a quandl subscription
qcAlgorithm.AddData <Quandl>("EURCAD");
var quandlSubscription = qcAlgorithm.SubscriptionManager.Subscriptions.FirstOrDefault(x => x.Type == typeof(Quandl));
Assert.AreEqual(quandlSubscription.Type, typeof(Quandl));
}
/// <summary>
/// Alpha model that uses the Interest rate released by Fed to create insights
/// </summary>
/// <param name="algorithm">The algorithm instance</param>
/// <param name="period">The prediction interval period</param>
/// <param name="resolution">The resolution of data</param>
public InterestReleaseAlphaModel(QCAlgorithm algorithm, int period = 30, Resolution resolution = Resolution.Daily)
{
_predictionInterval = Time.Multiply(Extensions.ToTimeSpan(resolution), period);
_calendar = algorithm.AddData <TradingEconomicsCalendar>(TradingEconomics.Calendar.UnitedStates.InterestRate).Symbol;
Name = $"{nameof(InterestReleaseAlphaModel)}({period}, {resolution})";
}
public void AddSecurityWithSymbol(Symbol symbol, Type type = null)
{
var security = type != null?_algo.AddData(type, symbol.Underlying) : _algo.AddSecurity(symbol);
Assert.AreEqual(security.Symbol, symbol);
Assert.IsTrue(_algo.Securities.ContainsKey(symbol));
Assert.DoesNotThrow(() =>
{
switch (symbol.SecurityType)
{
case SecurityType.Equity:
var equity = (Equity)security;
break;
case SecurityType.Option:
var option = (Option)security;
break;
case SecurityType.Forex:
var forex = (Forex)security;
break;
case SecurityType.Future:
var future = (Future)security;
break;
case SecurityType.Cfd:
var cfd = (Cfd)security;
break;
case SecurityType.Index:
var index = (Index)security;
break;
case SecurityType.IndexOption:
var indexOption = (IndexOption)security;
break;
case SecurityType.Crypto:
var crypto = (Crypto)security;
break;
case SecurityType.Base:
break;
default:
throw new Exception($"Invalid Security Type: {symbol.SecurityType}");
}
});
if (symbol.IsCanonical())
{
Assert.DoesNotThrow(() => _algo.OnEndOfTimeStep());
Assert.IsTrue(_algo.UniverseManager.ContainsKey(symbol));
}
}
public void CustomDataTypes_AreAddedToSubscriptions_Successfully()
{
var qcAlgorithm = new QCAlgorithm();
qcAlgorithm.SubscriptionManager.SetDataManager(new DataManagerStub(qcAlgorithm));
// Add a bitcoin subscription
qcAlgorithm.AddData <Bitcoin>("BTC");
var bitcoinSubscription = qcAlgorithm.SubscriptionManager.Subscriptions.FirstOrDefault(x => x.Type == typeof(Bitcoin));
Assert.AreEqual(bitcoinSubscription.Type, typeof(Bitcoin));
// Add a unlinkedData subscription
qcAlgorithm.AddData <UnlinkedData>("EURCAD");
var unlinkedDataSubscription = qcAlgorithm.SubscriptionManager.Subscriptions.FirstOrDefault(x => x.Type == typeof(UnlinkedData));
Assert.AreEqual(unlinkedDataSubscription.Type, typeof(UnlinkedData));
}
public void AddingInvalidDataTypeThrows()
{
var qcAlgorithm = new QCAlgorithm();
qcAlgorithm.SubscriptionManager.SetDataManager(new DataManagerStub(qcAlgorithm));
Assert.Throws <ArgumentException>(() => qcAlgorithm.AddData(typeof(double),
"double",
Resolution.Daily,
DateTimeZone.Utc));
}
public void SingleAlphaSinglePosition(Language language)
{
SetPortfolioConstruction(language);
var alpha = _algorithm.AddData <AlphaStreamsPortfolioState>("9fc8ef73792331b11dbd5429a").Symbol;
var data = _algorithm.History <AlphaStreamsPortfolioState>(alpha, TimeSpan.FromDays(2)).Last();
_algorithm.SetCurrentSlice(new Slice(_algorithm.UtcTime, new List <BaseData> {
data
}));
var targets = _algorithm.PortfolioConstruction.CreateTargets(_algorithm, Array.Empty <Insight>()).ToList();
Assert.AreEqual(1, targets.Count);
var position = data.PositionGroups.Single().Positions.Single();
Assert.AreEqual(position.Symbol, targets.Single().Symbol);
Assert.AreEqual(position.Quantity, targets.Single().Quantity);
}
public MortgageRateVolatilityAlphaModel(
QCAlgorithm algorithm,
int indicatorPeriod = 15,
double insightMagnitude = 0.0005,
int deviations = 2,
Resolution resolution = Resolution.Daily
)
{
// Add Quandl data for a Well's Fargo 30-year Fixed Rate mortgage
_mortgageRate = algorithm.AddData <QuandlMortgagePriceColumns>("WFC/PR_GOV_30YFIXEDVA_APR").Symbol;
_indicatorPeriod = indicatorPeriod;
_resolution = resolution;
_insightDuration = resolution.ToTimeSpan().Multiply(indicatorPeriod);
_insightMagnitude = insightMagnitude;
_deviations = deviations;
// Add indicators for the mortgage rate -- Standard Deviation and Simple Moving Average
_mortgageRateStd = algorithm.STD(_mortgageRate, _indicatorPeriod, resolution);
_mortgageRateSma = algorithm.SMA(_mortgageRate, _indicatorPeriod, resolution);
// Use a history call to warm-up the indicators
WarmUpIndicators(algorithm);
}
public void EmitsDailyQuandlFutureDataOverWeekends()
{
RemoteFileSubscriptionStreamReader.SetDownloadProvider(new Api.Api());
var tickers = new[] { "CHRIS/CME_ES1", "CHRIS/CME_ES2" };
var startDate = new DateTime(2018, 4, 1);
var endDate = new DateTime(2018, 4, 20);
// delete temp files
foreach (var ticker in tickers)
{
var fileName = TestableQuandlFuture.GetLocalFileName(ticker, "test");
File.Delete(fileName);
}
var algorithm = new QCAlgorithm();
CreateDataFeed();
var dataManager = new DataManagerStub(algorithm, _feed);
algorithm.SubscriptionManager.SetDataManager(dataManager);
var symbols = tickers.Select(ticker => algorithm.AddData <TestableQuandlFuture>(ticker, Resolution.Daily).Symbol).ToList();
var timeProvider = new ManualTimeProvider(TimeZones.NewYork);
timeProvider.SetCurrentTime(startDate);
var dataPointsEmitted = 0;
RunLiveDataFeed(algorithm, startDate, symbols, timeProvider, dataManager);
var cancellationTokenSource = new CancellationTokenSource();
var lastFileWriteDate = DateTime.MinValue;
// create a timer to advance time much faster than realtime and to simulate live Quandl data file updates
var timerInterval = TimeSpan.FromMilliseconds(20);
var timer = Ref.Create <Timer>(null);
timer.Value = new Timer(state =>
{
try
{
var currentTime = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork);
if (currentTime.Date > endDate.Date)
{
Log.Trace($"Total data points emitted: {dataPointsEmitted.ToStringInvariant()}");
_feed.Exit();
cancellationTokenSource.Cancel();
return;
}
if (currentTime.Date > lastFileWriteDate.Date)
{
foreach (var ticker in tickers)
{
var source = TestableQuandlFuture.GetLocalFileName(ticker, "csv");
// write new local file including only rows up to current date
var outputFileName = TestableQuandlFuture.GetLocalFileName(ticker, "test");
var sb = new StringBuilder();
{
using (var reader = new StreamReader(source))
{
var firstLine = true;
string line;
while ((line = reader.ReadLine()) != null)
{
if (firstLine)
{
sb.AppendLine(line);
firstLine = false;
continue;
}
var csv = line.Split(',');
var time = Parse.DateTimeExact(csv[0], "yyyy-MM-dd");
if (time.Date >= currentTime.Date)
{
break;
}
sb.AppendLine(line);
}
}
}
if (currentTime.Date.DayOfWeek != DayOfWeek.Saturday && currentTime.Date.DayOfWeek != DayOfWeek.Sunday)
{
var fileContent = sb.ToString();
try
{
File.WriteAllText(outputFileName, fileContent);
}
catch (IOException)
{
Log.Error("IOException: will sleep 200ms and retry once more");
// lets sleep 200ms and retry once more, consumer could be reading the file
// this exception happens in travis intermittently, GH issue 3273
Thread.Sleep(200);
File.WriteAllText(outputFileName, fileContent);
}
Log.Trace($"Time:{currentTime} - Ticker:{ticker} - Files written:{++_countFilesWritten}");
}
}
lastFileWriteDate = currentTime;
}
// 30 minutes is the check interval for daily remote files, so we choose a smaller one to advance time
timeProvider.Advance(TimeSpan.FromMinutes(20));
//Log.Trace($"Time advanced to: {timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork)}");
// restart the timer
timer.Value.Change(timerInterval.Milliseconds, Timeout.Infinite);
}
catch (Exception exception)
{
Log.Error(exception);
_feed.Exit();
cancellationTokenSource.Cancel();
}
}, null, timerInterval.Milliseconds, Timeout.Infinite);
try
{
foreach (var timeSlice in _synchronizer.StreamData(cancellationTokenSource.Token))
{
foreach (var dataPoint in timeSlice.Slice.Values)
{
Log.Trace($"Data point emitted at {timeSlice.Slice.Time.ToStringInvariant()}: " +
$"{dataPoint.Symbol.Value} {dataPoint.Value.ToStringInvariant()} " +
$"{dataPoint.EndTime.ToStringInvariant()}"
);
dataPointsEmitted++;
}
}
}
catch (Exception exception)
{
Log.Trace($"Error: {exception}");
}
timer.Value.Dispose();
dataManager.RemoveAllSubscriptions();
Assert.AreEqual(14 * tickers.Length, dataPointsEmitted);
}
public void RemoteDataDoesNotIncreaseNumberOfSlices()
{
Config.Set("quandl-auth-token", "QUANDL-TOKEN");
var startDate = new DateTime(2018, 4, 2);
var endDate = new DateTime(2018, 4, 19);
var algorithm = new QCAlgorithm();
var timeProvider = new ManualTimeProvider(TimeZones.NewYork);
timeProvider.SetCurrentTime(startDate);
var dataQueueHandler = new FuncDataQueueHandler(fdqh =>
{
var time = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork);
var tick = new Tick(time, Symbols.SPY, 1.3m, 1.2m, 1.3m)
{
TickType = TickType.Trade
};
var tick2 = new Tick(time, Symbols.AAPL, 1.3m, 1.2m, 1.3m)
{
TickType = TickType.Trade
};
return(new[] { tick, tick2 });
}, timeProvider);
CreateDataFeed(dataQueueHandler);
var dataManager = new DataManagerStub(algorithm, _feed);
algorithm.SubscriptionManager.SetDataManager(dataManager);
var symbols = new List <Symbol>
{
algorithm.AddData <Quandl>("CBOE/VXV", Resolution.Daily).Symbol,
algorithm.AddData <QuandlVix>("CBOE/VIX", Resolution.Daily).Symbol,
algorithm.AddEquity("SPY", Resolution.Daily).Symbol,
algorithm.AddEquity("AAPL", Resolution.Daily).Symbol
};
algorithm.PostInitialize();
var cancellationTokenSource = new CancellationTokenSource();
var dataPointsEmitted = 0;
var slicesEmitted = 0;
RunLiveDataFeed(algorithm, startDate, symbols, timeProvider, dataManager);
Thread.Sleep(5000); // Give remote sources a handicap, so the data is available in time
// create a timer to advance time much faster than realtime and to simulate live Quandl data file updates
var timerInterval = TimeSpan.FromMilliseconds(100);
var timer = Ref.Create <Timer>(null);
timer.Value = new Timer(state =>
{
// stop the timer to prevent reentrancy
timer.Value.Change(Timeout.Infinite, Timeout.Infinite);
var currentTime = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork);
if (currentTime.Date > endDate.Date)
{
_feed.Exit();
cancellationTokenSource.Cancel();
return;
}
timeProvider.Advance(TimeSpan.FromHours(3));
// restart the timer
timer.Value.Change(timerInterval, timerInterval);
}, null, TimeSpan.FromSeconds(2), timerInterval);
try
{
foreach (var timeSlice in _synchronizer.StreamData(cancellationTokenSource.Token))
{
if (timeSlice.Slice.HasData)
{
slicesEmitted++;
dataPointsEmitted += timeSlice.Slice.Values.Count;
Assert.IsTrue(timeSlice.Slice.Values.Any(x => x.Symbol == symbols[0]), $"Slice doesn't contain {symbols[0]}");
Assert.IsTrue(timeSlice.Slice.Values.Any(x => x.Symbol == symbols[1]), $"Slice doesn't contain {symbols[1]}");
Assert.IsTrue(timeSlice.Slice.Values.Any(x => x.Symbol == symbols[2]), $"Slice doesn't contain {symbols[2]}");
Assert.IsTrue(timeSlice.Slice.Values.Any(x => x.Symbol == symbols[3]), $"Slice doesn't contain {symbols[3]}");
}
}
}
catch (Exception exception)
{
Log.Trace($"Error: {exception}");
}
timer.Value.Dispose();
dataManager.RemoveAllSubscriptions();
dataQueueHandler.DisposeSafely();
Assert.AreEqual(14, slicesEmitted);
Assert.AreEqual(14 * symbols.Count, dataPointsEmitted);
}
public void AddDataSecurityTickerNoUnderlying(string ticker, Type customDataType, SecurityType securityType, bool securityShouldBeMapped, bool customDataShouldBeMapped)
{
SymbolCache.Clear();
var qcAlgorithm = new QCAlgorithm();
qcAlgorithm.SubscriptionManager.SetDataManager(new DataManagerStub(qcAlgorithm));
Security asset;
switch (securityType)
{
case SecurityType.Cfd:
asset = qcAlgorithm.AddCfd(ticker, Resolution.Daily);
break;
case SecurityType.Crypto:
asset = qcAlgorithm.AddCrypto(ticker, Resolution.Daily);
break;
case SecurityType.Equity:
asset = qcAlgorithm.AddEquity(ticker, Resolution.Daily);
break;
case SecurityType.Forex:
asset = qcAlgorithm.AddForex(ticker, Resolution.Daily);
break;
case SecurityType.Future:
asset = qcAlgorithm.AddFuture(ticker, Resolution.Daily);
break;
default:
throw new Exception($"SecurityType {securityType} is not valid for this test");
}
// Dummy here is meant to try to corrupt the SymbolCache. Ideally, SymbolCache should return non-custom data types with higher priority
// in case we want to add two custom data types, but still have them associated with the equity from the cache if we're using it.
// This covers the case where two idential data subscriptions are created.
var dummy = qcAlgorithm.AddData(customDataType, ticker, Resolution.Daily, qcAlgorithm.SubscriptionManager.Subscriptions.Where(x => x.SecurityType == securityType).First().DataTimeZone);
var customData = qcAlgorithm.AddData(customDataType, ticker, Resolution.Daily, qcAlgorithm.SubscriptionManager.Subscriptions.Where(x => x.SecurityType == securityType).First().DataTimeZone);
// Check to see if we have an underlying symbol when we shouldn't
Assert.IsFalse(customData.Symbol.HasUnderlying, $"{customDataType.Name} has underlying symbol for SecurityType {securityType} with ticker {ticker}");
Assert.AreEqual(customData.Symbol.Underlying, null, $"{customDataType.Name} - Custom data underlying Symbol for SecurityType {securityType} is not null");
var assetSubscription = qcAlgorithm.SubscriptionManager.Subscriptions.Where(x => x.SecurityType == securityType).First();
var customDataSubscription = qcAlgorithm.SubscriptionManager.Subscriptions.Where(x => x.SecurityType == SecurityType.Base).Single();
var assetShouldBeMapped = assetSubscription.TickerShouldBeMapped();
var customShouldBeMapped = customDataSubscription.TickerShouldBeMapped();
Assert.AreEqual(securityShouldBeMapped, assetShouldBeMapped);
Assert.AreEqual(customDataShouldBeMapped, customShouldBeMapped);
Assert.AreNotEqual(assetSubscription, customDataSubscription);
if (assetShouldBeMapped == customShouldBeMapped)
{
// Would fail with CL future without this check because MappedSymbol returns "/CL" for the Future symbol
if (assetSubscription.SecurityType == SecurityType.Future)
{
Assert.AreNotEqual(assetSubscription.MappedSymbol, customDataSubscription.MappedSymbol);
Assert.AreNotEqual(asset.Symbol.Value, customData.Symbol.Value.Split('.').First());
}
else
{
Assert.AreEqual(assetSubscription.MappedSymbol, customDataSubscription.MappedSymbol);
Assert.AreEqual(asset.Symbol.Value, customData.Symbol.Value.Split('.').First());
}
}
}
public void AddDataSecurityTickerWithUnderlying(string ticker, Type customDataType, SecurityType securityType, bool securityShouldBeMapped, bool customDataShouldBeMapped)
{
SymbolCache.Clear();
var qcAlgorithm = new QCAlgorithm();
qcAlgorithm.SubscriptionManager.SetDataManager(new DataManagerStub(qcAlgorithm));
Security asset;
switch (securityType)
{
case SecurityType.Cfd:
asset = qcAlgorithm.AddCfd(ticker, Resolution.Daily);
break;
case SecurityType.Crypto:
asset = qcAlgorithm.AddCrypto(ticker, Resolution.Daily);
break;
case SecurityType.Equity:
asset = qcAlgorithm.AddEquity(ticker, Resolution.Daily);
break;
case SecurityType.Forex:
asset = qcAlgorithm.AddForex(ticker, Resolution.Daily);
break;
case SecurityType.Future:
asset = qcAlgorithm.AddFuture(ticker, Resolution.Daily);
break;
default:
throw new Exception($"SecurityType {securityType} is not valid for this test");
}
// Aliased value for Futures contains a forward-slash, which causes the
// lookup in the SymbolCache to fail
if (securityType == SecurityType.Future)
{
ticker = asset.Symbol.Value;
}
// Dummy here is meant to try to corrupt the SymbolCache. Ideally, SymbolCache should return non-custom data types with higher priority
// in case we want to add two custom data types, but still have them associated with the equity from the cache if we're using it.
// This covers the case where two idential data subscriptions are created.
var dummy = qcAlgorithm.AddData(customDataType, ticker, Resolution.Daily, qcAlgorithm.SubscriptionManager.Subscriptions.Where(x => x.SecurityType == securityType).First().DataTimeZone);
var customData = qcAlgorithm.AddData(customDataType, ticker, Resolution.Daily, qcAlgorithm.SubscriptionManager.Subscriptions.Where(x => x.SecurityType == securityType).First().DataTimeZone);
Assert.IsTrue(customData.Symbol.HasUnderlying, $"Custom data added as {ticker} Symbol with SecurityType {securityType} does not have underlying");
Assert.AreEqual(customData.Symbol.Underlying, asset.Symbol, $"Custom data underlying does not match {securityType} Symbol for {ticker}");
var assetSubscription = qcAlgorithm.SubscriptionManager.Subscriptions.Where(x => x.SecurityType == securityType).First();
var customDataSubscription = qcAlgorithm.SubscriptionManager.Subscriptions.Where(x => x.SecurityType == SecurityType.Base).Single();
var assetShouldBeMapped = assetSubscription.TickerShouldBeMapped();
var customShouldBeMapped = customDataSubscription.TickerShouldBeMapped();
if (securityType == SecurityType.Equity)
{
Assert.AreEqual(securityShouldBeMapped, assetShouldBeMapped);
Assert.AreEqual(customDataShouldBeMapped, customShouldBeMapped);
Assert.AreNotEqual(assetSubscription, customDataSubscription);
if (assetShouldBeMapped == customShouldBeMapped)
{
Assert.AreEqual(assetSubscription.MappedSymbol, customDataSubscription.MappedSymbol);
Assert.AreEqual(asset.Symbol.Value, customData.Symbol.Value.Split('.').First());
}
}
}
public void EmitsDailyQuandlFutureDataOverWeekends()
{
var tickers = new[] { "CHRIS/CME_ES1", "CHRIS/CME_ES2" };
var startDate = new DateTime(2018, 4, 1);
var endDate = new DateTime(2018, 4, 20);
// delete temp files
foreach (var ticker in tickers)
{
var fileName = TestableQuandlFuture.GetLocalFileName(ticker, "test");
File.Delete(fileName);
}
var algorithm = new QCAlgorithm();
var dataManager = new DataManagerStub(algorithm);
algorithm.SubscriptionManager.SetDataManager(dataManager);
var symbols = tickers.Select(ticker => algorithm.AddData <TestableQuandlFuture>(ticker, Resolution.Daily).Symbol).ToList();
algorithm.PostInitialize();
var timeProvider = new ManualTimeProvider(TimeZones.NewYork);
timeProvider.SetCurrentTime(startDate);
var dataPointsEmitted = 0;
var feed = RunLiveDataFeed(algorithm, startDate, symbols, timeProvider, dataManager);
var lastFileWriteDate = DateTime.MinValue;
// create a timer to advance time much faster than realtime and to simulate live Quandl data file updates
var timerInterval = TimeSpan.FromMilliseconds(100);
var timer = Ref.Create <Timer>(null);
timer.Value = new Timer(state =>
{
// stop the timer to prevent reentrancy
timer.Value.Change(Timeout.Infinite, Timeout.Infinite);
var currentTime = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork);
if (currentTime.Date > endDate.Date)
{
Log.Trace($"Total data points emitted: {dataPointsEmitted}");
feed.Exit();
return;
}
if (currentTime.Date > lastFileWriteDate.Date)
{
foreach (var ticker in tickers)
{
var source = TestableQuandlFuture.GetLocalFileName(ticker, "csv");
// write new local file including only rows up to current date
var outputFileName = TestableQuandlFuture.GetLocalFileName(ticker, "test");
var sb = new StringBuilder();
{
using (var reader = new StreamReader(source))
{
var firstLine = true;
string line;
while ((line = reader.ReadLine()) != null)
{
if (firstLine)
{
sb.AppendLine(line);
firstLine = false;
continue;
}
var csv = line.Split(',');
var time = DateTime.ParseExact(csv[0], "yyyy-MM-dd", CultureInfo.InvariantCulture);
if (time.Date >= currentTime.Date)
{
break;
}
sb.AppendLine(line);
}
}
}
if (currentTime.Date.DayOfWeek != DayOfWeek.Saturday && currentTime.Date.DayOfWeek != DayOfWeek.Sunday)
{
File.WriteAllText(outputFileName, sb.ToString());
Log.Trace($"Time:{currentTime} - Ticker:{ticker} - Files written:{++_countFilesWritten}");
}
}
lastFileWriteDate = currentTime;
}
// 30 minutes is the check interval for daily remote files, so we choose a smaller one to advance time
timeProvider.Advance(TimeSpan.FromMinutes(15));
//Log.Trace($"Time advanced to: {timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork)}");
// restart the timer
timer.Value.Change(timerInterval, timerInterval);
}, null, TimeSpan.FromSeconds(2), timerInterval);
try
{
foreach (var timeSlice in feed)
{
foreach (var dataPoint in timeSlice.Slice.Values)
{
Log.Trace($"Data point emitted at {timeSlice.Slice.Time}: {dataPoint.Symbol.Value} {dataPoint.Value} {dataPoint.EndTime}");
dataPointsEmitted++;
}
}
}
catch (Exception exception)
{
Log.Trace($"Error: {exception}");
}
timer.Value.Dispose();
Assert.AreEqual(14 * tickers.Length, dataPointsEmitted);
}