public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode)
{
var csv = line.Split(',');
var data = new TestableCustomFuture
{
Symbol = config.Symbol,
Time = DateTime.ParseExact(csv[0], "yyyy-MM-dd", CultureInfo.InvariantCulture),
Value = csv[6].ToDecimal()
};
return(data);
}
public void EmitsDailyCustomFutureDataOverWeekends()
{
RemoteFileSubscriptionStreamReader.SetDownloadProvider(new Api.Api());
var tickers = new[] { "CHRIS/CME_ES1", "CHRIS/CME_ES2" };
var startDate = new DateTime(2018, 4, 1);
var endDate = new DateTime(2018, 4, 20);
// delete temp files
foreach (var ticker in tickers)
{
var fileName = TestableCustomFuture.GetLocalFileName(ticker, "test");
File.Delete(fileName);
}
var algorithm = new QCAlgorithm();
CreateDataFeed();
var dataManager = new DataManagerStub(algorithm, _feed);
algorithm.SubscriptionManager.SetDataManager(dataManager);
var symbols = tickers.Select(ticker => algorithm.AddData <TestableCustomFuture>(ticker, Resolution.Daily).Symbol).ToList();
var timeProvider = new ManualTimeProvider(TimeZones.NewYork);
timeProvider.SetCurrentTime(startDate);
var dataPointsEmitted = 0;
RunLiveDataFeed(algorithm, startDate, symbols, timeProvider, dataManager);
var cancellationTokenSource = new CancellationTokenSource();
var lastFileWriteDate = DateTime.MinValue;
// create a timer to advance time much faster than realtime and to simulate live Quandl data file updates
var timerInterval = TimeSpan.FromMilliseconds(20);
var timer = Ref.Create <Timer>(null);
timer.Value = new Timer(state =>
{
try
{
var currentTime = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork);
if (currentTime.Date > endDate.Date)
{
Log.Trace($"Total data points emitted: {dataPointsEmitted.ToStringInvariant()}");
_feed.Exit();
cancellationTokenSource.Cancel();
return;
}
if (currentTime.Date > lastFileWriteDate.Date)
{
foreach (var ticker in tickers)
{
var source = TestableCustomFuture.GetLocalFileName(ticker, "csv");
// write new local file including only rows up to current date
var outputFileName = TestableCustomFuture.GetLocalFileName(ticker, "test");
var sb = new StringBuilder();
{
using (var reader = new StreamReader(source))
{
var firstLine = true;
string line;
while ((line = reader.ReadLine()) != null)
{
if (firstLine)
{
sb.AppendLine(line);
firstLine = false;
continue;
}
var csv = line.Split(',');
var time = Parse.DateTimeExact(csv[0], "yyyy-MM-dd");
if (time.Date >= currentTime.Date)
{
break;
}
sb.AppendLine(line);
}
}
}
if (currentTime.Date.DayOfWeek != DayOfWeek.Saturday && currentTime.Date.DayOfWeek != DayOfWeek.Sunday)
{
var fileContent = sb.ToString();
try
{
File.WriteAllText(outputFileName, fileContent);
}
catch (IOException)
{
Log.Error("IOException: will sleep 200ms and retry once more");
// lets sleep 200ms and retry once more, consumer could be reading the file
// this exception happens in travis intermittently, GH issue 3273
Thread.Sleep(200);
File.WriteAllText(outputFileName, fileContent);
}
Log.Trace($"Time:{currentTime} - Ticker:{ticker} - Files written:{++_countFilesWritten}");
}
}
lastFileWriteDate = currentTime;
}
// 30 minutes is the check interval for daily remote files, so we choose a smaller one to advance time
timeProvider.Advance(TimeSpan.FromMinutes(20));
//Log.Trace($"Time advanced to: {timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork)}");
// restart the timer
timer.Value.Change(timerInterval.Milliseconds, Timeout.Infinite);
}
catch (Exception exception)
{
Log.Error(exception);
_feed.Exit();
cancellationTokenSource.Cancel();
}
}, null, timerInterval.Milliseconds, Timeout.Infinite);
try
{
foreach (var timeSlice in _synchronizer.StreamData(cancellationTokenSource.Token))
{
foreach (var dataPoint in timeSlice.Slice.Values)
{
Log.Trace($"Data point emitted at {timeSlice.Slice.Time.ToStringInvariant()}: " +
$"{dataPoint.Symbol.Value} {dataPoint.Value.ToStringInvariant()} " +
$"{dataPoint.EndTime.ToStringInvariant()}"
);
dataPointsEmitted++;
}
}
}
catch (Exception exception)
{
Log.Trace($"Error: {exception}");
}
timer.Value.Dispose();
dataManager.RemoveAllSubscriptions();
Assert.AreEqual(14 * tickers.Length, dataPointsEmitted);
}
