public NullSynchronizer(IAlgorithm algorithm, Dividend dividend)
{
_algorithm = algorithm;
_dividend = dividend;
_symbol = dividend.Symbol;
_timeSliceFactory = new TimeSliceFactory(TimeZones.NewYork);
}
public IEnumerable <Slice> GetHistory <T>(Symbol symbol, Resolution resolution, IEnumerable <T> data)
where T : IBaseData
{
var subscriptionDataConfig = GetSubscriptionDataConfig <T>(symbol, resolution);
var security = GetSecurity(subscriptionDataConfig);
var timeSliceFactory = new TimeSliceFactory(TimeZones.Utc);
return(data.Select(t => timeSliceFactory.Create(
t.Time,
new List <DataFeedPacket> {
new DataFeedPacket(security, subscriptionDataConfig, new List <BaseData>()
{
t as BaseData
})
},
new SecurityChanges(Enumerable.Empty <Security>(), Enumerable.Empty <Security>()),
new Dictionary <Universe, BaseDataCollection>()).Slice));
}
/// <summary>
/// Creates an enumerable of Slice to update the alpha model
/// </summary>
protected virtual IEnumerable <Slice> CreateSlices()
{
var timeSliceFactory = new TimeSliceFactory(TimeZones.NewYork);
var changes = SecurityChanges.None;
var sliceDateTimes = GetSliceDateTimes(MaxSliceCount);
for (var i = 0; i < sliceDateTimes.Count; i++)
{
var utcDateTime = sliceDateTimes[i];
var packets = new List <DataFeedPacket>();
// TODO : Give securities different values -- will require updating all derived types
var last = Convert.ToDecimal(100 + 10 * Math.Sin(Math.PI * i / 180.0));
var high = last * 1.005m;
var low = last / 1.005m;
foreach (var kvp in Algorithm.Securities)
{
var security = kvp.Value;
var exchange = security.Exchange.Hours;
var configs = Algorithm.SubscriptionManager.SubscriptionDataConfigService
.GetSubscriptionDataConfigs(security.Symbol);
var extendedMarket = configs.IsExtendedMarketHours();
var localDateTime = utcDateTime.ConvertFromUtc(exchange.TimeZone);
if (!exchange.IsOpen(localDateTime, extendedMarket))
{
continue;
}
var configuration = security.Subscriptions.FirstOrDefault();
var period = configs.GetHighestResolution().ToTimeSpan();
var time = (utcDateTime - period).ConvertFromUtc(configuration.DataTimeZone);
var tradeBar = new TradeBar(time, security.Symbol, last, high, low, last, 1000, period);
packets.Add(new DataFeedPacket(security, configuration, new List <BaseData> {
tradeBar
}));
}
if (packets.Count > 0)
{
yield return(timeSliceFactory.Create(utcDateTime, packets, changes, new Dictionary <Universe, BaseDataCollection>()).Slice);
}
}
}
/// <summary>
/// Gets the history for the requested securities
/// </summary>
/// <param name="requests">The historical data requests</param>
/// <param name="sliceTimeZone">The time zone used when time stamping the slice instances</param>
/// <returns>An enumerable of the slices of data covering the span specified in each request</returns>
public override IEnumerable <Slice> GetHistory(IEnumerable <HistoryRequest> requests, DateTimeZone sliceTimeZone)
{
var configsByDateTime = GetSubscriptionDataConfigByDateTime(requests);
var count = configsByDateTime.Count;
var i = 0;
var timeSliceFactory = new TimeSliceFactory(sliceTimeZone);
foreach (var kvp in configsByDateTime)
{
var utcDateTime = kvp.Key;
var configs = kvp.Value;
var last = Convert.ToDecimal(100 + 10 * Math.Sin(Math.PI * (360 - count + i) / 180.0));
var high = last * 1.005m;
var low = last / 1.005m;
var packets = new List <DataFeedPacket>();
foreach (var config in configs)
{
Security security;
if (!_securities.TryGetValue(config.Symbol, out security))
{
continue;
}
var period = config.Resolution.ToTimeSpan();
var time = (utcDateTime - period).ConvertFromUtc(config.DataTimeZone);
var data = new TradeBar(time, config.Symbol, last, high, last, last, 1000, period);
security.SetMarketPrice(data);
packets.Add(new DataFeedPacket(security, config, new List <BaseData> {
data
}));
}
i++;
yield return(timeSliceFactory.Create(utcDateTime, packets, _securityChanges, new Dictionary <Universe, BaseDataCollection>()).Slice);
}
}
private void TestSubscriptionSynchronizerSpeed(QCAlgorithm algorithm)
{
var feed = new MockDataFeed();
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder();
var securityService = new SecurityService(
algorithm.Portfolio.CashBook,
marketHoursDatabase,
symbolPropertiesDataBase,
algorithm,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCacheProvider(algorithm.Portfolio));
algorithm.Securities.SetSecurityService(securityService);
var dataPermissionManager = new DataPermissionManager();
var dataManager = new DataManager(feed,
new UniverseSelection(algorithm, securityService, dataPermissionManager, new DefaultDataProvider()),
algorithm,
algorithm.TimeKeeper,
marketHoursDatabase,
false,
RegisteredSecurityDataTypesProvider.Null,
dataPermissionManager);
algorithm.SubscriptionManager.SetDataManager(dataManager);
algorithm.Initialize();
algorithm.PostInitialize();
// set exchanges to be always open
foreach (var kvp in algorithm.Securities)
{
var security = kvp.Value;
security.Exchange = new SecurityExchange(SecurityExchangeHours.AlwaysOpen(security.Exchange.TimeZone));
}
var endTimeUtc = algorithm.EndDate.ConvertToUtc(TimeZones.NewYork);
var startTimeUtc = algorithm.StartDate.ConvertToUtc(TimeZones.NewYork);
var subscriptionBasedTimeProvider = new SubscriptionFrontierTimeProvider(startTimeUtc, dataManager);
var timeSliceFactory = new TimeSliceFactory(algorithm.TimeZone);
var synchronizer = new SubscriptionSynchronizer(dataManager.UniverseSelection);
synchronizer.SetTimeProvider(subscriptionBasedTimeProvider);
synchronizer.SetTimeSliceFactory(timeSliceFactory);
var totalDataPoints = 0;
var subscriptions = dataManager.DataFeedSubscriptions;
foreach (var kvp in algorithm.Securities)
{
int dataPointCount;
subscriptions.TryAdd(CreateSubscription(algorithm, kvp.Value, startTimeUtc, endTimeUtc, out dataPointCount));
totalDataPoints += dataPointCount;
}
// log what we're doing
Console.WriteLine($"Running {subscriptions.Count()} subscriptions with a total of {totalDataPoints} data points. Start: {algorithm.StartDate:yyyy-MM-dd} End: {algorithm.EndDate:yyyy-MM-dd}");
var count = 0;
DateTime currentTime = DateTime.MaxValue;
DateTime previousValue;
var stopwatch = Stopwatch.StartNew();
var enumerator = synchronizer.Sync(subscriptions, CancellationToken.None).GetEnumerator();
do
{
previousValue = currentTime;
enumerator.MoveNext();
var timeSlice = enumerator.Current;
currentTime = timeSlice.Time;
count += timeSlice.DataPointCount;
}while (currentTime != previousValue);
stopwatch.Stop();
enumerator.DisposeSafely();
var kps = count / 1000d / stopwatch.Elapsed.TotalSeconds;
Console.WriteLine($"Current Time: {currentTime:u} Elapsed time: {(int)stopwatch.Elapsed.TotalSeconds,4}s KPS: {kps,7:.00} COUNT: {count,10}");
Assert.GreaterOrEqual(count, 100); // this assert is for sanity purpose
dataManager.RemoveAllSubscriptions();
}
public void SetUp()
{
_timeSliceFactory = new TimeSliceFactory(TimeZones.Utc);
}
/// <summary>
/// Enumerates the subscriptions into slices
/// </summary>
protected IEnumerable <Slice> CreateSliceEnumerableFromSubscriptions(List <Subscription> subscriptions, DateTimeZone sliceTimeZone)
{
// required by TimeSlice.Create, but we don't need it's behavior
var frontier = DateTime.MinValue;
var timeSliceFactory = new TimeSliceFactory(sliceTimeZone);
while (true)
{
var earlyBirdTicks = long.MaxValue;
var data = new List <DataFeedPacket>();
foreach (var subscription in subscriptions)
{
if (subscription.EndOfStream)
{
continue;
}
var packet = new DataFeedPacket(subscription.Security, subscription.Configuration);
while (subscription.Current.EmitTimeUtc <= frontier)
{
packet.Add(subscription.Current.Data);
Interlocked.Increment(ref _dataPointCount);
if (!subscription.MoveNext())
{
break;
}
}
// only add if we have data
if (packet.Count != 0)
{
data.Add(packet);
}
// udate our early bird ticks (next frontier time)
if (subscription.Current != null)
{
// take the earliest between the next piece of data or the next tz discontinuity
earlyBirdTicks = Math.Min(earlyBirdTicks, subscription.Current.EmitTimeUtc.Ticks);
}
}
// end of subscriptions
if (earlyBirdTicks == long.MaxValue)
{
break;
}
if (data.Count != 0)
{
// reuse the slice construction code from TimeSlice.Create
yield return(timeSliceFactory.Create(frontier, data, SecurityChanges.None, new Dictionary <Universe, BaseDataCollection>()).Slice);
}
frontier = new DateTime(Math.Max(earlyBirdTicks, frontier.Ticks), DateTimeKind.Utc);
}
// make sure we clean up after ourselves
foreach (var subscription in subscriptions)
{
subscription.Dispose();
}
}
