public async Task <bool> Initialize(IStreamPersister persister, CancellationToken cancellationToken)
{
_persister = persister;
bool ret = false;
if (_persister != null)
{
ret = await _persister.Connect();
}
bool persistSynchronously = false;
#if false
_streamEntityPersisters[(int)StreamEntityType.Submarket] = new StreamEntityPersister(StreamEntityType.Submarket, persister, cancellationToken, 1, persistSynchronously, 50, 100);
_streamEntityPersisters[(int)StreamEntityType.Instrument] = new StreamEntityPersister(StreamEntityType.Instrument, persister, cancellationToken, 1, persistSynchronously, 50, 100);
_streamEntityPersisters[(int)StreamEntityType.Portfolio] = new StreamEntityPersister(StreamEntityType.Portfolio, persister, cancellationToken, 1, persistSynchronously, 50, 100);
_streamEntityPersisters[(int)StreamEntityType.Position] = new StreamEntityPersister(StreamEntityType.Position, persister, cancellationToken, 1, persistSynchronously, 50, 100);
_streamEntityPersisters[(int)StreamEntityType.Order] = new StreamEntityPersister(StreamEntityType.Order, persister, cancellationToken, 1, persistSynchronously, 50, 100);
_streamEntityPersisters[(int)StreamEntityType.Rule] = new StreamEntityPersister(StreamEntityType.Rule, persister, cancellationToken, 1, persistSynchronously, 50, 100);
#endif
_streamEntityPersisters[(int)StreamEntityType.Price] = new StreamEntityPersister(StreamEntityType.Price, persister, cancellationToken, 250, persistSynchronously, 50, 100);
#if false
_streamEntityPersisters[(int)StreamEntityType.Trade] = new StreamEntityPersister(StreamEntityType.Rule, persister, cancellationToken, 1, persistSynchronously, 50, 100);
#endif
return(ret);
}
public async Task RunSimulationAsync(CancellationToken cancellationToken, int startMarketNo = 0)
{
SimulationReceiver simulationReceiver = new SimulationReceiver(_dataLayer);
int numThreads = 8;
int numSubmarketsPerMarket = 4;
int numInstrumentsPerMarket = 1000;
int numPortfolios = 1000;
int maxPositionsPerPortfolio = 50;
int maxRulesPerPortfolio = 20;
int numPriceUpdatesPerSecond = 0; // this is per market (or thread). 0 means as fast as possible
_simulationLayer = new SimulationLayer(simulationReceiver);
// Persister Factory
IStreamPersisterFactory persisterFactory = new SqlStreamPersisterFactory();
//IStreamPersisterFactory persisterFactory = new CosmosDbStreamPersisterFactory();
//IStreamPersisterFactory persisterFactory = new RealtimePersister.Redis.StreamPersisterFactory("pb-syncweek-redis.redis.cache.windows.net:6380,password=IG1aBMjxzo0uE106LJT+Ceigc1AZldzwd9HYDDKIdBc=,ssl=True,abortConnect=False");
IStreamPersister persister = null;
if (persisterFactory != null)
{
persister = persisterFactory.CreatePersister("DataIngester");
}
// initialize the persistence layer
var ret = await _persistenceLayer.Initialize(persister, cancellationToken);
// initialize the data layer
await _dataLayer.Initialize(cancellationToken, _persistenceLayer);
for (int marketNo = startMarketNo; marketNo < (startMarketNo + numThreads); marketNo++)
{
await _simulationLayer.GenerateData(marketNo, numSubmarketsPerMarket, numInstrumentsPerMarket,
(marketNo == 0 ? numPortfolios : 0), (marketNo == 0 ? maxPositionsPerPortfolio : 0), (marketNo == 0 ? maxRulesPerPortfolio : 0));
}
var marketTasks = new Task[numThreads];
for (int marketNo = startMarketNo; marketNo < (startMarketNo + numThreads); marketNo++)
{
var marketNoCopy = marketNo;
marketTasks[marketNoCopy - startMarketNo] = Task.Run(async() =>
{
});
}
await Task.WhenAll(marketTasks);
for (int marketNo = 0; marketNo < numThreads; marketNo++)
{
var marketNoCopy = marketNo;
marketTasks[marketNoCopy] = Task.Run(async() =>
{
await _simulationLayer.SimulatePrices(cancellationToken, marketNoCopy, numPriceUpdatesPerSecond, persister);
});
}
await Task.WhenAll(marketTasks);
}
public SystemController(IStreamPersister streamPersister)
{
if (streamPersister == null)
{
throw new ArgumentNullException("streamPersister");
}
this.streamPersister = streamPersister;
}
public SystemController(IStreamPersister streamPersister)
{
if (streamPersister == null)
{
throw new ArgumentNullException("streamPersister");
}
this.streamPersister = streamPersister;
}
public StreamEntityPersisterPartitionAggregateQueueInMemory(IStreamPersister persister, StreamEntityType entityType, int partitionKey, int holdOffBusy = 0, int holdOffIdle = 100)
: base(persister, entityType, partitionKey, holdOffBusy, holdOffIdle)
{
var reportingRunning = Interlocked.Exchange(ref _reportingRunning, 1);
if (reportingRunning == 0)
{
ReportMetrics();
}
}
public IStreamPersister CreatePersister(string database)
{
if (_persister == null)
{
var uri = ConfigurationManager.AppSettings["EndpointUrl"];
var key = ConfigurationManager.AppSettings["PrimaryKey"];
_persister = new CosmosDbStreamPersister(uri, key, database, "syncweek", 1000_000);
}
return(_persister);
}
public StreamEntityPersister(StreamEntityType entityType, IStreamPersister persister, CancellationToken cancellationToken, int numPartitions, bool direct = false, int holdOffBusy = 0, int holdOffIdle = 100)
{
_numPartitions = numPartitions;
_partitions = new StreamEntityPersisterPartition[Math.Max(numPartitions, 1)];
for (int partitionKey = 0; partitionKey < _partitions.Count(); partitionKey++)
{
if (direct)
{
_partitions[partitionKey] = new StreamEntityPersisterPartitionDirect(persister, entityType, partitionKey, 100);
}
else
{
var persisterPartition = new StreamEntityPersisterPartitionAggregateQueueInMemory(persister, entityType, partitionKey, holdOffBusy, holdOffIdle);
_partitions[partitionKey] = persisterPartition;
persisterPartition.ProcessPendingItems(cancellationToken);
}
}
}
public Operation(IStreamProvider streamProvider, IStreamTransformer transformer, IStreamPersister persister)
{
if (streamProvider == null)
{
throw new ArgumentNullException(nameof(streamProvider));
}
if (transformer == null)
{
throw new ArgumentNullException(nameof(transformer));
}
if (persister == null)
{
throw new ArgumentNullException(nameof(persister));
}
_streamProvider = streamProvider;
_transformer = transformer;
_persister = persister;
}
public StreamEntityPersisterPartitionQueue(IStreamPersister persister, StreamEntityType entityType, int partitionKey, int holdOffBusy, int holdOffIdle)
: base(persister, entityType, partitionKey)
{
_holdOffBusy = holdOffBusy;
_holdOffIdle = holdOffIdle;
}
public StreamEntityPersisterPartition(IStreamPersister persister, StreamEntityType entityType, int partitionKey)
{
_persister = persister;
_entityType = entityType;
_partitionKey = partitionKey;
}
public Task SimulatePrices(CancellationToken cancellationToken, int marketNo, int priceUpdatesPerSecond, IStreamPersister persister)
{
return(Task.Run(async() =>
{
var rand = new Random();
int numSlotsPerSecond = (priceUpdatesPerSecond > 1000 ? 40 : 20) + 1;
int priceUpdatesPerSlot = (priceUpdatesPerSecond / numSlotsPerSecond) + 1;
// generate a flat list of submarkets to generate prices for
List <SubmarketInfo> submarkets = new List <SubmarketInfo>();
foreach (var market in MarketsById.Values)
{
int marketNoFromMarket = Int32.Parse(market.Name.Substring(6));
if (marketNoFromMarket == marketNo)
{
foreach (var submarket in market.SubmarketsById.Values)
{
int submarketNo = Int32.Parse(submarket.Name.Substring(11));
submarkets.Add(new SubmarketInfo()
{
MarketNo = marketNo,
SubmarketNo = submarketNo,
Submarket = submarket
});
}
}
}
// this loop repeats every second
do
{
int numSubmarkets = submarkets.Count;
int submarketNo = 0;
var swSecond = new Stopwatch();
swSecond.Start();
for (int currentSlotInSecond = 0; currentSlotInSecond < numSlotsPerSecond; currentSlotInSecond++)
{
var swSlot = new Stopwatch();
swSlot.Start();
for (int numUpdatesInSlot = 0; numUpdatesInSlot < priceUpdatesPerSlot; numUpdatesInSlot++)
{
var submarketInfo = submarkets[submarketNo];
var instrument = GetRandomInstrument(rand, submarketInfo.MarketNo, submarketInfo.SubmarketNo, submarketInfo.Submarket);
instrument.PriceLatest += (rand.Next(10) > 4 ? 0.05 : -0.05);
instrument.PriceDate = DateTime.UtcNow;
if (_simulationReceiver != null)
{
await _simulationReceiver.PriceUpdated(instrument.ToPriceStream(StreamOperation.Update));
}
ReportSend();
submarketNo++;
if (submarketNo >= numSubmarkets)
{
submarketNo = 0;
}
}
swSlot.Stop();
if (priceUpdatesPerSecond > 0)
{
int timeLeftBeforeNextSlot = (1000 / numSlotsPerSecond) - (int)swSlot.ElapsedMilliseconds - 2 /* need som extra time I think */;
if (timeLeftBeforeNextSlot > 0)
{
Thread.Sleep(timeLeftBeforeNextSlot);
}
}
}
swSecond.Stop();
if (priceUpdatesPerSecond > 0)
{
int timeLeftBeforeNextSecond = 1000 - (int)swSecond.ElapsedMilliseconds - 10 /* need some extra time I think */;
if (timeLeftBeforeNextSecond > 0)
{
Thread.Sleep(timeLeftBeforeNextSecond);
}
}
} while (!cancellationToken.IsCancellationRequested);
}));
}
public StreamEntityPersisterPartitionDirect(IStreamPersister persister, StreamEntityType entityType, int partitionKey, int numItemsPerBatch = 0)
: base(persister, entityType, partitionKey)
{
_numItemsPerBatch = numItemsPerBatch;
}
