public void AddOrMerge(MarketDataUpdate update)
{
var lockTaken = false;
try
{
_spinLock.Enter(ref lockTaken);
if (_currentEvent == null)
{
Add(update);
}
else
{
Merge(update);
}
}
finally
{
if (lockTaken)
{
_spinLock.Exit();
}
}
}
public void SetMarketDataForBatching(int securityId, MarketDataUpdate update)
{
EventType = XEventType.MarketData;
update.ApplyTo(MarketDataUpdate);
EventData.MarketDataForBatching.SecurityId = securityId;
EventData.MarketDataForBatching.Update = MarketDataUpdate;
}
public void AddOrMerge(MarketDataUpdate update)
{
var lockTaken = false;
try
{
_spinLock.Enter(ref lockTaken);
if (_currentEvent == null)
{
using (var acquire = _targetEngine.AcquireEvent())
{
_currentEvent = acquire.Event;
_currentEvent.SetMarketData(_securityId, this);
}
}
update.ApplyTo(_currentEvent.MarketDataUpdate);
}
finally
{
if (lockTaken)
{
_spinLock.Exit();
}
}
}
private List <MarketDataUpdate> Algo3MergeData(MarketDataUpdate[] inputSource)
{
var result = inputSource
.Reverse()
.GroupBy(c => c.InstrumentId)
.Select(g => {
var currentResult = new MarketDataUpdate
{
InstrumentId = g.Key,
Fields = new Dictionary <byte, long>()
};
g.SelectMany(p => p.Fields.Reverse()
.Select(f => new { FieldNo = f.Key, FieldVal = f.Value }))
.DistinctBy(c => c.FieldNo)
.Reverse()
.ForEach(c => currentResult.Fields.Add(c.FieldNo, c.FieldVal));
return(currentResult);
})
.OrderBy(c => c.InstrumentId)
.ToList();
return(result);
}
public void Reset()
{
Bid = null;
Ask = null;
Last = null;
UpdateCount = 0;
Next = null;
}
private void Add(MarketDataUpdate update)
{
using (var acquire = _engine.AcquireEvent())
{
_currentEvent = acquire.Event;
_currentEvent.SetMarketData(_securityId, this, update);
}
}
public void SetMarketData(int securityId, MarketDataUpdate update)
{
EventType = XEventType.MarketData;
EventData.MarketData.SecurityId = securityId;
EventData.MarketData.BidOrZero = update.Bid.GetValueOrDefault();
EventData.MarketData.AskOrZero = update.Ask.GetValueOrDefault();
EventData.MarketData.LastOrZero = update.Last.GetValueOrDefault();
EventData.MarketData.UpdateCount = update.UpdateCount;
}
public static void ProcessQueue()
{
var pause = TimeSpan.FromMilliseconds(100);
while (true)
{
if (UpdateQueue.IsEmpty)
{
// no pending updates available pause
Thread.Sleep(pause);
continue;
}
bool tryDQ = false;
do
{
MarketDataUpdate mdUpdate = new MarketDataUpdate();
tryDQ = UpdateQueue.TryDequeue(out mdUpdate);
if (tryDQ)
{
if (mdUpdate.Nounce <= Markets[mdUpdate.MarketName].Nounce || Markets[mdUpdate.MarketName] == null)
{
break;
}
else if (mdUpdate.Nounce > Markets[mdUpdate.MarketName].Nounce + 1)
{
//IF NOUNCE IS DE-SYNCED, WIPE BOOK AND RE-SNAP
Trace.WriteLine(string.Format(" !!!!ERR>> NOUNCE OUT OF ORDER! " + mdUpdate.MarketName + " BOOK-DSYNC. {0}, {1}", Markets[mdUpdate.MarketName].Nounce, mdUpdate.Nounce));
bool removed;
do
{
Market.Market m;
removed = Markets.TryRemove(mdUpdate.MarketName, out m);
} while (!removed);
//Request MarketQuery from websocket, and OpenBook() with new snapshot
MarketQueryResponse marketQuery = BtrexWS.WSSharpTransport.HubProxy.Invoke <MarketQueryResponse>("QueryExchangeState", mdUpdate.MarketName).Result;
marketQuery.MarketName = mdUpdate.MarketName;
OpenMarket(marketQuery).Wait();
Trace.WriteLine(" [BOOK RE-SYNCED]");
break;
}
else
{
Markets[mdUpdate.MarketName].SetUpdate(mdUpdate);
}
}
} while (!tryDQ);
}
}
public void ApplyTo(MarketDataUpdate other)
{
if (Bid != null)
{
other.Bid = Bid;
}
if (Ask != null)
{
other.Ask = Ask;
}
if (Last != null)
{
other.Last = Last;
}
other.UpdateCount += UpdateCount;
}
public void CheckToString_NoEmptyComma()
{
//// Arrange
var item = new MarketDataUpdate()
{
InstrumentId = "AAA",
Fields = new Dictionary <byte, long>
{
{ 1, 10 },
{ 2, 12 }
}
};
//// Act
var result = item.ToString();
//// Assert
Assert.IsFalse(result.Contains(", ]"));
}
internal void MergeWith(MarketDataUpdate update)
{
if (update.Bid != null)
{
BidOrZero = update.Bid.GetValueOrDefault();
}
if (update.Ask != null)
{
AskOrZero = update.Ask.GetValueOrDefault();
}
if (update.Last != null)
{
LastOrZero = update.Last.GetValueOrDefault();
}
UpdateCount += update.UpdateCount;
}
public static void Start(ISubject <MarketDataUpdate> source)
{
var _randomizer = new Random();
Task.Run(() =>
{
while (true)
{
Thread.Sleep(1);
var update = new MarketDataUpdate();
update.InstrumentId = "AAPL_" + _randomizer.Next(1, 100);
update.Fields = new Dictionary <byte, long>();
for (int i = 0; i < _randomizer.Next(1, 5); i++)
{
update.Fields[(byte)_randomizer.Next(1, 20)] = _randomizer.Next(1, 10000);
}
source.OnNext(update);
}
});
}
public void CheckToString_ValidOutput()
{
//// Arrange
string expected = "Instrument: AAA, FieldsNo: 2, Fields: [1: 10, 2: 12]";
var item = new MarketDataUpdate()
{
InstrumentId = "AAA", Fields = new Dictionary <byte, long>
{
{ 1, 10 },
{ 2, 12 }
}
};
//// Act
var result = item.ToString();
//// Assert
Assert.IsTrue(result.Equals(expected), $"Wrong ToString result. \n\tExpected: \t{expected}, \n\tActual: \t{result}\n");
Console.WriteLine(expected);
Console.WriteLine(result);
}
private List <MarketDataUpdate> Algo2MergeData(MarketDataUpdate[] inputSource)
{
var grouped = inputSource.ToLookup(p => p.InstrumentId);
var result = new List <MarketDataUpdate>();
foreach (var dataGroup in grouped)
{
var currentResult = new MarketDataUpdate
{
InstrumentId = dataGroup.Key,
Fields = new Dictionary <byte, long>()
};
var current = dataGroup.SelectMany(c => c.Fields).ToList();
current.ForEach(x => currentResult.Fields[x.Key] = x.Value);
result.Add(currentResult);
}
return(result.OrderBy(c => c.InstrumentId).ToList());
}
public void ThrottledMarketDataStreamTasksMultithreadingOnUpdateSecondTest()
{
var stream = new ThrottledMarketDataStream();
var randomizer = new Random(new Random().Next(1, int.MaxValue));
var tasks = new List <Task>();
List <MarketDataUpdate> list = new List <MarketDataUpdate>();
for (int i = 0; i < 1000; i++)
{
var update = new MarketDataUpdate()
{
InstrumentId = "AAPL_" + randomizer.Next(1, 100),
Fields = new Dictionary <byte, long>(),
};
for (int j = 0; j < randomizer.Next(1, 5); j++)
{
update.Fields[(byte)randomizer.Next(1, 20)] = randomizer.Next(1, 10000);
}
list.Add(update);
}
for (int i = 0; i < list.Count - 1; i++)
{
tasks.Add(new Task(() => stream.OnUpdate(list[i])));
}
Parallel.ForEach(tasks, task =>
{
task.Start();
});
Task.WhenAll(tasks).ContinueWith(done =>
{
// this will not print untill all tasks have completed.
System.Diagnostics.Debug.WriteLine("All tasks OnUpdate finished.");
});
}
/*private static dynamic ProcessingMarketDataStream(IList<MarketDataUpdate> data)
* {
* data.Reverse();
*
* var result = data.GroupBy(c => c.InstrumentId).Select(g => new {
* InstrumentId = g.Key,
* Data = g.SelectMany(p => p.Fields.Reverse()
* .Select(f => new { FieldNo = f.Key, FieldVal = f.Value }))
* .DistinctBy(c => c.FieldNo).Reverse().ToList()
* }).OrderBy(c => c.InstrumentId).ToList();
*
* return result;
* }
* private static void PrintData(dynamic data)
* {
* foreach (var current in data)
* {
* Console.WriteLine($"InstrumentId : {current.InstrumentId}");
* foreach (var c in current.Data)
* {
*
* Console.WriteLine($"FieldNo - {c.FieldNo}, FieldVal - {c.FieldVal}");
* }
* }
* }
*/
private static IList <MarketDataUpdate> ProcessingMarketDataStream(IList <MarketDataUpdate> data)
{
var result = data.OrderBy(c => c.Timestamp).Reverse().GroupBy(c => c.InstrumentId).Select(g => {
var currentResult = new MarketDataUpdate
{
InstrumentId = g.Key,
Timestamp = DateTime.Now,
Fields = new Dictionary <byte, long>()
};
g.SelectMany(p => p.Fields.Reverse()
.Select(f => new { FieldNo = f.Key, FieldVal = f.Value }))
.DistinctBy(c => c.FieldNo)
.Reverse()
.ForEach(c => currentResult.Fields.Add(c.FieldNo, c.FieldVal));
return(currentResult);
})
.OrderBy(c => c.InstrumentId)
.ToList();
return(result);
}
public void TestPipeline()
{
var tdata1 = new MarketDataUpdate()
{
InstrumentId = "AAPL_1",
Fields = new Dictionary <byte, long>()
{
{ 1, 10 },
{ 4, 200 },
{ 12, 187 }
},
Timestamp = DateTime.Now
};
source1.Post(tdata1);
var tdata2 = new MarketDataUpdate()
{
InstrumentId = "AAPL_2",
Fields = new Dictionary <byte, long>()
{
{ 1, 12 },
{ 4, 210 }
},
Timestamp = DateTime.Now
};
source2.Post(tdata2);
var tdata3 = new MarketDataUpdate()
{
InstrumentId = "AAPL_1",
Fields = new Dictionary <byte, long>()
{
{ 12, 189 }
},
Timestamp = DateTime.Now
};
source3.Post(tdata3);
var tdata4 = new MarketDataUpdate()
{
InstrumentId = "AAPL_1",
Fields = new Dictionary <byte, long>()
{
{ 2, 24 }
},
Timestamp = DateTime.Now
};
source4.Post(tdata4);
var tdata5 = new MarketDataUpdate()
{
InstrumentId = "AAPL_2",
Fields = new Dictionary <byte, long>()
{
{ 5, 120 }
}
,
Timestamp = DateTime.Now
};
source5.Post(tdata5);
var tdata6 = new MarketDataUpdate()
{
InstrumentId = "AAPL_3",
Fields = new Dictionary <byte, long>()
{
{ 22, 121 }
}
};
source1.Post(tdata6);
var tdata7 = new MarketDataUpdate()
{
InstrumentId = "AAPL_4",
Fields = new Dictionary <byte, long>()
{
{ 15, 122 }
}
};
source2.Post(tdata7);
var tdata8 = new MarketDataUpdate()
{
InstrumentId = "AAPL_5",
Fields = new Dictionary <byte, long>()
{
{ 17, 124 }
}
};
source3.Post(tdata8);
var tdata9 = new MarketDataUpdate()
{
InstrumentId = "AAPL_6",
Fields = new Dictionary <byte, long>()
{
{ 21, 125 }
}
};
source4.Post(tdata9);
var tdata10 = new MarketDataUpdate()
{
InstrumentId = "AAPL_7",
Fields = new Dictionary <byte, long>()
{
{ 24, 127 }
}
};
source5.Post(tdata10);
source1.Complete();
source2.Complete();
source3.Complete();
source4.Complete();
source5.Complete();
writer.Completion.Wait();
Console.ReadLine();
}
public void SetUpdate(MarketDataUpdate update)
{
if (update.Buys.Count > 0)
{
foreach (mdBuy bid in update.Buys)
{
switch (bid.Type)
{
case 0:
OrderBook.Bids[bid.Rate] = bid.Quantity;
break;
case 1:
bool removed = false;
while (!removed)
{
removed = OrderBook.Bids.TryRemove(bid.Rate, out decimal q);
}
break;
case 2:
OrderBook.Bids[bid.Rate] = bid.Quantity;
break;
}
}
}
if (update.Sells.Count > 0)
{
foreach (mdSell ask in update.Sells)
{
switch (ask.Type)
{
case 0:
OrderBook.Asks[ask.Rate] = ask.Quantity;
break;
case 1:
bool removed = false;
while (!removed)
{
removed = OrderBook.Asks.TryRemove(ask.Rate, out decimal q);
}
break;
case 2:
OrderBook.Asks[ask.Rate] = ask.Quantity;
break;
}
}
}
if (update.Fills.Count > 0)
{
update.Fills.Reverse();
foreach (mdFill fill in update.Fills)
{
TradeHistory.RecentFills.Add(fill);
//Trace.WriteLine("[{5}] {0} {1} == R:{2}...V:{3}...BV:{4}", fill.TimeStamp.ToShortTimeString(), fill.OrderType, fill.Rate, fill.Quantity, (fill.Quantity * fill.Rate), MarketDelta);
}
}
Nounce++;
}
private void ProcessMarketDataFullRefresh(Message msg)
{
Debug.Assert(MsgType.Market_Data_Snapshot_Full_Refresh == msg.Type);
MarketDataUpdate update = new MarketDataUpdate();
string symbol = msg.Get(Tags.Symbol);
bool isPriceChanged = false;
Group mdDEntries = msg.GetGroup(Tags.NoMDEntries);
for (int i = 0; i < mdDEntries.NumberOfInstances; ++i)
{
string priceValue = mdDEntries.Get(Tags.MDEntryPx, i);
if (null != priceValue)
{
double price;
if (double.TryParse(priceValue, NumberStyles.Number, CultureInfo.InvariantCulture, out price))
{
double size = 0;
string sizeValue = mdDEntries.Get(Tags.MDEntrySize, i);
if (!string.IsNullOrEmpty(sizeValue))
{
double.TryParse(sizeValue, NumberStyles.Number, CultureInfo.InvariantCulture, out size);
}
string entryType = mdDEntries.Get(Tags.MDEntryType, i);
if (MDEntryType.Bid == entryType)
{
if (update.Bid != price || update.BidSize != size)
{
update.Bid = price;
update.BidSize = size;
isPriceChanged = true;
}
}
else if (MDEntryType.Offer == entryType)
{
if (update.Ask != price || update.AskSize != size)
{
update.Ask = price;
update.AskSize = size;
isPriceChanged = true;
}
}
else if (MDEntryType.Trade == entryType)
{
TradeArgs rv = new TradeArgs(symbol, price, size);
if (TradeReceived != null)
{
TradeReceived(this, rv);
}
}
}
}
}
if (isPriceChanged)
{
PriceChangeArgs rv = new PriceChangeArgs(symbol, update.Bid, update.BidSize, update.Ask, update.AskSize);
if (PriceChange != null)
{
PriceChange(this, rv);
}
}
}
private void ProcessMarketDataIncrementalRefresh(Message msg)
{
Debug.Assert(MsgType.Market_Data_Incremental_Refresh == msg.Type);
string mdReqID = msg.Get(Tags.MDReqID);
MarketDataUpdate update;
lock (lastMD)
{
if (!lastMD.TryGetValue(mdReqID, out update))
{
update = new MarketDataUpdate();
lastMD.Add(mdReqID, update);
}
}
string symbol = null;
bool isPriceChanged = false;
Group mdDEntries = msg.GetGroup(Tags.NoMDEntries);
for (int i = 0; i < mdDEntries.NumberOfInstances; ++i)
{
if (null == symbol)
{
symbol = mdDEntries.Get(Tags.Symbol, i);
}
else
{
Debug.Assert(mdDEntries.Get(Tags.Symbol, i) == symbol);
}
string priceValue = mdDEntries.Get(Tags.MDEntryPx, i);
if (null != priceValue)
{
double price;
if (double.TryParse(priceValue, NumberStyles.Number, System.Globalization.CultureInfo.InvariantCulture, out price))
{
double size = 0;
string sizeValue = mdDEntries.Get(Tags.MDEntrySize, i);
if (!string.IsNullOrEmpty(sizeValue))
{
double.TryParse(sizeValue, NumberStyles.Number, CultureInfo.InvariantCulture, out size);
}
string entryType = mdDEntries.Get(Tags.MDEntryType, i);
if (MDEntryType.Bid == entryType)
{
if (update.Bid != price)
{
update.Bid = price;
update.BidSize = size;
isPriceChanged = true;
}
}
else if (MDEntryType.Offer == entryType)
{
if (update.Ask != price)
{
update.Ask = price;
update.AskSize = size;
isPriceChanged = true;
}
}
else if (MDEntryType.Trade == entryType)
{
TradeArgs rv = new TradeArgs(symbol, price, size);
if (TradeReceived != null)
{
TradeReceived(this, rv);
}
}
}
}
}
if (isPriceChanged)
{
PriceChangeArgs rv = new PriceChangeArgs(symbol, update.Bid, update.BidSize, update.Ask, update.AskSize);
if (PriceChange != null)
{
PriceChange(this, rv);
}
}
}
private void Merge(MarketDataUpdate update)
{
_currentEvent.EventData.MarketData.MergeWith(update);
}
static void Main()
{
var marketData = new List <MarketDataUpdate> {
new MarketDataUpdate()
{
InstrumentId = "AAPL_1", Timestamp = DateTime.Now, Fields = new Dictionary <byte, long>()
{
{ 1, 10 },
{ 4, 200 },
{ 12, 187 }
}
},
new MarketDataUpdate()
{
InstrumentId = "AAPL_2", Timestamp = DateTime.Now, Fields = new Dictionary <byte, long>()
{
{ 1, 12 },
{ 4, 210 }
}
},
new MarketDataUpdate()
{
InstrumentId = "AAPL_1", Timestamp = DateTime.Now, Fields = new Dictionary <byte, long>()
{
{ 12, 189 }
}
},
new MarketDataUpdate()
{
InstrumentId = "AAPL_1", Timestamp = DateTime.Now, Fields = new Dictionary <byte, long>()
{
{ 2, 24 }
}
},
new MarketDataUpdate()
{
InstrumentId = "AAPL_2", Timestamp = DateTime.Now, Fields = new Dictionary <byte, long>()
{
{ 5, 120 }
}
}
};
var data = ProcessingMarketDataStream(marketData);
Console.WriteLine("Test task control data.");
Print(data);
Console.WriteLine("-------------------------------------------------------------------------------");
// 4 per second - 1000/4, 5 per second - 1000/5
var source = new Subject <MarketDataUpdate>();
// накапливаем данные в буфере в течении 250 ms. - агрегация.
// источников может быть несколько Concat, Merge RX.
var result = source.AsObservable().Buffer(TimeSpan.FromMilliseconds(1000 / 4));
result.Subscribe(x =>
{
Console.WriteLine($"Buffered Count - {x.Count}");
var stopWatch = Stopwatch.StartNew();
// Do work - Processing aggregated data.
var data = ProcessingMarketDataStream(x);
stopWatch.Stop();
Console.WriteLine($"Time elapsed: {stopWatch.Elapsed} TotalMilliseconds : {stopWatch.Elapsed.TotalMilliseconds.ToString("0.0###")}");
Console.WriteLine($"Agregated Count - {data.Count}");
Console.WriteLine("Current Agregated Data Result:");
Print(data);
Console.WriteLine("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!");
},
() => Console.WriteLine("Completed"));
IObservable <long> observable = Observable.Interval(TimeSpan.FromMilliseconds(1), Scheduler.Immediate);
//IObservable<long> observable = Observable.Interval(TimeSpan.FromMilliseconds(1), Scheduler.CurrentThread);
var _randomizer = new Random();
using (observable.Subscribe(x => {
var update = new MarketDataUpdate();
update.InstrumentId = "AAPL_" + _randomizer.Next(1, 4);
update.Fields = new Dictionary <byte, long>();
for (int i = 0; i < _randomizer.Next(1, 5); i++)
{
update.Fields[(byte)_randomizer.Next(1, 6)] = _randomizer.Next(1, 10000);
}
source.OnNext(update);
}))
{
Console.WriteLine("Press Enter to unsubscribe.");
Console.ReadLine();
}
//Start(source);
Console.ReadLine();
}
