public void CouldWriteToStorage()
{
var repo = new SeriesStorage(SeriesStorage.GetDefaultConnectionString("../StorageTests.db"));
var test = new SortedMap <DateTime, double>();
for (int i = 0; i < 10; i++)
{
test.Add(DateTime.UtcNow.Date.AddSeconds(i), i);
}
test.Complete();
foreach (var kvp in test.Map(x => (decimal)x))
{
Console.WriteLine($"{kvp.Key} - {kvp.Key.Kind} - {kvp.Value}");
}
var storageSeries = repo.GetPersistentOrderedMap <DateTime, decimal>("test_series_CouldWriteToStorage");
var test2 = storageSeries.ToSortedMap();
foreach (var kvp in test2)
{
Console.WriteLine($"{kvp.Key} - {kvp.Key.Kind} - {kvp.Value}");
}
storageSeries.Append(test.Map(x => (decimal)x), AppendOption.RequireEqualOverlap);
storageSeries.Flush();
}
private void AsyncMergeTick() {
bool bBusy = false;
lock (__oLock) {
bBusy = __bBusy;
if (!bBusy) {
__bBusy = true;
}
}
if (!bBusy) {
Task.Factory.StartNew(() => {
while (__cQueue.Count > 0) {
QuoteEvent cQuoteEvent = null;
lock (__cQueue) {
cQuoteEvent = __cQueue.Dequeue();
}
SeriesStorage cStorage = null;
string sLSymbolId = cQuoteEvent.Quote.SymbolId.ToLower();
lock (__cStorages) {
__cStorages.TryGetValue(sLSymbolId, out cStorage);
}
if (cStorage != null) {
cStorage.MergeTick(cQuoteEvent.Tick);
}
}
lock (__oLock) {
__bBusy = false;
}
});
}
}
/// <summary>
/// 移除商品資訊(如果 GetSeries 不是使用 useCache 模式都需要移除)
/// </summary>
/// <param name="seriesSymbolDataRand">商品資訊類別</param>
internal void RemoveSeries(SeriesSymbolDataRand seriesSymbolDataRand) {
SeriesSymbolData cSeries = seriesSymbolDataRand.Source;
SeriesStorage cStorage = null;
string sLSymbolId = cSeries.DataRequest.Symbol.ToLower();
lock (__cStorages) {
__cStorages.TryGetValue(sLSymbolId, out cStorage);
}
if (cStorage != null) {
if (cSeries.Id > 0x40000000) { //Id 編號從 0x40000001 開始編號(如果低於表示使用時間週期總秒數當作 Hash, 使用時間週期總秒數都是 Cache 資料所以不能移除)
cStorage.Remove(cSeries.Id);
}
}
}
public void CouldCreateSeriesStorage()
{
var folder = Bootstrap.Bootstrapper.Instance.DataFolder;
Console.WriteLine(folder);
var storage = SeriesStorage.GetDefault("storage_test.db");
storage.GetPersistentOrderedMap <int, int>("int_map");
Assert.Throws(typeof(ArgumentException), () => {
try {
var map = storage.GetPersistentOrderedMap <int, double>("int_map");
} catch (Exception e) {
Console.WriteLine(e.Message);
throw;
}
});
}
private static void Main2(string[] args)
{
// from Ractor.Persistence
var store = new SeriesStorage(SeriesStorage.GetDefaultConnectionString("TAQSample.db"));
var aapl = store.GetPersistentOrderedMap <DateTime, TaqTrade>("aapl").Map(t => t.TradePrice / 10000.0);
Console.WriteLine("Count: " + aapl.Count());
Console.WriteLine("Open: " + aapl.First.Value);
Console.WriteLine("High: " + aapl.Values.Max());
Console.WriteLine("Low: " + aapl.Values.Min());
Console.WriteLine("Close: " + aapl.Last.Value);
Console.WriteLine("Average price: " + aapl.Values.Average());
Console.WriteLine("Total volume: " + aapl.Values.Sum());
//https://uk.finance.yahoo.com/q/hp?s=AAPL&b=5&a=07&c=2015&e=5&d=07&f=2015&g=d
var msft = store.GetPersistentOrderedMap <DateTime, TaqTrade>("msft").Map(t => t.TradePrice / 10000.0);;
var spread = (aapl.Repeat() / msft.Repeat() - 1.0).ToSortedMap();
Console.ReadLine();
}
public void CouldAddValuesByKey()
{
var repo = new SeriesStorage(SeriesStorage.GetDefaultConnectionString("../StorageTests.db"));
var series = repo.GetPersistentOrderedMap <DateTime, decimal>("test_series_CouldAddValuesByKey");
var test2 = series.Map(x => (double)x);
series.RemoveAll();
//series.RemoveMany(DateTime.Today.AddHours(-6), Lookup.GE);
for (int i = 0; i < 10; i++)
{
series.Add(DateTime.Today.AddMinutes(i), i);
}
series.Flush();
for (int i = 10; i < 100; i++)
{
series[DateTime.Today.AddMinutes(i)] = i;
}
series.Flush();
series[DateTime.Today.AddMinutes(100)] = 100;
Console.WriteLine(test2.Last.Key + " " + test2.Last.Key);
series[DateTime.Today.AddMinutes(1000)] = 1000;
}
static unsafe void Main(string[] args)
{
GC.Collect(3, GCCollectionMode.Forced, true);
var store = new SeriesStorage(SeriesStorage.GetDefaultConnectionString("TAQSample2.db"));
var date = new DateTime(2015, 8, 5);
var tsize = Marshal.SizeOf(typeof(TaqTrade));
Console.WriteLine(tsize);
var zip = ZipFile.OpenRead(path);
var stream = zip.Entries.Single().Open();
var seriesDictionary = new Dictionary <string, IPersistentOrderedMap <DateTime, TaqTrade> >();
using (BufferedStream bs = new BufferedStream(stream, 2 * 1024 * 1024))
using (var reader = new StreamReader(bs, Encoding.ASCII)) {
byte[] compressedBuffer = null;
var byteBuffer = new byte[106];
int len;
var line = reader.ReadLine();
len = bs.ReadLineIntoBuffer(byteBuffer);
Console.WriteLine(line);
Console.WriteLine("Press enter to continue");
Console.ReadLine();
var sw = new Stopwatch();
sw.Start();
var c = 0;
while ((len = bs.ReadLineIntoBuffer(byteBuffer)) != 0) // && c < 100
{
var fb = new FixedBuffer(byteBuffer, 0, len);
var trade = new TaqTrade(date, fb);
var symbol = trade.Symbol.ToLowerInvariant().Trim();
IPersistentOrderedMap <DateTime, TaqTrade> series;
if (!seriesDictionary.TryGetValue(symbol, out series))
{
series = store.GetPersistentOrderedMap <DateTime, TaqTrade>(symbol);
seriesDictionary[symbol] = series;
}
series[trade.Time] = trade;
c++;
if (c % 100000 == 0)
{
Console.WriteLine($"Read so far: {c}");
foreach (var s in seriesDictionary)
{
s.Value.Flush();
}
}
}
sw.Stop();
foreach (var series in seriesDictionary)
{
series.Value.Flush();
}
Console.WriteLine($"Lines read: ${c} in msecs: {sw.ElapsedMilliseconds}");
}
Console.WriteLine("Finished");
GC.Collect(3, GCCollectionMode.Forced, true);
Console.WriteLine($"Total memory: {GC.GetTotalMemory(true)}");
Console.ReadLine();
}
public void CouldCRUDSeriesStorage()
{
var storage = new SeriesStorage("Filename=../benchmark.db"); // SeriesStorage.Default;
var timeseries = storage.GetPersistentOrderedMap <DateTime, double>("test_timeseries");
Console.WriteLine(storage.Connection.DataSource);
var start = DateTime.UtcNow;
Console.WriteLine($"Started at: {start}");
if (!timeseries.IsEmpty)
{
// Remove all values
timeseries.RemoveMany(timeseries.First.Key, Lookup.GE);
}
var sw = new Stopwatch();
var count = 10000000L;
Console.WriteLine($"Count: {count}");
var date = DateTime.UtcNow.Date;
var rng = new Random();
sw.Start();
for (long i = 0; i < count; i++)
{
timeseries.Add(date, Math.Round(i + rng.NextDouble(), 2));
date = date.AddTicks(rng.Next(1, 100));
if (i % 1000000 == 0)
{
var msec = (DateTime.UtcNow - start).TotalMilliseconds;
var mops = i * 0.001 / msec;
Console.WriteLine($"Wrote: {i} - {Math.Round((i * 1.0) / (count * 1.0), 4) * 100.0}% in {msec/1000} sec, Mops: {mops}");
}
}
timeseries.Flush();
Console.WriteLine($"Wrote: {count} - 100%");
Console.WriteLine($"Finished at: {DateTime.UtcNow}");
sw.Stop();
Console.WriteLine($"Writes, Mops: {count * 0.001 / sw.ElapsedMilliseconds}");
sw.Restart();
var sum = 0.0;
var storage2 = new SeriesStorage("Filename=../benchmark.db"); // $"Filename={Path.Combine(Bootstrap.Bootstrapper.Instance.DataFolder, "default.db")}");
var timeseries2 = storage2.GetPersistentOrderedMap <DateTime, double>("test_timeseries");
foreach (var kvp in timeseries2)
{
sum += kvp.Value;
}
Assert.IsTrue(sum > 0);
sw.Stop();
Console.WriteLine($"Reads, Mops: {count * 0.001 / sw.ElapsedMilliseconds}");
var _connection =
new SqliteConnection("Filename=../benchmark.db");
//$"Filename={Path.Combine(Bootstrap.Bootstrapper.Instance.DataFolder, "default.db")}");
var sqlCount = _connection.ExecuteScalar <long>($"SELECT sum(count) FROM {storage.ChunkTableName} where id = (SELECT id from {storage.IdTableName} where TextId = 'test_timeseries'); ");
Console.WriteLine($"Count in SQLite: {sqlCount}");
Assert.AreEqual(count, sqlCount);
var sqlSize = _connection.ExecuteScalar <long>($"SELECT sum(length(ChunkValue)) FROM {storage.ChunkTableName} where id = (SELECT id from {storage.IdTableName} where TextId = 'test_timeseries'); ");
Console.WriteLine($"Memory size: {count * 16L}; SQLite net blob size: {sqlSize}; comp ratio: {Math.Round(count * 16.0 / sqlSize * 1.0, 2)}");
}
private SeriesSymbolDataRand InternalGetSeries(InstrumentDataRequest dataRequest, bool useCache) {
SeriesStorage cStorage = null;
string sLSymbolId = dataRequest.Symbol.ToLower();
lock (__cStorages) {
if (!__cStorages.TryGetValue(sLSymbolId, out cStorage)) {
cStorage = new SeriesStorage(16);
__cStorages.Add(sLSymbolId, cStorage);
}
}
SeriesSymbolDataRand cSeriesRand = null;
int iTotalSeconds = dataRequest.Resolution.TotalSeconds;
if (useCache) { //是否使用快取
lock (cStorage) { //需要 lock 這個區塊(避免非同步讀取資料時發生問題)
int iBaseSeconds = (iTotalSeconds < Resolution.MAX_BASE_TOTALSECONDS) ? Resolution.MIN_BASE_TOTALSECONDS : Resolution.MAX_BASE_TOTALSECONDS;
SeriesSymbolData cSeries = cStorage.GetSeries(iTotalSeconds);
if (cSeries == null) {
cSeries = cStorage.GetSeries(iBaseSeconds);
if (cSeries == null) {
DataAdapter cAdapter = LoadAdapter(ref dataRequest);
cSeries = cAdapter.Series;
cStorage.Add(cSeries);
}
}
if (iBaseSeconds == iTotalSeconds) {
dataRequest = cSeries.DataRequest;
goto exit;
} else {
cSeries = cSeries.CreateSeries(dataRequest); //利用基礎周期建立其他的資料周期
cStorage.Add(cSeries); //加入至 SeriesStorage
}
dataRequest.Resolution = cSeries.DataRequest.Resolution; //將目標的週期結構更新至傳入的 InstrumentDataRequest 週期結構
DataRequestEvent cRequestEvent = new DataRequestEvent(dataRequest);
cSeries.OnRequest(cRequestEvent); //如果已經存在則請求使用者需要的歷史資料區間(請求方法會檢查目前已下載的歷史資料區間是否足夠, 如果使用者需要的歷史資料區間比較大會向伺服器請求)
dataRequest.Range.Count = cRequestEvent.Count; //將請求後的正確數量傳入至結構內
exit:
cSeriesRand = new SeriesSymbolDataRand(cSeries, dataRequest);
}
} else {
DataAdapter cAdapter = LoadAdapter(ref dataRequest, false); //重新建立新的基礎週期序列資料(不使用快取, 不保存至快取內, 使用完畢之後立即 Dispose)
SeriesSymbolData cSeries = cAdapter.Series; //取得新的基礎周期序列資料
int iBaseSeconds = (iTotalSeconds < Resolution.MAX_BASE_TOTALSECONDS) ? Resolution.MIN_BASE_TOTALSECONDS : Resolution.MAX_BASE_TOTALSECONDS;
if (iBaseSeconds == iTotalSeconds) {
dataRequest = cSeries.DataRequest;
} else {
SeriesSymbolData cTargetSeries = cSeries.CreateSeries(dataRequest); //使用 InstrumentDataRequest 建立新的其他週期序列資料
cSeries.Merge(cTargetSeries); //將基礎周期序列資料合併至新的其他週期序列資料
cSeries.Dispose(); //釋放基礎周期序列資料
cSeries = cTargetSeries;
}
cSeriesRand = new SeriesSymbolDataRand(cSeries, dataRequest);
cStorage.Add(cSeries, true); //保存序列資料(存放在 SeriesStorage 內的序列資料才會自動合併最新的即時資訊報價)
cAdapter.Dispose(); //釋放資料配置者類別
}
return cSeriesRand;
}
