public ServiceResult <QuoteBasicBase> GetQuoteBasic(string symbol, int interval, long stime, int num)
{
try
{
var q = new QuoteBasicBase(symbol, interval);
var q1 = _fileStore.Load(symbol, interval, stime, num);
var q2 = _qbStore.GetQuoteBasic(symbol, interval);
if (q1 != null)
{
q.Append(q1, false);
}
if (q2 != null)
{
q.Append(q2, false);
}
return(new ServiceResult <QuoteBasicBase> {
Result = true, Data = q
});
}
catch (Exception ex)
{
return(new ServiceResult <QuoteBasicBase> {
Result = false
});
}
}
public static IQuoteBasicBase LoadFile(string filename)
{
var readtream = new FileStream(filename, FileMode.Open);
var quote = QuoteBasicBase.InitByStream(readtream);
readtream.Close();
return(quote);
}
public QuoteBasicBase Load(string symbol, int interval, long?startTime, int maxCount = 500)
{
lock (this)
{
var files = this.FindQuoteFiles(symbol, interval);
if (files == null || files.Count <= 0)
{
return(null);
}
var quotes = new List <QuoteBasicBase>();
var quote = new QuoteBasicBase(symbol, interval);
files.Sort();
var count = 0;
for (int i = files.Count - 1; i >= 0; i--)
{
var q = _cache.GetItem(files[i]);
if (q == null)
{
q = QuoteBasicFileStore.LoadQuote(files[i]);
_cache.AddItem(files[i], q);
}
quotes.Add(q);
count += q.Count;
if (startTime.HasValue && q.FirstTime <= startTime)
{
break;
}
if (count >= maxCount)
{
break;
}
}
for (int i = 0; i < quotes.Count; i++)
{
quote.Append(quotes[i], false);
}
var ind1 = startTime.HasValue ? quote.FindIndexForGivenTime(startTime.Value) : 0;
var ind2 = quote.Count - maxCount;
var index = Math.Max(0, Math.Max(ind1, ind2));
if (index != 0)
{
return(quote.Extract(index, quote.Count - 1) as QuoteBasicBase);
}
else
{
return(quote);
}
}
}
private IQuoteBasicBase CreateQuote(long starttime, int num)
{
var q = new QuoteBasicBase(_symbol, _interval);
for (int j = 0; j < num; j++)
{
q.AddUpdate(starttime + j * _interval, 0, 0, 0, 0, 0);
}
return(q);
}
public void AddCandle(List <OHLC> ohlc, bool isAddToAll, bool isTriggerEvent)
{
if (ohlc == null || ohlc.Count == 0)
{
return;
}
var q = new QuoteBasicBase(ohlc);
this.AddQuoteBasic(q, isAddToAll, isTriggerEvent);
}
QuoteBasicBase CreateQuoteBasic(string symbol, int interval, long stime, int num)
{
var q = new QuoteBasicBase(symbol, interval);
var count = 0;
while (count < num)
{
q.AddUpdate(stime + interval * count++, 1, 1, 1, 1, 1, false);
}
return(q);
}
private string CreateQuote(string symbol, int interval)
{
var quoteId = this.GetQuoteId(symbol, interval);
if (!Quotes.ContainsKey(quoteId))
{
var q = new QuoteBasicBase(symbol, interval);
Quotes.TryAdd(quoteId, q as QuoteBasicBase);
Quotes[quoteId].OnDataAddedOrUpdated += QuoteStore_OnDataAddedOrUpdated; //trigger add to other intervals
}
return(quoteId);
}
public static QuoteBasicBase LoadQuote(string filename)
{
try
{
Console.WriteLine($"QuoteBasicFileStore: load... {filename}");
var readtream = new FileStream(filename, FileMode.Open);
var q1 = QuoteBasicBase.InitByStream(readtream);
readtream.Close();
return(q1);
}
catch (FileFormatNotSupportedException ex)
{
Console.WriteLine("remove last line from file");
var lines = File.ReadAllLines(filename);
lines = lines.Take(lines.Length - 1).ToArray();
File.WriteAllLines(filename, lines);
throw ex;
}
}
public void TestUpdateHistoricalDataWithExistingQuoteFilesButTimeDiffTooSmall()
{
//create quote file and save to folder
var filename = this.GetQuoteFileName(_symbol, _interval, 0);
var q = new QuoteBasicBase(_symbol, _interval);
var ts = _timenow - 1000 * _interval;
for (int j = 0; j < 1000; j++)
{
q.AddUpdate(ts + j * _interval, 0, 0, 0, 0, 0);
}
q.Time[q.Count - 1] = DateTime.UtcNow.GetUnixTimeFromUTC();
q.SaveToFile(filename);
//time interval too small => no quote updated
var r = _hist.Update(_moqDownload.Object, _symbol, _interval).Result;
Assert.IsFalse(r);
}
public void Initialize()
{
if (Directory.Exists(_folder))
{
Directory.Delete(_folder, true);
}
Directory.CreateDirectory(_folder);
_timenow = DateTime.UtcNow.GetUnixTimeFromUTC() / _interval * _interval;
_moqDownload.Setup(d => d.Exchange).Returns(_exchange);
_moqDownload.Setup(d => d.Download(It.IsAny <string>(), It.IsAny <int>(), It.IsAny <int>())).Returns <string, int, int>((s, i, t) =>
{
//if (_count == 0)
// _lasttime = _timenow - _startInterval * _interval;
if (_count < 0)
{
return(Task.FromResult <IQuoteBasicBase>(null));
}
else
{
++_count;
//create quote file and save to folder
var q1 = new QuoteBasicBase(_symbol, _interval);
var ts = _lasttime + _interval;
for (int j = 0; j < 700; j++)
{
q1.AddUpdate(ts + j * _interval, 0, 0, 0, 0, 0);
}
_lasttime = q1.LastTime;
return(Task.FromResult <IQuoteBasicBase>(q1));
}
});
_hist = new QuoteBasicFileStore(_exchange, _folder, _numBarsInFile);
}
public void TestUpdateHistoricalDataWithExistingQuoteFilesDownloadSuccess()
{
//create quote file and save to folder
var filename = this.GetQuoteFileName(_symbol, _interval, 0);
var q = new QuoteBasicBase(_symbol, _interval) as IQuoteBasicBase;
var ts1 = _timenow - 10000 * _interval;
for (int j = 0; j < 700; j++)
{
q.AddUpdate(ts1 + j * _interval, 0, 0, 0, 0, 0);
}
q.SaveToFile(filename);
_lasttime = q.LastTime + _interval;
_hist = new QuoteBasicFileStore(_exchange, _folder, _numBarsInFile);
var fn = string.Empty;
_hist.OnQuoteSaved += (object sender, string exch, string file) => fn = file;
//append to the existing quote file
var r = _hist.Update(_moqDownload.Object, _symbol, _interval).Result;
Assert.IsTrue(r);
Assert.IsTrue(fn == this.GetQuoteFileName(_symbol, _interval, 0));
q = q.LoadFile(fn);
Assert.IsTrue(q.Count == 1400);
//since the number is larger than maxNumbars, a new file is created
r = _hist.Update(_moqDownload.Object, _symbol, _interval).Result;
Assert.IsTrue(r);
Assert.IsTrue(fn == this.GetQuoteFileName(_symbol, _interval, 1));
q = q.LoadFile(fn);
Assert.IsTrue(q.Count == 700);
r = _hist.Update(_moqDownload.Object, _symbol, _interval).Result;
Assert.IsTrue(r);
Assert.IsTrue(fn == this.GetQuoteFileName(_symbol, _interval, 1));
q = q.LoadFile(fn);
Assert.IsTrue(q.Count == 1400);
}
//file naming convention: exchange_symbol_interval_index.txt
public async Task <bool> Update(IQuoteBasicDownloader download, string symbol, int interval, int timeout = 50000)
{
//lock (this)
{
var files = this.FindQuoteFiles(symbol, interval);
if (files != null && files.Count > 0)
{
files.Sort();
var readtream = new FileStream(files.Last(), FileMode.Open);
var q1 = QuoteBasicBase.InitByStream(readtream);
readtream.Close();
//if time difference is too small => skip download
var utcNow = DateTime.UtcNow.GetUnixTimeFromUTC();
if (utcNow - q1.LastTime <= interval)
{
return(false);
}
var q = await download.Download(symbol, interval, timeout);
if (q == null || q.Count == 0)
{
return(false);
}
if (q1.Count < _maxNumBarsInFile)
{
q1.Append(q, false);
var writestream = new FileStream(files.Last(), FileMode.Truncate);
q1.AppendStream(writestream);
writestream.Close();
OnQuoteSaved?.Invoke(this, this.Exchange, files.Last());
}
else
{
var index = int.Parse(files.Last().Split('_')[4].Split('.')[0]);
var fn = this.GetQuoteFileName(symbol, interval, index + 1);
var writestream = new FileStream(fn, FileMode.OpenOrCreate);
q.AppendStream(writestream);
writestream.Close();
OnQuoteSaved?.Invoke(this, this.Exchange, fn);
}
}
else
{
var q = await download.Download(symbol, interval, timeout);
if (q == null || q.Count == 0)
{
return(false);
}
var fn = this.GetQuoteFileName(symbol, interval, 0);
var writestream = new FileStream(fn, FileMode.OpenOrCreate);
q.AppendStream(writestream);
writestream.Close();
OnQuoteSaved?.Invoke(this, this.Exchange, fn);
}
return(true);
}
}
//file naming convention: exchange_symbol_interval_index.txt
public bool Save(IQuoteBasicBase quote, int numBarsToRemoveGap = -1)
{
lock (this)
{
if (quote == null || quote.Count == 0)
{
return(false);
}
var files = this.FindQuoteFiles(quote.Symbol, quote.Interval);
if (files != null && files.Count > 0)
{
files.Sort();
var q1 = QuoteBasicExension.LoadFile(files.Last());
if (numBarsToRemoveGap > 0)
{
for (int i = Math.Max(0, q1.Count - numBarsToRemoveGap); i < q1.Count - 1; i++)
{
if (q1.Time[i + 1] - q1.Time[i] > q1.Interval)
{
q1.Clear(i + 1, q1.Count - 1);
break;
}
}
}
//var readtream = new FileStream(files.Last(), FileMode.Open);
//var q1 = QuoteBasicBase.InitByStream(readtream);
//readtream.Close();
if (quote.LastTime <= q1.LastTime)
{
return(false);
}
if (q1.Count < _maxNumBarsInFile)
{
var fn = files.Last();
q1.Append(quote);
q1.SaveToFile(fn);
//var writestream = new FileStream(files.Last(), FileMode.Truncate);
//q1.AppendStream(writestream);
//writestream.Close();
OnQuoteSaved?.Invoke(this, this.Exchange, fn);
}
else
{
var last = files.Last().Split('_').Last();
var index = int.Parse(last.Split('.')[0]);
var fn = this.GetQuoteFileName(quote.Symbol, quote.Interval, index + 1);
//var writestream = new FileStream(fn, FileMode.OpenOrCreate);
if (q1.LastTime >= quote.FirstTime) //remove redundant sticks from new files
{
var sind = quote.FindIndexForGivenTime(q1.LastTime);
var q2 = new QuoteBasicBase(quote);
q2.Clear(0, sind);
q2.SaveToFile(fn);
//q2.AppendStream(writestream);
}
else
{
quote.SaveToFile(fn);
//quote.AppendStream(writestream);
}
//writestream.Close();
OnQuoteSaved?.Invoke(this, this.Exchange, fn);
}
}
else
{
var fn = this.GetQuoteFileName(quote.Symbol, quote.Interval, 0);
quote.SaveToFile(fn);
//var writestream = new FileStream(fn, FileMode.OpenOrCreate);
//quote.AppendStream(writestream);
//writestream.Close();
OnQuoteSaved?.Invoke(this, this.Exchange, fn);
}
return(true);
}
}
private void InitQuoteBasic2(string symbol)
{
if (_symbolsInitialized.Contains(symbol))
{
return;
}
Console.WriteLine($"initializing {symbol}...");
var minInterval = 60;
var interval = minInterval;
var q = _fileStore.Load(symbol, interval, null, _limit);
if (q != null)
{
for (int i = q.Count - 1; i >= Math.Max(1, q.Count - _numBarsFillGap); i--)
{
if (q.Time[i] - q.Time[i - 1] > q.Interval)
{
q.Clear(i, q.Count - 1);
break;
}
}
_qbStore.AddQuoteBasic(q, false, false);
}
var missingNum = q == null ? _limit : (int)CFacility.Clip((DateTime.UtcNow.GetUnixTimeFromUTC() - q.LastTime) / interval + 10, 0, _limit);
if (missingNum > 0)
{
var retry = 0;
while (retry++ < 2)
{
var r = _download.Download(symbol, interval, missingNum);
if (r.Result)
{
var qb = r.Data;
_qbStore.AddQuoteBasic(qb, false, true);
_fileStore.Save(qb, _numBarsFillGap);
break;
}
Console.WriteLine($"\ntried count: {retry}, waiting to retry download {symbol} {interval}");
Thread.Sleep(1000);
}
}
//////////////////////////////////////////////////////////////////////////
/// initialize other intervals without making request to server
q = _qbStore.GetQuoteBasic(symbol, interval);
if (q != null && q.Count > 0)
{
_quoteIdInitialized.Add($"{symbol}_{interval}");
var q60 = new QuoteBasicBase(symbol, interval);
var q2 = _fileStore.Load(symbol, interval, null, 3000);
if (q2 != null)
{
q60.Append(q2);
}
q60.Append(q);
foreach (var intv in _qbStore.Intervals)
{
if (intv <= interval || intv % interval != 0)
{
continue;
}
var qb = new QuoteBasicBase(symbol, intv);
var q1 = _fileStore.Load(symbol, intv, null, 500);
if (q1 != null)
{
qb.Append(q1);
}
qb.Append(q60);
_qbStore.AddQuoteBasic(qb, false, true);
_fileStore.Save(qb, _numBarsFillGap);
_quoteIdInitialized.Add($"{symbol}_{intv}");
}
}
_symbolsInitialized.Add(symbol);
Console.WriteLine($"{symbol} initialized");
}
