public void SendHistoricalDataRequest(HistoricalDataRequest request)
{
WebRequest webRequest = WebRequest.Create(string.Format(RbFKKlTxInQUoAZCSj.q6GyF96n8(96), (object)RbFKKlTxInQUoAZCSj.q6GyF96n8(268), (object)(request.Instrument as Instrument).GetSymbol(this.Name), (object)(request.BeginDate.Month - 1), (object)request.BeginDate.Day, (object)request.BeginDate.Year, (object)(request.EndDate.Month - 1), (object)request.EndDate.Day, (object)request.EndDate.Year));
this.EjDBtJJBo.Add(request.RequestId, webRequest);
webRequest.BeginGetResponse(new AsyncCallback(this.LEn1aOXgs), (object)request);
}
private void iWVkkmJrHP([In] object obj0)
{
this.zw8lbdiSvG = false;
ThreadPool.QueueUserWorkItem(new WaitCallback(this.IjmklsnVLm));
List <HistoricalDataRequest> list = new List <HistoricalDataRequest>((IEnumerable <HistoricalDataRequest>) this.NLklDLbERp.Values);
int num = (int)obj0;
int index = 0;
while (true)
{
while (this.hRhl1skFVG.Count == num)
{
Thread.Sleep(1);
}
if (index < this.NLklDLbERp.Count)
{
HistoricalDataRequest historicalDataRequest = list[index];
++index;
this.hRhl1skFVG.Add(historicalDataRequest.RequestId, historicalDataRequest);
WaitCallback callBack = (WaitCallback)(obj1 => this.tlNlxfyMLv.SendHistoricalDataRequest((HistoricalDataRequest)obj0));
this.zYhk9pJVyB(this.V43ldlVmhs[historicalDataRequest.RequestId], nB29ckVlruqARiYysQ.Downloading);
ThreadPool.QueueUserWorkItem(callBack, (object)historicalDataRequest);
}
else
{
break;
}
}
while (this.hRhl1skFVG.Count > 0)
{
Thread.Sleep(1);
}
this.zw8lbdiSvG = true;
this.CgEkTZcADa();
}
/// <summary>
/// Sends off a historical data reques to the datasource that needs to fulfill it
/// </summary>
private void ForwardHistoricalRequest(HistoricalDataRequest request)
{
string timezone = request.Instrument.Exchange == null ? "UTC" : request.Instrument.Exchange.Timezone;
var exchangeTZ = TimeZoneInfo.FindSystemTimeZoneById(timezone);
//limit the ending date to the present
var now = TimeZoneInfo.ConvertTime(DateTime.Now, TimeZoneInfo.Local, exchangeTZ);
DateTime endDate = request.EndingDate > now ? now : request.EndingDate;
request.EndingDate = endDate;
//make sure this doesn't result in endDate < startDate
request.StartingDate = request.StartingDate > request.EndingDate
? request.EndingDate.AddMinutes(-1)
: request.StartingDate;
if (request.Instrument.IsContinuousFuture)
{
DataSources["ContinuousFuturesBroker"].RequestHistoricalData(request);
}
else
{
DataSources[request.Instrument.Datasource.Name].RequestHistoricalData(request);
}
}
/// <summary>
/// Processes incoming historical data requests.
/// </summary>
public void RequestHistoricalData(HistoricalDataRequest request)
{
//assign an ID to the request
request.AssignedID = GetUniqueRequestID();
_logger.Info(
"Received request: {6} {0} @ {1} from {2} to {3} Location: {4} {5:;;SaveToLocal}",
request.Instrument.Symbol,
Enum.GetName(typeof(BarSize), request.Frequency),
request.StartingDate,
request.EndingDate,
request.DataLocation,
request.SaveDataToStorage ? 0 : 1,
request.Instrument.Datasource.Name);
//make sure data source is present and available
try
{
CheckDataSource(request); //todo if requesting local data, ds does not need to be connected
}
catch (Exception ex)
{
Log(LogLevel.Error, string.Format("Could not fulfill request ID {0}, error: {1}", request.AssignedID, ex.Message));
throw;
}
_originalRequests.TryAdd(request.AssignedID, request);
PendingRequests.TryAdd(request);
_requestQueue.Enqueue(request);
}
public void DataArrivedEventIsRaisedWhenDataSourceReturnsData()
{
bool eventRaised = false;
_broker.HistoricalDataArrived += (sender, e) => eventRaised = true;
var request = new HistoricalDataRequest(_instrument, BarSize.OneDay, new DateTime(2012, 1, 1), new DateTime(2013, 1, 1),
dataLocation: DataLocation.ExternalOnly,
saveToLocalStorage: false,
rthOnly: true);
var data = new List <OHLCBar>
{
new OHLCBar {
Open = 1, High = 2, Low = 3, Close = 4, DT = new DateTime(2000, 1, 1)
}
};
//we need to set up a callback with the request after it has had an AssignedID assigned to it.
HistoricalDataRequest newRequest = new HistoricalDataRequest();
_dataSourceMock
.Setup(x => x.RequestHistoricalData(It.IsAny <HistoricalDataRequest>()))
.Callback <HistoricalDataRequest>(req => newRequest = req);
_broker.RequestHistoricalData(request);
Thread.Sleep(50);
_dataSourceMock.Raise(x => x.HistoricalDataArrived += null, new HistoricalDataEventArgs(newRequest, data));
Assert.IsTrue(eventRaised);
}
public void LocalStorageOnlyFlagIsObeyed()
{
var request = new HistoricalDataRequest(_instrument, BarSize.OneDay, new DateTime(2012, 1, 1), new DateTime(2013, 1, 1),
dataLocation: DataLocation.LocalOnly,
saveToLocalStorage: false,
rthOnly: true);
_broker.RequestHistoricalData(request);
Thread.Sleep(50);
_dataSourceMock.Verify(x => x.RequestHistoricalData(It.IsAny <HistoricalDataRequest>()), Times.Never);
_localStorageMock.Verify(x => x.RequestHistoricalData(
It.Is <HistoricalDataRequest>(
i =>
i.Instrument.ID == 1 &&
i.Frequency == BarSize.OneDay &&
i.StartingDate.Year == 2012 &&
i.StartingDate.Month == 1 &&
i.StartingDate.Day == 1 &&
i.EndingDate.Year == 2013 &&
i.EndingDate.Month == 1 &&
i.EndingDate.Day == 1 &&
i.DataLocation == DataLocation.LocalOnly &&
i.SaveDataToStorage == false &&
i.RTHOnly == true)));
}
//for IHistoricalDataSource, we just grab data and send it back with the event
public void RequestHistoricalData(HistoricalDataRequest request)
{
var data = GetData(request.Instrument, request.StartingDate, request.EndingDate, request.Frequency);
//TODO: multithread this
RaiseEvent(HistoricalDataArrived, this, new HistoricalDataEventArgs(request, data));
}
public void HistoricalDataRequestsAreForwardedToTheCorrectDataSource()
{
var request = new HistoricalDataRequest(_instrument, BarSize.OneDay, new DateTime(2012, 1, 1), new DateTime(2013, 1, 1),
dataLocation: DataLocation.ExternalOnly,
saveToLocalStorage: false,
rthOnly: true);
_broker.RequestHistoricalData(request);
Thread.Sleep(50);
_dataSourceMock.Verify(x => x.RequestHistoricalData(
It.Is <HistoricalDataRequest>(
i =>
i.Instrument.ID == 1 &&
i.Frequency == BarSize.OneDay &&
i.StartingDate.Year == 2012 &&
i.StartingDate.Month == 1 &&
i.StartingDate.Day == 1 &&
i.EndingDate.Year == 2013 &&
i.EndingDate.Month == 1 &&
i.EndingDate.Day == 1 &&
i.DataLocation == DataLocation.ExternalOnly &&
i.SaveDataToStorage == false &&
i.RTHOnly == true)), Times.Once);
}
public void HistoricalRequestsAreCorrectlyForwardedToTheIBClient()
{
var exchange = new Exchange {
ID = 1, Name = "Ex", Timezone = "Pacific Standard Time"
};
var req = new HistoricalDataRequest
{
Instrument = new Instrument {
ID = 1, Symbol = "SPY", Exchange = exchange
},
Frequency = QDMS.BarSize.OneDay,
StartingDate = new DateTime(2014, 1, 14),
EndingDate = new DateTime(2014, 1, 15),
RTHOnly = true
};
_ibDatasource.RequestHistoricalData(req);
_ibClientMock.Verify(x => x.RequestHistoricalData(
It.IsAny <int>(),
It.IsAny <Contract>(),
It.IsAny <string>(),
It.IsAny <string>(),
It.IsAny <QDMSIBClient.BarSize>(),
It.IsAny <HistoricalDataType>(),
It.IsAny <bool>(),
It.IsAny <bool>(),
It.IsAny <List <TagValue> >()),
Times.Once);
}
public void SendHistoricalDataRequest(HistoricalDataRequest request)
{
Instrument inst = request.Instrument as Instrument;
string altSymbol = inst.GetSymbol(Name);
string altExchange = inst.GetSecurityExchange(Name);
string market;
if (!_dictCode2Market.TryGetValue(altSymbol, out market))
{
EmitHistoricalDataError(request, "找不到此合约！");
return;
}
switch (request.DataType)
{
case HistoricalDataType.Trade:
case HistoricalDataType.Quote:
SendHistoricalDataRequestTick(request, market);
return;
case HistoricalDataType.Bar:
case HistoricalDataType.Daily:
SendHistoricalDataRequestBar(request, market);
return;
case HistoricalDataType.MarketDepth:
default:
break;
}
EmitHistoricalDataError(request, "不支持的请求类型！");
}
public void WhenDataSourceSymbolIsSetThatIsTheValueSentInTheHistoricalRequest()
{
var exchange = new Exchange {
ID = 1, Name = "Ex", Timezone = "Pacific Standard Time"
};
var req = new HistoricalDataRequest
{
Instrument = new Instrument {
ID = 1, Symbol = "SPY", UnderlyingSymbol = "SPY", DatasourceSymbol = "TestMe!", Exchange = exchange, Currency = "USD", Type = InstrumentType.Stock
},
Frequency = QDMS.BarSize.OneDay,
StartingDate = new DateTime(2014, 1, 14),
EndingDate = new DateTime(2014, 1, 15),
RTHOnly = true
};
_ibDatasource.RequestHistoricalData(req);
_ibClientMock.Verify(x => x.RequestHistoricalData(
It.IsAny <int>(),
It.Is <Contract>(y => y.Symbol == "TestMe!"),
It.IsAny <string>(),
It.IsAny <string>(),
It.IsAny <BarSize>(),
It.IsAny <HistoricalDataType>(),
It.IsAny <bool>(),
It.IsAny <bool>(),
It.IsAny <List <TagValue> >()));
}
/// <summary>
/// Splits a historical data request into multiple pieces so that they obey the request limits
/// </summary>
private List <HistoricalDataRequest> SplitRequest(HistoricalDataRequest request)
{
var requests = new List <HistoricalDataRequest>();
//start at the end, and work backward in increments
int step = 495; //max bars returned per request
var freqInterval = request.Frequency.ToTimeSpan().TotalSeconds *step;
DateTime currentDate = request.EndingDate;
while (currentDate > request.StartingDate)
{
var newReq = (HistoricalDataRequest)request.Clone();
newReq.EndingDate = currentDate;
newReq.StartingDate = newReq.EndingDate.AddSeconds(-freqInterval);
if (newReq.StartingDate < request.StartingDate)
{
newReq.StartingDate = request.StartingDate;
}
currentDate = newReq.StartingDate;
requests.Add(newReq);
}
requests.Reverse();
return(requests);
}
/// <summary>
/// Simple check for any data abnormalities
/// </summary>
private void CheckDataForOutliers(HistoricalDataRequest request)
{
if (!_settings.Outliers)
{
return;
}
double abnormalLimit = 0.20;
double abnormalStDevRange = 3;
var inst = request.Instrument;
var freq = request.Frequency;
//Grab the data plus some slightly older for comparison
List <OHLCBar> data = _localStorage.GetData(
inst,
request.StartingDate.Add(-TimeSpan.FromMinutes(freq.ToTimeSpan().Minutes * 5)),
request.EndingDate,
freq);
if (data == null || data.Count <= 1)
{
return;
}
//count how many bars are not newly updated
int toSkip = data.Count(x => x.DT < request.StartingDate);
CheckForAbnormalReturns(data, toSkip, abnormalLimit, inst, freq);
CheckForAbnormalRanges(data, toSkip, abnormalStDevRange, inst, freq);
CheckForMissingDays(request, data, inst);
CheckForZeroPrices(request, data, inst);
}
private static BarSeries DownloadBars(DataManager manager, Instrument inst, long barSize, DateTime dateTime1, DateTime dateTime2)
{
var bars = new BarSeries();
var request = new HistoricalDataRequest()
{
RequestId = Guid.NewGuid().ToString("N"),
DataType = DataObjectType.Bar,
BarSize = barSize,
Instrument = inst,
DateTime1 = dateTime1,
DateTime2 = dateTime2
};
DownloadDataRequest(manager.GetFramework(), request, OnHistoricalData);
return(bars);
void OnHistoricalData(HistoricalData data)
{
foreach (var item in data.Objects)
{
switch (item.TypeId)
{
case DataObjectType.Bar:
bars.Add((Bar)item);
break;
default:
break;
}
}
}
}
public void HistoricalDataRequestsAreForwardedToTheBroker()
{
var instrument = new Instrument
{
ID = 1,
Symbol = "SPY",
Datasource = new Datasource {
ID = 1, Name = "MockSource"
},
Exchange = new Exchange
{
ID = 1,
Name = "Exchange",
Timezone = "Eastern Standard Time"
}
};
var request = new HistoricalDataRequest(instrument, BarSize.OneDay, new DateTime(2012, 1, 1), new DateTime(2013, 1, 1));
_client.RequestHistoricalData(request);
// TODO: Think about delay amount
Thread.Sleep(1500);
_historicalDataBrokerMock.Verify(
x => x.RequestHistoricalData(
It.Is <HistoricalDataRequest>(
r =>
r.Instrument.ID == 1 &&
r.Frequency == BarSize.OneDay &&
r.StartingDate == new DateTime(2012, 1, 1) &&
r.EndingDate == new DateTime(2013, 1, 1))),
Times.Once);
}
private void SendHistoricalRequest(int id, HistoricalDataRequest request)
{
Log(LogLevel.Info, string.Format("Sent historical data request to TWS. ID: {0}, Symbol: {1}, {2} back from {3}",
id,
request.Instrument.Symbol,
TWSUtils.TimespanToDurationString((request.EndingDate - request.StartingDate), request.Frequency),
request.EndingDate.ToString("yyyy-MM-dd hh:mm:ss")));
TimeZoneInfo exchangeTZ = request.Instrument.GetTZInfo();
//we need to convert time from the exchange TZ to Local...the ib client then converts it to UTC
var startingDate = TimeZoneInfo.ConvertTime(request.StartingDate, exchangeTZ, TimeZoneInfo.Local);
var endingDate = TimeZoneInfo.ConvertTime(request.EndingDate, exchangeTZ, TimeZoneInfo.Local);
try
{
_client.RequestHistoricalData
(
id,
TWSUtils.InstrumentToContract(request.Instrument),
endingDate,
TWSUtils.TimespanToDurationString((endingDate - startingDate), request.Frequency),
TWSUtils.BarSizeConverter(request.Frequency),
GetDataType(request.Instrument),
request.RTHOnly ? 1 : 0
);
}
catch (Exception ex)
{
Log(LogLevel.Error, "IB: Could not send historical data request: " + ex.Message);
RaiseEvent(Error, this, new ErrorArgs(-1, "Could not send historical data request: " + ex.Message, id));
}
}
// 请求历史数据
public void SendHistoricalDataRequestBar(HistoricalDataRequest request, string market)
{
Instrument inst = request.Instrument as Instrument;
string altSymbol = inst.GetSymbol(Name);
PeriodType pt = GetPeriodTypeFromDataType(request);
if (pt == PeriodType.MAX_PERIOD_TYPE)
{
EmitHistoricalDataError(request, "不支持的时间周期！");
return;
}
string key = GetKeyFromSTKHISDATA(market, altSymbol, pt);
// 先发请求再存下来
int nRet = QuotApi.QT_RequestHistory(m_pQuotApi, market, altSymbol, pt);
ehlog.Info("-->RequestHistory:{0},{1} Return:{2}", market, altSymbol, nRet);
if (0 == nRet)
{
SaveRequest(request, key, market, altSymbol, DateTime.MaxValue);
}
else
{
EmitHistoricalDataError(request, "API返回错误:" + nRet);
}
}
public void RequestsAreGivenAUniqueAssignedIDGreaterThanZero()
{
var request = new HistoricalDataRequest(_instrument, BarSize.OneDay, new DateTime(2012, 1, 1), new DateTime(2013, 1, 1),
dataLocation: DataLocation.ExternalOnly,
saveToLocalStorage: false,
rthOnly: true);
var req2 = (HistoricalDataRequest)request.Clone();
var req3 = (HistoricalDataRequest)request.Clone();
var assignedIDs = new List <int>();
_dataSourceMock
.Setup(x => x.RequestHistoricalData(It.IsAny <HistoricalDataRequest>()))
.Callback <HistoricalDataRequest>(req => assignedIDs.Add(req.AssignedID));
_broker.RequestHistoricalData(request);
_broker.RequestHistoricalData(req2);
_broker.RequestHistoricalData(req3);
Thread.Sleep(50);
Assert.AreEqual(3, assignedIDs.Count);
Assert.AreEqual(3, assignedIDs.Distinct().Count());
Assert.AreEqual(0, assignedIDs.Count(x => x < 0));
}
/// <summary>
/// Given a historical data request and the data that fill it,
/// send the reply to the client who made the request.
/// </summary>
private void SendFilledHistoricalRequest(HistoricalDataRequest request, List <OHLCBar> data)
{
lock (socketLock)
{
if (socket != null)
{
using (var ms = new MemoryStream())
{
// This is a 5 part message
// 1st message part: the identity string of the client that we're routing the data to
var clientIdentity = request.RequesterIdentity;
socket.SendMoreFrame(clientIdentity ?? string.Empty);
// 2nd message part: the type of reply we're sending
socket.SendMoreFrame(BitConverter.GetBytes((byte)DataRequestMessageType.HistReply));
// 3rd message part: the HistoricalDataRequest object that was used to make the request
socket.SendMoreFrame(MyUtils.ProtoBufSerialize(request, ms));
// 4th message part: the size of the uncompressed, serialized data. Necessary for decompression on the client end.
var uncompressed = MyUtils.ProtoBufSerialize(data, ms);
socket.SendMoreFrame(BitConverter.GetBytes(uncompressed.Length));
// 5th message part: the compressed serialized data.
var compressed = LZ4Codec.EncodeHC(uncompressed, 0, uncompressed.Length); // compress
socket.SendFrame(compressed);
}
}
}
}
public void DoesNotSaveToLocalStorageWhenSaveToLocalStorageFlagIsNotSet()
{
var request = new HistoricalDataRequest(_instrument, BarSize.OneDay, new DateTime(2012, 1, 1), new DateTime(2013, 1, 1),
dataLocation: DataLocation.ExternalOnly,
saveToLocalStorage: false,
rthOnly: true);
var data = new List <OHLCBar>
{
new OHLCBar {
Open = 1, High = 2, Low = 3, Close = 4, DT = new DateTime(2000, 1, 1)
}
};
//we need to set up a callback with the request after it has had an AssignedID assigned to it.
HistoricalDataRequest newRequest = new HistoricalDataRequest();
_dataSourceMock
.Setup(x => x.RequestHistoricalData(It.IsAny <HistoricalDataRequest>()))
.Callback <HistoricalDataRequest>(req => newRequest = req);
_broker.RequestHistoricalData(request);
Thread.Sleep(50);
_dataSourceMock.Raise(x => x.HistoricalDataArrived += null, new HistoricalDataEventArgs(newRequest, data));
_localStorageMock.Verify(
x => x.AddData(It.IsAny <List <OHLCBar> >(), It.IsAny <Instrument>(), It.IsAny <BarSize>(), It.IsAny <bool>(), It.IsAny <bool>()), Times.Never);
}
private List <DataObject> RequestTrade(HistoricalDataRequest request)
{
string query_str = string.Format("select from (select datetime, sym, price, size:{{first[x] -': x}} volume, openint from `volume xasc select from `trade where sym = `$\"{0}\") where size >0", request.Instrument.GetSymbol(base.id));
Log.Info(query_str);
Flip flip = (Flip)c.k(query_str);
int nRows = c.n(flip.y[0]); // flip.y is an array of columns. Get the number of rows from the first column.
int nColumns = c.n(flip.x);
var y_datetime = (DateTime[])flip.y[0];
var y_sym = (string[])flip.y[1];
var y_price = (double[])flip.y[2];
var y_size = (long[])flip.y[3];
var y_openint = (long[])flip.y[4];
var ts = new List <DataObject>();
for (int i = 0; i < nRows; ++i)
{
var t = new Trade()
{
ProviderId = base.id,
InstrumentId = request.Instrument.Id,
DateTime = y_datetime[i],
Price = y_price[i],
Size = (int)y_size[i],
};
ts.Add(t);
}
return(ts);
}
public void RequestEndingTimesAreCorrectlyConstrainedToThePresentTimeInTheInstrumentsTimeZone()
{
var request = new HistoricalDataRequest(_instrument, BarSize.OneDay, new DateTime(2012, 1, 1), new DateTime(2100, 1, 1),
dataLocation: DataLocation.ExternalOnly,
saveToLocalStorage: false,
rthOnly: true);
var est = TimeZoneInfo.FindSystemTimeZoneById("Eastern Standard Time");
var now = TimeZoneInfo.ConvertTime(DateTime.Now, TimeZoneInfo.Local, est);
var modifiedRequest = new HistoricalDataRequest();
_dataSourceMock.Setup(x => x
.RequestHistoricalData(It.IsAny <HistoricalDataRequest>()))
.Callback <HistoricalDataRequest>(req => modifiedRequest = req);
_broker.RequestHistoricalData(request);
Thread.Sleep(50);
Assert.AreEqual(now.Year, modifiedRequest.EndingDate.Year, string.Format("Expected: {0} Was: {1}", modifiedRequest.EndingDate.Year, now.Year));
Assert.AreEqual(now.Month, modifiedRequest.EndingDate.Month, string.Format("Expected: {0} Was: {1}", modifiedRequest.EndingDate.Month, now.Month));
Assert.AreEqual(now.Day, modifiedRequest.EndingDate.Day, string.Format("Expected: {0} Was: {1}", modifiedRequest.EndingDate.Day, now.Day));
Assert.AreEqual(now.Hour, modifiedRequest.EndingDate.Hour, string.Format("Expected: {0} Was: {1}", modifiedRequest.EndingDate.Hour, now.Hour));
Assert.AreEqual(now.Minute, modifiedRequest.EndingDate.Minute, string.Format("Expected: {0} Was: {1}", modifiedRequest.EndingDate.Minute, now.Minute));
}
private void btnDownload_Click(object sender, EventArgs e)
{
if (!this.ValidateDownloadSettings())
{
return;
}
this.requests.Clear();
this.items.Clear();
this.activeRequests.Clear();
foreach (DownloadViewItem downloadViewItem in this.ltvDownload.Items)
{
downloadViewItem.Reset();
HistoricalDataRequest historicalDataRequest = new HistoricalDataRequest();
historicalDataRequest.Instrument = (IFIXInstrument)downloadViewItem.Instrument;
historicalDataRequest.DataType = (HistoricalDataType)this.cbxDataTypes.SelectedItem;
historicalDataRequest.BeginDate = this.dtpBegin.Value.Date;
historicalDataRequest.EndDate = this.dtpEnd.Value.Date;
if (historicalDataRequest.DataType == HistoricalDataType.Bar)
{
historicalDataRequest.BarSize = (this.cbxBarSize.SelectedItem as BarSizeItem).BarSize;
}
this.requests.Add(historicalDataRequest.RequestId, historicalDataRequest);
this.items.Add(historicalDataRequest.RequestId, downloadViewItem);
}
this.gbxSettings.Enabled = false;
this.gbxAdvanced.Enabled = false;
this.btnDownload.Enabled = false;
this.btnStop.Enabled = true;
this.btnClose.Enabled = false;
this.progressBar.Value = 0;
this.progressBar.Minimum = 0;
this.progressBar.Maximum = this.requests.Count;
this.progressBar.Step = 1;
ThreadPool.QueueUserWorkItem(new WaitCallback(this.DownloadThread), this.trbMaxRequests.Value);
}
/// <summary>
/// Processes incoming historical data requests.
/// </summary>
public void RequestHistoricalData(HistoricalDataRequest request)
{
request.AssignedID = GetUniqueRequestID();
originalRequests.TryAdd(request.AssignedID, request);
if (RequestDataFromLocalStorateOnly(request))
{
return;
}
CheckDataSource(request);
if (RequestDataFromExternalSourceOnly(request))
{
return;
}
lock (localStorageLock)
{
var localDataInfo = dataStorage.GetStoredDataInfo(request.Instrument.ID, request.Frequency);
if (CheckIfLocalStorageCanSatisfyThisRequest(request, localDataInfo))
{
return;
}
SendNewHistoricalDataRequest(request, localDataInfo);
}
}
//TODO historical tick data! aggTrades
private async Task <List <OHLCBar> > GetData(HistoricalDataRequest req)
{
string symbol = string.IsNullOrEmpty(req.Instrument.DatasourceSymbol)
? req.Instrument.Symbol
: req.Instrument.DatasourceSymbol;
string interval = BarSizetoInterval(req.Frequency);
long startTime = MyUtils.ConvertToMillisecondTimestamp(DateTime.SpecifyKind(req.StartingDate, DateTimeKind.Utc));
long endTime = MyUtils.ConvertToMillisecondTimestamp(DateTime.SpecifyKind(req.EndingDate, DateTimeKind.Utc));
string url = $"https://www.binance.com/api/v1/klines?symbol={symbol}&interval={interval}&startTime={startTime}&endTime={endTime}";
_logger.Info("Binance filling historical req from URL: " + url);
var result = await _httpClient.GetAsync(url);
if (429 == (int)result.StatusCode)
{
//pacing violation, grab Retry-After, wait, and re-request
var retryHeader = result.Headers.RetryAfter;
await Task.Delay((int)retryHeader.Delta.Value.TotalSeconds * 1000 + 1000);
return(await GetData(req));
}
else
{
result.EnsureSuccessStatusCode();
}
string contents = await result.Content.ReadAsStringAsync();
return(ParseHistoricalData(JArray.Parse(contents)));
}
/// <summary>
/// Given a historical data request and the data that fill it,
/// send the reply to the client who made the request.
/// </summary>
private void SendFilledHistoricalRequest(HistoricalDataRequest request, List <OHLCBar> data)
{
lock (_socketLock)
{
if (_socket != null)
{
using (var ms = new MemoryStream())
{
// This is a 5 part message
// 1st message part: the identity string of the client that we're routing the data to
var clientIdentity = request.RequesterIdentity;
_socket.SendMoreFrame(clientIdentity ?? string.Empty);
// 2nd message part: the type of reply we're sending
_socket.SendMoreFrame(MessageType.HistReply);
// 3rd message part: the HistoricalDataRequest object that was used to make the request
_socket.SendMoreFrame(MyUtils.ProtoBufSerialize(request, ms));
// 4th message part: the size of the uncompressed, serialized data. Necessary for decompression on the client end.
var uncompressed = MyUtils.ProtoBufSerialize(data, ms);
_socket.SendMoreFrame(BitConverter.GetBytes(uncompressed.Length));
// 5th message part: the compressed serialized data.
var compressed = new byte[LZ4Codec.MaximumOutputSize(uncompressed.Length)];
var targetSpan = new Span <byte>(compressed);
int compressedLength = LZ4Codec.Encode(new Span <byte>(uncompressed), targetSpan, LZ4Level.L03_HC);
//var compressed = LZ4Codec.EncodeHC(uncompressed, 0, uncompressed.Length); // compress
_socket.SendFrame(targetSpan.Slice(0, compressedLength).ToArray());
}
}
}
}
public void DataRequestsOnContinuousFuturesAreForwardedToTheCFBroker()
{
_instrument.IsContinuousFuture = true;
var request = new HistoricalDataRequest(_instrument, BarSize.OneDay, new DateTime(2012, 1, 1), new DateTime(2013, 1, 1),
dataLocation: DataLocation.ExternalOnly,
saveToLocalStorage: false,
rthOnly: true);
_broker.RequestHistoricalData(request);
_cfBrokerMock.Verify(x => x.RequestHistoricalData(
It.Is <HistoricalDataRequest>(
i =>
i.Instrument.ID == 1 &&
i.Frequency == BarSize.OneDay &&
i.StartingDate.Year == 2012 &&
i.StartingDate.Month == 1 &&
i.StartingDate.Day == 1 &&
i.EndingDate.Year == 2013 &&
i.EndingDate.Month == 1 &&
i.EndingDate.Day == 1 &&
i.DataLocation == DataLocation.ExternalOnly &&
i.SaveDataToStorage == false &&
i.RTHOnly == true)), Times.Once);
}
private myFrame GetData_myDB_EquityVolatility(HistoricalDataRequest myRequest, Type ContainerType)
{
myDB_Connector myConnect = new myDB_Connector();
List <EquityVolatility_Line> myRes = myConnect.Select <EquityVolatility_Line>(myRequest.id.DBID, myRequest.fields, myRequest.startDate, myRequest.endDate);
return(new myFrame(myRes));
}
public void SendsErrorMessageWhenExceptionIsRaisedByBrokerOnHistoricalDataRequest()
{
var errorRaised = false;
_client.Error += (sender, e) => errorRaised = true;
var instrument = new Instrument
{
ID = 1,
Symbol = "SPY",
Datasource = new Datasource {
ID = 1, Name = "MockSource"
},
Exchange = new Exchange
{
ID = 1,
Name = "Exchange",
Timezone = "Eastern Standard Time"
}
};
var request = new HistoricalDataRequest(instrument, BarSize.OneDay, new DateTime(2012, 1, 1), new DateTime(2013, 1, 1));
_historicalDataBrokerMock.Setup(x => x.RequestHistoricalData(It.IsAny <HistoricalDataRequest>())).Throws(new Exception("error message"));
_client.RequestHistoricalData(request);
// TODO: Think about delay amount
Thread.Sleep(1500);
Assert.IsTrue(errorRaised);
}
public override void Send(HistoricalDataRequest request)
{
if (!IsApiConnected(_HdApi))
{
EmitHistoricalDataEnd(request.RequestId, RequestResult.Error, "Provider is not connected.");
xlog.Error("历史行情服务器没有连接");
return;
}
int iRet = 1;
switch (request.DataType)
{
case DataObjectType.Bid:
case DataObjectType.Ask:
case DataObjectType.Trade:
case DataObjectType.Quote:
iRet = GetHistoricalTicks(ToStruct(request));
break;
case DataObjectType.Bar:
iRet = GetHistoricalBars(ToStruct(request));
break;
}
historicalDataRecords.Add(iRet, new HistoricalDataRecord(request));
historicalDataIds.Add(request.RequestId, iRet);
}
private static void DownloadDataRequest(Framework framework, HistoricalDataRequest request, Action <HistoricalData> action)
{
var wait = new AutoResetEvent(false);
var hdata = framework.ProviderManager.GetHistoricalDataProvider(QuantBoxConst.PIdHisData);
if (hdata != null)
{
framework.EventManager.Dispatcher.HistoricalData += OnHistoricalData;
framework.EventManager.Dispatcher.HistoricalDataEnd += OnHistoricalDataEnd;
hdata.Send(request);
while (!wait.WaitOne(0))
{
#if NETFRAMEWORK
Application.DoEvents();
#endif
}
framework.EventManager.Dispatcher.HistoricalData -= OnHistoricalData;
framework.EventManager.Dispatcher.HistoricalDataEnd -= OnHistoricalDataEnd;
}
void OnHistoricalData(object sender, HistoricalDataEventArgs args)
{
action(args.Data);
}
void OnHistoricalDataEnd(object sender, HistoricalDataEndEventArgs args)
{
wait.Set();
}
}
public void SendHistoricalDataRequest(HistoricalDataRequest request)
{
Instrument inst = request.Instrument as Instrument;
string altSymbol = inst.GetSymbol(Name);
string altExchange = inst.GetSecurityExchange(Name);
historicalDataIds.Add(request.RequestId, request);
if (true)
{
historicalDataIds.Remove(request.RequestId);
EmitHistoricalDataCompleted(request);
}
else
{
EmitHistoricalDataError(request, "Error");
}
}
public void SendHistoricalDataRequestTick(HistoricalDataRequest request, string market)
{
Instrument inst = request.Instrument as Instrument;
string altSymbol = inst.GetSymbol(Name);
string beginDate = request.BeginDate.ToString("yyyyMMdd");
string endDate = request.EndDate.ToString("yyyyMMdd");
string date = beginDate;
string key = GetKeyFromSTKTRACEDATA(market, altSymbol);
historicalDataIds[request.RequestId] = request;
historicalDataRecords[key] = request.RequestId;
int nRet = QuotApi.QT_RequestTrace(m_pQuotApi, market, altSymbol, date);
if (0 == nRet)
{
historicalDataIds[request.RequestId] = request;
historicalDataRecords[key] = request.RequestId;
}
else
{
EmitHistoricalDataError(request, "API返回错误:" + nRet);
}
}
private void EmitNewHistoricalTrade(HistoricalDataRequest request, DateTime datetime, double price, int size)
{
if (NewHistoricalTrade != null)
{
Trade trade = new Trade(datetime, price, size);
NewHistoricalTrade(this,
new HistoricalTradeEventArgs(trade, request.RequestId, request.Instrument, this, -1));
}
}
public static List<OHLCBar> ParseJson(JObject data, HistoricalDataRequest request)
{
var bars = new List<OHLCBar>();
var barTimeSpan = request.Frequency.ToTimeSpan();
Dictionary<int, InstrumentSession> sessionStartTimesByDay =
request.Instrument.SessionStartTimesByDay();
var exchangeTz = TimeZoneInfo.FindSystemTimeZoneById(request.Instrument.Exchange.Timezone);
JToken jsonBars = data["results"];
foreach (JToken jsonBar in jsonBars)
{
var bar = new OHLCBar
{
Open = decimal.Parse(jsonBar["open"].ToString()),
High = decimal.Parse(jsonBar["high"].ToString()),
Low = decimal.Parse(jsonBar["low"].ToString()),
Close = decimal.Parse(jsonBar["close"].ToString())
};
long volume;
if (long.TryParse(jsonBar.Value<string>("volume"), out volume))
{
bar.Volume = volume;
}
if (request.Frequency < BarSize.OneDay)
{
//The timezone in which the data is delivered is NOT the exchange TZ
//it seems to change depending on the whim of the api and/or location of user?!?!
//anyway, we make sure the time is in the exchange's timezone
bar.DTOpen = TimeZoneInfo.ConvertTime(DateTimeOffset.Parse(jsonBar["timestamp"].ToString()), exchangeTz).DateTime;
bar.DT = bar.DTOpen.Value + barTimeSpan;
//For intraday bars, the time is the bar's OPENING time
//But it fails to "fit" properly...if you ask for hourly bars
//it'll start at 9AM instead of 9:30AM open. So we use the instrument sessions to correct this.
//closing time stays the same
int dayOfWeek = (int)bar.DTOpen.Value.DayOfWeek -1;
if (dayOfWeek < 0) dayOfWeek = 6; //small fix due to DayOfWeek vs DayOfTheWeek different format
if (sessionStartTimesByDay.ContainsKey(dayOfWeek) &&
bar.DTOpen.Value.TimeOfDay < sessionStartTimesByDay[dayOfWeek].OpeningTime)
{
bar.DTOpen = bar.DTOpen.Value.Date.Add(sessionStartTimesByDay[dayOfWeek].OpeningTime);
}
}
else
{
//daily bars or lower frequencies - here the time is provided as "2015-08-17T00:00:00-04:00",
//i.e. just the day, not the actual closing time. We only need the date portion, so we parse the tradingDay field.
bar.DT = DateTime.Parse(jsonBar["tradingDay"].ToString());
}
string openInterest = jsonBar.Value<string>("openInterest");
if (!string.IsNullOrEmpty(openInterest))
{
bar.OpenInterest = int.Parse(openInterest);
}
bars.Add(bar);
}
return bars;
}
private void EmitHistoricalDataCompleted(HistoricalDataRequest request)
{
if (HistoricalDataRequestCompleted != null)
HistoricalDataRequestCompleted(this,
new HistoricalDataEventArgs(request.RequestId, request.Instrument, this, -1));
}
private void EmitNewHistoricalBar(HistoricalDataRequest request, DateTime datetime, double open, double high, double low, double close, long volume, long openInt)
{
if (NewHistoricalBar != null)
{
Bar bar = new Bar(BarType.Time, request.BarSize, datetime, datetime.AddSeconds(request.BarSize), open, high, low, close, volume, openInt);
NewHistoricalBar(this,
new HistoricalBarEventArgs(bar, request.RequestId, request.Instrument, this, -1));
}
}
public void EmitHistoricalQuote(HistoricalDataRequest request, DateTime datetime, double bid, int bidSize, double ask, int askSize)
{
if (this.NewHistoricalQuote == null)
return;
this.NewHistoricalQuote((object) this, new HistoricalQuoteEventArgs(new FreeQuant.Data.Quote(datetime, bid, bidSize, ask, askSize), request.request.RequestId, request.request.Instrument, (IHistoricalDataProvider) this, 0));
}
public void SendHistoricalDataRequest(FreeQuant.Providers.HistoricalDataRequest request)
{
HistoricalDataRequest request1 = new HistoricalDataRequest(request);
this.historicalDataRequests.Add(request.RequestId, request1);
this.provider.CallRequestHistoricalData(request1);
}
protected void EmitHistoricalDataCancelled(HistoricalDataRequest request)
{
this.provider.EmitHistoricalDataCancelled(request);
}
protected void EmitNewHistoricalBar(HistoricalDataRequest request, DateTime datetime, double open, double high, double low, double close, long volume)
{
this.provider.EmitHistoricalBar(request, datetime, open, high, low, close, volume);
}
internal void CallRequestHistoricalData(HistoricalDataRequest request)
{
this.RequestHistoricalData(request);
}
protected void EmitHistoricalDataError(HistoricalDataRequest request, string message)
{
this.provider.EmitHistoricalDataError(request, message);
}
internal void CallCancelHistoricalData(HistoricalDataRequest request)
{
this.CancelHistoricalData(request);
}
protected virtual void RequestHistoricalData(HistoricalDataRequest request)
{
}
protected virtual void CancelHistoricalData(HistoricalDataRequest request)
{
}
PeriodType GetPeriodTypeFromDataType(HistoricalDataRequest request)
{
if (request.DataType == HistoricalDataType.Daily)
{
return PeriodType.Daily;
}
else if (request.DataType == HistoricalDataType.Bar)
{
switch (request.BarSize)
{
case 60:
return PeriodType.Min1;
case 300:
return PeriodType.Min5;
case 3600:
return PeriodType.Min60;
default:
break;
}
}
return PeriodType.MAX_PERIOD_TYPE;
}
protected void EmitNewHistoricalQuote(HistoricalDataRequest request, DateTime datetime, double bid, int bidSize, double ask, int askSize)
{
this.provider.EmitHistoricalQuote(request, datetime, bid, bidSize, ask, askSize);
}
public void EmitHistoricalBar(HistoricalDataRequest request, DateTime datetime, double open, double high, double low, double close, long volume)
{
if (this.NewHistoricalBar == null)
return;
this.NewHistoricalBar((object) this, new HistoricalBarEventArgs(request.request.DataType != HistoricalDataType.Daily ? new FreeQuant.Data.Bar(datetime, open, high, low, close, volume, request.request.BarSize) : (FreeQuant.Data.Bar) new Daily(datetime, open, high, low, close, volume, 0L), request.request.RequestId, request.request.Instrument, (IHistoricalDataProvider) this, 0));
}
protected void EmitNewHistoricalTrade(HistoricalDataRequest request, DateTime datetime, double price, int size)
{
this.provider.EmitHistoricalTrade(request, datetime, price, size);
}
public void EmitHistoricalTrade(HistoricalDataRequest request, DateTime datetime, double price, int size)
{
if (this.NewHistoricalTrade == null)
return;
this.NewHistoricalTrade((object) this, new HistoricalTradeEventArgs(new FreeQuant.Data.Trade(datetime, price, size), request.request.RequestId, request.request.Instrument, (IHistoricalDataProvider) this, 0));
}
public void EmitHistoricalDataError(HistoricalDataRequest request, string message)
{
if (this.HistoricalDataRequestError == null)
return;
this.HistoricalDataRequestError((object) this, new HistoricalDataErrorEventArgs(request.request.RequestId, request.request.Instrument, (IHistoricalDataProvider) this, 0, message));
}
// 请求历史数据
public void SendHistoricalDataRequestBar(HistoricalDataRequest request, string market)
{
Instrument inst = request.Instrument as Instrument;
string altSymbol = inst.GetSymbol(Name);
PeriodType pt = GetPeriodTypeFromDataType(request);
if (pt == PeriodType.MAX_PERIOD_TYPE)
{
EmitHistoricalDataError(request, "不支持的时间周期！");
return;
}
string key = GetKeyFromSTKHISDATA(market, altSymbol, pt);
// 先发请求再存下来
int nRet = QuotApi.QT_RequestHistory(m_pQuotApi, market, altSymbol, pt);
ehlog.Info("-->RequestHistory:{0},{1} Return:{2}", market, altSymbol, nRet);
if (0 == nRet)
{
SaveRequest(request, key, market, altSymbol, DateTime.MaxValue);
}
else
{
EmitHistoricalDataError(request, "API返回错误:" + nRet);
}
}
public void EmitHistoricalDataCancelled(HistoricalDataRequest request)
{
if (this.HistoricalDataRequestCancelled == null)
return;
this.HistoricalDataRequestCancelled((object) this, new HistoricalDataEventArgs(request.request.RequestId, request.request.Instrument, (IHistoricalDataProvider) this, 0));
}
private void EmitHistoricalDataError(HistoricalDataRequest request, string message)
{
if (HistoricalDataRequestError != null)
HistoricalDataRequestError(this,
new HistoricalDataErrorEventArgs(request.RequestId, request.Instrument, this, -1, message));
}
public void SendHistoricalDataRequest(HistoricalDataRequest request)
{
Instrument inst = request.Instrument as Instrument;
string altSymbol = inst.GetSymbol(Name);
string altExchange = inst.GetSecurityExchange(Name);
}
private void EmitNewHistoricalQuote(HistoricalDataRequest request, DateTime datetime, double bid, int bidSize, double ask, int askSize)
{
if (NewHistoricalQuote != null)
{
Quote quote = new Quote(datetime, bid, bidSize, ask, askSize);
NewHistoricalQuote(this,
new HistoricalQuoteEventArgs(quote, request.RequestId, request.Instrument, this, -1));
}
}
public void SendHistoricalDataRequestBar(HistoricalDataRequest request, string market)
{
Instrument inst = request.Instrument as Instrument;
string altSymbol = inst.GetSymbol(Name);
PeriodType pt = GetPeriodTypeFromDataType(request);
if (pt == PeriodType.MAX_PERIOD_TYPE)
{
EmitHistoricalDataError(request, "不支持的时间周期！");
return;
}
string key = GetKeyFromSTKHISDATA(market, altSymbol, pt);
int nRet = QuotApi.QT_RequestHistory(m_pQuotApi, market, altSymbol, pt);
if (0 == nRet)
{
historicalDataIds[request.RequestId] = request;
historicalDataRecords[key] = request.RequestId;
}
else
{
EmitHistoricalDataError(request, "API返回错误:" + nRet);
}
}
private void SaveRequest(HistoricalDataRequest request, string key, string market, string symbol, DateTime date)
{
DataRecord dr;
if (!historicalDataRecords_key.TryGetValue(key, out dr))
{
dr = new DataRecord();
dr.key = key;
dr.market = market;
dr.symbol = symbol;
dr.request = request;
}
historicalDataRecords_key[key] = dr;
historicalDataRecords_requestId[request.RequestId] = dr;
dr.date = date;
}
// 同一合约的一次只能查一天，如果同时查多天就没法区分哪天返回的为0
public void SendHistoricalDataRequestTick(HistoricalDataRequest request, string market)
{
TimeSpan ts = request.EndDate - request.BeginDate;
if (ts.Days <= 0)
{
EmitHistoricalDataError(request, "开始与结束为同一天");
return;
}
else if(ts.Days>7)
{
EmitHistoricalDataError(request, "为减少服务器负担，Tick数据一次只能取一周");
return;
}
Instrument inst = request.Instrument as Instrument;
string altSymbol = inst.GetSymbol(Name);
string key = GetKeyFromSTKTRACEDATA(market, altSymbol);
// 先存下来再发请求
SaveRequest(request, key, market, altSymbol, request.BeginDate.AddDays(-1));
SendRequest_Tick(key);
}