public List <OHLCBar> GetData(EntityModel.Instrument instrument, DateTime from, DateTime to, BarSize frequency = BarSize.OneDay)
{
if (!_client.Connected)
{
return(null);
}
RefreshInstrumentsList();
var qdmsInst = instrument.GetQDMSInstrument(_instrumentsList);
if (qdmsInst == null) //nothing like this in QDMS, just grab local data
{
return(null);
}
StoredDataInfo dataInfo = TryGetStorageInfo(qdmsInst);
//Here we check if there's is absolutely no
if ((dataInfo == null || dataInfo.LatestDate < from || dataInfo.EarliestDate > to) &&
!_allowFreshData)
{
return(null);
}
//grab the data
return(RequestData(qdmsInst, from, to, frequency));
}
private void DataClient_LocallyAvailableDataInfoReceived(object sender, LocallyAvailableDataInfoReceivedEventArgs e)
{
if (e.Instrument.ID == null)
{
throw new Exception("Null instrument ID return wtf");
}
int id = e.Instrument.ID.Value;
lock (_storageInfoLock)
{
StoredDataInfo info = e.StorageInfo.FirstOrDefault(x => x.Frequency == BarSize.OneDay);
if (info == null)
{
return;
}
if (_storageInfo.ContainsKey(id))
{
_storageInfo[id] = info;
}
else
{
_storageInfo.Add(id, info);
}
}
}
public List <OHLCBar> GetAllData(EntityModel.Instrument instrument, BarSize frequency = BarSize.OneDay)
{
if (!_client.Connected)
{
return(new List <OHLCBar>());
}
RefreshInstrumentsList();
//find instrument
var qdmsInst = instrument.GetQDMSInstrument(_instrumentsList);
if (qdmsInst == null) //nothing like this in QDMS, just grab local data
{
return(new List <OHLCBar>());
}
StoredDataInfo dataInfo = TryGetStorageInfo(qdmsInst);
if (dataInfo == null)
{
return(new List <OHLCBar>());
}
return(GetData(
instrument,
dataInfo.EarliestDate,
dataInfo.LatestDate,
frequency));
}
public List <StoredDataInfo> GetStorageInfo(int instrumentID)
{
SqlConnection connection;
if (!TryConnect(out connection))
{
throw new Exception("Could not connect to database");
}
var instrumentInfos = new List <StoredDataInfo>();
using (var cmd = new SqlCommand("", connection))
{
cmd.CommandText = "SELECT * FROM instrumentinfo WHERE InstrumentID = " + instrumentID;
using (var reader = cmd.ExecuteReader())
{
while (reader.Read())
{
var info = new StoredDataInfo
{
InstrumentID = instrumentID,
Frequency = (BarSize)reader.GetInt32(1),
EarliestDate = reader.GetDateTime(2),
LatestDate = reader.GetDateTime(3)
};
instrumentInfos.Add(info);
}
}
}
connection.Close();
return(instrumentInfos);
}
public StoredDataInfo GetStorageInfo(int instrumentID, BarSize frequency)
{
SqlConnection connection;
if (!TryConnect(out connection))
{
throw new Exception("Could not connect to database");
}
var instrumentInfo = new StoredDataInfo();
using (var cmd = new SqlCommand("", connection))
{
cmd.CommandText = string.Format("SELECT * FROM instrumentinfo WHERE InstrumentID = {0} AND Frequency = {1}",
instrumentID,
(int)frequency);
using (var reader = cmd.ExecuteReader())
{
if (reader.Read())
{
instrumentInfo.InstrumentID = instrumentID;
instrumentInfo.Frequency = (BarSize)reader.GetInt32(1);
instrumentInfo.EarliestDate = reader.GetDateTime(2);
instrumentInfo.LatestDate = reader.GetDateTime(3);
}
else
{
return(null); //return null if nothing is found that matches these criteria
}
}
}
connection.Close();
return(instrumentInfo);
}
public decimal?GetLastPrice(EntityModel.Instrument instrument, out DateTime lastDate)
{
lastDate = new DateTime(1, 1, 1);
var qdmsInst = instrument.GetQDMSInstrument(_instrumentsList);
if (qdmsInst == null || !qdmsInst.ID.HasValue)
{
return(null);
}
StoredDataInfo dataInfo = TryGetStorageInfo(qdmsInst);
if (dataInfo == null)
{
return(null);
}
var lastAvailableDate = dataInfo.LatestDate;
//Send out the request for the data
var req = new HistoricalDataRequest
{
Instrument = qdmsInst,
StartingDate = lastAvailableDate.AddDays(-1),
EndingDate = lastAvailableDate,
Frequency = BarSize.OneDay,
DataLocation = _allowFreshData ? DataLocation.Both : DataLocation.LocalOnly,
RTHOnly = true,
SaveDataToStorage = false
};
var id = _client.RequestHistoricalData(req);
_requestIDs.Add(id, false);
//Wait until the data arrives
int i = 0;
while (i < 100)
{
Thread.Sleep(20);
lock (_arrivedDataLock)
{
if (_requestIDs[id])
{
var data = _arrivedData[qdmsInst.ID.Value].Last();
_arrivedData.Remove(qdmsInst.ID.Value);
lastDate = data.DT;
return(data.Close);
}
}
i++;
}
return(null);
}
private void SendNewHistoricalDataRequest(HistoricalDataRequest request, StoredDataInfo localDataInfo)
{
if (localDataInfo == null)
{
ForwardHistoricalRequest(request);
}
else
{
GenerateSubRequests(request, localDataInfo);
}
}
/// <summary>
/// When a data request can be partly filled by the local db,
/// we need to split it into one or two sub-requests for the parts that
/// are not locally available. This method does that and forwards the sub-requests.
/// </summary>
private void GenerateSubRequests(HistoricalDataRequest request, StoredDataInfo localDataInfo)
{
subRequests.TryAdd(request.AssignedID, new List <HistoricalDataRequest>());
HistoricalDataRequest newBackRequest = CheckIfEarlierDataMayBeNeeded(request, localDataInfo);
HistoricalDataRequest newForwardRequest = CheckIfLaterDataMayBeNeeded(request, localDataInfo);
//we send these together, because too large of a delay between the two requests can cause problems
if (newBackRequest != null)
{
ForwardHistoricalRequest(newBackRequest);
}
if (newForwardRequest != null)
{
ForwardHistoricalRequest(newForwardRequest);
}
}
/// <summary>
/// Called when we get a reply on a request for available data in local storage.
/// </summary>
private void HandleAvailabledataReply()
{
// First the instrument
byte[] buffer = historicalDataSocket.ReceiveFrameBytes();
Instrument instrument;
using (MemoryStream ms = new MemoryStream(buffer))
{
instrument = MyUtils.ProtoBufDeserialize <Instrument>(ms);
}
// Second the number of items
buffer = historicalDataSocket.ReceiveFrameBytes();
int count = BitConverter.ToInt32(buffer, 0);
// Then actually get the items, if any
if (count == 0)
{
RaiseEvent(LocallyAvailableDataInfoReceived, this,
new LocallyAvailableDataInfoReceivedEventArgs(instrument, new List <StoredDataInfo>()));
}
else
{
List <StoredDataInfo> storageInfo = new List <StoredDataInfo>();
for (int i = 0; i < count; i++)
{
buffer = historicalDataSocket.ReceiveFrameBytes(out bool hasMore);
using (MemoryStream ms = new MemoryStream(buffer))
{
StoredDataInfo info = MyUtils.ProtoBufDeserialize <StoredDataInfo>(ms);
storageInfo.Add(info);
}
if (!hasMore)
{
break;
}
}
RaiseEvent(LocallyAvailableDataInfoReceived, this,
new LocallyAvailableDataInfoReceivedEventArgs(instrument, storageInfo));
}
}
//check the latest date we have available in local storage, then request historical data from that date to the current time
private void UpdateHistoricalDataBtn_ItemClick(object sender, RoutedEventArgs routedEventArgs)
{
var frequency = (BarSize)((MenuItem)sender).Tag;
List <Instrument> selectedInstruments = InstrumentsGrid.SelectedItems.Cast <Instrument>().ToList();
int requestCount = 0;
using (IDataStorage localStorage = DataStorageFactory.Get())
{
foreach (Instrument i in selectedInstruments)
{
if (!i.ID.HasValue)
{
continue;
}
//TODO: add GetStorageInfo to client through REST, deprecate zmq, then remove dependency on DataStorageFactory here
List <StoredDataInfo> storageInfo = localStorage.GetStorageInfo(i.ID.Value);
if (storageInfo.Any(x => x.Frequency == frequency))
{
StoredDataInfo relevantStorageInfo = storageInfo.First(x => x.Frequency == frequency);
_client.RequestHistoricalData(new HistoricalDataRequest(
i,
frequency,
relevantStorageInfo.LatestDate + frequency.ToTimeSpan(),
DateTime.Now,
DataLocation.ExternalOnly,
true));
requestCount++;
}
}
}
if (_progressBar.Value >= _progressBar.Maximum)
{
_progressBar.Maximum = requestCount;
_progressBar.Value = 0;
}
else
{
_progressBar.Maximum += requestCount;
}
}
/// <summary>
/// When a data request can be partly filled by the local db,
/// we need to split it into one or two sub-requests for the parts that
/// are not locally available. This method does that and forwards the sub-requests.
/// </summary>
private void GenerateSubRequests(HistoricalDataRequest request, StoredDataInfo localDataInfo)
{
_subRequests.TryAdd(request.AssignedID, new List <HistoricalDataRequest>());
//earlier data that may be needed
HistoricalDataRequest newBackRequest = null;
if (localDataInfo.EarliestDate > request.StartingDate)
{
newBackRequest = (HistoricalDataRequest)request.Clone();
newBackRequest.EndingDate = localDataInfo.EarliestDate.AddMilliseconds(-request.Frequency.ToTimeSpan().TotalMilliseconds / 2);
newBackRequest.IsSubrequestFor = request.AssignedID;
newBackRequest.AssignedID = GetUniqueRequestID();
_subRequests[newBackRequest.IsSubrequestFor.Value].Add(newBackRequest);
}
//later data that may be needed
HistoricalDataRequest newForwardRequest = null;
if (localDataInfo.LatestDate < request.EndingDate)
{
//the local storage is insufficient, so we save the original request, make a copy,
//modify it, and pass it to the external data source
newForwardRequest = (HistoricalDataRequest)request.Clone();
newForwardRequest.StartingDate = localDataInfo.LatestDate.AddMilliseconds(request.Frequency.ToTimeSpan().TotalMilliseconds / 2);
newForwardRequest.IsSubrequestFor = request.AssignedID;
newForwardRequest.AssignedID = GetUniqueRequestID();
_subRequests[newForwardRequest.IsSubrequestFor.Value].Add(newForwardRequest);
}
//we send these together, because too large of a delay between the two requests can cause problems
if (newBackRequest != null)
{
ForwardHistoricalRequest(newBackRequest);
}
if (newForwardRequest != null)
{
ForwardHistoricalRequest(newForwardRequest);
}
}
public void RequestsAreCorrectlySplitIntoSubrequestsWhenOnlyPartOfTheDataIsAvailable()
{
var request = new HistoricalDataRequest(_instrument, BarSize.OneDay, new DateTime(2012, 1, 1), new DateTime(2013, 1, 1),
dataLocation: DataLocation.Both,
saveToLocalStorage: false,
rthOnly: true);
StoredDataInfo sdInfo = new StoredDataInfo()
{
EarliestDate = new DateTime(2012, 6, 1),
LatestDate = new DateTime(2012, 9, 1),
Frequency = BarSize.OneDay,
InstrumentID = 1
};
_localStorageMock.Setup(x => x.GetStorageInfo(1, BarSize.OneDay)).Returns(sdInfo);
_broker.RequestHistoricalData(request);
Thread.Sleep(50);
//first subrequest
_dataSourceMock.Verify(x => x.RequestHistoricalData(
It.Is <HistoricalDataRequest>(y =>
y.StartingDate.Month == 1 &&
y.StartingDate.Day == 1 &&
y.EndingDate.Month == 5 &&
y.EndingDate.Day == 31
)), Times.Once);
//second subrequest
_dataSourceMock.Verify(x => x.RequestHistoricalData(
It.Is <HistoricalDataRequest>(y =>
y.StartingDate.Month == 9 &&
y.StartingDate.Day == 1 &&
y.EndingDate.Month == 1 &&
y.EndingDate.Day == 1
)), Times.Once);
}
private HistoricalDataRequest CheckIfLaterDataMayBeNeeded(HistoricalDataRequest request, StoredDataInfo localDataInfo)
{
HistoricalDataRequest newForwardRequest = null;
if (localDataInfo.LatestDate < request.EndingDate)
{
//the local storage is insufficient, so we save the original request, make a copy,
//modify it, and pass it to the external data source
newForwardRequest = (HistoricalDataRequest)request;
newForwardRequest.StartingDate =
localDataInfo.LatestDate.AddMilliseconds(request.Frequency.ToTimeSpan().TotalMilliseconds / 2);
newForwardRequest.IsSubrequestFor = request.AssignedID;
newForwardRequest.AssignedID = GetUniqueRequestID();
subRequests[newForwardRequest.IsSubrequestFor.Value].Add(newForwardRequest);
}
return(newForwardRequest);
}
private HistoricalDataRequest CheckIfEarlierDataMayBeNeeded(HistoricalDataRequest request, StoredDataInfo localDataInfo)
{
HistoricalDataRequest newBackRequest = null;
if (localDataInfo.EarliestDate > request.StartingDate)
{
newBackRequest = (HistoricalDataRequest)request;
newBackRequest.EndingDate =
localDataInfo.EarliestDate.AddMilliseconds(-request.Frequency.ToTimeSpan().TotalMilliseconds / 2);
newBackRequest.IsSubrequestFor = request.AssignedID;
newBackRequest.AssignedID = GetUniqueRequestID();
subRequests[newBackRequest.IsSubrequestFor.Value].Add(newBackRequest);
}
return(newBackRequest);
}
private bool CheckIfLocalStorageCanSatisfyThisRequest(HistoricalDataRequest request, StoredDataInfo localDataInfo)
{
if (localDataInfo != null &&
localDataInfo.LatestDate >= request.EndingDate &&
localDataInfo.EarliestDate <= request.StartingDate)
{
dataStorage.RequestHistoricalData(request);
return(true);
}
if (localDataInfo != null)
{
int dotw = request.Instrument.Expiration.Date.DayOfWeek.ToInt();
InstrumentSession session = request.Instrument.Sessions?.First(x => (int)x.ClosingDay == dotw && x.IsSessionEnd);
if (session != null &&
localDataInfo.LatestDate >= request.Instrument.Expiration.Date + session.ClosingTime)
{
dataStorage.RequestHistoricalData(request);
return(true);
}
}
return(false);
}