public void DataArrivedEventIsRaisedWhenDataSourceReturnsData()
{
bool eventRaised = false;
_broker.HistoricalDataArrived += (sender, e) => eventRaised = true;
var request = new HistoricalDataRequest(_instrument, BarSize.OneDay, new DateTime(2012, 1, 1), new DateTime(2013, 1, 1),
forceFreshData: true,
localStorageOnly: false,
saveToLocalStorage: false,
rthOnly: true);
var data = new List<OHLCBar>
{
new OHLCBar {Open = 1, High = 2, Low = 3, Close = 4, DT = new DateTime(2000, 1, 1) }
};
//we need to set up a callback with the request after it has had an AssignedID assigned to it.
HistoricalDataRequest newRequest = new HistoricalDataRequest();
_dataSourceMock
.Setup(x => x.RequestHistoricalData(It.IsAny<HistoricalDataRequest>()))
.Callback<HistoricalDataRequest>(req => newRequest = req);
_broker.RequestHistoricalData(request);
_dataSourceMock.Raise(x => x.HistoricalDataArrived += null, new HistoricalDataEventArgs(newRequest, data));
Assert.IsTrue(eventRaised);
}
public void HistoricalDataRequestsAreForwardedToTheBroker()
{
var instrument = new Instrument
{
ID = 1,
Symbol = "SPY",
Datasource = new Datasource { ID = 1, Name = "MockSource" },
Exchange = new Exchange
{
ID = 1,
Name = "Exchange",
Timezone = "Eastern Standard Time"
}
};
var request = new HistoricalDataRequest(instrument, BarSize.OneDay, new DateTime(2012, 1, 1), new DateTime(2013, 1, 1));
_client.RequestHistoricalData(request);
// TODO: Think about delay amount
Thread.Sleep(1500);
_historicalDataBrokerMock.Verify(
x => x.RequestHistoricalData(
It.Is<HistoricalDataRequest>(
r =>
r.Instrument.ID == 1 &&
r.Frequency == BarSize.OneDay &&
r.StartingDate == new DateTime(2012, 1, 1) &&
r.EndingDate == new DateTime(2013, 1, 1))),
Times.Once);
}
/// <summary>
/// Creates a new object that is a copy of the current instance.
/// </summary>
/// <returns>
/// A new object that is a copy of this instance.
/// </returns>
public object Clone()
{
var clone = new HistoricalDataRequest(Instrument, Frequency, StartingDate, EndingDate, DataLocation, SaveDataToStorage, RTHOnly, RequestID);
clone.AssignedID = AssignedID;
clone.IsSubrequestFor = IsSubrequestFor;
clone.RequesterIdentity = RequesterIdentity;
return(clone);
}
public void BarTimeAdjustmentsHappenCorrectly()
{
string json = @"{
""status"": {
""code"": 200,
""message"": ""Success.""
},
""results"": [{
""symbol"": ""IBM"",
""timestamp"": ""2016-08-15T09:00:00-04:00"",
""tradingDay"": ""2016-08-15"",
""open"": 162.4,
""high"": 162.97,
""low"": 162.38,
""close"": 162.77,
""volume"": 215371
},
{
""symbol"": ""IBM"",
""timestamp"": ""2016-08-15T10:00:00-04:00"",
""tradingDay"": ""2016-08-15"",
""open"": 162.8,
""high"": 162.95,
""low"": 162.61,
""close"": 162.63,
""volume"": 222815
}]
}";
var exchange = new Exchange
{
Timezone = "Eastern Standard Time"
};
var instrument = new Instrument()
{
Symbol = "IBM",
Exchange = exchange,
Sessions = new List<InstrumentSession>{
new InstrumentSession{ OpeningDay = DayOfTheWeek.Monday, OpeningTime = new TimeSpan(9, 30, 0) } }
};
var request = new HistoricalDataRequest(instrument, BarSize.OneHour, DateTime.Now, DateTime.Now);
List<OHLCBar> bars = BarChartUtils.ParseJson(JObject.Parse(json), request);
//opening time set to the session opening instead of earlier
Assert.AreEqual(new DateTime(2016, 8, 15, 9, 30, 0), bars[0].DTOpen.Value);
//Bar closing time set correctly
Assert.AreEqual(new DateTime(2016, 8, 15, 10, 0, 0), bars[0].DT);
//timezone conversion has happened properly
Assert.AreEqual(new DateTime(2016, 8, 15, 10, 0, 0), bars[1].DTOpen.Value);
}
/// <summary>
/// Check if a request for historical data obeys the duration limits of TWS.
/// </summary>
/// <returns>True if the request obeys the limits, false otherwise.</returns>
public static bool RequestObeysLimits(HistoricalDataRequest request)
{
//The limitations are laid out here: https://www.interactivebrokers.com/en/software/api/apiguide/api/historical_data_limitations.htm
TimeSpan period = (request.EndingDate - request.StartingDate);
double periodSeconds = period.TotalSeconds;
double freqSeconds = request.Frequency.ToTimeSpan().TotalSeconds;
if(periodSeconds / freqSeconds > 2000) return false;
return periodSeconds < MaxRequestLength(request.Frequency);
}
public void RequestHistoricalDataRaisesErrorEventAndReturnsMinusOneWhenNotConnected()
{
var req = new HistoricalDataRequest
{
Instrument = new Instrument { ID = 1, Symbol = "SPY" },
StartingDate = new DateTime(2012, 1, 1),
EndingDate = new DateTime(2013, 1, 1),
Frequency = BarSize.OneDay
};
bool errorTriggered = false;
_client.Error += (sender, e) => errorTriggered = true;
Assert.AreEqual(-1, _client.RequestHistoricalData(req));
Assert.IsTrue(errorTriggered);
}
public void ArrivedHistoricalDataCorrectlyRaisesEvent()
{
var exchange = new Exchange { ID = 1, Name = "Ex", Timezone = "Pacific Standard Time" };
var req = new HistoricalDataRequest
{
Instrument = new Instrument { ID = 1, Symbol = "SPY", UnderlyingSymbol = "SPY", Exchange = exchange, Currency = "USD", Type = InstrumentType.Stock },
Frequency = QDMS.BarSize.OneDay,
StartingDate = new DateTime(2014, 1, 14),
EndingDate = new DateTime(2014, 1, 15),
RTHOnly = true
};
int requestID = -1;
_ibClientMock
.Setup(x => x.RequestHistoricalData(It.IsAny<int>(), It.IsAny<Contract>(), It.IsAny<DateTime>(), It.IsAny<string>(), It.IsAny<BarSize>(), It.IsAny<HistoricalDataType>(), It.IsAny<int>()))
.Callback<Int32, Contract, DateTime, String, BarSize, HistoricalDataType, Int32>((y, a, b, c, d, e, f) => requestID = y);
_ibDatasource.RequestHistoricalData(req);
bool received = false;
_ibDatasource.HistoricalDataArrived += (sender, e) => received = true;
_ibClientMock.Raise(x => x.HistoricalData += null,
new HistoricalDataEventArgs(
requestID,
new DateTime(2014, 1, 15),
1,
2,
3,
4,
5,
5,
3,
false,
1,
1));
Assert.IsTrue(received);
}
public void SetUp()
{
_clientMock = new Mock<IDataClient>();
_instrumentMgrMock = new Mock<IInstrumentSource>();
_broker = new ContinuousFuturesBroker(_clientMock.Object, _instrumentMgrMock.Object);
_cfInst = new Instrument();
_cfInst.Type = InstrumentType.Future;
_cfInst.DatasourceID = 1;
_cfInst.Datasource = new Datasource { ID = 1, Name = "Interactive Brokers" };
var cf = new ContinuousFuture();
cf.Month = 1;
_cfInst.IsContinuousFuture = true;
_cfInst.ContinuousFuture = cf;
_cfInst.ContinuousFuture.AdjustmentMode = ContinuousFuturesAdjustmentMode.NoAdjustment;
var vix = new UnderlyingSymbol();
vix.Rule = new ExpirationRule
{
DaysBefore = 30,
DayType = DayType.Calendar,
ReferenceRelativeMonth = RelativeMonth.NextMonth,
ReferenceUsesDays = false,
ReferenceWeekDay = DayOfTheWeek.Friday,
ReferenceWeekDayCount = WeekDayCount.Third,
ReferenceDayMustBeBusinessDay = true
};
vix.Symbol = "VIX";
_cfInst.ContinuousFuture.UnderlyingSymbol = vix;
_req = new HistoricalDataRequest(
_cfInst,
BarSize.OneDay,
new DateTime(2013, 1, 1),
new DateTime(2013, 2, 1));
}
public void SetUp()
{
_clientMock = new Mock<IDataClient>();
_instrumentMgrMock = new Mock<IInstrumentSource>();
_broker = new ContinuousFuturesBroker(_clientMock.Object, _instrumentMgrMock.Object);
_cfInst = new Instrument();
_cfInst.Type = InstrumentType.Future;
_cfInst.DatasourceID = 1;
_cfInst.Datasource = new Datasource { ID = 1, Name = "Interactive Brokers" };
var cf = new ContinuousFuture();
cf.Month = 1;
_cfInst.ContinuousFuture = cf;
_cfInst.ContinuousFuture.AdjustmentMode = ContinuousFuturesAdjustmentMode.NoAdjustment;
var underlying = new UnderlyingSymbol();
underlying.Symbol = "VIX";
_cfInst.ContinuousFuture.UnderlyingSymbol = underlying;
_req = new HistoricalDataRequest(
_cfInst,
BarSize.OneDay,
new DateTime(2013, 1, 1),
new DateTime(2013, 2, 1));
}
/// <summary>
/// Creates a new object that is a copy of the current instance.
/// </summary>
/// <returns>
/// A new object that is a copy of this instance.
/// </returns>
public object Clone()
{
var clone = new HistoricalDataRequest(Instrument, Frequency, StartingDate, EndingDate, DataLocation, SaveDataToStorage, RTHOnly, RequestID);
clone.AssignedID = AssignedID;
clone.IsSubrequestFor = IsSubrequestFor;
clone.RequesterIdentity = RequesterIdentity;
return clone;
}
/// <summary>
/// Creates a new object that is a copy of the current instance.
/// </summary>
/// <returns>
/// A new object that is a copy of this instance.
/// </returns>
public object Clone()
{
var clone = new HistoricalDataRequest(Instrument, Frequency, StartingDate, EndingDate, ForceFreshData, LocalStorageOnly, SaveDataToStorage, RTHOnly);
clone.AssignedID = AssignedID;
return clone;
}
/// <summary>
/// Called by the <see cref="T:Quartz.IScheduler"/> when a <see cref="T:Quartz.ITrigger"/>
/// fires that is associated with the <see cref="T:Quartz.IJob"/>.
/// </summary>
/// <remarks>
/// The implementation may wish to set a result object on the
/// JobExecutionContext before this method exits. The result itself
/// is meaningless to Quartz, but may be informative to
/// <see cref="T:Quartz.IJobListener"/>s or
/// <see cref="T:Quartz.ITriggerListener"/>s that are watching the job's
/// execution.
/// </remarks>
/// <param name="context">The execution context.</param>
public void Execute(IJobExecutionContext context)
{
_logger = LogManager.GetCurrentClassLogger();
if(_broker == null)
{
Log(LogLevel.Error, "Data Update Job failed: broker not set.");
return;
}
JobDataMap dataMap = context.JobDetail.JobDataMap;
var details = (DataUpdateJobDetails)dataMap["details"];
Log(LogLevel.Info, string.Format("Data Update job {0} triggered.", details.Name));
//Multiple jobs may be called simultaneously, so what we do is seed the Random based on the job name
byte[] bytes = new byte[details.Name.Length * sizeof(char)];
Buffer.BlockCopy(details.Name.ToCharArray(), 0, bytes, 0, bytes.Length);
Random r = new Random((int)DateTime.Now.TimeOfDay.TotalSeconds ^ BitConverter.ToInt32(bytes, 0));
_requesterID = "DataUpdateJob" + r.Next(); //we use this ID to identify this particular data update job
List<Instrument> instruments = details.UseTag
? _instrumentManager.FindInstruments(pred: x => x.Tags.Any(y => y.ID == details.TagID))
: _instrumentManager.FindInstruments(pred: x => x.ID == details.InstrumentID);
if (instruments.Count == 0)
{
Log(LogLevel.Error, string.Format("Aborting data update job {0}: no instruments found.", details.Name));
return;
}
_broker.HistoricalDataArrived += _broker_HistoricalDataArrived;
_broker.Error += _broker_Error;
int counter = 1;
//What we do here: we check what we have available locally..
//If there is something, we send a query to grab data between the last stored time and "now"
//Otherwise we send a query to grab everything since 1900
foreach (Instrument i in instruments)
{
if (!i.ID.HasValue) continue;
//don't request data on expired securities unless the expiration was recent
if (i.Expiration.HasValue && (DateTime.Now - i.Expiration.Value).TotalDays > 15)
{
Log(LogLevel.Trace, string.Format("Data update job {0}: ignored instrument w/ ID {1} due to expiration date.", details.Name, i.ID));
continue;
}
DateTime startingDT = new DateTime(1900, 1, 1);
var storageInfo = _localStorage.GetStorageInfo(i.ID.Value);
if (storageInfo.Any(x => x.Frequency == details.Frequency))
{
var relevantStorageInfo = storageInfo.First(x => x.Frequency == details.Frequency);
startingDT = relevantStorageInfo.LatestDate;
}
var req = new HistoricalDataRequest(
i,
details.Frequency,
startingDT,
DateTime.Now, //TODO this should be in the instrument's timezone...
dataLocation: DataLocation.ExternalOnly,
saveToLocalStorage: true,
rthOnly: true,
requestID: counter)
{
RequesterIdentity = _requesterID
};
try
{
_broker.RequestHistoricalData(req);
lock (_reqIDLock)
{
_pendingRequests.Add(req);
}
}
catch(Exception ex)
{
_errors.Add(ex.Message);
}
counter++;
}
Stopwatch sw = new Stopwatch();
sw.Start();
//loop until time runs out or all requests are completed
while (_pendingRequests.Count > 0 &&
sw.ElapsedMilliseconds < _settings.Timeout * 1000)
{
Thread.Sleep(100);
}
JobComplete();
Log(LogLevel.Info, string.Format("Data Update job {0} completed.", details.Name));
}
public void ForceFreshDataFlagIsObeyed()
{
var request = new HistoricalDataRequest(_instrument, BarSize.OneDay, new DateTime(2012, 1, 1), new DateTime(2013, 1, 1),
forceFreshData: true,
localStorageOnly: false,
saveToLocalStorage: false,
rthOnly: true);
_broker.RequestHistoricalData(request);
_localStorageMock.Verify(x => x.GetStorageInfo(It.IsAny<int>(), It.IsAny<BarSize>()), Times.Never);
_localStorageMock.Verify(x => x.GetData(It.IsAny<Instrument>(), It.IsAny<DateTime>(), It.IsAny<DateTime>(), It.IsAny<BarSize>()), Times.Never);
}
public void WhenDataSourceSymbolIsSetThatIsTheValueSentInTheHistoricalRequest()
{
var exchange = new Exchange { ID = 1, Name = "Ex", Timezone = "Pacific Standard Time" };
var req = new HistoricalDataRequest
{
Instrument = new Instrument { ID = 1, Symbol = "SPY", UnderlyingSymbol = "SPY", DatasourceSymbol = "TestMe!", Exchange = exchange, Currency = "USD", Type = InstrumentType.Stock },
Frequency = QDMS.BarSize.OneDay,
StartingDate = new DateTime(2014, 1, 14),
EndingDate = new DateTime(2014, 1, 15),
RTHOnly = true
};
_ibDatasource.RequestHistoricalData(req);
_ibClientMock.Verify(x => x.RequestHistoricalData(
It.IsAny<int>(),
It.Is<Contract>(y => y.Symbol == "TestMe!"),
It.IsAny<DateTime>(),
It.IsAny<string>(),
It.IsAny<BarSize>(),
It.IsAny<HistoricalDataType>(),
It.IsAny<int>()));
}
public void HistoricalRequestsAreReSentAfterARealTimeDataPacingViolation()
{
var exchange = new Exchange { ID = 1, Name = "Ex", Timezone = "Pacific Standard Time" };
var req = new HistoricalDataRequest
{
Instrument = new Instrument { ID = 1, Symbol = "SPY", Exchange = exchange },
Frequency = QDMS.BarSize.OneDay,
StartingDate = new DateTime(2014, 1, 14),
EndingDate = new DateTime(2014, 1, 15),
RTHOnly = true
};
int requestID = 0;
_ibClientMock
.Setup(x => x.RequestHistoricalData(
It.IsAny<int>(),
It.IsAny<Contract>(),
It.IsAny<DateTime>(),
It.IsAny<string>(),
It.IsAny<Krs.Ats.IBNet.BarSize>(),
It.IsAny<HistoricalDataType>(),
It.IsAny<int>()))
.Callback<Int32, Contract, DateTime, String, BarSize, HistoricalDataType, Int32>((y, a, b, c, d, e, f) => requestID = y);
_ibDatasource.RequestHistoricalData(req);
_ibClientMock.Raise(x => x.Error += null, new ErrorEventArgs(requestID, (ErrorMessage) 162, ""));
Thread.Sleep(25000);
_ibClientMock.Verify(x => x.RequestHistoricalData(
It.IsAny<int>(),
It.IsAny<Contract>(),
It.IsAny<DateTime>(),
It.IsAny<string>(),
It.IsAny<Krs.Ats.IBNet.BarSize>(),
It.IsAny<HistoricalDataType>(),
It.IsAny<int>()),
Times.Exactly(2));
}
public void ThrowsExceptionWhenMakingRequestForInstrumentWithDataSourceThatIsDisconnected()
{
var request = new HistoricalDataRequest(_instrument, BarSize.OneDay, new DateTime(2012, 1, 1), new DateTime(2013, 1, 1),
forceFreshData: true,
localStorageOnly: false,
saveToLocalStorage: false,
rthOnly: true);
_dataSourceMock.SetupGet(x => x.Connected).Returns(false);
_broker.RequestHistoricalData(request);
}
public void ThrowsExceptionWhenMakingRequestForInstrumentWithDataSourceThatDoesNotExist()
{
_instrument.Datasource.Name = "ASDfasdf___________________aasdf";
var request = new HistoricalDataRequest(_instrument, BarSize.OneDay, new DateTime(2012, 1, 1), new DateTime(2013, 1, 1),
forceFreshData: true,
localStorageOnly: false,
saveToLocalStorage: false,
rthOnly: true);
_broker.RequestHistoricalData(request);
}
public void SavesToLocalStorageWhenSaveToLocalStorageFlagIsSet()
{
var request = new HistoricalDataRequest(_instrument, BarSize.OneDay, new DateTime(2012, 1, 1), new DateTime(2013, 1, 1),
forceFreshData: true,
localStorageOnly: false,
saveToLocalStorage: true,
rthOnly: true);
var data = new List<OHLCBar>
{
new OHLCBar {Open = 1, High = 2, Low = 3, Close = 4, DT = new DateTime(2000, 1, 1) }
};
//we need to set up a callback with the request after it has had an AssignedID assigned to it.
HistoricalDataRequest newRequest = new HistoricalDataRequest();
_dataSourceMock
.Setup(x => x.RequestHistoricalData(It.IsAny<HistoricalDataRequest>()))
.Callback<HistoricalDataRequest>(req => newRequest = req);
_broker.RequestHistoricalData(request);
_dataSourceMock.Raise(x => x.HistoricalDataArrived += null, new HistoricalDataEventArgs(newRequest, data));
_localStorageMock.Verify(
x => x.AddData(
It.Is<List<OHLCBar>>(y => y.Count == 1),
It.Is<Instrument>(y => y.ID == 1),
It.Is<BarSize>(y => y == BarSize.OneDay),
It.Is<bool>(y => y == true),
It.IsAny<bool>()),
Times.Once);
}
public void RequestsAreGivenAUniqueAssignedIDGreaterThanZero()
{
var request = new HistoricalDataRequest(_instrument, BarSize.OneDay, new DateTime(2012, 1, 1), new DateTime(2013, 1, 1),
forceFreshData: true,
localStorageOnly: false,
saveToLocalStorage: false,
rthOnly: true);
var assignedIDs = new List<int>();
_dataSourceMock
.Setup(x => x.RequestHistoricalData(It.IsAny<HistoricalDataRequest>()))
.Callback<HistoricalDataRequest>(req => assignedIDs.Add(req.AssignedID));
for (int i = 0; i < 3; i++)
{
_broker.RequestHistoricalData(request);
}
Assert.AreEqual(3, assignedIDs.Count);
Assert.AreEqual(3, assignedIDs.Distinct().Count());
Assert.AreEqual(0, assignedIDs.Count(x => x < 0));
}
public void RequestsAreCorrectlySplitIntoSubrequestsWhenOnlyPartOfTheDataIsAvailable()
{
var request = new HistoricalDataRequest(_instrument, BarSize.OneDay, new DateTime(2012, 1, 1), new DateTime(2013, 1, 1),
forceFreshData: false,
localStorageOnly: false,
saveToLocalStorage: false,
rthOnly: true);
StoredDataInfo sdInfo = new StoredDataInfo()
{
EarliestDate = new DateTime(2012, 6, 1),
LatestDate = new DateTime(2012, 9, 1),
Frequency = BarSize.OneDay,
InstrumentID = 1
};
_localStorageMock.Setup(x => x.GetStorageInfo(1, BarSize.OneDay)).Returns(sdInfo);
_broker.RequestHistoricalData(request);
//first subrequest
_dataSourceMock.Verify(x => x.RequestHistoricalData(
It.Is<HistoricalDataRequest>(y =>
y.StartingDate.Month == 1 &&
y.StartingDate.Day == 1 &&
y.EndingDate.Month == 5 &&
y.EndingDate.Day == 31
)), Times.Once);
//second subrequest
_dataSourceMock.Verify(x => x.RequestHistoricalData(
It.Is<HistoricalDataRequest>(y =>
y.StartingDate.Month == 9 &&
y.StartingDate.Day == 1 &&
y.EndingDate.Month == 1 &&
y.EndingDate.Day == 1
)), Times.Once);
}
public void RequestEndingTimesAreCorrectlyConstrainedToThePresentTimeInTheInstrumentsTimeZone()
{
var request = new HistoricalDataRequest(_instrument, BarSize.OneDay, new DateTime(2012, 1, 1), new DateTime(2100, 1, 1),
forceFreshData: true,
localStorageOnly: false,
saveToLocalStorage: false,
rthOnly: true);
var est = TimeZoneInfo.FindSystemTimeZoneById("Eastern Standard Time");
var now = TimeZoneInfo.ConvertTime(DateTime.Now, TimeZoneInfo.Local, est);
var modifiedRequest = new HistoricalDataRequest();
_dataSourceMock.Setup(x => x
.RequestHistoricalData(It.IsAny<HistoricalDataRequest>()))
.Callback<HistoricalDataRequest>(req => modifiedRequest = req);
_broker.RequestHistoricalData(request);
Assert.AreEqual(now.Year, modifiedRequest.EndingDate.Year, string.Format("Expected: {0} Was: {1}", modifiedRequest.EndingDate.Year, now.Year));
Assert.AreEqual(now.Month, modifiedRequest.EndingDate.Month, string.Format("Expected: {0} Was: {1}", modifiedRequest.EndingDate.Month, now.Month));
Assert.AreEqual(now.Day, modifiedRequest.EndingDate.Day, string.Format("Expected: {0} Was: {1}", modifiedRequest.EndingDate.Day, now.Day));
Assert.AreEqual(now.Hour, modifiedRequest.EndingDate.Hour, string.Format("Expected: {0} Was: {1}", modifiedRequest.EndingDate.Hour, now.Hour));
Assert.AreEqual(now.Minute, modifiedRequest.EndingDate.Minute, string.Format("Expected: {0} Was: {1}", modifiedRequest.EndingDate.Minute, now.Minute));
}
public void LocalStorageOnlyFlagIsObeyed()
{
var request = new HistoricalDataRequest(_instrument, BarSize.OneDay, new DateTime(2012, 1, 1), new DateTime(2013, 1, 1),
forceFreshData: false,
localStorageOnly: true,
saveToLocalStorage: false,
rthOnly: true);
_broker.RequestHistoricalData(request);
_dataSourceMock.Verify(x => x.RequestHistoricalData(It.IsAny<HistoricalDataRequest>()), Times.Never);
_localStorageMock.Verify(x => x.RequestHistoricalData(
It.Is<HistoricalDataRequest>(
i =>
i.Instrument.ID == 1 &&
i.Frequency == BarSize.OneDay &&
i.StartingDate.Year == 2012 &&
i.StartingDate.Month == 1 &&
i.StartingDate.Day == 1 &&
i.EndingDate.Year == 2013 &&
i.EndingDate.Month == 1 &&
i.EndingDate.Day == 1 &&
i.ForceFreshData == false &&
i.LocalStorageOnly == true &&
i.SaveDataToStorage == false &&
i.RTHOnly == true)));
}
/// <summary>
/// Given a historical data request and the data that fill it,
/// send the reply to the client who made the request.
/// </summary>
private void SendFilledHistoricalRequest(HistoricalDataRequest request, List<OHLCBar> data)
{
lock (_socketLock)
{
if (_socket != null)
{
using (var ms = new MemoryStream())
{
// This is a 5 part message
// 1st message part: the identity string of the client that we're routing the data to
var clientIdentity = request.RequesterIdentity;
_socket.SendMoreFrame(clientIdentity ?? string.Empty);
// 2nd message part: the type of reply we're sending
_socket.SendMoreFrame(MessageType.HistReply);
// 3rd message part: the HistoricalDataRequest object that was used to make the request
_socket.SendMoreFrame(MyUtils.ProtoBufSerialize(request, ms));
// 4th message part: the size of the uncompressed, serialized data. Necessary for decompression on the client end.
var uncompressed = MyUtils.ProtoBufSerialize(data, ms);
_socket.SendMoreFrame(BitConverter.GetBytes(uncompressed.Length));
// 5th message part: the compressed serialized data.
var compressed = LZ4Codec.EncodeHC(uncompressed, 0, uncompressed.Length); // compress
_socket.SendFrame(compressed);
}
}
}
}
public void RequestHistoricalData(HistoricalDataRequest request)
{
throw new NotImplementedException();
}
public void HistoricalRequestsAreCorrectlyForwardedToTheIBClient()
{
var exchange = new Exchange { ID = 1, Name = "Ex", Timezone = "Pacific Standard Time" };
var req = new HistoricalDataRequest
{
Instrument = new Instrument { ID = 1, Symbol = "SPY", Exchange = exchange },
Frequency = QDMS.BarSize.OneDay,
StartingDate = new DateTime(2014, 1, 14),
EndingDate = new DateTime(2014, 1, 15),
RTHOnly = true
};
_ibDatasource.RequestHistoricalData(req);
_ibClientMock.Verify(
x => x.RequestHistoricalData(
It.IsAny<int>(),
It.IsAny<Contract>(),
It.IsAny<DateTime>(),
It.Is<string>(y => y == "1 D"),
It.Is<Krs.Ats.IBNet.BarSize>(y => y == Krs.Ats.IBNet.BarSize.OneDay),
It.Is<HistoricalDataType>(y => y == HistoricalDataType.Trades),
It.Is<int>(y => y == 1))
, Times.Once);
}
public void HistoricalDataRequestsAreForwardedToTheCorrectDataSource()
{
var request = new HistoricalDataRequest(_instrument, BarSize.OneDay, new DateTime(2012, 1, 1), new DateTime(2013, 1, 1),
dataLocation: DataLocation.ExternalOnly,
saveToLocalStorage: false,
rthOnly: true);
_broker.RequestHistoricalData(request);
_dataSourceMock.Verify(x => x.RequestHistoricalData(
It.Is<HistoricalDataRequest>(
i =>
i.Instrument.ID == 1 &&
i.Frequency == BarSize.OneDay &&
i.StartingDate.Year == 2012 &&
i.StartingDate.Month == 1 &&
i.StartingDate.Day == 1 &&
i.EndingDate.Year == 2013 &&
i.EndingDate.Month == 1 &&
i.EndingDate.Day == 1 &&
i.DataLocation == DataLocation.ExternalOnly &&
i.SaveDataToStorage == false &&
i.RTHOnly == true)), Times.Once);
}
static void Main()
{
//create the client, assuming the default port settings
QDMSClient.QDMSClient client = new QDMSClient.QDMSClient(
"SampleClient",
"127.0.0.1",
5556,
5557,
5558,
5555);
//hook up the events needed to receive data & error messages
client.HistoricalDataReceived += client_HistoricalDataReceived;
client.RealTimeDataReceived += client_RealTimeDataReceived;
client.LocallyAvailableDataInfoReceived += client_LocallyAvailableDataInfoReceived;
client.Error += client_Error;
//connect to the server
client.Connect();
//make sure the connection was succesful before we continue
if (!client.Connected)
{
Console.WriteLine("Could not connect.");
Console.WriteLine("Press enter to exit.");
Console.ReadLine();
return;
}
//request the list of available instruments
List<Instrument> instruments = client.FindInstruments();
foreach (Instrument i in instruments)
{
Console.WriteLine("Instrument ID {0}: {1} ({2}), Datasource: {3}",
i.ID,
i.Symbol,
i.Type,
i.Datasource.Name);
}
Thread.Sleep(3000);
//then we grab some historical data from Yahoo
//start by finding the SPY instrument
var spy = client.FindInstruments(x => x.Symbol == "SPY" && x.Datasource.Name == "Yahoo").FirstOrDefault();
if (spy != null)
{
var req = new HistoricalDataRequest(
spy,
BarSize.OneDay,
new DateTime(2013, 1, 1),
new DateTime(2013, 1, 15),
dataLocation: DataLocation.Both,
saveToLocalStorage: true,
rthOnly: true);
client.RequestHistoricalData(req);
Thread.Sleep(3000);
//now that we downloaded the data, let's make a request to see what is stored locally
client.GetLocallyAvailableDataInfo(spy);
Thread.Sleep(3000);
//finally send a real time data request (from the simulated data datasource)
spy.Datasource.Name = "SIM";
var rtReq = new RealTimeDataRequest(spy, BarSize.OneSecond);
client.RequestRealTimeData(rtReq);
Thread.Sleep(3000);
//And then cancel the real time data stream
client.CancelRealTimeData(spy);
}
Console.WriteLine("Press enter to exit.");
Console.ReadLine();
client.Disconnect();
client.Dispose();
}
/// <summary>
/// Simple check for any data abnormalities
/// </summary>
private void CheckDataForOutliers(HistoricalDataRequest request)
{
if (!_settings.Outliers) return;
double abnormalLimit = 0.20;
double abnormalStDevRange = 3;
var inst = request.Instrument;
var freq = request.Frequency;
//Grab the data plus some slightly older for comparison
List<OHLCBar> data = _localStorage.GetData(
inst,
request.StartingDate.Add(-TimeSpan.FromMinutes(freq.ToTimeSpan().Minutes * 5)),
request.EndingDate,
freq);
if(data == null || data.Count <= 1) return;
//count how many bars are not newly updated
int toSkip = data.Count(x => x.DT < request.StartingDate);
var closePrices = data
.Select(x => x.AdjClose.HasValue ? (double)x.AdjClose.Value : (double)x.Close);
var absRets = closePrices.Zip(closePrices.Skip(1), (x, y) => Math.Abs(y / x - 1));
if(absRets.Skip(Math.Max(0, toSkip - 1)).Max() >= abnormalLimit)
{
_errors.Add(string.Format("Possible dirty data detected, abnormally large returns in instrument {0} at frequency {1}.",
inst,
freq));
}
//Check for abnormally large ranges
var highs = data.Select(x => x.AdjHigh.HasValue ? x.AdjHigh.Value : x.High);
var lows = data.Select(x => x.AdjLow.HasValue ? x.AdjLow.Value : x.Low);
var ranges = highs.Zip(lows, (h, l) => (double) (h - l));
double stDev = ranges.QDMSStandardDeviation();
double mean = ranges.Average();
if(ranges.Skip(toSkip).Any(x => x > mean + stDev * abnormalStDevRange))
{
_errors.Add(string.Format("Possible dirty data detected, abnormally large range in instrument {0} at frequency {1}.",
inst,
freq));
}
}
/// <summary>
/// Historical data event args.
/// </summary>
public HistoricalDataEventArgs(HistoricalDataRequest request, List <OHLCBar> data)
{
Request = request;
Data = data;
}
public void ThrowsExceptionWhenMakingRequestForInstrumentWithDataSourceThatIsDisconnected()
{
var request = new HistoricalDataRequest(_instrument, BarSize.OneDay, new DateTime(2012, 1, 1), new DateTime(2013, 1, 1),
dataLocation: DataLocation.ExternalOnly,
saveToLocalStorage: false,
rthOnly: true);
_dataSourceMock.SetupGet(x => x.Connected).Returns(false);
Assert.Throws<Exception>(() =>_broker.RequestHistoricalData(request));
}
/// <summary>
/// Historical data event args.
/// </summary>
public HistoricalDataEventArgs(HistoricalDataRequest request, List<OHLCBar> data)
{
Request = request;
Data = data;
}
public void DataRequestsOnContinuousFuturesAreForwardedToTheCFBroker()
{
_instrument.IsContinuousFuture = true;
var request = new HistoricalDataRequest(_instrument, BarSize.OneDay, new DateTime(2012, 1, 1), new DateTime(2013, 1, 1),
forceFreshData: true,
localStorageOnly: false,
saveToLocalStorage: false,
rthOnly: true);
_broker.RequestHistoricalData(request);
_cfBrokerMock.Verify(x => x.RequestHistoricalData(
It.Is<HistoricalDataRequest>(
i =>
i.Instrument.ID == 1 &&
i.Frequency == BarSize.OneDay &&
i.StartingDate.Year == 2012 &&
i.StartingDate.Month == 1 &&
i.StartingDate.Day == 1 &&
i.EndingDate.Year == 2013 &&
i.EndingDate.Month == 1 &&
i.EndingDate.Day == 1 &&
i.ForceFreshData == true &&
i.LocalStorageOnly == false &&
i.SaveDataToStorage == false &&
i.RTHOnly == true)), Times.Once);
}
