public void HistoricalDataRequestsAreForwardedToTheBroker()
{
var instrument = new Instrument
{
ID = 1,
Symbol = "SPY",
Datasource = new Datasource { ID = 1, Name = "MockSource" },
Exchange = new Exchange
{
ID = 1,
Name = "Exchange",
Timezone = "Eastern Standard Time"
}
};
var request = new HistoricalDataRequest(instrument, BarSize.OneDay, new DateTime(2012, 1, 1), new DateTime(2013, 1, 1));
_client.RequestHistoricalData(request);
// TODO: Think about delay amount
Thread.Sleep(1500);
_historicalDataBrokerMock.Verify(
x => x.RequestHistoricalData(
It.Is<HistoricalDataRequest>(
r =>
r.Instrument.ID == 1 &&
r.Frequency == BarSize.OneDay &&
r.StartingDate == new DateTime(2012, 1, 1) &&
r.EndingDate == new DateTime(2013, 1, 1))),
Times.Once);
}
public void EmailReportSentOnBrokerError()
{
var settings = new UpdateJobSettings(errors: true, timeout: 1, toEmail: "*****@*****.**", fromEmail: "*****@*****.**");
var job = new DataUpdateJob(_brokerMock.Object, _mailMock.Object, settings, _localStorageMock.Object, _instrumentManagerMock.Object);
_brokerMock
.Setup(x => x.RequestHistoricalData(It.IsAny<HistoricalDataRequest>()))
.Throws(new Exception("TestException123"));
Instrument inst = new Instrument() { ID = 1, Symbol = "SPY", Currency = "USD", Type = InstrumentType.Stock };
_instrumentManagerMock
.Setup(x => x.FindInstruments(It.IsAny<Expression<Func<Instrument, bool>>>(), It.IsAny<MyDBContext>()))
.Returns(new List<Instrument>() { inst });
_localStorageMock
.Setup(x => x.GetStorageInfo(It.IsAny<int>()))
.Returns(new List<StoredDataInfo>() {
new StoredDataInfo()
{
Frequency = BarSize.OneDay,
InstrumentID = inst.ID.Value,
LatestDate = DateTime.Now.AddDays(-2)
} });
job.Execute(_contextMock.Object);
_mailMock.Verify(x =>
x.Send(
It.IsAny<string>(),
It.Is<string>(y => y == "*****@*****.**"),
It.IsAny<string>(),
It.Is<string>(y => y.Contains("TestException123"))));
}
public DataAdditionRequest(BarSize frequency, Instrument instrument, List<OHLCBar> data, bool overwrite = true)
{
Data = data;
Frequency = frequency;
Instrument = instrument;
Overwrite = overwrite;
}
public HistoricalRequestWindow(Instrument instrument)
{
InitializeComponent();
DataContext = this;
Random r = new Random(); //we have to randomize the name of the client, can't reuse the identity
Title = string.Format("Data Request - {0} @ {1}", instrument.Symbol, instrument.Datasource.Name);
_client = new QDMSClient.QDMSClient(
string.Format("DataRequestClient-{0}", r.Next()),
"localhost",
Properties.Settings.Default.rtDBReqPort,
Properties.Settings.Default.rtDBPubPort,
Properties.Settings.Default.instrumentServerPort,
Properties.Settings.Default.hDBPort);
_client.HistoricalDataReceived += _client_HistoricalDataReceived;
_client.Error += _client_Error;
_client.Connect();
Data = new ObservableCollection<OHLCBar>();
TheInstrument = instrument;
StartTime = new DateTime(1950, 1, 1, 0, 0, 0, 0);
EndTime = new DateTime(DateTime.Now.Year, DateTime.Now.Month, DateTime.Now.Day, 0, 0, 0, 0);
if (!TheInstrument.ID.HasValue) return;
ShowDialog();
}
/// <summary>
/// Delete an instrument and all locally stored data.
/// </summary>
public static void RemoveInstrument(Instrument instrument)
{
using (var entityContext = new MyDBContext())
{
//hacking around the circular reference issue
if (instrument.IsContinuousFuture)
{
entityContext.Instruments.Attach(instrument);
var tmpCF = instrument.ContinuousFuture;
instrument.ContinuousFuture = null;
instrument.ContinuousFutureID = null;
entityContext.SaveChanges();
entityContext.ContinuousFutures.Attach(tmpCF);
entityContext.ContinuousFutures.Remove(tmpCF);
entityContext.SaveChanges();
}
entityContext.Instruments.Attach(instrument);
entityContext.Instruments.Remove(instrument);
entityContext.SaveChanges();
}
using (var localStorage = DataStorageFactory.Get())
{
localStorage.Connect();
localStorage.DeleteAllInstrumentData(instrument);
}
}
public static Contract InstrumentToContract(Instrument instrument)
{
string symbol = string.IsNullOrEmpty(instrument.DatasourceSymbol) ? instrument.Symbol : instrument.DatasourceSymbol;
var contract = new Contract(
0,
symbol,
SecurityTypeConverter(instrument.Type),
instrument.Expiration.HasValue ? instrument.Expiration.Value.ToString("yyyyMM", CultureInfo.InvariantCulture) : "",
0,
OptionTypeToRightType(instrument.OptionType),
instrument.Multiplier.ToString(),
"",
instrument.Currency,
null,
null,
SecurityIdType.None,
string.Empty);
if (instrument.Strike.HasValue)
contract.Strike = (double)instrument.Strike.Value;
if (instrument.Exchange != null)
contract.Exchange = instrument.Exchange.Name;
return contract;
}
public RealTimeDataRequest(Instrument instrument, BarSize frequency, bool rthOnly = true, bool savetoLocalStorage = false)
{
Instrument = instrument;
Frequency = frequency;
RTHOnly = rthOnly;
SaveToLocalStorage = savetoLocalStorage;
}
public static Instrument ContractDetailsToInstrument(ContractDetails contract)
{
var instrument = new Instrument
{
Symbol = contract.Summary.LocalSymbol,
UnderlyingSymbol = contract.Summary.Symbol,
Name = contract.LongName,
OptionType = RightTypeToOptionType(contract.Summary.Right),
Type = InstrumentTypeConverter(contract.Summary.SecurityType),
Multiplier = contract.Summary.Multiplier == null ? 1 : int.Parse(contract.Summary.Multiplier),
Expiration = string.IsNullOrEmpty(contract.Summary.Expiry) ? (DateTime?)null : DateTime.ParseExact(contract.Summary.Expiry, "yyyyMMdd", CultureInfo.InvariantCulture),
Strike = (decimal)contract.Summary.Strike,
Currency = contract.Summary.Currency,
MinTick = (decimal)contract.MinTick,
Industry = contract.Industry,
Category = contract.Category,
Subcategory = contract.Subcategory,
IsContinuousFuture = false,
ValidExchanges = contract.ValidExchanges
};
if (!string.IsNullOrEmpty(contract.Summary.PrimaryExchange))
instrument.PrimaryExchange = new Exchange { Name = contract.Summary.PrimaryExchange };
return instrument;
}
public void SetUp()
{
_instrument = new Instrument
{
ID = 1,
Symbol = "SPY",
Datasource = new Datasource { ID = 1, Name = "MockSource" }
};
_instrument.Exchange = new Exchange()
{
ID = 1,
Name = "Exchange",
Timezone = "Eastern Standard Time"
};
_dataSourceMock = new Mock<IHistoricalDataSource>();
_dataSourceMock.SetupGet(x => x.Name).Returns("MockSource");
_dataSourceMock.SetupGet(x => x.Connected).Returns(false);
_localStorageMock = new Mock<IDataStorage>();
_cfBrokerMock = new Mock<IContinuousFuturesBroker>();
_cfBrokerMock.SetupGet(x => x.Connected).Returns(true);
_broker = new HistoricalDataBroker(_cfBrokerMock.Object, _localStorageMock.Object, new List<IHistoricalDataSource> { _dataSourceMock.Object });
_dataSourceMock.SetupGet(x => x.Connected).Returns(true);
}
public DataImportWindow(Instrument instrument)
{
InitializeComponent();
//reload the instrument first to make sure we have up-to-date data
using (var context = new MyDBContext())
{
context.Instruments.Attach(instrument);
context.Entry(instrument).Reload();
_instrument = instrument;
}
Title += " - " + _instrument.Symbol;
//fill frequency combo box
var values = MyUtils.GetEnumValues<BarSize>();
foreach (BarSize s in values)
{
FrequencyComboBox.Items.Add(s);
}
FrequencyComboBox.SelectedItem = BarSize.OneDay;
MinDT.Value = new DateTime(1950, 1, 1);
MaxDT.Value = DateTime.Now;
}
public void DataPushRequestIsForwardedToLocalStorage()
{
var instrument = new Instrument
{
ID = 1,
Symbol = "SPY",
Datasource = new Datasource { ID = 1, Name = "MockSource" }
};
var data = new List<OHLCBar>
{
new OHLCBar {Open = 1, High = 2, Low = 3, Close = 4, DT = new DateTime(2013, 1, 1) }
};
var req = new DataAdditionRequest(BarSize.OneDay, instrument, data, true);
_client.PushData(req);
Thread.Sleep(50);
_brokerMock.Verify(x => x.AddData(
It.Is<DataAdditionRequest>(y =>
y.Frequency == BarSize.OneDay &&
y.Instrument.ID == 1 &&
y.Data.Count == 1)
), Times.Once);
}
public void ProtoBufSerializeAndDeserializeResultsInIdenticalObject()
{
var inst = new Instrument
{
ID = 5,
Symbol = "test",
UnderlyingSymbol = "123",
Exchange = new Exchange
{
ID = 1,
Name = "CBOE"
},
Type = InstrumentType.Stock
};
var ms = new MemoryStream();
var instDeserialized = MyUtils.ProtoBufDeserialize <Instrument>(MyUtils.ProtoBufSerialize(inst, ms), ms);
Assert.AreEqual(inst.ID, instDeserialized.ID);
Assert.AreEqual(inst.Symbol, instDeserialized.Symbol);
Assert.AreEqual(inst.UnderlyingSymbol, instDeserialized.UnderlyingSymbol);
Assert.AreEqual(inst.Type, instDeserialized.Type);
Assert.AreEqual(inst.Exchange.ID, instDeserialized.Exchange.ID);
Assert.AreEqual(inst.Exchange.Name, instDeserialized.Exchange.Name);
}
public void ServerCorrectlyForwardsRealTimeData()
{
var ds = new Datasource() { ID = 1, Name = "TestDS" };
var inst = new Instrument() { ID = 15, Datasource = ds, DatasourceID = 1, Symbol = "SPY", Type = InstrumentType.Stock };
var req = new RealTimeDataRequest(inst, BarSize.FiveSeconds, rthOnly: false, savetoLocalStorage: false);
_brokerMock.Setup(x => x.RequestRealTimeData(It.IsAny<RealTimeDataRequest>())).Returns(true);
_client.RequestRealTimeData(req);
Thread.Sleep(50);
RealTimeDataEventArgs receivedData = null;
_client.RealTimeDataReceived += (s,e) => receivedData = e;
long dt = DateTime.Now.ToBinary();
_brokerMock.Raise(x => x.RealTimeDataArrived += null, new RealTimeDataEventArgs(15, dt, 100m, 105m, 95m, 99m, 10000000, 101, 500, 1));
Thread.Sleep(50);
Assert.IsNotNull(receivedData);
Assert.AreEqual(15, receivedData.InstrumentID);
Assert.AreEqual(dt, receivedData.Time);
Assert.AreEqual(100m, receivedData.Open);
Assert.AreEqual(105m, receivedData.High);
Assert.AreEqual(95m, receivedData.Low);
Assert.AreEqual(99m, receivedData.Close);
Assert.AreEqual(10000000, receivedData.Volume);
Assert.AreEqual(500, receivedData.Count);
Assert.AreEqual(101, receivedData.Wap);
}
public void GetTZInfoThrowsExceptionIfNonExistingTimezoneIsSpecified()
{
var inst = new Instrument { Exchange = new Exchange { Timezone = "asdf____" } };
Assert.Throws<TimeZoneNotFoundException>(() =>
{
TimeZoneInfo tz = inst.GetTZInfo();
});
}
public RealTimeStreamInfo(Instrument instrument, int requestID, string datasource, BarSize frequency, bool rthOnly) : this()
{
Instrument = instrument;
RequestID = requestID;
Datasource = datasource;
Frequency = frequency;
RTHOnly = rthOnly;
}
public void ServerCorrectlyForwardsCancellationRequestsToBroker()
{
var ds = new Datasource() { ID = 1, Name = "TestDS" };
var inst = new Instrument() { ID = 15, Datasource = ds, DatasourceID = 1, Symbol = "SPY", Type = InstrumentType.Stock };
_client.CancelRealTimeData(inst);
_brokerMock.Verify(x => x.CancelRTDStream(It.Is<int>(y => y == 15)));
}
//utility function to get the contracts used for the continuous futures calculations
public static List<Instrument> GetVIXFutures()
{
var nov12 = new Instrument
{
ID = 1,
Type = InstrumentType.Future,
Expiration = new DateTime(2012, 11, 21),
UnderlyingSymbol = "VIX",
Symbol = "VIXX12"
};
var dec12 = new Instrument
{
ID = 2,
Type = InstrumentType.Future,
Expiration = new DateTime(2012, 12, 19),
UnderlyingSymbol = "VIX",
Symbol = "VIXZ12"
};
var jan13 = new Instrument
{
ID = 3,
Type = InstrumentType.Future,
Expiration = new DateTime(2013, 1, 16),
UnderlyingSymbol = "VIX",
Symbol = "VIXF13"
};
var feb13 = new Instrument
{
ID = 4,
Type = InstrumentType.Future,
Expiration = new DateTime(2013, 2, 13),
UnderlyingSymbol = "VIX",
Symbol = "VIXG13"
};
var mar13 = new Instrument
{
ID = 5,
Type = InstrumentType.Future,
Expiration = new DateTime(2013, 3, 20),
UnderlyingSymbol = "VIX",
Symbol = "VIXH13"
};
var futures = new List<Instrument>
{
nov12,
dec12,
jan13,
feb13,
mar13
};
return futures;
}
/// <summary>
///
/// </summary>
/// <param name="continuousFuture"></param>
/// <returns>The current front future for this continuous futures contract. Null if it is not found.</returns>
public Instrument GetCurrentFrontFuture(Instrument continuousFuture)
{
var searchInstrument = new Instrument { UnderlyingSymbol = continuousFuture.UnderlyingSymbol, Type = InstrumentType.Future };
var futures = InstrumentManager.FindInstruments(search: searchInstrument);
if (futures.Count == 0) return null;
return new Instrument();
}
public void BarTimeAdjustmentsHappenCorrectly()
{
string json = @"{
""status"": {
""code"": 200,
""message"": ""Success.""
},
""results"": [{
""symbol"": ""IBM"",
""timestamp"": ""2016-08-15T09:00:00-04:00"",
""tradingDay"": ""2016-08-15"",
""open"": 162.4,
""high"": 162.97,
""low"": 162.38,
""close"": 162.77,
""volume"": 215371
},
{
""symbol"": ""IBM"",
""timestamp"": ""2016-08-15T10:00:00-04:00"",
""tradingDay"": ""2016-08-15"",
""open"": 162.8,
""high"": 162.95,
""low"": 162.61,
""close"": 162.63,
""volume"": 222815
}]
}";
var exchange = new Exchange
{
Timezone = "Eastern Standard Time"
};
var instrument = new Instrument()
{
Symbol = "IBM",
Exchange = exchange,
Sessions = new List<InstrumentSession>{
new InstrumentSession{ OpeningDay = DayOfTheWeek.Monday, OpeningTime = new TimeSpan(9, 30, 0) } }
};
var request = new HistoricalDataRequest(instrument, BarSize.OneHour, DateTime.Now, DateTime.Now);
List<OHLCBar> bars = BarChartUtils.ParseJson(JObject.Parse(json), request);
//opening time set to the session opening instead of earlier
Assert.AreEqual(new DateTime(2016, 8, 15, 9, 30, 0), bars[0].DTOpen.Value);
//Bar closing time set correctly
Assert.AreEqual(new DateTime(2016, 8, 15, 10, 0, 0), bars[0].DT);
//timezone conversion has happened properly
Assert.AreEqual(new DateTime(2016, 8, 15, 10, 0, 0), bars[1].DTOpen.Value);
}
public HistoricalDataRequest(Instrument instrument, BarSize frequency, DateTime startingDate, DateTime endingDate, bool forceFreshData = false, bool localStorageOnly = false, bool saveToLocalStorage = true, bool rthOnly = false)
{
Frequency = frequency;
Instrument = instrument;
StartingDate = startingDate;
EndingDate = endingDate;
ForceFreshData = forceFreshData;
LocalStorageOnly = localStorageOnly;
SaveDataToStorage = saveToLocalStorage;
RTHOnly = rthOnly;
}
/// <summary>
/// Add a new instrument or update an existing instrument in the database.
/// </summary>
/// <param name="instrument"></param>
/// <param name="updateIfExists"></param>
/// <param name="saveChanges">Set to true if saving to db should be done.</param>
/// <returns>True if the insertion or update succeeded. False if it did not.</returns>
public static bool AddInstrument(Instrument instrument, bool updateIfExists = false, bool saveChanges = true)
{
if (instrument.IsContinuousFuture)
{
throw new Exception("Cannot add continuous futures using this method");
}
using (var context = new MyDBContext())
{
var existingInstrument = context.Instruments.SingleOrDefault(x =>
(x.ID == instrument.ID) ||
(x.Symbol == instrument.Symbol && x.DatasourceID == instrument.DatasourceID && x.ExchangeID == instrument.ExchangeID));
if (existingInstrument == null) //object doesn't exist, so we add it
{
//if necessary, load sessions from teplate or exchange
if (instrument.SessionsSource == SessionsSource.Exchange)
{
instrument.Sessions = new List<InstrumentSession>();
var exchange = context.Exchanges.Include("Sessions").First(x => x.ID == instrument.ExchangeID);
foreach (ExchangeSession s in exchange.Sessions)
{
instrument.Sessions.Add(MyUtils.SessionConverter(s));
}
}
else if (instrument.SessionsSource == SessionsSource.Template)
{
instrument.Sessions = new List<InstrumentSession>();
var template = context.SessionTemplates.Include("Sessions").First(x => x.ID == instrument.SessionTemplateID);
foreach (TemplateSession s in template.Sessions)
{
instrument.Sessions.Add(MyUtils.SessionConverter(s));
}
}
context.Instruments.Add(instrument);
context.Database.Connection.Open();
if (saveChanges) context.SaveChanges();
return true;
}
else if (updateIfExists) //object exist, but we want to update it
{
context.Entry(existingInstrument).CurrentValues.SetValues(instrument);
if (saveChanges) context.SaveChanges();
return true;
}
}
return false; //object exists and we don't update it
}
public void ServerCorrectlyForwardsRealTimeDataRequestsToBroker()
{
var ds = new Datasource() { ID = 1, Name = "TestDS" };
var inst = new Instrument() { ID = 15, Datasource = ds, DatasourceID = 1, Symbol = "SPY", Type = InstrumentType.Stock };
var req = new RealTimeDataRequest(inst, BarSize.FiveSeconds, rthOnly: false, savetoLocalStorage: false);
_client.RequestRealTimeData(req);
_brokerMock.Verify(x => x.RequestRealTimeData(It.Is<RealTimeDataRequest>(
y => y.Frequency == BarSize.FiveSeconds &&
y.RTHOnly == false &&
y.SaveToLocalStorage == false &&
y.Instrument.ID == 15 &&
y.Instrument.Symbol == "SPY" &&
y.Instrument.Datasource.Name == "TestDS")));
}
public DataEditWindow(Instrument instrument)
{
InitializeComponent();
DataContext = this;
Data = new ObservableCollection<OHLCBar>();
//grab and update the instrument
using (var context = new MyDBContext())
{
context.Instruments.Attach(instrument);
context.Entry(instrument).Reload();
TheInstrument = instrument;
}
string timezone = TheInstrument.Exchange == null ? "UTC" : TheInstrument.Exchange.Timezone;
_tzInfo = TimeZoneInfo.FindSystemTimeZoneById(timezone);
StartTime = new DateTime(1950, 1, 1, 0, 0, 0, 0);
EndTime = new DateTime(DateTime.Now.Year, DateTime.Now.Month, DateTime.Now.Day, 0, 0, 0, 0);
if (!TheInstrument.ID.HasValue) return;
//grab the data info
using (var localStorage = DataStorageFactory.Get())
{
var storageInfo = localStorage.GetStorageInfo(TheInstrument.ID.Value);
if (storageInfo.Count == 0) //if it doesn't have any data, we just exit
{
MessageBox.Show("This instrument has no data.");
Hide();
}
else
{
foreach (StoredDataInfo s in storageInfo) //fill the resolution box
{
ResolutionComboBox.Items.Add(s.Frequency);
}
}
}
}
public void InstrumentAdditionRequestsAreSentCorrectly()
{
var instrumentSourceMock = new Mock<IInstrumentSource>();
var instrumentsServer = new InstrumentsServer(5555, instrumentSourceMock.Object);
instrumentsServer.StartServer();
var rtdBrokerMock = new Mock<IRealTimeDataBroker>();
var rtdServer = new RealTimeDataServer(5554, 5553, rtdBrokerMock.Object);
rtdServer.StartServer();
_client.Connect();
var exchange = new Exchange() { ID = 1, Name = "NYSE", Sessions = new List<ExchangeSession>(), Timezone = "Eastern Standard Time" };
var datasource = new Datasource() { ID = 1, Name = "Yahoo" };
var instrument = new Instrument() { Symbol = "SPY", UnderlyingSymbol = "SPY", Type = InstrumentType.Stock, Currency = "USD", Exchange = exchange, Datasource = datasource, Multiplier = 1 };
instrumentSourceMock.Setup(x => x.AddInstrument(It.IsAny<Instrument>(), It.IsAny<bool>(), It.IsAny<bool>())).Returns(instrument);
Instrument result = _client.AddInstrument(instrument);
Thread.Sleep(50);
Assert.IsTrue(result != null);
instrumentSourceMock.Verify(x => x.AddInstrument(
It.Is<Instrument>(y =>
y.Symbol == "SPY" &&
y.Exchange != null &&
y.Exchange.Name == "NYSE" &&
y.Datasource != null &&
y.Datasource.Name == "Yahoo" &&
y.Type == InstrumentType.Stock &&
y.Currency == "USD" &&
y.Multiplier == 1),
It.Is<bool>(y => y == false),
It.Is<bool>(y => y == true)));
rtdServer.StopServer();
rtdServer.Dispose();
instrumentsServer.StopServer();
instrumentsServer.Dispose();
}
public void AcceptAvailableDataRequestsAreForwardedToTheHistoricalDataBroker()
{
var instrument = new Instrument
{
ID = 1,
Symbol = "SPY",
Datasource = new Datasource { ID = 1, Name = "MockSource" }
};
_client.GetLocallyAvailableDataInfo(instrument);
Thread.Sleep(50);
_brokerMock.Verify(x => x.GetAvailableDataInfo(
It.Is<Instrument>(y =>
y.ID == 1 &&
y.Symbol == "SPY")
), Times.Once);
}
/// <summary>
/// Finds the currently active futures contract for a continuous futures instrument.
/// The contract is returned asynchronously through the FoundFrontContract event.
/// </summary>
/// <returns>Returns an ID uniquely identifying this request.</returns>
public int RequestFrontContract(Instrument cfInstrument, DateTime?date = null)
{
if (!cfInstrument.IsContinuousFuture)
{
throw new Exception("Not a continuous future instrument.");
}
_lastFrontDontractRequestID++;
var req = new FrontContractRequest
{
ID = _lastFrontDontractRequestID,
Instrument = cfInstrument,
Date = date
};
ProcessFrontContractRequest(req);
return(_lastFrontDontractRequestID);
}
public void HistoricalRequestsAreSplitToRespectRequestLimits()
{
int[] requestCount = {0};
_ibClientMock.Setup(
x => x.RequestHistoricalData(
It.IsAny<int>(),
It.IsAny<Contract>(),
It.IsAny<DateTime>(),
It.IsAny<string>(),
It.IsAny<BarSize>(),
It.IsAny<HistoricalDataType>(),
It.IsAny<int>()))
.Callback<int, Contract, DateTime, string, BarSize, HistoricalDataType, int>(
(a, b, c, d, e, f, g) => requestCount[0]++);
var requests = new Dictionary<KeyValuePair<BarSize, int>, int> //left side is barsize/seconds, right side is expected splits
{
{ new KeyValuePair<BarSize, int>(BarSize.OneDay, 500 * 24 * 3600), 2},
{ new KeyValuePair<BarSize, int>(BarSize.OneHour, 75 * 24 * 3600), 3},
{ new KeyValuePair<BarSize, int>(BarSize.ThirtyMinutes, 22 * 24 * 3600), 4},
{ new KeyValuePair<BarSize, int>(BarSize.OneMinute, 9 * 24 * 3600), 5},
{ new KeyValuePair<BarSize, int>(BarSize.ThirtySeconds, 40 * 3600), 2},
{ new KeyValuePair<BarSize, int>(BarSize.FifteenSeconds, 4 * 14400), 5},
{ new KeyValuePair<BarSize, int>(BarSize.FiveSeconds, 2 * 7200), 3},
{ new KeyValuePair<BarSize, int>(BarSize.OneSecond, 10 * 1800), 11}
};
var inst = new Instrument();
foreach (var kvp in requests)
{
_ibDatasource.RequestHistoricalData(new HistoricalDataRequest(
inst,
TWSUtils.BarSizeConverter(kvp.Key.Key),
DateTime.Now.AddSeconds(-kvp.Key.Value),
DateTime.Now,
dataLocation: DataLocation.ExternalOnly));
Assert.AreEqual(kvp.Value, requestCount[0], kvp.Key.Key.ToString());
requestCount[0] = 0;
}
}
public void DoesNotRepeatRequestToSourceWhenARealTimeStreamAlreadyExists()
{
var inst = new Instrument
{
ID = 1,
Symbol = "SPY",
Datasource = new Datasource { ID = 999, Name = "MockSource" }
};
var req = new RealTimeDataRequest(inst, BarSize.FiveSeconds);
_broker.RequestRealTimeData(req);
Thread.Sleep(100);
_broker.RequestRealTimeData(req);
_dataSourceMock.Verify(x => x.RequestRealTimeData(
It.IsAny<RealTimeDataRequest>()),
Times.Once);
}
public void RealTimeDataRequestsForwardedToDataSource()
{
var inst = new Instrument
{
ID = 1,
Symbol = "SPY",
Datasource = new Datasource { ID = 999, Name = "MockSource" }
};
var req = new RealTimeDataRequest(inst, BarSize.FiveSeconds);
_broker.RequestRealTimeData(req);
_dataSourceMock.Verify(x => x.RequestRealTimeData(
It.Is<RealTimeDataRequest>(
r => r.Instrument.ID == 1 &&
r.Instrument.Symbol == "SPY" &&
r.Frequency == BarSize.FiveSeconds)),
Times.Once);
}
public static Contract InstrumentToContract(Instrument instrument)
{
string symbol = string.IsNullOrEmpty(instrument.DatasourceSymbol) ? instrument.UnderlyingSymbol : instrument.DatasourceSymbol;
string expirationString = "";
//multiple options expire each month so the string needs to be more specific there
if(instrument.Type == InstrumentType.Option && instrument.Expiration.HasValue)
{
expirationString = instrument.Expiration.Value.ToString("yyyyMMdd", CultureInfo.InvariantCulture);
}
else if (instrument.Expiration.HasValue)
{
expirationString = instrument.Expiration.Value.ToString("yyyyMM", CultureInfo.InvariantCulture);
}
var contract = new Contract(
0,
symbol,
SecurityTypeConverter(instrument.Type),
expirationString,
0,
OptionTypeToRightType(instrument.OptionType),
instrument.Multiplier.ToString(),
"",
instrument.Currency,
null,
instrument.PrimaryExchange == null ? null : instrument.PrimaryExchange.Name,
SecurityIdType.None,
string.Empty);
contract.IncludeExpired = instrument.Expiration.HasValue; //only set IncludeExpired to true if the contract can actually expire
if (instrument.Strike.HasValue && instrument.Strike.Value != 0)
contract.Strike = (double)instrument.Strike.Value;
if (instrument.Exchange != null)
contract.Exchange = instrument.Exchange.Name;
return contract;
}
public void ServerReturnsErrorToClientIfNoInstrumentIdIsSet()
{
var ds = new Datasource { ID = 1, Name = "TestDS" };
var inst = new Instrument { ID = null, Datasource = ds, DatasourceID = 1, Symbol = "SPY", Type = InstrumentType.Stock };
var req = new RealTimeDataRequest(inst, BarSize.FiveSeconds, false);
string error = null;
int? requestId = null;
_client.Error += (s, e) =>
{
error = e.ErrorMessage;
requestId = e.RequestID;
};
_client.RequestRealTimeData(req);
Thread.Sleep(DefaultDelayInMilliseconds);
Assert.IsTrue(!string.IsNullOrEmpty(error));
Assert.IsTrue(requestId.HasValue);
}
public void SetUp()
{
_clientMock = new Mock<IDataClient>();
_instrumentMgrMock = new Mock<IInstrumentSource>();
_broker = new ContinuousFuturesBroker(_clientMock.Object, _instrumentMgrMock.Object);
_cfInst = new Instrument();
_cfInst.Type = InstrumentType.Future;
_cfInst.DatasourceID = 1;
_cfInst.Datasource = new Datasource { ID = 1, Name = "Interactive Brokers" };
var cf = new ContinuousFuture();
cf.Month = 1;
_cfInst.IsContinuousFuture = true;
_cfInst.ContinuousFuture = cf;
_cfInst.ContinuousFuture.AdjustmentMode = ContinuousFuturesAdjustmentMode.NoAdjustment;
var vix = new UnderlyingSymbol();
vix.Rule = new ExpirationRule
{
DaysBefore = 30,
DayType = DayType.Calendar,
ReferenceRelativeMonth = RelativeMonth.NextMonth,
ReferenceUsesDays = false,
ReferenceWeekDay = DayOfTheWeek.Friday,
ReferenceWeekDayCount = WeekDayCount.Third,
ReferenceDayMustBeBusinessDay = true
};
vix.Symbol = "VIX";
_cfInst.ContinuousFuture.UnderlyingSymbol = vix;
_req = new HistoricalDataRequest(
_cfInst,
BarSize.OneDay,
new DateTime(2013, 1, 1),
new DateTime(2013, 2, 1));
}
/// <summary>
/// Historical data has arrived.
/// Add it to our data store, then check if all requests have been
/// fulfilled for a particular continuous futures request. If they
/// have, go do the calculations.
/// </summary>
private void _client_HistoricalDataReceived(object sender, HistoricalDataEventArgs e)
{
if (e.Request.Instrument.ID == null)
{
return;
}
int id = e.Request.Instrument.ID.Value;
var kvpID = new KeyValuePair <int, BarSize>(id, e.Request.Frequency);
lock (_data)
{
if (_data.ContainsKey(kvpID))
{
//We already have data on this instrument ID/Frequency combo.
//Add the arrived data, then discard any doubles, then order.
_data[kvpID].AddRange(e.Data);
_data[kvpID] = _data[kvpID].Distinct((x, y) => x.DT == y.DT).ToList();
_data[kvpID] = _data[kvpID].OrderBy(x => x.DT).ToList();
}
else
{
//We have nothing on this instrument ID/Frequency combo.
//Just add a new entry in the dictionary.
_data.Add(kvpID, e.Data);
}
}
//Here we check if all necessary requests have arrived. If they have, do some work.
lock (_reqCountLock)
{
//get the request id of the continuous futures request that caused this contract request
int cfReqID = _histReqIDMap[e.Request.RequestID];
_histReqIDMap.Remove(e.Request.RequestID);
_requestCounts[cfReqID]--;
if (_requestCounts[cfReqID] == 0)
{
//we have received all the data we asked for
_requestCounts.Remove(e.Request.RequestID);
HistoricalDataRequest req;
lock (_requestsLock)
{
req = _requests[cfReqID];
_requests.Remove(cfReqID);
}
if (_requestTypes[cfReqID])
{
//This request originates from a CF data request
//so now we want to generate the continuous prices
GetContFutData(req);
}
else
{
//This request originates from a front contract request
Instrument frontContract = GetContFutData(req, false);
FrontContractRequest originalReq = _frontContractRequestMap[cfReqID];
lock (_frontContractReturnLock)
{
RaiseEvent(FoundFrontContract, this, new FoundFrontContractEventArgs(originalReq.ID, frontContract, originalReq.Date == null ? DateTime.Now : originalReq.Date.Value));
}
}
_requestTypes.Remove(e.Request.AssignedID);
}
}
}
/// <summary>
/// Finds the front contract for continuous futures with time-based roll.
/// </summary>
private void ProcessTimeBasedFrontContractRequest(FrontContractRequest request)
{
DateTime currentDate = request.Date ?? DateTime.Now;
var cf = request.Instrument.ContinuousFuture;
//if the roll-over is time based, we can find the appropriate contract programmatically
DateTime selectedDate = currentDate;
while (!cf.MonthIsUsed(selectedDate.Month))
{
selectedDate = selectedDate.AddMonths(1);
}
DateTime currentMonthsExpirationDate = cf.UnderlyingSymbol.ExpirationDate(selectedDate.Year, selectedDate.Month);
DateTime switchOverDate = currentMonthsExpirationDate;
Calendar calendar = MyUtils.GetCalendarFromCountryCode("US");
//the front contract
//find the switchover date
int daysBack = cf.RolloverDays;
while (daysBack > 0)
{
switchOverDate = switchOverDate.AddDays(-1);
if (calendar.isBusinessDay(switchOverDate))
{
daysBack--;
}
}
//this month's contract has already been switched to the next one
int monthsLeft = 1;
int count = 0;
if (currentDate >= switchOverDate)
{
while (monthsLeft > 0)
{
count++;
if (cf.MonthIsUsed(selectedDate.AddMonths(count).Month))
{
monthsLeft--;
}
}
selectedDate = selectedDate.AddMonths(count);
}
//we found the "front" month, no go back the required number of months
//while skipping unused months
monthsLeft = cf.Month - 1;
count = 0;
while (monthsLeft > 0)
{
if (cf.MonthIsUsed(selectedDate.AddMonths(count).Month))
{
monthsLeft--;
}
count++;
}
selectedDate = selectedDate.AddMonths(count);
string tradingClass = request.Instrument.TradingClass;
//we got the month we want! find the contract
Instrument contract = _instrumentRepo.FindInstruments(x => x.Expiration.HasValue &&
x.Expiration.Value.Month == selectedDate.Month &&
x.Expiration.Value.Year == selectedDate.Year &&
x.UnderlyingSymbol == cf.UnderlyingSymbol.Symbol &&
(string.IsNullOrEmpty(tradingClass) || x.TradingClass == tradingClass)).Result.FirstOrDefault();
var timer = new Timer(50)
{
AutoReset = false
};
timer.Elapsed += (sender, e) =>
{
lock (_frontContractReturnLock)
{
RaiseEvent(FoundFrontContract, this, new FoundFrontContractEventArgs(request.ID, contract, currentDate));
}
};
timer.Start();
}
/// <summary>
/// Taking a historical data request for a continuous futures instrument,
/// it returns a list of requests for the underlying contracts that are needed to fulfill it.
/// </summary>
private List <HistoricalDataRequest> GetRequiredRequests(HistoricalDataRequest request)
{
var requests = new List <HistoricalDataRequest>();
var cf = request.Instrument.ContinuousFuture;
var futures = _instrumentRepo.FindInstruments(x =>
x.UnderlyingSymbol == request.Instrument.ContinuousFuture.UnderlyingSymbol.Symbol &&
x.Type == InstrumentType.Future &&
x.DatasourceID == request.Instrument.DatasourceID).Result;
if (futures == null)
{
Log(LogLevel.Error, "CFB: Error in GetRequiredRequests, failed to return any contracts to historical request ID: " + request.AssignedID);
return(requests);
}
//remove any continuous futures
futures = futures.Where(x => !x.IsContinuousFuture).ToList();
//order them by ascending expiration date
futures = futures.OrderBy(x => x.Expiration).ToList();
//filter the futures months, we may not want all of them.
for (int i = 1; i <= 12; i++)
{
if (cf.MonthIsUsed(i))
{
continue;
}
int i1 = i;
futures = futures.Where(x => x.Expiration.HasValue && x.Expiration.Value.Month != i1).ToList();
}
//nothing found, return with empty hands
if (futures.Count == 0)
{
return(requests);
}
//the first contract we need is the first one expiring before the start of the request period
var expiringBeforeStart = futures
.Where(x => x.Expiration != null && x.Expiration.Value < request.StartingDate)
.Select(x => x.Expiration.Value).ToList();
DateTime firstExpiration =
expiringBeforeStart.Count > 0
? expiringBeforeStart.Max()
: futures.Select(x => x.Expiration.Value).Min();
futures = futures.Where(x => x.Expiration != null && x.Expiration.Value >= firstExpiration).ToList();
//I think the last contract we need is the one that is N months after the second contract that expires after the request period end
//where N is the number of months away from the front contract that the CF uses
var firstExpAfterEnd = futures.Where(x => x.Expiration > request.EndingDate).ElementAtOrDefault(1);
if (firstExpAfterEnd != null)
{
DateTime limitDate = firstExpAfterEnd.Expiration.Value.AddMonths(request.Instrument.ContinuousFuture.Month - 1);
futures = futures.Where(x => x.Expiration.Value.Year < limitDate.Year ||
(x.Expiration.Value.Year == limitDate.Year && x.Expiration.Value.Month <= limitDate.Month)).ToList();
}
//Make sure each month's contract is allowed only once, even if there are multiple copies in the db
//Sometimes you might get two versions of the same contract with 1 day difference in expiration date
//so this step is necessary to clean that up
futures = futures.Distinct(x => x.Expiration.Value.ToString("yyyyMM").GetHashCode()).ToList();
//save the number of requests we're gonna make
lock (_reqCountLock)
{
_requestCounts.Add(request.AssignedID, futures.Count);
}
//save the contracts used, we need them later
_contracts.Add(request.AssignedID, futures);
Log(LogLevel.Info, string.Format("CFB: fulfilling historical request ID {0}, requested data on contracts: {1}",
request.AssignedID,
string.Join(", ", futures.Select(x => x.Symbol))));
Instrument prevInst = null;
//request the data for all futures left
foreach (Instrument i in futures)
{
//the question of how much data, exactly, to ask for is complicated...
//I'm going with: difference of expiration dates (unless it's the first one, in which case 30 days)
//plus a month
//plus the CF selected month
int daysBack = 30;
if (prevInst == null)
{
daysBack += 30;
}
else
{
daysBack += (int)(i.Expiration.Value - prevInst.Expiration.Value).TotalDays;
}
daysBack += 30 * cf.Month;
var endDate = i.Expiration.Value > DateTime.Now.Date ? DateTime.Now.Date : i.Expiration.Value;
var req = new HistoricalDataRequest(
i,
request.Frequency,
endDate.AddDays(-daysBack),
endDate,
rthOnly: request.RTHOnly,
dataLocation: request.DataLocation == DataLocation.LocalOnly ? DataLocation.LocalOnly : DataLocation.Both);
requests.Add(req);
prevInst = i;
}
return(requests);
}
/// <summary>
///
/// </summary>
/// <returns>The last contract used in the construction of this continuous futures instrument.</returns>
private Instrument GetContFutData(HistoricalDataRequest request, bool raiseDataEvent = true)
{
//copy over the list of contracts that we're gonna be using
List <Instrument> futures = new List <Instrument>(_contracts[request.AssignedID]);
//start by cleaning up the data, it is possible that some of the futures may not have had ANY data returned!
lock (_dataLock)
{
futures = futures.Where(x => _data[new KeyValuePair <int, BarSize>(x.ID.Value, request.Frequency)].Count > 0).ToList();
}
if (futures.Count == 0)
{
Log(LogLevel.Warn, "No data found");
if (raiseDataEvent)
{
RaiseEvent(HistoricalDataArrived, this, new HistoricalDataEventArgs(request, new List <OHLCBar>()));
}
return(null);
}
var cf = request.Instrument.ContinuousFuture;
Instrument frontFuture = futures.FirstOrDefault();
Instrument backFuture = futures.ElementAt(1);
//sometimes the contract will be based on the Xth month
//this is where we keep track of the actual contract currently being used
Instrument selectedFuture = futures.ElementAt(cf.Month - 1);
Instrument lastUsedSelectedFuture = selectedFuture;
//final date is the earliest of: the last date of data available, or the request's endingdate
DateTime lastDateAvailable = new DateTime(1, 1, 1);
TimeSeries frontData, backData, selectedData;
lock (_dataLock)
{
frontData = new TimeSeries(_data[new KeyValuePair <int, BarSize>(frontFuture.ID.Value, request.Frequency)]);
backData = new TimeSeries(_data[new KeyValuePair <int, BarSize>(backFuture.ID.Value, request.Frequency)]);
selectedData = new TimeSeries(_data[new KeyValuePair <int, BarSize>(selectedFuture.ID.Value, request.Frequency)]);
lastDateAvailable = _data[new KeyValuePair <int, BarSize>(futures.Last().ID.Value, request.Frequency)].Last().DT;
}
DateTime finalDate = request.EndingDate < lastDateAvailable ? request.EndingDate : lastDateAvailable;
//This is a super dirty hack to make non-time based rollovers actually work.
//The reason is that the starting point will otherwise be a LONG time before the date we're interested in.
//And at that time both the front and back futures are really far from expiration.
//As such volumes can be wonky, and thus result in a rollover far before we ACTUALLY would
//want to roll over if we had access to even earlier data.
DateTime currentDate = frontFuture.Expiration.Value.AddDays(-20 - cf.RolloverDays);
frontData.AdvanceTo(currentDate);
backData.AdvanceTo(currentDate);
selectedData.AdvanceTo(currentDate);
List <OHLCBar> cfData = new List <OHLCBar>();
Calendar calendar = MyUtils.GetCalendarFromCountryCode("US");
bool switchContract = false;
int counter = 0; //some rollover rules require multiple consecutive days of greater vol/OI...this keeps track of that
List <long> frontDailyVolume = new List <long>(); //keeps track of how much volume has occured in each day
List <int> frontDailyOpenInterest = new List <int>(); //keeps track of open interest on a daily basis
List <long> backDailyVolume = new List <long>();
List <int> backDailyOpenInterest = new List <int>();
long frontTodaysVolume = 0, backTodaysVolume = 0;
//add the first piece of data we have available, and start looping
cfData.Add(selectedData[0]);
//the first time we go from one day to the next we don't want to check for switching conditions
//because we need to ensure that we use an entire day's worth of volume data.
bool firstDaySwitchover = true;
while (currentDate < finalDate)
{
//keep track of total volume "today"
if (frontData[0].Volume.HasValue)
{
frontTodaysVolume += frontData[0].Volume.Value;
}
if (backData != null && backData[0].Volume.HasValue)
{
backTodaysVolume += backData[0].Volume.Value;
}
if (frontData.CurrentBar > 0 && frontData[0].DT.Day != frontData[1].DT.Day)
{
if (firstDaySwitchover)
{
firstDaySwitchover = false;
frontTodaysVolume = 0;
backTodaysVolume = 0;
}
frontDailyVolume.Add(frontTodaysVolume);
backDailyVolume.Add(backTodaysVolume);
if (frontData[0].OpenInterest.HasValue)
{
frontDailyOpenInterest.Add(frontData[0].OpenInterest.Value);
}
if (backData != null && backData[0].OpenInterest.HasValue)
{
backDailyOpenInterest.Add(backData[0].OpenInterest.Value);
}
frontTodaysVolume = 0;
backTodaysVolume = 0;
//do we need to switch contracts?
switch (cf.RolloverType)
{
case ContinuousFuturesRolloverType.Time:
if (MyUtils.BusinessDaysBetween(currentDate, frontFuture.Expiration.Value, calendar) <= cf.RolloverDays)
{
switchContract = true;
}
break;
case ContinuousFuturesRolloverType.Volume:
if (backData != null && backDailyVolume.Last() > frontDailyVolume.Last())
{
counter++;
}
else
{
counter = 0;
}
switchContract = counter >= cf.RolloverDays;
break;
case ContinuousFuturesRolloverType.OpenInterest:
if (backData != null && backDailyOpenInterest.Last() > frontDailyOpenInterest.Last())
{
counter++;
}
else
{
counter = 0;
}
switchContract = counter >= cf.RolloverDays;
break;
case ContinuousFuturesRolloverType.VolumeAndOpenInterest:
if (backData != null && backDailyOpenInterest.Last() > frontDailyOpenInterest.Last() &&
backDailyVolume.Last() > frontDailyVolume.Last())
{
counter++;
}
else
{
counter = 0;
}
switchContract = counter >= cf.RolloverDays;
break;
case ContinuousFuturesRolloverType.VolumeOrOpenInterest:
if (backData != null && backDailyOpenInterest.Last() > frontDailyOpenInterest.Last() ||
backDailyVolume.Last() > frontDailyVolume.Last())
{
counter++;
}
else
{
counter = 0;
}
switchContract = counter >= cf.RolloverDays;
break;
}
}
if (frontFuture.Expiration.Value <= currentDate)
{
//no matter what, obviously we need to switch if the contract expires
switchContract = true;
}
//finally if we have simply run out of data, we're forced to switch
if (frontData.ReachedEndOfSeries)
{
switchContract = true;
}
//finally advance the time and indices...keep moving forward until the selected series has moved
frontData.NextBar();
currentDate = frontData[0].DT;
if (backData != null)
{
backData.AdvanceTo(currentDate);
}
selectedData.AdvanceTo(currentDate);
//this next check here is necessary for the time-based switchover to work after weekends or holidays
if (cf.RolloverType == ContinuousFuturesRolloverType.Time &&
(frontFuture.Expiration.Value < currentDate ||
MyUtils.BusinessDaysBetween(currentDate, frontFuture.Expiration.Value, calendar) <= cf.RolloverDays))
{
switchContract = true;
}
//we switch to the next contract
if (switchContract)
{
//make any required price adjustments
decimal adjustmentFactor;
if (cf.AdjustmentMode == ContinuousFuturesAdjustmentMode.Difference)
{
adjustmentFactor = backData[0].Close - frontData[0].Close;
foreach (OHLCBar bar in cfData)
{
AdjustBar(bar, adjustmentFactor, cf.AdjustmentMode);
}
}
else if (cf.AdjustmentMode == ContinuousFuturesAdjustmentMode.Ratio)
{
adjustmentFactor = backData[0].Close / frontData[0].Close;
foreach (OHLCBar bar in cfData)
{
AdjustBar(bar, adjustmentFactor, cf.AdjustmentMode);
}
}
//update the contracts
var prevFront = frontFuture;
frontFuture = backFuture;
backFuture = futures.FirstOrDefault(x => x.Expiration > backFuture.Expiration);
var prevSelected = selectedFuture;
selectedFuture = futures.Where(x => x.Expiration >= frontFuture.Expiration).ElementAtOrDefault(cf.Month - 1);
Log(LogLevel.Info,
string.Format("CFB Filling request for {0}: switching front contract from {1} to {2} (selected contract from {3} to {4}) at {5}",
request.Instrument.Symbol,
prevFront.Symbol,
frontFuture.Symbol,
prevSelected.Symbol,
selectedFuture == null
? ""
: selectedFuture.Symbol,
currentDate.ToString("yyyy-MM-dd")));
if (frontFuture == null)
{
break; //no other futures left, get out
}
if (selectedFuture == null)
{
break;
}
lock (_dataLock)
{
frontData = new TimeSeries(_data[new KeyValuePair <int, BarSize>(frontFuture.ID.Value, request.Frequency)]);
backData = backFuture != null ? new TimeSeries(_data[new KeyValuePair <int, BarSize>(backFuture.ID.Value, request.Frequency)]) : null;
selectedData = new TimeSeries(_data[new KeyValuePair <int, BarSize>(selectedFuture.ID.Value, request.Frequency)]);
}
frontData.AdvanceTo(currentDate);
if (backData != null)
{
backData.AdvanceTo(currentDate);
}
selectedData.AdvanceTo(currentDate);
//TODO make sure that the data series actually cover the current date
switchContract = false;
lastUsedSelectedFuture = selectedFuture;
}
cfData.Add(selectedData[0]);
}
//clean up
_contracts.Remove(request.AssignedID);
//throw out any data from before the start of the request
cfData = cfData.Where(x => x.DT >= request.StartingDate && x.DT <= request.EndingDate).ToList();
//we're done, so just raise the event
if (raiseDataEvent)
{
RaiseEvent(HistoricalDataArrived, this, new HistoricalDataEventArgs(request, cfData));
}
//clean up some data!
lock (_dataUsesLock)
{
foreach (Instrument i in futures)
{
var kvp = new KeyValuePair <int, BarSize>(i.ID.Value, request.Frequency);
if (_dataUsesPending[kvp] == 1) //this data isn't needed anywhere else, we can delete it
{
_dataUsesPending.Remove(kvp);
lock (_dataLock)
{
_data.Remove(kvp);
}
}
else
{
_dataUsesPending[kvp]--;
}
}
}
return(lastUsedSelectedFuture);
}
