/// <summary>
/// Converts a Tradier symbol to a Lean symbol instance
/// </summary>
/// <param name="brokerageSymbol">The Tradier symbol</param>
/// <returns>A new Lean Symbol instance</returns>
public Symbol GetLeanSymbol(string brokerageSymbol)
{
Symbol symbol;
if (brokerageSymbol.Length > 15)
{
// convert the Tradier option symbol to OSI format
var underlying = brokerageSymbol.Substring(0, brokerageSymbol.Length - 15);
var ticker = underlying.PadRight(6, ' ') + brokerageSymbol.Substring(underlying.Length);
symbol = SymbolRepresentation.ParseOptionTickerOSI(ticker);
}
else
{
symbol = Symbol.Create(brokerageSymbol, SecurityType.Equity, Market.USA);
}
return(symbol);
}
/// <summary>
/// Private method loads all option or future contracts for a particular underlying
/// symbol (placeholder) on demand by walking through the IQFeed universe file
/// </summary>
/// <param name="placeholder">Underlying symbol</param>
/// <returns></returns>
private IEnumerable <SymbolData> LoadSymbolOnDemand(SymbolData placeholder)
{
var dayOfWeek = DateTimeFormatInfo.CurrentInfo.Calendar.GetWeekOfYear(DateTime.Today, CalendarWeekRule.FirstDay, DayOfWeek.Monday);
var thisYearWeek = $"{DateTime.Today.ToStringInvariant("yyyy")}-{dayOfWeek.ToStringInvariant()}";
var todayCsvFileName = "mktsymbols_v2.txt";
var todayFullCsvName = Path.Combine(Globals.Cache, todayCsvFileName);
var reader = new LocalFileSubscriptionStreamReader(_dataCacheProvider, todayFullCsvName, placeholder.StartPosition);
Log.Trace("Loading data on demand for {0}...", placeholder.Symbol.ID);
var symbolUniverse = new List <SymbolData>();
long currentPosition = placeholder.StartPosition;
while (!reader.EndOfStream && currentPosition <= placeholder.EndPosition)
{
var line = reader.ReadLine();
currentPosition += line.Length + NewLine.Length;
var columns = line.Split(Tabulation);
if (columns.Length != totalColumns)
{
continue;
}
switch (columns[columnSecurityType])
{
case "IEOPTION":
var ticker = columns[columnSymbol];
var result = SymbolRepresentation.ParseOptionTickerIQFeed(ticker);
symbolUniverse.Add(new SymbolData
{
Symbol = Symbol.CreateOption(result.Underlying,
Market.USA,
OptionStyle.American,
result.OptionRight,
result.OptionStrike,
result.ExpirationDate),
SecurityCurrency = Currencies.USD,
SecurityExchange = Market.USA,
Ticker = columns[columnSymbol]
});
break;
case "FUTURE":
if (columns[columnSymbol].EndsWith("#"))
{
continue;
}
var futuresTicker = columns[columnSymbol].TrimStart(new[] { '@' });
var parsed = SymbolRepresentation.ParseFutureTicker(futuresTicker);
var underlyingString = parsed.Underlying;
var market = Market.USA;
if (_iqFeedNameMap.ContainsKey(underlyingString))
{
underlyingString = _iqFeedNameMap[underlyingString];
}
if (underlyingString != placeholder.Symbol.Value)
{
continue;
}
// Futures contracts have different idiosyncratic expiration dates that IQFeed symbol universe file doesn't contain
// We request IQFeed explicitly for the exact expiration data of each contract
var expirationDate = _symbolFundamentalData.Request(columns[columnSymbol]).Item1;
if (expirationDate == DateTime.MinValue)
{
// contract is outdated
continue;
}
symbolUniverse.Add(new SymbolData
{
Symbol = Symbol.CreateFuture(underlyingString,
market,
expirationDate),
SecurityCurrency = Currencies.USD,
SecurityExchange = market,
Ticker = columns[columnSymbol]
});
break;
default:
continue;
}
}
return(symbolUniverse);
}
/// <summary>
/// Private method performs initial loading of data from IQFeed universe:
/// - method loads FX,equities, indices as is into memory
/// - method prepares ondemand data for options and futures and stores it to disk
/// - method updates futures mapping files if required
/// </summary>
/// <returns></returns>
private IEnumerable <SymbolData> LoadSymbols()
{
// default URI
const string uri = "http://www.dtniq.com/product/mktsymbols_v2.zip";
if (!Directory.Exists(Globals.Cache))
{
Directory.CreateDirectory(Globals.Cache);
}
// we try to check if we already downloaded the file and it is in cache. If yes, we use it. Otherwise, download new file.
IStreamReader reader;
// we update the files every week
var dayOfWeek = DateTimeFormatInfo.CurrentInfo.Calendar.GetWeekOfYear(DateTime.Today, CalendarWeekRule.FirstDay, DayOfWeek.Monday);
var thisYearWeek = $"{DateTime.Today.ToStringInvariant("yyyy")}-{dayOfWeek.ToStringInvariant()}";
var todayZipFileName = "IQFeed-symbol-universe-" + thisYearWeek + ".zip";
var todayFullZipName = Path.Combine(Globals.Cache, todayZipFileName);
var todayCsvFileName = "mktsymbols_v2.txt";
var todayFullCsvName = Path.Combine(Globals.Cache, todayCsvFileName);
var iqfeedNameMapFileName = "IQFeed-symbol-map.json";
var iqfeedNameMapFullName = Path.Combine("IQFeed", iqfeedNameMapFileName);
var mapExists = File.Exists(iqfeedNameMapFullName);
var universeExists = File.Exists(todayFullZipName);
if (mapExists)
{
Log.Trace("Loading IQFeed futures symbol map file...");
_iqFeedNameMap = JsonConvert.DeserializeObject <Dictionary <string, string> >(File.ReadAllText(iqfeedNameMapFullName));
}
if (!universeExists)
{
Log.Trace("Loading and unzipping IQFeed symbol universe file ({0})...", uri);
using (var client = new WebClient())
{
client.Proxy = WebRequest.GetSystemWebProxy();
client.DownloadFile(uri, todayFullZipName);
}
Compression.Unzip(todayFullZipName, Globals.Cache, true);
}
else
{
Log.Trace("Found up-to-date IQFeed symbol universe file in local cache. Loading it...");
}
var symbolCache = new Dictionary <Symbol, SymbolData>();
var symbolUniverse = new List <SymbolData>();
long currentPosition = 0;
long prevPosition = 0;
long lineCounter = 0;
reader = new LocalFileSubscriptionStreamReader(_dataCacheProvider, todayFullCsvName);
while (!reader.EndOfStream)
{
lineCounter++;
if (lineCounter % 100000 == 0)
{
Log.Trace($"{lineCounter} lines read.");
}
prevPosition = currentPosition;
var line = reader.ReadLine();
currentPosition += line.Length + NewLine.Length; // file position 'estimator' for ASCII file of IQFeed universe
var columns = line.Split(Tabulation);
if (columns[columnSymbol] == "TST$Y")
{
continue;
}
if (columns.Length != totalColumns)
{
Log.Trace("Discrepancy found while parsing IQFeed symbol universe file. Expected 8 columns, but arrived {0}. Line: {1}", columns.Length, line);
continue;
}
switch (columns[columnSecurityType])
{
case "INDEX":
case "EQUITY":
// we load equities/indices in memory
symbolUniverse.Add(new SymbolData
{
Symbol = Symbol.Create(columns[columnSymbol], SecurityType.Equity, Market.USA),
SecurityCurrency = Currencies.USD,
SecurityExchange = Market.USA,
Ticker = columns[columnSymbol]
});
break;
case "IEOPTION":
var ticker = columns[columnSymbol];
var result = SymbolRepresentation.ParseOptionTickerIQFeed(ticker);
var optionUnderlying = result.Underlying;
var canonicalSymbol = Symbol.Create(optionUnderlying, SecurityType.Option, Market.USA);
if (!symbolCache.ContainsKey(canonicalSymbol))
{
var placeholderSymbolData = new SymbolData
{
Symbol = canonicalSymbol,
SecurityCurrency = Currencies.USD,
SecurityExchange = Market.USA,
StartPosition = prevPosition,
EndPosition = currentPosition
};
symbolCache.Add(canonicalSymbol, placeholderSymbolData);
}
else
{
symbolCache[canonicalSymbol].EndPosition = currentPosition;
}
break;
case "FOREX":
// we use FXCM symbols only
if (columns[columnSymbol].EndsWith(".FXCM"))
{
var symbol = columns[columnSymbol].Replace(".FXCM", string.Empty);
symbolUniverse.Add(new SymbolData
{
Symbol = Symbol.Create(symbol, SecurityType.Forex, Market.FXCM),
SecurityCurrency = Currencies.USD,
SecurityExchange = Market.FXCM,
Ticker = columns[columnSymbol]
});
}
break;
case "FUTURE":
// we are not interested in designated continuous contracts
if (columns[columnSymbol].EndsWith("#") || columns[columnSymbol].EndsWith("#C"))
{
continue;
}
var futuresTicker = columns[columnSymbol].TrimStart(new[] { '@' });
var parsed = SymbolRepresentation.ParseFutureTicker(futuresTicker);
var underlyingString = parsed.Underlying;
if (_iqFeedNameMap.ContainsKey(underlyingString))
{
underlyingString = _iqFeedNameMap[underlyingString];
}
else
{
if (!mapExists)
{
if (!_iqFeedNameMap.ContainsKey(underlyingString))
{
// if map is not created yet, we request this information from IQFeed
var exchangeSymbol = _symbolFundamentalData.Request(columns[columnSymbol]).Item2;
if (!string.IsNullOrEmpty(exchangeSymbol))
{
_iqFeedNameMap[underlyingString] = exchangeSymbol;
underlyingString = exchangeSymbol;
}
}
}
}
var market = _futuresExchanges.ContainsKey(columns[columnExchange]) ? _futuresExchanges[columns[columnExchange]] : Market.USA;
canonicalSymbol = Symbol.Create(underlyingString, SecurityType.Future, market);
if (!symbolCache.ContainsKey(canonicalSymbol))
{
var placeholderSymbolData = new SymbolData
{
Symbol = canonicalSymbol,
SecurityCurrency = Currencies.USD,
SecurityExchange = market,
StartPosition = prevPosition,
EndPosition = currentPosition
};
symbolCache.Add(canonicalSymbol, placeholderSymbolData);
}
else
{
symbolCache[canonicalSymbol].EndPosition = currentPosition;
}
break;
default:
continue;
}
}
if (!mapExists)
{
Log.Trace("Saving IQFeed futures symbol map file...");
File.WriteAllText(iqfeedNameMapFullName, JsonConvert.SerializeObject(_iqFeedNameMap));
}
symbolUniverse.AddRange(symbolCache.Values);
Log.Trace("Finished loading IQFeed symbol universe file.");
return(symbolUniverse);
}
/// <summary>
/// Parse a string line into a future tick.
/// </summary>
/// <param name="line"></param>
/// <returns></returns>
private Tick Parse(string line)
{
try
{
const int TradeMask = 2;
const int QuoteMask = 1;
const int OpenInterestMask = 11;
const int MessageTypeMask = 15;
// parse csv check column count
var csv = line.ToCsv();
if (csv.Count - 1 < _columnsCount)
{
return(null);
}
var ticker = csv[_columnTicker];
// we filter out options and spreads
if (ticker.IndexOfAny(new [] { ' ', '-' }) != -1)
{
return(null);
}
ticker = ticker.Trim('"');
if (string.IsNullOrEmpty(ticker))
{
return(null);
}
// ignoring time zones completely -- this is all in the 'data-time-zone'
var timeString = csv[_columnTimestamp];
var time = DateTime.ParseExact(timeString, "yyyyMMddHHmmssFFF", CultureInfo.InvariantCulture);
var symbol = SymbolRepresentation.ParseFutureSymbol(ticker, time.Year);
if (symbol == null || !_symbolMultipliers.ContainsKey(symbol.ID.Symbol) ||
_symbolFilter != null && !_symbolFilter.Contains(symbol.ID.Symbol, StringComparer.InvariantCultureIgnoreCase))
{
return(null);
}
// detecting tick type (trade or quote)
TickType tickType;
bool isAsk = false;
var type = csv[_columnType].ConvertInvariant <int>();
if ((type & MessageTypeMask) == TradeMask)
{
tickType = TickType.Trade;
}
else if ((type & MessageTypeMask) == OpenInterestMask)
{
tickType = TickType.OpenInterest;
}
else if ((type & MessageTypeMask) == QuoteMask)
{
tickType = TickType.Quote;
switch (csv[_columnSide])
{
case "B":
isAsk = false;
break;
case "S":
isAsk = true;
break;
default:
{
return(null);
}
}
}
else
{
return(null);
}
// All futures but VIX are delivered with a scale factor of 10000000000.
var scaleFactor = symbol.ID.Symbol == "VX" ? decimal.One : 10000000000m;
var price = csv[_columnPrice].ToDecimal() / scaleFactor;
var quantity = csv[_columnQuantity].ToInt32();
price *= _symbolMultipliers[symbol.ID.Symbol];
switch (tickType)
{
case TickType.Quote:
var tick = new Tick
{
Symbol = symbol,
Time = time,
TickType = tickType,
Value = price
};
if (isAsk)
{
tick.AskPrice = price;
tick.AskSize = quantity;
}
else
{
tick.BidPrice = price;
tick.BidSize = quantity;
}
return(tick);
case TickType.Trade:
tick = new Tick
{
Symbol = symbol,
Time = time,
TickType = tickType,
Value = price,
Quantity = quantity
};
return(tick);
case TickType.OpenInterest:
tick = new Tick
{
Symbol = symbol,
Time = time,
TickType = tickType,
Exchange = symbol.ID.Market,
Value = quantity
};
return(tick);
}
return(null);
}
catch (Exception err)
{
Log.Error(err);
Log.Trace("Line: {0}", line);
return(null);
}
}
public void ParseInvalidFuturesTickers()
{
var result = SymbolRepresentation.ParseFutureTicker("invalid");
Assert.AreEqual(result, null);
}
/// <summary>
/// Parse a string line into a future tick.
/// </summary>
/// <param name="line"></param>
/// <returns></returns>
private Tick Parse(string line)
{
try
{
const int TradeMask = 2;
const int QuoteMask = 1;
const int OpenInterestMask = 11;
const int MessageTypeMask = 15;
// parse csv check column count
var csv = line.ToCsv();
if (csv.Count - 1 < _columnsCount)
{
return(null);
}
var ticker = csv[_columnTicker];
// we filter out options and spreads
if (ticker.IndexOfAny(new [] { ' ', '-' }) != -1)
{
return(null);
}
// detecting tick type (trade or quote)
TickType tickType;
bool isAsk = false;
var type = Convert.ToInt32(csv[_columnType]);
if ((type & MessageTypeMask) == TradeMask)
{
tickType = TickType.Trade;
}
else if ((type & MessageTypeMask) == OpenInterestMask)
{
tickType = TickType.OpenInterest;
}
else if ((type & MessageTypeMask) == QuoteMask)
{
tickType = TickType.Quote;
switch (csv[_columnSide])
{
case "B":
isAsk = false;
break;
case "S":
isAsk = true;
break;
default:
{
return(null);
}
}
}
else
{
return(null);
}
ticker = ticker.Trim(new char[] { '"' });
if (_symbolFilter != null && !_symbolFilter.Contains(ticker))
{
return(null);
}
if (string.IsNullOrEmpty(ticker))
{
return(null);
}
// ignoring time zones completely -- this is all in the 'data-time-zone'
var timeString = csv[_columnTimestamp];
var time = DateTime.ParseExact(timeString, "yyyyMMddHHmmssFFF", CultureInfo.InvariantCulture);
var parsed = SymbolRepresentation.ParseFutureTicker(ticker);
if (parsed == null)
{
return(null);
}
var underlying = parsed.Underlying;
var expirationYearShort = parsed.ExpirationYearShort;
var expirationMonth = parsed.ExpirationMonth;
var expirationYear = GetExpirationYear(time, expirationYearShort);
var expirationYearMonth = new DateTime(expirationYear, expirationMonth, DateTime.DaysInMonth(expirationYear, expirationMonth));
var symbol = Symbol.CreateFuture(underlying, Market.USA, expirationYearMonth);
var price = csv[_columnPrice].ToDecimal() / 10000000000m;
var quantity = csv[_columnQuantity].ToInt32();
price *= _symbolMultipliers.ContainsKey(underlying) ? _symbolMultipliers[underlying] : 1.0m;
switch (tickType)
{
case TickType.Quote:
var tick = new Tick
{
Symbol = symbol,
Time = time,
TickType = tickType,
Exchange = Market.USA,
Value = price
};
if (isAsk)
{
tick.AskPrice = price;
tick.AskSize = quantity;
}
else
{
tick.BidPrice = price;
tick.BidSize = quantity;
}
return(tick);
case TickType.Trade:
tick = new Tick
{
Symbol = symbol,
Time = time,
TickType = tickType,
Exchange = Market.USA,
Value = price,
Quantity = quantity
};
return(tick);
case TickType.OpenInterest:
tick = new Tick
{
Symbol = symbol,
Time = time,
TickType = tickType,
Exchange = Market.USA,
Value = quantity
};
return(tick);
}
return(null);
}
catch (Exception err)
{
Log.Error(err);
Log.Trace("Line: {0}", line);
return(null);
}
}
public void GenerateFutureTickerExpiringInNextMonth(string ticker, int year, int month, int day, string expectedValue, bool doubleDigitsYear)
{
var result = SymbolRepresentation.GenerateFutureTicker(ticker, new DateTime(year, month, day), doubleDigitsYear);
Assert.AreEqual(expectedValue, result);
}
public void GenerateFutureSymbolFromTickerMissingDecadeInfo(string ticker)
{
var result = SymbolRepresentation.ParseFutureSymbol(ticker, 2012);
Assert.AreEqual(new DateTime(2012, 12, 19), result.ID.Date.Date);
}
