/// <summary> /// Converts a Tradier symbol to a Lean symbol instance /// </summary> /// <param name="brokerageSymbol">The Tradier symbol</param> /// <returns>A new Lean Symbol instance</returns> public Symbol GetLeanSymbol(string brokerageSymbol) { Symbol symbol; if (brokerageSymbol.Length > 15) { // convert the Tradier option symbol to OSI format var underlying = brokerageSymbol.Substring(0, brokerageSymbol.Length - 15); var ticker = underlying.PadRight(6, ' ') + brokerageSymbol.Substring(underlying.Length); symbol = SymbolRepresentation.ParseOptionTickerOSI(ticker); } else { symbol = Symbol.Create(brokerageSymbol, SecurityType.Equity, Market.USA); } return(symbol); }
/// <summary> /// Private method loads all option or future contracts for a particular underlying /// symbol (placeholder) on demand by walking through the IQFeed universe file /// </summary> /// <param name="placeholder">Underlying symbol</param> /// <returns></returns> private IEnumerable <SymbolData> LoadSymbolOnDemand(SymbolData placeholder) { var dayOfWeek = DateTimeFormatInfo.CurrentInfo.Calendar.GetWeekOfYear(DateTime.Today, CalendarWeekRule.FirstDay, DayOfWeek.Monday); var thisYearWeek = $"{DateTime.Today.ToStringInvariant("yyyy")}-{dayOfWeek.ToStringInvariant()}"; var todayCsvFileName = "mktsymbols_v2.txt"; var todayFullCsvName = Path.Combine(Globals.Cache, todayCsvFileName); var reader = new LocalFileSubscriptionStreamReader(_dataCacheProvider, todayFullCsvName, placeholder.StartPosition); Log.Trace("Loading data on demand for {0}...", placeholder.Symbol.ID); var symbolUniverse = new List <SymbolData>(); long currentPosition = placeholder.StartPosition; while (!reader.EndOfStream && currentPosition <= placeholder.EndPosition) { var line = reader.ReadLine(); currentPosition += line.Length + NewLine.Length; var columns = line.Split(Tabulation); if (columns.Length != totalColumns) { continue; } switch (columns[columnSecurityType]) { case "IEOPTION": var ticker = columns[columnSymbol]; var result = SymbolRepresentation.ParseOptionTickerIQFeed(ticker); symbolUniverse.Add(new SymbolData { Symbol = Symbol.CreateOption(result.Underlying, Market.USA, OptionStyle.American, result.OptionRight, result.OptionStrike, result.ExpirationDate), SecurityCurrency = Currencies.USD, SecurityExchange = Market.USA, Ticker = columns[columnSymbol] }); break; case "FUTURE": if (columns[columnSymbol].EndsWith("#")) { continue; } var futuresTicker = columns[columnSymbol].TrimStart(new[] { '@' }); var parsed = SymbolRepresentation.ParseFutureTicker(futuresTicker); var underlyingString = parsed.Underlying; var market = Market.USA; if (_iqFeedNameMap.ContainsKey(underlyingString)) { underlyingString = _iqFeedNameMap[underlyingString]; } if (underlyingString != placeholder.Symbol.Value) { continue; } // Futures contracts have different idiosyncratic expiration dates that IQFeed symbol universe file doesn't contain // We request IQFeed explicitly for the exact expiration data of each contract var expirationDate = _symbolFundamentalData.Request(columns[columnSymbol]).Item1; if (expirationDate == DateTime.MinValue) { // contract is outdated continue; } symbolUniverse.Add(new SymbolData { Symbol = Symbol.CreateFuture(underlyingString, market, expirationDate), SecurityCurrency = Currencies.USD, SecurityExchange = market, Ticker = columns[columnSymbol] }); break; default: continue; } } return(symbolUniverse); }
/// <summary> /// Private method performs initial loading of data from IQFeed universe: /// - method loads FX,equities, indices as is into memory /// - method prepares ondemand data for options and futures and stores it to disk /// - method updates futures mapping files if required /// </summary> /// <returns></returns> private IEnumerable <SymbolData> LoadSymbols() { // default URI const string uri = "http://www.dtniq.com/product/mktsymbols_v2.zip"; if (!Directory.Exists(Globals.Cache)) { Directory.CreateDirectory(Globals.Cache); } // we try to check if we already downloaded the file and it is in cache. If yes, we use it. Otherwise, download new file. IStreamReader reader; // we update the files every week var dayOfWeek = DateTimeFormatInfo.CurrentInfo.Calendar.GetWeekOfYear(DateTime.Today, CalendarWeekRule.FirstDay, DayOfWeek.Monday); var thisYearWeek = $"{DateTime.Today.ToStringInvariant("yyyy")}-{dayOfWeek.ToStringInvariant()}"; var todayZipFileName = "IQFeed-symbol-universe-" + thisYearWeek + ".zip"; var todayFullZipName = Path.Combine(Globals.Cache, todayZipFileName); var todayCsvFileName = "mktsymbols_v2.txt"; var todayFullCsvName = Path.Combine(Globals.Cache, todayCsvFileName); var iqfeedNameMapFileName = "IQFeed-symbol-map.json"; var iqfeedNameMapFullName = Path.Combine("IQFeed", iqfeedNameMapFileName); var mapExists = File.Exists(iqfeedNameMapFullName); var universeExists = File.Exists(todayFullZipName); if (mapExists) { Log.Trace("Loading IQFeed futures symbol map file..."); _iqFeedNameMap = JsonConvert.DeserializeObject <Dictionary <string, string> >(File.ReadAllText(iqfeedNameMapFullName)); } if (!universeExists) { Log.Trace("Loading and unzipping IQFeed symbol universe file ({0})...", uri); using (var client = new WebClient()) { client.Proxy = WebRequest.GetSystemWebProxy(); client.DownloadFile(uri, todayFullZipName); } Compression.Unzip(todayFullZipName, Globals.Cache, true); } else { Log.Trace("Found up-to-date IQFeed symbol universe file in local cache. Loading it..."); } var symbolCache = new Dictionary <Symbol, SymbolData>(); var symbolUniverse = new List <SymbolData>(); long currentPosition = 0; long prevPosition = 0; long lineCounter = 0; reader = new LocalFileSubscriptionStreamReader(_dataCacheProvider, todayFullCsvName); while (!reader.EndOfStream) { lineCounter++; if (lineCounter % 100000 == 0) { Log.Trace($"{lineCounter} lines read."); } prevPosition = currentPosition; var line = reader.ReadLine(); currentPosition += line.Length + NewLine.Length; // file position 'estimator' for ASCII file of IQFeed universe var columns = line.Split(Tabulation); if (columns[columnSymbol] == "TST$Y") { continue; } if (columns.Length != totalColumns) { Log.Trace("Discrepancy found while parsing IQFeed symbol universe file. Expected 8 columns, but arrived {0}. Line: {1}", columns.Length, line); continue; } switch (columns[columnSecurityType]) { case "INDEX": case "EQUITY": // we load equities/indices in memory symbolUniverse.Add(new SymbolData { Symbol = Symbol.Create(columns[columnSymbol], SecurityType.Equity, Market.USA), SecurityCurrency = Currencies.USD, SecurityExchange = Market.USA, Ticker = columns[columnSymbol] }); break; case "IEOPTION": var ticker = columns[columnSymbol]; var result = SymbolRepresentation.ParseOptionTickerIQFeed(ticker); var optionUnderlying = result.Underlying; var canonicalSymbol = Symbol.Create(optionUnderlying, SecurityType.Option, Market.USA); if (!symbolCache.ContainsKey(canonicalSymbol)) { var placeholderSymbolData = new SymbolData { Symbol = canonicalSymbol, SecurityCurrency = Currencies.USD, SecurityExchange = Market.USA, StartPosition = prevPosition, EndPosition = currentPosition }; symbolCache.Add(canonicalSymbol, placeholderSymbolData); } else { symbolCache[canonicalSymbol].EndPosition = currentPosition; } break; case "FOREX": // we use FXCM symbols only if (columns[columnSymbol].EndsWith(".FXCM")) { var symbol = columns[columnSymbol].Replace(".FXCM", string.Empty); symbolUniverse.Add(new SymbolData { Symbol = Symbol.Create(symbol, SecurityType.Forex, Market.FXCM), SecurityCurrency = Currencies.USD, SecurityExchange = Market.FXCM, Ticker = columns[columnSymbol] }); } break; case "FUTURE": // we are not interested in designated continuous contracts if (columns[columnSymbol].EndsWith("#") || columns[columnSymbol].EndsWith("#C")) { continue; } var futuresTicker = columns[columnSymbol].TrimStart(new[] { '@' }); var parsed = SymbolRepresentation.ParseFutureTicker(futuresTicker); var underlyingString = parsed.Underlying; if (_iqFeedNameMap.ContainsKey(underlyingString)) { underlyingString = _iqFeedNameMap[underlyingString]; } else { if (!mapExists) { if (!_iqFeedNameMap.ContainsKey(underlyingString)) { // if map is not created yet, we request this information from IQFeed var exchangeSymbol = _symbolFundamentalData.Request(columns[columnSymbol]).Item2; if (!string.IsNullOrEmpty(exchangeSymbol)) { _iqFeedNameMap[underlyingString] = exchangeSymbol; underlyingString = exchangeSymbol; } } } } var market = _futuresExchanges.ContainsKey(columns[columnExchange]) ? _futuresExchanges[columns[columnExchange]] : Market.USA; canonicalSymbol = Symbol.Create(underlyingString, SecurityType.Future, market); if (!symbolCache.ContainsKey(canonicalSymbol)) { var placeholderSymbolData = new SymbolData { Symbol = canonicalSymbol, SecurityCurrency = Currencies.USD, SecurityExchange = market, StartPosition = prevPosition, EndPosition = currentPosition }; symbolCache.Add(canonicalSymbol, placeholderSymbolData); } else { symbolCache[canonicalSymbol].EndPosition = currentPosition; } break; default: continue; } } if (!mapExists) { Log.Trace("Saving IQFeed futures symbol map file..."); File.WriteAllText(iqfeedNameMapFullName, JsonConvert.SerializeObject(_iqFeedNameMap)); } symbolUniverse.AddRange(symbolCache.Values); Log.Trace("Finished loading IQFeed symbol universe file."); return(symbolUniverse); }
/// <summary> /// Parse a string line into a future tick. /// </summary> /// <param name="line"></param> /// <returns></returns> private Tick Parse(string line) { try { const int TradeMask = 2; const int QuoteMask = 1; const int OpenInterestMask = 11; const int MessageTypeMask = 15; // parse csv check column count var csv = line.ToCsv(); if (csv.Count - 1 < _columnsCount) { return(null); } var ticker = csv[_columnTicker]; // we filter out options and spreads if (ticker.IndexOfAny(new [] { ' ', '-' }) != -1) { return(null); } ticker = ticker.Trim('"'); if (string.IsNullOrEmpty(ticker)) { return(null); } // ignoring time zones completely -- this is all in the 'data-time-zone' var timeString = csv[_columnTimestamp]; var time = DateTime.ParseExact(timeString, "yyyyMMddHHmmssFFF", CultureInfo.InvariantCulture); var symbol = SymbolRepresentation.ParseFutureSymbol(ticker, time.Year); if (symbol == null || !_symbolMultipliers.ContainsKey(symbol.ID.Symbol) || _symbolFilter != null && !_symbolFilter.Contains(symbol.ID.Symbol, StringComparer.InvariantCultureIgnoreCase)) { return(null); } // detecting tick type (trade or quote) TickType tickType; bool isAsk = false; var type = csv[_columnType].ConvertInvariant <int>(); if ((type & MessageTypeMask) == TradeMask) { tickType = TickType.Trade; } else if ((type & MessageTypeMask) == OpenInterestMask) { tickType = TickType.OpenInterest; } else if ((type & MessageTypeMask) == QuoteMask) { tickType = TickType.Quote; switch (csv[_columnSide]) { case "B": isAsk = false; break; case "S": isAsk = true; break; default: { return(null); } } } else { return(null); } // All futures but VIX are delivered with a scale factor of 10000000000. var scaleFactor = symbol.ID.Symbol == "VX" ? decimal.One : 10000000000m; var price = csv[_columnPrice].ToDecimal() / scaleFactor; var quantity = csv[_columnQuantity].ToInt32(); price *= _symbolMultipliers[symbol.ID.Symbol]; switch (tickType) { case TickType.Quote: var tick = new Tick { Symbol = symbol, Time = time, TickType = tickType, Value = price }; if (isAsk) { tick.AskPrice = price; tick.AskSize = quantity; } else { tick.BidPrice = price; tick.BidSize = quantity; } return(tick); case TickType.Trade: tick = new Tick { Symbol = symbol, Time = time, TickType = tickType, Value = price, Quantity = quantity }; return(tick); case TickType.OpenInterest: tick = new Tick { Symbol = symbol, Time = time, TickType = tickType, Exchange = symbol.ID.Market, Value = quantity }; return(tick); } return(null); } catch (Exception err) { Log.Error(err); Log.Trace("Line: {0}", line); return(null); } }
public void ParseInvalidFuturesTickers() { var result = SymbolRepresentation.ParseFutureTicker("invalid"); Assert.AreEqual(result, null); }
/// <summary> /// Parse a string line into a future tick. /// </summary> /// <param name="line"></param> /// <returns></returns> private Tick Parse(string line) { try { const int TradeMask = 2; const int QuoteMask = 1; const int OpenInterestMask = 11; const int MessageTypeMask = 15; // parse csv check column count var csv = line.ToCsv(); if (csv.Count - 1 < _columnsCount) { return(null); } var ticker = csv[_columnTicker]; // we filter out options and spreads if (ticker.IndexOfAny(new [] { ' ', '-' }) != -1) { return(null); } // detecting tick type (trade or quote) TickType tickType; bool isAsk = false; var type = Convert.ToInt32(csv[_columnType]); if ((type & MessageTypeMask) == TradeMask) { tickType = TickType.Trade; } else if ((type & MessageTypeMask) == OpenInterestMask) { tickType = TickType.OpenInterest; } else if ((type & MessageTypeMask) == QuoteMask) { tickType = TickType.Quote; switch (csv[_columnSide]) { case "B": isAsk = false; break; case "S": isAsk = true; break; default: { return(null); } } } else { return(null); } ticker = ticker.Trim(new char[] { '"' }); if (_symbolFilter != null && !_symbolFilter.Contains(ticker)) { return(null); } if (string.IsNullOrEmpty(ticker)) { return(null); } // ignoring time zones completely -- this is all in the 'data-time-zone' var timeString = csv[_columnTimestamp]; var time = DateTime.ParseExact(timeString, "yyyyMMddHHmmssFFF", CultureInfo.InvariantCulture); var parsed = SymbolRepresentation.ParseFutureTicker(ticker); if (parsed == null) { return(null); } var underlying = parsed.Underlying; var expirationYearShort = parsed.ExpirationYearShort; var expirationMonth = parsed.ExpirationMonth; var expirationYear = GetExpirationYear(time, expirationYearShort); var expirationYearMonth = new DateTime(expirationYear, expirationMonth, DateTime.DaysInMonth(expirationYear, expirationMonth)); var symbol = Symbol.CreateFuture(underlying, Market.USA, expirationYearMonth); var price = csv[_columnPrice].ToDecimal() / 10000000000m; var quantity = csv[_columnQuantity].ToInt32(); price *= _symbolMultipliers.ContainsKey(underlying) ? _symbolMultipliers[underlying] : 1.0m; switch (tickType) { case TickType.Quote: var tick = new Tick { Symbol = symbol, Time = time, TickType = tickType, Exchange = Market.USA, Value = price }; if (isAsk) { tick.AskPrice = price; tick.AskSize = quantity; } else { tick.BidPrice = price; tick.BidSize = quantity; } return(tick); case TickType.Trade: tick = new Tick { Symbol = symbol, Time = time, TickType = tickType, Exchange = Market.USA, Value = price, Quantity = quantity }; return(tick); case TickType.OpenInterest: tick = new Tick { Symbol = symbol, Time = time, TickType = tickType, Exchange = Market.USA, Value = quantity }; return(tick); } return(null); } catch (Exception err) { Log.Error(err); Log.Trace("Line: {0}", line); return(null); } }
public void GenerateFutureTickerExpiringInNextMonth(string ticker, int year, int month, int day, string expectedValue, bool doubleDigitsYear) { var result = SymbolRepresentation.GenerateFutureTicker(ticker, new DateTime(year, month, day), doubleDigitsYear); Assert.AreEqual(expectedValue, result); }
public void GenerateFutureSymbolFromTickerMissingDecadeInfo(string ticker) { var result = SymbolRepresentation.ParseFutureSymbol(ticker, 2012); Assert.AreEqual(new DateTime(2012, 12, 19), result.ID.Date.Date); }