/// <summary>
/// Reads the specified <paramref name="source"/>
/// </summary>
/// <param name="source">The source to be read</param>
/// <returns>An <see cref="IEnumerable{BaseData}"/> that contains the data in the source</returns>
public IEnumerable <BaseData> Read(SubscriptionDataSource source)
{
SubscriptionDataSourceReader.CheckRemoteFileCache();
IStreamReader reader = null;
try
{
switch (source.TransportMedium)
{
default:
case SubscriptionTransportMedium.Rest:
reader = new RestSubscriptionStreamReader(source.Source);
break;
case SubscriptionTransportMedium.LocalFile:
reader = new LocalFileSubscriptionStreamReader(_dataCacheProvider, source.Source);
break;
case SubscriptionTransportMedium.RemoteFile:
reader = new RemoteFileSubscriptionStreamReader(_dataCacheProvider, source.Source, Globals.Cache);
break;
}
var raw = "";
while (!reader.EndOfStream)
{
BaseDataCollection instances;
try
{
raw = reader.ReadLine();
var result = _factory.Reader(_config, raw, _date, _isLiveMode);
instances = result as BaseDataCollection;
if (instances == null)
{
OnInvalidSource(source, new Exception("Reader must generate a BaseDataCollection with the FileFormat.Collection"));
continue;
}
}
catch (Exception err)
{
OnReaderError(raw, err);
continue;
}
yield return(instances);
}
}
finally
{
if (reader != null)
{
reader.Dispose();
}
}
}
public void ReadsFromSpecificZipEntry()
{
var source = Path.Combine("TestData", "multizip.zip");
const string entryName = "multizip/three.txt";
using (var reader = new LocalFileSubscriptionStreamReader(source, entryName))
{
var line = reader.ReadLine();
Assert.AreEqual("3", line);
}
}
/// <summary>
/// Creates a new <see cref="IStreamReader"/> for the specified <paramref name="subscriptionDataSource"/>
/// </summary>
/// <param name="subscriptionDataSource">The source to produce an <see cref="IStreamReader"/> for</param>
/// <returns>A new instance of <see cref="IStreamReader"/> to read the source, or null if there was an error</returns>
protected IStreamReader CreateStreamReader(SubscriptionDataSource subscriptionDataSource)
{
IStreamReader reader = null;
try
{
switch (subscriptionDataSource.TransportMedium)
{
case SubscriptionTransportMedium.LocalFile:
reader = new LocalFileSubscriptionStreamReader(DataCacheProvider, subscriptionDataSource.Source);
break;
case SubscriptionTransportMedium.RemoteFile:
reader = HandleRemoteSourceFile(subscriptionDataSource);
break;
case SubscriptionTransportMedium.Rest:
reader = new RestSubscriptionStreamReader(subscriptionDataSource.Source, subscriptionDataSource.Headers, IsLiveMode);
break;
default:
throw new InvalidEnumArgumentException("Unexpected SubscriptionTransportMedium specified: " + subscriptionDataSource.TransportMedium);
}
}
catch (Exception e)
{
OnInvalidSource(subscriptionDataSource, e);
return(reader);
}
if (reader == null || reader.EndOfStream)
{
OnInvalidSource(subscriptionDataSource, new Exception($"The reader was empty for source: ${subscriptionDataSource.Source}"));
return(null);
}
return(reader);
}
/// <summary>
/// Private method loads all option or future contracts for a particular underlying
/// symbol (placeholder) on demand by walking through the IQFeed universe file
/// </summary>
/// <param name="placeholder">Underlying symbol</param>
/// <returns></returns>
private IEnumerable <SymbolData> LoadSymbolOnDemand(SymbolData placeholder)
{
var dayOfWeek = DateTimeFormatInfo.CurrentInfo.Calendar.GetWeekOfYear(DateTime.Today, CalendarWeekRule.FirstDay, DayOfWeek.Monday);
var thisYearWeek = $"{DateTime.Today.ToStringInvariant("yyyy")}-{dayOfWeek.ToStringInvariant()}";
var todayCsvFileName = "mktsymbols_v2.txt";
var todayFullCsvName = Path.Combine(Globals.Cache, todayCsvFileName);
var reader = new LocalFileSubscriptionStreamReader(_dataCacheProvider, todayFullCsvName, placeholder.StartPosition);
Log.Trace("Loading data on demand for {0}...", placeholder.Symbol.ID);
var symbolUniverse = new List <SymbolData>();
long currentPosition = placeholder.StartPosition;
while (!reader.EndOfStream && currentPosition <= placeholder.EndPosition)
{
var line = reader.ReadLine();
currentPosition += line.Length + NewLine.Length;
var columns = line.Split(Tabulation);
if (columns.Length != totalColumns)
{
continue;
}
switch (columns[columnSecurityType])
{
case "IEOPTION":
var ticker = columns[columnSymbol];
var result = SymbolRepresentation.ParseOptionTickerIQFeed(ticker);
symbolUniverse.Add(new SymbolData
{
Symbol = Symbol.CreateOption(result.Underlying,
Market.USA,
OptionStyle.American,
result.OptionRight,
result.OptionStrike,
result.ExpirationDate),
SecurityCurrency = Currencies.USD,
SecurityExchange = Market.USA,
Ticker = columns[columnSymbol]
});
break;
case "FUTURE":
if (columns[columnSymbol].EndsWith("#"))
{
continue;
}
var futuresTicker = columns[columnSymbol].TrimStart(new[] { '@' });
var parsed = SymbolRepresentation.ParseFutureTicker(futuresTicker);
var underlyingString = parsed.Underlying;
var market = Market.USA;
if (_iqFeedNameMap.ContainsKey(underlyingString))
{
underlyingString = _iqFeedNameMap[underlyingString];
}
if (underlyingString != placeholder.Symbol.Value)
{
continue;
}
// Futures contracts have different idiosyncratic expiration dates that IQFeed symbol universe file doesn't contain
// We request IQFeed explicitly for the exact expiration data of each contract
var expirationDate = _symbolFundamentalData.Request(columns[columnSymbol]).Item1;
if (expirationDate == DateTime.MinValue)
{
// contract is outdated
continue;
}
symbolUniverse.Add(new SymbolData
{
Symbol = Symbol.CreateFuture(underlyingString,
market,
expirationDate),
SecurityCurrency = Currencies.USD,
SecurityExchange = market,
Ticker = columns[columnSymbol]
});
break;
default:
continue;
}
}
return(symbolUniverse);
}
/// <summary>
/// Private method performs initial loading of data from IQFeed universe:
/// - method loads FX,equities, indices as is into memory
/// - method prepares ondemand data for options and futures and stores it to disk
/// - method updates futures mapping files if required
/// </summary>
/// <returns></returns>
private IEnumerable <SymbolData> LoadSymbols()
{
// default URI
const string uri = "http://www.dtniq.com/product/mktsymbols_v2.zip";
if (!Directory.Exists(Globals.Cache))
{
Directory.CreateDirectory(Globals.Cache);
}
// we try to check if we already downloaded the file and it is in cache. If yes, we use it. Otherwise, download new file.
IStreamReader reader;
// we update the files every week
var dayOfWeek = DateTimeFormatInfo.CurrentInfo.Calendar.GetWeekOfYear(DateTime.Today, CalendarWeekRule.FirstDay, DayOfWeek.Monday);
var thisYearWeek = $"{DateTime.Today.ToStringInvariant("yyyy")}-{dayOfWeek.ToStringInvariant()}";
var todayZipFileName = "IQFeed-symbol-universe-" + thisYearWeek + ".zip";
var todayFullZipName = Path.Combine(Globals.Cache, todayZipFileName);
var todayCsvFileName = "mktsymbols_v2.txt";
var todayFullCsvName = Path.Combine(Globals.Cache, todayCsvFileName);
var iqfeedNameMapFileName = "IQFeed-symbol-map.json";
var iqfeedNameMapFullName = Path.Combine("IQFeed", iqfeedNameMapFileName);
var mapExists = File.Exists(iqfeedNameMapFullName);
var universeExists = File.Exists(todayFullZipName);
if (mapExists)
{
Log.Trace("Loading IQFeed futures symbol map file...");
_iqFeedNameMap = JsonConvert.DeserializeObject <Dictionary <string, string> >(File.ReadAllText(iqfeedNameMapFullName));
}
if (!universeExists)
{
Log.Trace("Loading and unzipping IQFeed symbol universe file ({0})...", uri);
using (var client = new WebClient())
{
client.Proxy = WebRequest.GetSystemWebProxy();
client.DownloadFile(uri, todayFullZipName);
}
Compression.Unzip(todayFullZipName, Globals.Cache, true);
}
else
{
Log.Trace("Found up-to-date IQFeed symbol universe file in local cache. Loading it...");
}
var symbolCache = new Dictionary <Symbol, SymbolData>();
var symbolUniverse = new List <SymbolData>();
long currentPosition = 0;
long prevPosition = 0;
long lineCounter = 0;
reader = new LocalFileSubscriptionStreamReader(_dataCacheProvider, todayFullCsvName);
while (!reader.EndOfStream)
{
lineCounter++;
if (lineCounter % 100000 == 0)
{
Log.Trace($"{lineCounter} lines read.");
}
prevPosition = currentPosition;
var line = reader.ReadLine();
currentPosition += line.Length + NewLine.Length; // file position 'estimator' for ASCII file of IQFeed universe
var columns = line.Split(Tabulation);
if (columns[columnSymbol] == "TST$Y")
{
continue;
}
if (columns.Length != totalColumns)
{
Log.Trace("Discrepancy found while parsing IQFeed symbol universe file. Expected 8 columns, but arrived {0}. Line: {1}", columns.Length, line);
continue;
}
switch (columns[columnSecurityType])
{
case "INDEX":
case "EQUITY":
// we load equities/indices in memory
symbolUniverse.Add(new SymbolData
{
Symbol = Symbol.Create(columns[columnSymbol], SecurityType.Equity, Market.USA),
SecurityCurrency = Currencies.USD,
SecurityExchange = Market.USA,
Ticker = columns[columnSymbol]
});
break;
case "IEOPTION":
var ticker = columns[columnSymbol];
var result = SymbolRepresentation.ParseOptionTickerIQFeed(ticker);
var optionUnderlying = result.Underlying;
var canonicalSymbol = Symbol.Create(optionUnderlying, SecurityType.Option, Market.USA);
if (!symbolCache.ContainsKey(canonicalSymbol))
{
var placeholderSymbolData = new SymbolData
{
Symbol = canonicalSymbol,
SecurityCurrency = Currencies.USD,
SecurityExchange = Market.USA,
StartPosition = prevPosition,
EndPosition = currentPosition
};
symbolCache.Add(canonicalSymbol, placeholderSymbolData);
}
else
{
symbolCache[canonicalSymbol].EndPosition = currentPosition;
}
break;
case "FOREX":
// we use FXCM symbols only
if (columns[columnSymbol].EndsWith(".FXCM"))
{
var symbol = columns[columnSymbol].Replace(".FXCM", string.Empty);
symbolUniverse.Add(new SymbolData
{
Symbol = Symbol.Create(symbol, SecurityType.Forex, Market.FXCM),
SecurityCurrency = Currencies.USD,
SecurityExchange = Market.FXCM,
Ticker = columns[columnSymbol]
});
}
break;
case "FUTURE":
// we are not interested in designated continuous contracts
if (columns[columnSymbol].EndsWith("#") || columns[columnSymbol].EndsWith("#C"))
{
continue;
}
var futuresTicker = columns[columnSymbol].TrimStart(new[] { '@' });
var parsed = SymbolRepresentation.ParseFutureTicker(futuresTicker);
var underlyingString = parsed.Underlying;
if (_iqFeedNameMap.ContainsKey(underlyingString))
{
underlyingString = _iqFeedNameMap[underlyingString];
}
else
{
if (!mapExists)
{
if (!_iqFeedNameMap.ContainsKey(underlyingString))
{
// if map is not created yet, we request this information from IQFeed
var exchangeSymbol = _symbolFundamentalData.Request(columns[columnSymbol]).Item2;
if (!string.IsNullOrEmpty(exchangeSymbol))
{
_iqFeedNameMap[underlyingString] = exchangeSymbol;
underlyingString = exchangeSymbol;
}
}
}
}
var market = _futuresExchanges.ContainsKey(columns[columnExchange]) ? _futuresExchanges[columns[columnExchange]] : Market.USA;
canonicalSymbol = Symbol.Create(underlyingString, SecurityType.Future, market);
if (!symbolCache.ContainsKey(canonicalSymbol))
{
var placeholderSymbolData = new SymbolData
{
Symbol = canonicalSymbol,
SecurityCurrency = Currencies.USD,
SecurityExchange = market,
StartPosition = prevPosition,
EndPosition = currentPosition
};
symbolCache.Add(canonicalSymbol, placeholderSymbolData);
}
else
{
symbolCache[canonicalSymbol].EndPosition = currentPosition;
}
break;
default:
continue;
}
}
if (!mapExists)
{
Log.Trace("Saving IQFeed futures symbol map file...");
File.WriteAllText(iqfeedNameMapFullName, JsonConvert.SerializeObject(_iqFeedNameMap));
}
symbolUniverse.AddRange(symbolCache.Values);
Log.Trace("Finished loading IQFeed symbol universe file.");
return(symbolUniverse);
}
/// <summary>
/// Does not attempt to retrieve any data
/// </summary>
public IStreamReader Fetch(Symbol symbol, SubscriptionDataSource source, DateTime date, Resolution resolution, TickType tickType)
{
string entryName = null; // default to all entries
var filename = source.Source;
var hashIndex = source.Source.LastIndexOf("#", StringComparison.Ordinal);
if (hashIndex != -1)
{
entryName = source.Source.Substring(hashIndex + 1);
filename = source.Source.Substring(0, hashIndex);
}
if (!File.Exists(filename))
{
return(null);
}
// handles zip files
if (filename.GetExtension() == ".zip")
{
IStreamReader reader = null;
try
{
// cleaning the outdated cache items
if (lastDate == DateTime.MinValue || lastDate < date.Date)
{
// clean all items that that are older than _cachePeriodBars bars than the current date
foreach (var zip in _zipFileCache.Where(x => x.Value.Value.Item1 < date.Date.AddDays(-_cachePeriodBars)))
{
// disposing zip archive
zip.Value.Value.Item2.Dispose();
// removing it from the cache
Lazy <CacheEntry> removed;
_zipFileCache.TryRemove(zip.Key, out removed);
}
lastDate = date.Date;
}
_zipFileCache.AddOrUpdate(filename,
x =>
{
var newItem = Tuple.Create(date.Date, new ZipFile(filename));
reader = new LocalFileSubscriptionStreamReader(newItem.Item2, entryName);
return(newItem);
},
(x, existingEntry) =>
{
reader = new LocalFileSubscriptionStreamReader(existingEntry.Item2, entryName);
return(existingEntry);
});
return(reader);
}
catch (Exception err)
{
Log.Error(err, "Inner try/catch");
if (reader != null)
{
reader.Dispose();
}
return(null);
}
}
else
{
// handles text files
return(new LocalFileSubscriptionStreamReader(filename, entryName));
}
}
/// <summary>
/// Reads the specified <paramref name="source"/>
/// </summary>
/// <param name="source">The source to be read</param>
/// <returns>An <see cref="IEnumerable{BaseData}"/> that contains the data in the source</returns>
public IEnumerable <BaseData> Read(SubscriptionDataSource source)
{
SubscriptionDataSourceReader.CheckRemoteFileCache();
IStreamReader reader = null;
try
{
try
{
switch (source.TransportMedium)
{
default:
case SubscriptionTransportMedium.Rest:
reader = new RestSubscriptionStreamReader(source.Source, source.Headers, _isLiveMode);
break;
case SubscriptionTransportMedium.LocalFile:
reader = new LocalFileSubscriptionStreamReader(_dataCacheProvider, source.Source);
break;
case SubscriptionTransportMedium.RemoteFile:
reader = new RemoteFileSubscriptionStreamReader(_dataCacheProvider, source.Source, Globals.Cache, source.Headers);
break;
}
}
catch (Exception e)
{
OnInvalidSource(source, e);
yield break;
}
var raw = "";
while (!reader.EndOfStream)
{
BaseDataCollection instances = null;
try
{
raw = reader.ReadLine();
var result = _factory.Reader(_config, raw, _date, _isLiveMode);
instances = result as BaseDataCollection;
if (instances == null && !reader.ShouldBeRateLimited)
{
OnInvalidSource(source, new Exception("Reader must generate a BaseDataCollection with the FileFormat.Collection"));
continue;
}
}
catch (Exception err)
{
OnReaderError(raw, err);
if (!reader.ShouldBeRateLimited)
{
continue;
}
}
if (_isLiveMode
// this shouldn't happen, rest reader is the only one to be rate limited
// and in live mode, but just in case...
|| instances == null && reader.ShouldBeRateLimited)
{
yield return(instances);
}
else
{
foreach (var instance in instances.Data)
{
if (instance != null && instance.EndTime != default(DateTime))
{
yield return(instance);
}
}
}
}
}
finally
{
if (reader != null)
{
reader.Dispose();
}
}
}
