/// <summary>
/// Helper method to resolve the mapping file to use.
/// </summary>
/// <remarks>This method is aware of the data type being added for <see cref="SecurityType.Base"/>
/// to the <see cref="SecurityIdentifier.Symbol"/> value</remarks>
/// <param name="mapFileResolver">The map file resolver</param>
/// <param name="symbol">The symbol that we want to map</param>
/// <param name="dataType">The string data type name if any</param>
/// <param name="mappingMode">The mapping mode to use if any</param>
/// <returns>The mapping file to use</returns>
public static MapFile ResolveMapFile(this MapFileResolver mapFileResolver,
Symbol symbol,
string dataType = null,
DataMappingMode?mappingMode = null)
{
// Load the symbol and date to complete the mapFile checks in one statement
var symbolID = symbol.HasUnderlying ? symbol.Underlying.ID.Symbol : symbol.ID.Symbol;
if (dataType == null && symbol.SecurityType == SecurityType.Base)
{
symbol.ID.Symbol.TryGetCustomDataType(out dataType);
}
symbolID = symbol.SecurityType == SecurityType.Base && dataType != null?symbolID.RemoveFromEnd($".{dataType}") : symbolID;
MapFile result;
if (ReferenceEquals(mapFileResolver, MapFileResolver.Empty))
{
result = mapFileResolver.ResolveMapFile(symbol.Value, Time.BeginningOfTime);
}
else
{
var date = symbol.HasUnderlying ? symbol.Underlying.ID.Date : symbol.ID.Date;
result = mapFileResolver.ResolveMapFile(symbolID, date);
}
result.DataMappingMode = mappingMode;
return(result);
}
/// <summary>
/// Helper method to resolve the mapping file to use.
/// </summary>
/// <remarks>This method is aware of the data type being added for <see cref="SecurityType.Base"/>
/// to the <see cref="SecurityIdentifier.Symbol"/> value</remarks>
/// <param name="mapFileResolver">The map file resolver</param>
/// <param name="symbol">The symbol that we want to map</param>
/// <param name="dataType">The configuration data type <see cref="SubscriptionDataConfig.Type"/></param>
/// <returns>The mapping file to use</returns>
public static MapFile ResolveMapFile(this MapFileResolver mapFileResolver,
Symbol symbol,
Type dataType)
{
// Load the symbol and date to complete the mapFile checks in one statement
var symbolID = symbol.HasUnderlying ? symbol.Underlying.ID.Symbol : symbol.ID.Symbol;
var date = symbol.HasUnderlying ? symbol.Underlying.ID.Date : symbol.ID.Date;
return(mapFileResolver.ResolveMapFile(
symbol.SecurityType == SecurityType.Base ? symbolID.RemoveFromEnd($".{dataType.Name}") : symbolID,
date));
}
/// <summary>
/// Helper method to resolve the mapping file to use.
/// </summary>
/// <remarks>This method is aware of the data type being added for <see cref="SecurityType.Base"/>
/// to the <see cref="SecurityIdentifier.Symbol"/> value</remarks>
/// <param name="mapFileResolver">The map file resolver</param>
/// <param name="symbol">The symbol that we want to map</param>
/// <param name="dataType">The string data type name if any</param>
/// <returns>The mapping file to use</returns>
public static MapFile ResolveMapFile(this MapFileResolver mapFileResolver,
Symbol symbol,
string dataType = null)
{
// Load the symbol and date to complete the mapFile checks in one statement
var symbolID = symbol.HasUnderlying ? symbol.Underlying.ID.Symbol : symbol.ID.Symbol;
var date = symbol.HasUnderlying ? symbol.Underlying.ID.Date : symbol.ID.Date;
if (dataType == null && symbol.SecurityType == SecurityType.Base)
{
symbol.ID.Symbol.TryGetCustomDataType(out dataType);
}
return(mapFileResolver.ResolveMapFile(
symbol.SecurityType == SecurityType.Base && dataType != null ? symbolID.RemoveFromEnd($".{dataType}") : symbolID,
date));
}
private MapFileResolver GetMapFileResolver(string market)
{
if (market != QuantConnect.Market.USA.ToLowerInvariant())
{
// don't explode for other markets which request map files and we don't have
return(MapFileResolver.Empty);
}
var timestamp = DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork);
var todayNewYork = timestamp.Date;
var yesterdayNewYork = todayNewYork.AddDays(-1);
// start the search with yesterday, today's file will be available tomorrow
var count = 0;
var date = yesterdayNewYork;
do
{
var zipFileName = Path.Combine(Globals.DataFolder, MapFileZipHelper.GetMapFileZipFileName(market, date));
// Fetch a stream for our zip from our data provider
var stream = _dataProvider.Fetch(zipFileName);
// If we found a file we can read it
if (stream != null)
{
Log.Trace("LocalZipMapFileProvider.Get({0}): Fetched map files for: {1} NY", market, date.ToShortDateString());
var result = new MapFileResolver(MapFileZipHelper.ReadMapFileZip(stream));
stream.DisposeSafely();
return(result);
}
// prevent infinite recursion if something is wrong
if (count++ > 30)
{
throw new InvalidOperationException($"LocalZipMapFileProvider couldn't find any map files going all the way back to {date}");
}
date = date.AddDays(-1);
}while (true);
}
private MapFileResolver GetMapFileResolver(AuxiliaryDataKey auxiliaryDataKey)
{
var market = auxiliaryDataKey.Market;
var timestamp = DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork);
var todayNewYork = timestamp.Date;
var yesterdayNewYork = todayNewYork.AddDays(-1);
// start the search with yesterday, today's file will be available tomorrow
var count = 0;
var date = yesterdayNewYork;
do
{
var zipFileName = MapFileZipHelper.GetMapFileZipFileName(market, date, auxiliaryDataKey.SecurityType);
// Fetch a stream for our zip from our data provider
var stream = _dataProvider.Fetch(zipFileName);
// If we found a file we can read it
if (stream != null)
{
Log.Trace("LocalZipMapFileProvider.Get({0}): Fetched map files for: {1} NY", market, date.ToShortDateString());
var result = new MapFileResolver(MapFileZipHelper.ReadMapFileZip(stream, market, auxiliaryDataKey.SecurityType));
stream.DisposeSafely();
return(result);
}
// prevent infinite recursion if something is wrong
if (count++ > 30)
{
throw new InvalidOperationException($"LocalZipMapFileProvider couldn't find any map files going all the way back to {date}");
}
date = date.AddDays(-1);
}while (true);
}
/// <summary>
/// Reads the zip bytes as text and parses as FactorFileRows to create FactorFiles
/// </summary>
public static IEnumerable <KeyValuePair <Symbol, IFactorProvider> > ReadFactorFileZip(Stream file, MapFileResolver mapFileResolver, string market, SecurityType securityType)
{
if (file == null || file.Length == 0)
{
return(new Dictionary <Symbol, IFactorProvider>());
}
var keyValuePairs = (
from kvp in Compression.Unzip(file)
let filename = kvp.Key
let lines = kvp.Value
let factorFile = PriceScalingExtensions.SafeRead(Path.GetFileNameWithoutExtension(filename), lines, securityType)
let mapFile = mapFileResolver.GetByPermtick(factorFile.Permtick)
where mapFile != null
select new KeyValuePair <Symbol, IFactorProvider>(GetSymbol(mapFile, market, securityType), factorFile)
);
return(keyValuePairs);
}
/// <summary>
/// Reads the zip bytes as text and parses as FactorFileRows to create FactorFiles
/// </summary>
public static IEnumerable <KeyValuePair <Symbol, FactorFile> > ReadFactorFileZip(Stream file, MapFileResolver mapFileResolver, string market)
{
if (file == null || file.Length == 0)
{
return(new Dictionary <Symbol, FactorFile>());
}
var keyValuePairs = (
from kvp in Compression.Unzip(file)
let filename = kvp.Key
let lines = kvp.Value
let factorFile = SafeRead(filename, lines)
let mapFile = mapFileResolver.GetByPermtick(factorFile.Permtick)
where mapFile != null
let sid = SecurityIdentifier.GenerateEquity(mapFile.FirstDate, mapFile.FirstTicker, market)
let symbol = new Symbol(sid, mapFile.Permtick)
select new KeyValuePair <Symbol, FactorFile>(symbol, factorFile)
);
return(keyValuePairs);
}
/// <summary>
/// Helper method to resolve the mapping file to use.
/// </summary>
/// <remarks>This method is aware of the data type being added for <see cref="SecurityType.Base"/>
/// to the <see cref="SecurityIdentifier.Symbol"/> value</remarks>
/// <param name="mapFileResolver">The map file resolver</param>
/// <param name="symbol">The symbol that we want to map</param>
/// <param name="dataType">The configuration data type <see cref="SubscriptionDataConfig.Type"/></param>
/// <returns>The mapping file to use</returns>
public static MapFile ResolveMapFile(this MapFileResolver mapFileResolver,
Symbol symbol,
Type dataType)
{
return(mapFileResolver.ResolveMapFile(symbol, dataType.Name));
}
