public void CachesValueAndReturnsSameReference()
{
var provider = new LocalDiskFactorFileProvider();
var factorFile1 = provider.Get(Symbols.SPY);
var factorFile2 = provider.Get(Symbols.SPY);
Assert.IsTrue(ReferenceEquals(factorFile1, factorFile2));
}
public void DividendsDistribution(string exDividendDateStr, decimal expectedDistribution)
{
var dividendProvider = new DividendEventProvider();
var config = new SubscriptionDataConfig(typeof(TradeBar), Symbols.AAPL, Resolution.Second, TimeZones.NewYork, TimeZones.NewYork,
false, false, false);
var factorFile = new LocalDiskFactorFileProvider();
var mapFile = new LocalDiskMapFileProvider();
var start = new DateTime(1998, 01, 02);
dividendProvider.Initialize(config, factorFile.Get(Symbols.AAPL), mapFile.Get(Market.USA).ResolveMapFile(Symbols.AAPL, start), start);
var exDividendDate = DateTime.ParseExact(exDividendDateStr, DateFormat.EightCharacter, CultureInfo.InvariantCulture);
var events = dividendProvider
.GetEvents(new NewTradableDateEventArgs(exDividendDate, null, Symbols.AAPL, null))
.ToList();
// ex dividend date does not emit anything
Assert.AreEqual(0, events.Count);
events = dividendProvider
.GetEvents(new NewTradableDateEventArgs(exDividendDate.AddDays(1), null, Symbols.AAPL, null))
.ToList();
Assert.AreEqual(1, events.Count);
var dividend = events[0] as Dividend;
Assert.IsNotNull(dividend);
Assert.AreEqual(expectedDistribution, dividend.Distribution);
}
public override void Initialize()
{
SetStartDate(2014, 3, 25); //Set Start Date
SetEndDate(2014, 4, 7); //Set End Date
SetCash(100000); //Set Strategy Cash
// Set our DataNormalizationMode to raw
UniverseSettings.DataNormalizationMode = DataNormalizationMode.Raw;
_googl = AddEquity(Ticker, Resolution.Daily).Symbol;
// Get our factor file for this regression
var dataProvider =
Composer.Instance.GetExportedValueByTypeName <IDataProvider>(Config.Get("data-provider",
"DefaultDataProvider"));
var mapFileProvider = new LocalDiskMapFileProvider();
mapFileProvider.Initialize(dataProvider);
var factorFileProvider = new LocalDiskFactorFileProvider();
factorFileProvider.Initialize(mapFileProvider, dataProvider);
_factorFile = factorFileProvider.Get(_googl) as CorporateFactorProvider;
// Prime our expected values
_expectedRawPrices.MoveNext();
}
public void ReturnsNullForNotFound()
{
var provider = new LocalDiskFactorFileProvider();
var factorFile = provider.Get(Symbol.Create("not-a-ticker", SecurityType.Equity, QuantConnect.Market.USA));
Assert.IsNull(factorFile);
}
public void RetrievesFromDisk()
{
var provider = new LocalDiskFactorFileProvider();
var factorFile = provider.Get(Symbols.SPY);
Assert.IsNotNull(factorFile);
}
public void FindsFactorFilesWithErrors()
{
var provider = new LocalDiskFactorFileProvider();
var factorFileFolder = Path.Combine(Globals.DataFolder, "equity", QuantConnect.Market.USA, "factor_files");
foreach (var fileName in Directory.EnumerateFiles(factorFileFolder))
{
var ticker = Path.GetFileNameWithoutExtension(fileName).ToUpper(CultureInfo.InvariantCulture);
var symbol = Symbol.Create(ticker, SecurityType.Equity, QuantConnect.Market.USA);
try
{
provider.Get(symbol);
}
catch (Exception exception)
{
Console.WriteLine(ticker + ": " + exception.Message);
}
}
}
public override void Initialize()
{
SetStartDate(2014, 6, 5); //Set Start Date
SetEndDate(2014, 6, 5); //Set End Date
UniverseSettings.DataNormalizationMode = DataNormalizationMode.SplitAdjusted;
_aapl = AddEquity(Ticker, Resolution.Minute).Symbol;
var dataProvider =
Composer.Instance.GetExportedValueByTypeName <IDataProvider>(Config.Get("data-provider",
"DefaultDataProvider"));
var mapFileProvider = new LocalDiskMapFileProvider();
mapFileProvider.Initialize(dataProvider);
var factorFileProvider = new LocalDiskFactorFileProvider();
factorFileProvider.Initialize(mapFileProvider, dataProvider);
_factorFile = factorFileProvider.Get(_aapl) as CorporateFactorProvider;
}
/// <summary>
/// Iterates each daily file in the specified <paramref name="dailyFolder"/> and adds a line for each
/// day to the approriate coarse file
/// </summary>
/// <param name="dailyFolder">The folder with daily data</param>
/// <param name="coarseFolder">The coarse output folder</param>
/// <param name="mapFileResolver"></param>
/// <param name="exclusions">The symbols to be excluded from processing</param>
/// <param name="ignoreMapless">Ignore the symbols without a map file.</param>
/// <param name="startDate">The starting date for processing</param>
/// <param name="symbolResolver">Function used to provide symbol resolution. Default resolution uses the zip file name to resolve
/// the symbol, specify null for this behavior.</param>
/// <returns>A collection of the generated coarse files</returns>
public static ICollection <string> ProcessDailyFolder(string dailyFolder, string coarseFolder, MapFileResolver mapFileResolver, HashSet <string> exclusions, bool ignoreMapless, DateTime startDate, Func <string, string> symbolResolver = null)
{
const decimal scaleFactor = 10000m;
Log.Trace("Processing: {0}", dailyFolder);
var start = DateTime.UtcNow;
// load map files into memory
var symbols = 0;
var maplessCount = 0;
var dates = new HashSet <DateTime>();
// instead of opening/closing these constantly, open them once and dispose at the end (~3x speed improvement)
var writers = new Dictionary <string, StreamWriter>();
var dailyFolderDirectoryInfo = new DirectoryInfo(dailyFolder).Parent;
if (dailyFolderDirectoryInfo == null)
{
throw new Exception("Unable to resolve market for daily folder: " + dailyFolder);
}
var market = dailyFolderDirectoryInfo.Name.ToLower();
var fundamentalDirectoryInfo = new DirectoryInfo(coarseFolder).Parent;
if (fundamentalDirectoryInfo == null)
{
throw new Exception("Unable to resolve fundamental path for coarse folder: " + coarseFolder);
}
var fineFundamentalFolder = Path.Combine(fundamentalDirectoryInfo.FullName, "fine");
var mapFileProvider = new LocalDiskMapFileProvider();
var factorFileProvider = new LocalDiskFactorFileProvider(mapFileProvider);
// open up each daily file to get the values and append to the daily coarse files
foreach (var file in Directory.EnumerateFiles(dailyFolder))
{
try
{
var symbol = Path.GetFileNameWithoutExtension(file);
if (symbol == null)
{
Log.Trace("CoarseGenerator.ProcessDailyFolder(): Unable to resolve symbol from file: {0}", file);
continue;
}
if (symbolResolver != null)
{
symbol = symbolResolver(symbol);
}
symbol = symbol.ToUpper();
if (exclusions.Contains(symbol))
{
Log.Trace("Excluded symbol: {0}", symbol);
continue;
}
// check if symbol has any fine fundamental data
var firstFineSymbolDate = DateTime.MaxValue;
if (Directory.Exists(fineFundamentalFolder))
{
var fineSymbolFolder = Path.Combine(fineFundamentalFolder, symbol.ToLower());
var firstFineSymbolFileName = Directory.Exists(fineSymbolFolder) ? Directory.GetFiles(fineSymbolFolder).OrderBy(x => x).FirstOrDefault() : string.Empty;
if (firstFineSymbolFileName.Length > 0)
{
firstFineSymbolDate = DateTime.ParseExact(Path.GetFileNameWithoutExtension(firstFineSymbolFileName), "yyyyMMdd", CultureInfo.InvariantCulture);
}
}
ZipFile zip;
using (var reader = Compression.Unzip(file, out zip))
{
var checkedForMapFile = false;
symbols++;
string line;
while ((line = reader.ReadLine()) != null)
{
//20150625.csv
var csv = line.Split(',');
var date = DateTime.ParseExact(csv[0], DateFormat.TwelveCharacter, CultureInfo.InvariantCulture);
// spin past old data
if (date < startDate)
{
continue;
}
if (ignoreMapless && !checkedForMapFile)
{
checkedForMapFile = true;
if (!mapFileResolver.ResolveMapFile(symbol, date).Any())
{
// if the resolved map file has zero entries then it's a mapless symbol
maplessCount++;
break;
}
}
var close = decimal.Parse(csv[4]) / scaleFactor;
var volume = long.Parse(csv[5]);
var dollarVolume = close * volume;
var coarseFile = Path.Combine(coarseFolder, date.ToString("yyyyMMdd") + ".csv");
dates.Add(date);
// try to resolve a map file and if found, regen the sid
var sid = SecurityIdentifier.GenerateEquity(SecurityIdentifier.DefaultDate, symbol, market);
var mapFile = mapFileResolver.ResolveMapFile(symbol, date);
if (!mapFile.IsNullOrEmpty())
{
// if available, us the permtick in the coarse files, because of this, we need
// to update the coarse files each time new map files are added/permticks change
sid = SecurityIdentifier.GenerateEquity(mapFile.FirstDate, mapFile.OrderBy(x => x.Date).First().MappedSymbol, market);
}
if (mapFile == null && ignoreMapless)
{
// if we're ignoring mapless files then we should always be able to resolve this
Log.Error(string.Format("CoarseGenerator.ProcessDailyFolder(): Unable to resolve map file for {0} as of {1}", symbol, date.ToShortDateString()));
continue;
}
// check if symbol has fine fundamental data for the current date
var hasFundamentalDataForDate = date >= firstFineSymbolDate;
// get price and split factors from factor files
var leanSymbol = new Symbol(sid, symbol);
var factorFile = factorFileProvider.Get(leanSymbol);
var factorFileRow = factorFile?.GetScalingFactors(date);
var priceFactor = factorFileRow?.PriceFactor ?? 1m;
var splitFactor = factorFileRow?.SplitFactor ?? 1m;
// sid,symbol,close,volume,dollar volume,has fundamental data,price factor,split factor
var coarseFileLine = $"{sid},{symbol},{close},{volume},{Math.Truncate(dollarVolume)},{hasFundamentalDataForDate},{priceFactor},{splitFactor}";
StreamWriter writer;
if (!writers.TryGetValue(coarseFile, out writer))
{
writer = new StreamWriter(new FileStream(coarseFile, FileMode.Create, FileAccess.Write, FileShare.Write));
writers[coarseFile] = writer;
}
writer.WriteLine(coarseFileLine);
}
}
if (symbols % 1000 == 0)
{
Log.Trace("CoarseGenerator.ProcessDailyFolder(): Completed processing {0} symbols. Current elapsed: {1} seconds", symbols, (DateTime.UtcNow - start).TotalSeconds.ToString("0.00"));
}
}
catch (Exception err)
{
// log the error and continue with the process
Log.Error(err.ToString());
}
}
Log.Trace("CoarseGenerator.ProcessDailyFolder(): Saving {0} coarse files to disk", dates.Count);
// dispose all the writers at the end of processing
foreach (var writer in writers)
{
writer.Value.Dispose();
}
var stop = DateTime.UtcNow;
Log.Trace("CoarseGenerator.ProcessDailyFolder(): Processed {0} symbols into {1} coarse files in {2} seconds", symbols, dates.Count, (stop - start).TotalSeconds.ToString("0.00"));
Log.Trace("CoarseGenerator.ProcessDailyFolder(): Excluded {0} mapless symbols.", maplessCount);
return(writers.Keys);
}
