public void Execute()
{
var mapfiles = new MapFileCollection(this.Path);
foreach (((var src, var trg), var i) in mapfiles.GetFiles().Select((x, i) => (x, i)))
{
Log.Info($"{(i + 1).ToString().PadRight(5)} {trg}");
}
}
/// <summary>
/// Iterates each daily file in the specified <paramref name="dailyFolder"/> and adds a line for each
/// day to the approriate coarse file
/// </summary>
/// <param name="dailyFolder">The folder with daily data</param>
/// <param name="coarseFolder">The coarse output folder</param>
/// <param name="mapFileFolder">Location of the map file for the data</param>
/// <param name="start">The start time, this is resolve by finding the most recent written coarse file</param>
/// <param name="exclusions">The symbols to be excluded from processing</param>
/// <param name="ignoreMapless">Ignore the symbols without a map file.</param>
private static void ProcessDailyFolder(string dailyFolder, string coarseFolder, string mapFileFolder, DateTime start, HashSet<string> exclusions, bool ignoreMapless)
{
const decimal scaleFactor = 10000m;
Log.Trace("Processing: {0}", dailyFolder);
var stopwatch = Stopwatch.StartNew();
// load map files into memory
var mapFileCollection = new MapFileCollection(mapFileFolder);
var symbols = 0;
var maplessCount = 0;
var dates = new HashSet<DateTime>();
// instead of opening/closing these constantly, open them once and dispose at the end (~3x speed improvement)
var writers = new Dictionary<string, StreamWriter>();
// open up each daily file to get the values and append to the daily coarse files
foreach (var file in Directory.EnumerateFiles(dailyFolder))
{
try
{
var symbol = Path.GetFileNameWithoutExtension(file);
if (symbol == null)
{
Log.Trace("Unable to resolve symbol from file: {0}", file);
continue;
}
symbol = symbol.ToUpper();
if (exclusions.Contains(symbol))
{
Log.Trace("Excluded symbol: {0}", symbol);
continue;
}
if (ignoreMapless && !File.Exists(Path.Combine(mapFileFolder, symbol.ToLower() + ".csv")))
{
maplessCount++;
// Too verbose.
//Log.Trace("Excluded mapless symbol: " + symbol);
continue;
}
ZipFile zip;
using (var reader = Compression.Unzip(file, out zip))
{
// 30 period EMA constant
const decimal k = 2m / (30 + 1);
var seeded = false;
var runningAverageVolume = 0m;
symbols++;
string line;
while ((line = reader.ReadLine()) != null)
{
//20150625.csv
var csv = line.Split(',');
var date = DateTime.ParseExact(csv[0], DateFormat.TwelveCharacter, CultureInfo.InvariantCulture);
if (date <= start) continue;
var close = decimal.Parse(csv[4])/scaleFactor;
var volume = long.Parse(csv[5]);
// compute the current volume EMA for dollar volume calculations
runningAverageVolume = seeded
? volume*k + runningAverageVolume*(1 - k)
: volume;
seeded = true;
var dollarVolume = close*runningAverageVolume;
var coarseFile = Path.Combine(coarseFolder, date.ToString("yyyyMMdd") + ".csv");
dates.Add(date);
// this is kind of wierd, but is done to keep symbol resolution in the engine simpler
// we're going to map this symbol to the correct map file symbol for the date.
// for example, in 2013, GOOG was trading, but today it's GOOGL, and so it's
// map file is GOOGL.csv, so we'll map the symbol to GOOGL so the engine can
// easily resolve it
var mappedSymbol = symbol;
var mapFile = mapFileCollection.ResolveMapFile(symbol, date);
if (mapFile == null && ignoreMapless)
{
// if we're ignoring mapless files then we should always be able to resolve this
Log.Error(string.Format("Unable to resolve map file for {0} as of {1}", symbol, date.ToShortDateString()));
}
if (mapFile != null)
{
mappedSymbol = Path.GetFileNameWithoutExtension(mapFile).ToUpper();
}
// symbol,close,volume,dollar volume
var coarseFileLine = mappedSymbol + "," + close + "," + volume + "," + dollarVolume;
StreamWriter writer;
if (!writers.TryGetValue(coarseFile, out writer))
{
writer = new StreamWriter(new FileStream(coarseFile, FileMode.Append, FileAccess.Write, FileShare.Write));
writers[coarseFile] = writer;
}
writer.WriteLine(coarseFileLine);
}
}
if (symbols%1000 == 0)
{
Log.Trace("Completed processing {0} symbols. Current elapsed: {1} seconds", symbols, stopwatch.Elapsed.TotalSeconds.ToString("0.00"));
}
}
catch (Exception err)
{
// log the error and continue with the process
Log.Error(err.ToString());
}
}
Log.Trace("Saving {0} coarse files to disk", dates.Count);
// dispose all the writers at the end of processing
foreach (var writer in writers)
{
writer.Value.Dispose();
}
stopwatch.Stop();
Log.Trace("Processed {0} symbols into {1} coarse files in {2} seconds", symbols, dates.Count, stopwatch.Elapsed.TotalSeconds.ToString("0.00"));
Log.Trace("Excluded {0} mapless symbols.", maplessCount);
}
public static void PackPk3(
string path,
MapFileCollection files,
HashSet <Pk3File> stockFiles)
{
var folder = FileUtil.NewTempFolder;
var stock = new HashSet <string>(stockFiles.Select(x => x.Path));
foreach ((string source, string target) in files.GetFiles())
{
// If exact match found in stock files, skip
if (stock.Contains(target))
{
Log.Debug($"Skipping '{target}', exists in stock pk3s.");
continue;
}
string sourceFile = null;
string targetFile = null;
// Exact match
if (File.Exists(source))
{
sourceFile = source;
targetFile = target;
}
else if (Path.GetExtension(source) == "" ||
Path.GetExtension(source) == ".tga" ||
Path.GetExtension(source) == ".jpg")
{
string ambiguous = null;
if (File.Exists(ambiguous = Path.ChangeExtension(source, "tga")))
{
Log.Debug($"Using found tga version for file {target}");
sourceFile = ambiguous;
targetFile = Path.ChangeExtension(target, "tga");
}
else if (stock.Contains(ambiguous))
{
Log.Debug($"TGA for missing file '{target}' found in stock pk3s");
continue;
}
else if (File.Exists(ambiguous = Path.ChangeExtension(source, "jpg")))
{
Log.Debug($"Using found jpg version for file {target}");
sourceFile = ambiguous;
targetFile = Path.ChangeExtension(target, "jpg");
}
else if (stock.Contains(ambiguous))
{
Log.Debug($"JPG for missing file '{target}' found in stock pk3s");
continue;
}
}
if (sourceFile == null || targetFile == null)
{
Log.Error($"Cannot find '{target}' in filesystem or stock pk3s.");
continue;
}
var fullPathToTarget = Path.Combine(folder, targetFile);
Directory.CreateDirectory(Path.GetDirectoryName(fullPathToTarget));
File.Copy(sourceFile, fullPathToTarget, false);
Log.Debug($" -> {targetFile}");
}
ZipFile.CreateFromDirectory(folder, path);
}