public static void AlgoSeekOptionsConverter(string date)
{
// There are practical file limits we need to override for this to work.
// By default programs are only allowed 1024 files open; for options parsing we need 100k
Environment.SetEnvironmentVariable("MONO_MANAGED_WATCHER", "disabled");
Environment.SetEnvironmentVariable("MONO_GC_PARAMS", "max-heap-size=4g");
Log.LogHandler = new CompositeLogHandler(new ILogHandler[] { new ConsoleLogHandler(), new FileLogHandler("log.txt") });
// Directory for the data, output and processed cache:
var remoteMask = Config.Get("options-remote-file-mask", "*.bz2").Replace("{0}", date);
var remoteDirectory = Config.Get("options-remote-directory").Replace("{0}", date);
var sourceDirectory = Config.Get("options-source-directory").Replace("{0}", date);
var dataDirectory = Config.Get("data-directory").Replace("{0}", date);
var cleanSourceDirectory = Config.GetBool("clean-source-directory", false);
Log.Trace("CONFIGURATION:");
Log.Trace("Processor Count: " + Environment.ProcessorCount);
Log.Trace("Remote Directory: " + remoteDirectory);
Log.Trace("Source Directory: " + sourceDirectory);
Log.Trace("Destination Directory: " + dataDirectory);
// Date for the option bz files.
var referenceDate = DateTime.ParseExact(date, DateFormat.EightCharacter, CultureInfo.InvariantCulture);
Log.Trace($"DateTime: {referenceDate.Date.ToStringInvariant()}");
// checking if remote folder exists
if (!Directory.Exists(remoteDirectory))
{
Log.Error("Remote Directory doesn't exist: " + remoteDirectory);
return;
}
// Convert the date:
var timer = Stopwatch.StartNew();
var converter = new AlgoSeekOptionsConverter(Resolution.Minute, referenceDate, remoteDirectory, remoteMask, sourceDirectory, dataDirectory);
converter.Clean(referenceDate);
Log.Trace($"AlgoSeekOptionConverter.Main(): {referenceDate.ToStringInvariant()} Cleaning finished in time: {timer.Elapsed.ToStringInvariant(null)}");
timer.Restart();
converter.Convert();
Log.Trace($"AlgoSeekOptionConverter.Main(): {referenceDate.ToStringInvariant()} Conversion finished in time: {timer.Elapsed.ToStringInvariant(null)}");
timer.Restart();
converter.Package(referenceDate);
Log.Trace($"AlgoSeekOptionConverter.Main(): {referenceDate.ToStringInvariant()} Compression finished in time: {timer.Elapsed.ToStringInvariant(null)}");
if (cleanSourceDirectory)
{
Log.Trace($"AlgoSeekOptionConverter.Main(): Cleaning source directory: {sourceDirectory}");
try
{
Directory.Delete(sourceDirectory, true);
}
catch (Exception err)
{
Log.Trace($"AlgoSeekOptionConverter.Main(): Error while cleaning source directory {err.Message}");
}
}
}
public static void Main(string[] args)
{
var date = args[0];
//var startdate = DateTime.ParseExact("20150101", "yyyyMMdd",null);
//var enddate = DateTime.ParseExact("20161231", "yyyyMMdd", null);
var dirs = Directory.EnumerateDirectories("E:\\data\\optiondata\\option\\sse\\minute\\510050\\");
//var dirs = GetAllDays(startdate, enddate);
//foreach (DateTime d in dirs) {
foreach (string d in dirs)
{
//date = d.ToString("yyyyMMdd");
date = d.Substring(d.Length - 8);
// There are practical file limits we need to override for this to work.
// By default programs are only allowed 1024 files open; for options parsing we need 100k
Environment.SetEnvironmentVariable("MONO_MANAGED_WATCHER", "disabled");
Environment.SetEnvironmentVariable("MONO_GC_PARAMS", "max-heap-size=4g");
Log.LogHandler = new CompositeLogHandler(new ILogHandler[] { new ConsoleLogHandler() /*, new FileLogHandler("log.txt")*/ });
// Directory for the data, output and processed cache:
var remoteMask = Config.Get("options-remote-file-mask", "*.csv").Replace("{0}", date);
var remoteDirectory = Config.Get("options-remote-directory", "E:\\data\\optiondata\\option\\sse\\minute\\510050\\{0}").Replace("{0}", date);
var sourceDirectory = Config.Get("options-source-directory", "E:\\data\\optiondata\\option\\sse\\minute\\510050\\{0}").Replace("{0}", date);
var dataDirectory = Config.Get("data-directory", "option/output").Replace("{0}", date);
var cleanSourceDirectory = Config.GetBool("clean-source-directory", false);
Log.DebuggingLevel = -1;
Log.Trace("CONFIGURATION:");
Log.Trace("Processor Count: " + Environment.ProcessorCount);
Log.Trace("Remote Directory: " + remoteDirectory);
Log.Trace("Source Directory: " + sourceDirectory);
Log.Trace("Destination Directory: " + dataDirectory);
// Date for the option bz files.
var referenceDate = DateTime.ParseExact(date, DateFormat.EightCharacter, CultureInfo.InvariantCulture);
Log.Trace("DateTime: " + referenceDate.Date.ToString());
//// checking if remote folder exists
//if (!Directory.Exists(remoteDirectory))
//{
// Log.Error("Remote Directory doesn't exist: " + remoteDirectory);
// return;
//}
// Convert the date:
var timer = Stopwatch.StartNew();
var converter = new AlgoSeekOptionsConverter(Resolution.Minute, referenceDate, remoteDirectory, remoteMask, sourceDirectory, dataDirectory);
converter.Clean(referenceDate);
Log.Trace(string.Format("AlgoSeekOptionConverter.Main(): {0} Cleaning finished in time: {1}", referenceDate, timer.Elapsed));
timer.Restart();
converter.Convert();
Log.Trace(string.Format("AlgoSeekOptionConverter.Main(): {0} Conversion finished in time: {1}", referenceDate, timer.Elapsed));
timer.Restart();
converter.Package(referenceDate);
Log.Trace(string.Format("AlgoSeekOptionConverter.Main(): {0} Compression finished in time: {1}", referenceDate, timer.Elapsed));
if (cleanSourceDirectory)
{
Log.Trace(string.Format("AlgoSeekOptionConverter.Main(): Cleaning source directory: {0}", sourceDirectory));
try
{
Directory.Delete(sourceDirectory, true);
}
catch (Exception err)
{
Log.Trace(string.Format("AlgoSeekOptionConverter.Main(): Error while cleaning source directory {0}", err.Message));
}
}
}
}
