private static void Main(string[] args)
{
string[] codes = null;
string[] codesLimited = null;
List <string> ignoreList = new List <string>();
Dictionary <string, string> config =
args.Length == 0
? Utils.ReadConfigFile()
: Utils.ReadConfigFile(args[0]);
bool disableTrmf = false;
bool useBatchCd = false;
foreach (KeyValuePair <string, string> entry in config)
{
switch (entry.Key)
{
case "PlotsFolder":
DataWorks.FolderPlotsRemoteBase = entry.Value.Trim().Replace("~", Environment.GetFolderPath(Environment.SpecialFolder.Personal));
DataWorks.FolderPlotsRemote = DataWorks.FolderPlotsRemoteBase;
break;
case "AlgorithmsLocation":
AlgoPack.GlobalAlgorithmsLocation = entry.Value.Trim().Replace("~", Environment.GetFolderPath(Environment.SpecialFolder.Personal));
break;
case "Datasets":
codes = entry.Value.Split(',').Select(c => c.Trim()).ToArray();
codesLimited = codes
.Where(c => DataWorks.CountMatrixColumns($"{c}/{c}_normal.txt") > 4)
.ToArray();
break;
case "EnableStreaming":
EnumMethods.EnableStreaming = Convert.ToBoolean(entry.Value);
break;
case "EnableContinuous":
EnumMethods.EnableContinuous = Convert.ToBoolean(entry.Value);
break;
case "EnableMulticolumn":
EnumMethods.EnableMulticolumn = Convert.ToBoolean(entry.Value);
break;
case "DisableTrmf":
disableTrmf = Convert.ToBoolean(entry.Value);
break;
case "UseBatchCd":
useBatchCd = Convert.ToBoolean(entry.Value);
break;
case "IgnoreAlgorithms":
ignoreList.AddRange(entry.Value.Split(',').Select(x => x.Trim().ToLower()));
break;
default:
Utils.DelayedWarnings.Enqueue($"Warning: unknown config entry with the key {entry.Key}");
break;
}
}
// verificaiton that all necessary entries are provided
if (DataWorks.FolderPlotsRemoteBase == null)
{
throw new InvalidProgramException("Invalid config file: plots folder has to be supplied (PlotsFolder=)");
}
if (AlgoPack.GlobalAlgorithmsLocation == null)
{
throw new InvalidProgramException("Invalid config file: algorithms folder has to be supplied (AlgorithmsLocation=)");
}
if (codes == null || codes.Length == 0)
{
throw new InvalidProgramException("Invalid config file: datasets are not supplied (Datasets=) or the list is empty");
}
Algorithm cdVersion = useBatchCd ? AlgoPack.Cd : AlgoPack.InCd;
ignoreList = useBatchCd
? ignoreList.Select(x => x == "cdrec" ? "cd" : x).ToList()
: ignoreList.Select(x => x == "cd" ? "cdrec" : x).ToList();
var listStd = new List <Algorithm> {
cdVersion
};
listStd.AddRange(new[] { AlgoPack.Stmvl, AlgoPack.Tkcm, AlgoPack.Spirit, AlgoPack.Nnmf, AlgoPack.Grouse });
listStd.AddRange(new[] { AlgoPack.ArImp, AlgoPack.Ssa, AlgoPack.Mrnn, AlgoPack.DynaMMo, AlgoPack.MdIsvd, AlgoPack.SvdImp });// very new batch
if (!disableTrmf)
{
listStd.Add(AlgoPack.Trmf);
}
listStd = listStd.Where(x => !ignoreList.Contains(x.AlgCode.ToLower())).ToList();
AlgoPack.ListAlgorithms = listStd.ToArray();
listStd.Remove(AlgoPack.Tkcm);
listStd.Remove(AlgoPack.Spirit);
listStd.Remove(AlgoPack.Ssa);
AlgoPack.ListAlgorithmsMulticolumn = listStd.ToArray();
AlgoPack.ListAlgorithmsStreaming = AlgoPack.ListAlgorithmsStreaming.Where(x => !ignoreList.Contains(x.AlgCode.ToLower())).ToArray();
AlgoPack.CleanUncollectedResults();
AlgoPack.EnsureFolderStructure();
void FullPrecision()
{
if (EnumMethods.EnableContinuous)
{
codes.ForEach(c => TestRoutines.PrecisionTest(ExperimentType.Continuous, ExperimentScenario.Missing, c, 1000));
codes.ForEach(c => TestRoutines.PrecisionTest(ExperimentType.Continuous, ExperimentScenario.Length, c, 2000));
codesLimited.ForEach(c => TestRoutines.PrecisionTest(ExperimentType.Continuous, ExperimentScenario.Columns, c, 1000));
codesLimited.ForEach(c => TestRoutines.PrecisionTest(ExperimentType.Continuous, ExperimentScenario.Fullrow, c, 1000));
}
codes.ForEach(c => TestRoutines.PrecisionTest(ExperimentType.Recovery, ExperimentScenario.Missing, c, 1000));
codes.ForEach(c => TestRoutines.PrecisionTest(ExperimentType.Recovery, ExperimentScenario.Length, c, 2000));
codesLimited.ForEach(c => TestRoutines.PrecisionTest(ExperimentType.Recovery, ExperimentScenario.Columns, c, 1000));
codesLimited.ForEach(c => TestRoutines.PrecisionTest(ExperimentType.Recovery, ExperimentScenario.Fullrow, c, 1000));
codes.ForEach(c => TestRoutines.PrecisionTest(ExperimentType.Recovery, ExperimentScenario.BlockSlide, c, 1000));
codes.ForEach(c => TestRoutines.PrecisionTest(ExperimentType.Recovery, ExperimentScenario.McarElement, c, 1000));
codes.ForEach(c => TestRoutines.PrecisionTest(ExperimentType.Recovery, ExperimentScenario.McarBlock, c, 1000));
}
void FullRuntime()
{
if (EnumMethods.EnableContinuous)
{
codes.ForEach(c => TestRoutines.RuntimeTest(ExperimentType.Continuous, ExperimentScenario.Missing, c, 1000));
codes.ForEach(c => TestRoutines.RuntimeTest(ExperimentType.Continuous, ExperimentScenario.Length, c, 2000));
codesLimited.ForEach(c => TestRoutines.RuntimeTest(ExperimentType.Continuous, ExperimentScenario.Columns, c, 1000));
codesLimited.ForEach(c => TestRoutines.RuntimeTest(ExperimentType.Continuous, ExperimentScenario.Fullrow, c, 1000));
}
codes.ForEach(c => TestRoutines.RuntimeTest(ExperimentType.Recovery, ExperimentScenario.Missing, c, 1000));
codes.ForEach(c => TestRoutines.RuntimeTest(ExperimentType.Recovery, ExperimentScenario.Length, c, 2000));
codesLimited.ForEach(c => TestRoutines.RuntimeTest(ExperimentType.Recovery, ExperimentScenario.Columns, c, 1000));
codesLimited.ForEach(c => TestRoutines.RuntimeTest(ExperimentType.Recovery, ExperimentScenario.Fullrow, c, 1000));
codes.ForEach(c => TestRoutines.RuntimeTest(ExperimentType.Recovery, ExperimentScenario.BlockSlide, c, 1000));
codes.ForEach(c => TestRoutines.RuntimeTest(ExperimentType.Recovery, ExperimentScenario.McarElement, c, 1000));
codes.ForEach(c => TestRoutines.RuntimeTest(ExperimentType.Recovery, ExperimentScenario.McarBlock, c, 1000));
}
void FullRuntimeReplot() //service method
{
if (EnumMethods.EnableContinuous)
{
codes.ForEach(c => TestRoutines.RuntimeTestReplot(ExperimentType.Continuous, ExperimentScenario.Missing, c, 1000));
codes.ForEach(c => TestRoutines.RuntimeTestReplot(ExperimentType.Continuous, ExperimentScenario.Length, c, 2000));
codesLimited.ForEach(c => TestRoutines.RuntimeTestReplot(ExperimentType.Continuous, ExperimentScenario.Columns, c, 1000));
codesLimited.ForEach(c => TestRoutines.RuntimeTestReplot(ExperimentType.Continuous, ExperimentScenario.Fullrow, c, 1000));
}
codes.ForEach(c => TestRoutines.RuntimeTestReplot(ExperimentType.Recovery, ExperimentScenario.Missing, c, 1000));
codes.ForEach(c => TestRoutines.RuntimeTestReplot(ExperimentType.Recovery, ExperimentScenario.Length, c, 2000));
codesLimited.ForEach(c => TestRoutines.RuntimeTestReplot(ExperimentType.Recovery, ExperimentScenario.Columns, c, 1000));
codesLimited.ForEach(c => TestRoutines.RuntimeTestReplot(ExperimentType.Recovery, ExperimentScenario.Fullrow, c, 1000));
codes.ForEach(c => TestRoutines.RuntimeTestReplot(ExperimentType.Recovery, ExperimentScenario.BlockSlide, c, 1000));
}
void FullStreaming()
{
codes.ForEach(c => TestRoutines.RuntimeTest(ExperimentType.Streaming, ExperimentScenario.Missing, c, 1000));
codes.ForEach(c => TestRoutines.RuntimeTest(ExperimentType.Streaming, ExperimentScenario.Length, c, 2000));
codesLimited.ForEach(c => TestRoutines.RuntimeTest(ExperimentType.Streaming, ExperimentScenario.Columns, c, 1000));
}
void FullMulticolumn()
{
codes.ForEach(c => TestRoutines.PrecisionTest(ExperimentType.Recovery, ExperimentScenario.MultiColumnDisjoint, c, 1000));
codes.ForEach(c => TestRoutines.PrecisionTest(ExperimentType.Recovery, ExperimentScenario.MultiColumnOverlap, c, 1000));
codes.ForEach(c => TestRoutines.RuntimeTest(ExperimentType.Recovery, ExperimentScenario.MultiColumnDisjoint, c, 1000));
codes.ForEach(c => TestRoutines.RuntimeTest(ExperimentType.Recovery, ExperimentScenario.MultiColumnOverlap, c, 1000));
}
if (EnumMethods.EnableStreaming)
{
FullStreaming();
}
FullPrecision();
FullRuntime();
if (EnumMethods.EnableMulticolumn)
{
FullMulticolumn();
}
//codes.ForEach(c => TestRoutines.RuntimeTest(ExperimentType.Recovery, ExperimentScenario.MissingMultiColumn, c, 1000));
//codes.ForEach(c => TestRoutines.RuntimeTest(ExperimentType.Continuous, ExperimentScenario.MissingMultiColumn, c, 2000));
//codes.ForEach(c => TestRoutines.PrecisionTest(ExperimentType.Recovery, ExperimentScenario.MissingMultiColumn, c, 1000));
//codes.ForEach(c => TestRoutines.PrecisionTest(ExperimentType.Continuous, ExperimentScenario.MissingMultiColumn, c, 1000));
//
// multi-run for 1...N runtime tests and averaging the results from them
//
#if false
{
for (int i = 1; i <= 5; i++)
{
DataWorks.FolderPlotsRemote = DataWorks.FolderPlotsRemoteBase + i + "/";
if (!Directory.Exists(DataWorks.FolderPlotsRemote))
{
Directory.CreateDirectory(DataWorks.FolderPlotsRemote);
AlgoPack.EnsureFolderStructure();
}
FullRuntime();
//FullStreaming();
}
DataWorks.FolderPlotsRemote = DataWorks.FolderPlotsRemoteBase;
}
//SingularExperiments.AverageRTRuns(codes, codesLimited, 5);
#endif
//SingularExperiments.MsePerformanceReport(codes, codesLimited);
//SingularExperiments.RMSE("~/MVR/CD-RMV/incCD/_data/out/cdmvr200_k3.txt", "~/Downloads/bafu_int.csv", (0, 50, 100));
#if false
{
string cmptype = "meaninit_vert";
var sw_prec = new StreamWriter(File.Open($"prec_report_{cmptype}.txt", FileMode.Create));
var sw_rt = new StreamWriter(File.Open($"rt_report_{cmptype}.txt", FileMode.Create));
SingularExperiments.MsePerformanceReport(codes, codesLimited, sw_prec.WriteLine, cmptype);
SingularExperiments.SSVIterPerformanceReport(codes, codesLimited, sw_rt.WriteLine, cmptype);
sw_prec.Close();
sw_rt.Close();
}
#endif
//
// time series
//
#if false
{
//var data = DataWorks.TimeSeries("BAFU", "*.asc", 3, Utils.Specific.ParseWasserstand, new DateTime(2005, 1, 1), true);
//DataWorks.TimeSeriesMerge(data, "BAFU_total.txt");
}
#endif
FinalSequence();
}
private static void Main(string[] args)
{
// technical
Console.CancelKeyPress += InterruptHandler;
string[] codes = null;
string[] codesLimited = null;
List <string> ignoreList = new List <string>();
List <string> scenarios = new List <string>();
Dictionary <string, string> config =
args.Length == 0
? Utils.ReadConfigFile()
: Utils.ReadConfigFile(args[0]);
bool runPrecision = true;
bool runRuntime = true;
foreach (KeyValuePair <string, string> entry in config)
{
switch (entry.Key)
{
case "PlotsFolder":
DataWorks.FolderPlotsRemoteBase = entry.Value.Trim().Replace("~", Environment.GetFolderPath(Environment.SpecialFolder.Personal));
DataWorks.FolderPlotsRemote = DataWorks.FolderPlotsRemoteBase;
break;
case "AlgorithmsLocation":
AlgoPack.GlobalAlgorithmsLocation = entry.Value.Trim().Replace("~", Environment.GetFolderPath(Environment.SpecialFolder.Personal));
break;
case "Datasets":
codes = entry.Value.Split(',').Select(c => c.Trim()).ToArray();
codesLimited = codes
.Where(c => DataWorks.CountMatrixColumns($"{c}/{c}_normal.txt") > 4)
.ToArray();
break;
case "Scenarios":
scenarios.AddRange(entry.Value.Split(',').Select(x => x.Trim().ToLower()));
break;
case "EnableStreaming":
EnumMethods.EnableStreaming = Convert.ToBoolean(entry.Value);
break;
case "EnableContinuous":
EnumMethods.EnableContinuous = Convert.ToBoolean(entry.Value);
break;
case "EnableBatchMid":
EnumMethods.EnableBatchMid = Convert.ToBoolean(entry.Value);
break;
case "EnabledAlgorithms":
ignoreList.AddRange(entry.Value.Split(',').Select(x => x.Trim().ToLower()));
break;
case "DisablePrecision":
runPrecision = !Convert.ToBoolean(entry.Value);
break;
case "DisableRuntime":
runRuntime = !Convert.ToBoolean(entry.Value);
break;
default:
Utils.DelayedWarnings.Enqueue($"Warning: unknown config entry with the key {entry.Key}");
break;
}
}
// verification that all necessary entries are provided
if (DataWorks.FolderPlotsRemoteBase == null)
{
throw new InvalidProgramException("Invalid config file: plots folder has to be supplied (PlotsFolder=)");
}
if (AlgoPack.GlobalAlgorithmsLocation == null)
{
throw new InvalidProgramException("Invalid config file: algorithms folder has to be supplied (AlgorithmsLocation=)");
}
if (codes == null || codes.Length == 0)
{
throw new InvalidProgramException("Invalid config file: datasets are not supplied (Datasets=) or the list is empty");
}
AlgoPack.ListAlgorithms = AlgoPack.ListAlgorithms.Where(x => ignoreList.Contains(x.AlgCode.ToLower())).ToArray();
AlgoPack.ListAlgorithmsStreaming = AlgoPack.ListAlgorithms.Where(x => x.IsStreaming).ToArray();
AlgoPack.ListAlgorithmsMulticolumn = AlgoPack.ListAlgorithms.Where(x => x.IsMulticolumn).ToArray();
AlgoPack.CleanUncollectedResults();
List <ExperimentScenario> activeScenarios = new List <ExperimentScenario>();
foreach (string scen in scenarios)
{
bool found = false;
foreach (ExperimentScenario es in EnumMethods.AllExperimentScenarios())
{
if (scen == es.ToLongString())
{
found = true;
activeScenarios.Add(es);
}
}
if (!found)
{
throw new InvalidProgramException("Invalid config file: list of scenarios contains entries which are not supported by the testing framework");
}
}
void FullRun(bool enablePrecision, bool enableRuntime)
{
foreach (string code in codes)
{
if (EnumMethods.EnableContinuous)
{
AlgoPack.EnsureFolderStructure(ExperimentType.Continuous, activeScenarios.Where(EnumMethods.IsContinuous).Select(es => es.ToLongString()).ToList());
foreach (ExperimentScenario es in activeScenarios.Where(EnumMethods.IsContinuous))
{
if (es.IsLimited() && !codesLimited.Contains(code))
{
continue;
}
if (enablePrecision)
{
TestRoutines.PrecisionTest(ExperimentType.Continuous, es, code);
}
if (enableRuntime)
{
TestRoutines.RuntimeTest(ExperimentType.Continuous, es, code);
}
}
}
if (EnumMethods.EnableBatchMid)
{
AlgoPack.EnsureFolderStructure(ExperimentType.Recovery, activeScenarios.Where(EnumMethods.IsBatchMid).Select(es => es.ToLongString()).ToList());
foreach (ExperimentScenario es in activeScenarios.Where(EnumMethods.IsBatchMid))
{
if (es.IsLimited() && !codesLimited.Contains(code))
{
continue;
}
if (enablePrecision)
{
TestRoutines.PrecisionTest(ExperimentType.Recovery, es, code);
}
if (enableRuntime)
{
TestRoutines.RuntimeTest(ExperimentType.Recovery, es, code);
}
}
}
if (EnumMethods.EnableStreaming)
{
AlgoPack.EnsureFolderStructure(ExperimentType.Streaming, activeScenarios.Where(EnumMethods.IsStreaming).Select(es => es.ToLongString()).ToList());
foreach (ExperimentScenario es in activeScenarios.Where(EnumMethods.IsStreaming))
{
if (es.IsLimited() && !codesLimited.Contains(code))
{
continue;
}
if (enablePrecision)
{
TestRoutines.PrecisionTest(ExperimentType.Streaming, es, code);
}
if (enableRuntime)
{
TestRoutines.RuntimeTest(ExperimentType.Streaming, es, code);
}
}
}
}
}
void FullRuntimeReplot() //service method
{
if (EnumMethods.EnableContinuous)
{
codes.ForEach(c => TestRoutines.RuntimeTestReplot(ExperimentType.Continuous, ExperimentScenario.Missing, c));
codes.ForEach(c => TestRoutines.RuntimeTestReplot(ExperimentType.Continuous, ExperimentScenario.Length, c));
codesLimited.ForEach(c => TestRoutines.RuntimeTestReplot(ExperimentType.Continuous, ExperimentScenario.Columns, c));
}
codes.ForEach(c => TestRoutines.RuntimeTestReplot(ExperimentType.Recovery, ExperimentScenario.Missing, c));
codes.ForEach(c => TestRoutines.RuntimeTestReplot(ExperimentType.Recovery, ExperimentScenario.Length, c));
codesLimited.ForEach(c => TestRoutines.RuntimeTestReplot(ExperimentType.Recovery, ExperimentScenario.Columns, c));
}
FullRun(runPrecision, runRuntime);
FinalSequence();
}