public static void EnsureFolderStructure(List <string> scenarios)
{
string root = DataWorks.FolderPlotsRemote;
foreach (var es in EnumMethods.AllExperimentScenarios().Where(x => scenarios.Contains(x.ToLongString())))
{
string esDir = $"{root}{es.ToLongString()}/";
if (!Directory.Exists(esDir))
{
Directory.CreateDirectory(esDir);
}
}
}
public static void EnsureFolderStructure()
{
string root = DataWorks.FolderPlotsRemote;
foreach (var ex in EnumMethods.AllExperiments())
{
string exDir = $"{root}{ex.ToLongString()}/";
if (!Directory.Exists(exDir))
{
Directory.CreateDirectory(exDir);
}
foreach (var et in EnumMethods.AllExperimentTypes())
{
// exception
if (ex == Experiment.Precision && et == ExperimentType.Streaming)
{
continue;
}
string etDir = $"{exDir}{et.ToLongString()}/";
if (!Directory.Exists(etDir))
{
Directory.CreateDirectory(etDir);
}
foreach (var es in EnumMethods.AllExperimentScenarios())
{
string esDir = $"{etDir}{es.ToLongString()}/";
if (!Directory.Exists(esDir))
{
Directory.CreateDirectory(esDir);
}
}
}
}
}
private static void Main(string[] args)
{
DataWorks.FolderPlotsRemote = DataWorks.FolderPlotsRemoteBase = "Results/";
// technical
Console.CancelKeyPress += InterruptHandler;
// cli
List <string> codesinit = new List <string>();
List <Algorithm> algos = new List <Algorithm>();
List <string> scenarios = new List <string>();
bool doPrecision = true;
bool doRuntime = true;
CLIParse(args, ref codesinit, ref algos, ref scenarios, ref doPrecision, ref doRuntime);
if (!(doPrecision || doRuntime))
{
Console.WriteLine("Both precision and runtime tests are disabled, no results will be produced");
Environment.Exit(-1);
}
if (codesinit == null || codesinit.Count == 0)
{
codesinit = Directory.GetDirectories(DataWorks.FolderData)
.Where(x => !x.EndsWith("_.temp"))
.Select(x => x.Replace(DataWorks.FolderData, ""))
.ToList();
}
string[] codes = codesinit.Distinct().ToArray();
string[] codesLimited = codes
.Where(c => DataWorks.CountMatrixColumns($"{c}/{c}_normal.txt") > 4)
.ToArray();
if (algos == null || algos.Count == 0)
{
algos = AlgoPack.ListAlgorithms.ToList();
}
if (scenarios == null || scenarios.Count == 0)
{
scenarios = EnumMethods.AllExperimentScenarios().Select(EnumMethods.ToLongString).ToList();
}
AlgoPack.GlobalAlgorithmsLocation = "../../../Algorithms/";
AlgoPack.GlobalNewAlgorithmsLocation = "../../../NewAlgorithms/";
// verificaiton that all necessary entries are provided
if (codes == null || codes.Length == 0)
{
throw new InvalidProgramException($"Invalid program state: no datasets found in {DataWorks.FolderData} folder");
}
AlgoPack.ListAlgorithms = algos.ToArray();
AlgoPack.ListAlgorithmsMulticolumn = algos.Where(alg => alg.IsMultiColumn).ToArray();
AlgoPack.CleanUncollectedResults();
AlgoPack.EnsureFolderStructure(scenarios);
void FullRun(bool enablePrecision, bool enableRuntime)
{
foreach (string code in codes)
{
foreach (ExperimentScenario es in EnumMethods.AllExperimentScenarios().Where(x => scenarios.Contains(x.ToLongString()) || scenarios.Contains(x.ToShortString())))
{
if (es.IsLimited() && !codesLimited.Contains(code))
{
continue;
}
if (enablePrecision)
{
TestRoutines.PrecisionTest(ExperimentType.Recovery, es, code);
}
if (enableRuntime)
{
TestRoutines.RuntimeTest(ExperimentType.Recovery, es, code);
}
}
}
}
FullRun(doPrecision, doRuntime);
FinalSequence();
}
private static void CLIParse(
string[] args, ref List <string> codes, ref List <Algorithm> algos, ref List <string> scenarios,
ref bool doPrecision, ref bool doRuntime)
{
string subCmd;
bool allAlgosInvoked = false;
if (args.Length == 0)
{
CLIPrintHelp();
Environment.Exit(0);
}
for (int i = 0; i < args.Length; i++)
{
switch (args[i].ToLower())
{
case "[arguments]":
case "--help":
case "-h":
CLIPrintHelp();
Environment.Exit(0);
break;
case "--dataset":
case "-d":
CLIVerifyInput(i + 1, args.Length, args[i], "dataset name(s)");
subCmd = args[++i];
if (subCmd == "all")
{
}
else if (subCmd.Contains(","))
{
codes.AddRange(subCmd.Split(',').Select(x => x.Trim()).Distinct());
}
else
{
codes.Add(subCmd);
}
break;
case "--algorithm":
case "-alg":
CLIAllAgosIntegrity(allAlgosInvoked);
CLIVerifyInput(i + 1, args.Length, args[i], "algorithm name(s)");
subCmd = args[++i];
if (subCmd == "all")
{
if (algos.Count() != 0)
{
Console.WriteLine("Algorithms were already added with --algorithm or --algorithm-param");
Console.WriteLine("`--algorithm all` can't be invoked any more");
Environment.Exit(-1);
}
// static list for vldb bench
algos.AddRange(new[] { AlgoPack.Stmvl, AlgoPack.CdRec, AlgoPack.Tkcm, AlgoPack.Spirit, AlgoPack.Trmf, AlgoPack.Nnmf, AlgoPack.Grouse, AlgoPack.Svt, AlgoPack.SoftImpute, AlgoPack.ROSL, AlgoPack.DynaMMo, AlgoPack.SvdI });
allAlgosInvoked = true;
}
else if (subCmd == "full")
{
if (algos.Count() != 0)
{
Console.WriteLine("Algorithms were already added with --algorithm or --algorithm-param");
Console.WriteLine("`--algorithm full` can't be invoked any more");
Environment.Exit(-1);
}
// left null and will be filled with ALL algorithms
allAlgosInvoked = true;
}
else if (subCmd.Contains(","))
{
DataWorks.PlottableOverride = true;
algos.AddRange(subCmd.Split(',').Select(x => x.Trim()).Select(CLIFindAlgorithmByName));
}
else
{
DataWorks.PlottableOverride = true;
algos.Add(CLIFindAlgorithmByName(subCmd)); //map into Algorithm instance
}
break;
case "--algorithm-param":
case "-algx":
CLIAllAgosIntegrity(allAlgosInvoked);
CLIVerifyInput(i + 1, args.Length, args[i], "algorithm name");
subCmd = args[++i];
CLIVerifyInput(i + 1, args.Length, args[i], "algorithm parameter");
string param = args[++i];
DataWorks.PlottableOverride = true;
Algorithm alg = CLIFindAlgorithmByName(subCmd);
algos.Add(alg);
CLISetAlgoParam(alg, param);
break;
case "--scenario":
case "-scen":
CLIVerifyInput(i + 1, args.Length, args[i], "scenario name(s)");
subCmd = args[++i];
if (subCmd == "all")
{
}
else if (subCmd.Contains(","))
{
scenarios.AddRange(subCmd.Split(',').Select(x => x.Trim()).Distinct());
}
else
{
scenarios.Add(subCmd);
}
break;
case "--no-runtime":
case "-nort":
doRuntime = false;
break;
case "--no-precision":
case "-noprec":
doPrecision = false;
break;
case "--no-visualization":
case "-novis":
DataWorks.DisableVisualization = true;
break;
case "--set-output":
case "-out":
CLIVerifyInput(i + 1, args.Length, args[i], "output folder");
subCmd = args[++i];
if (!subCmd.EndsWith("/"))
{
subCmd = subCmd + "/";
}
DataWorks.FolderPlotsRemote = DataWorks.FolderPlotsRemoteBase = subCmd;
break;
default:
Console.WriteLine($"{args[i]} is not a recognized CLI parameter for the program");
Environment.Exit(-1);
break;
}
}
if (algos.Count() != algos.Distinct().Count())
{
Console.WriteLine($"One or more algorithms were supplied multiple times. This can cause configuration conflicts.");
Console.WriteLine($"Please ensure every algorithm is given only once.");
Environment.Exit(-1);
}
codes.ForEach(c =>
{
if (!File.Exists(DataWorks.FolderData + $"{c}/{c}_normal.txt"))
{
Console.WriteLine($"Dataset with name {c} is not found. Please make sure dataset with this name exists and the file is named correctly");
Environment.Exit(-1);
}
});
var allScen = EnumMethods.AllExperimentScenarios().SelectMany(x => new[] { x.ToLongString(), x.ToShortString() });
var invalidScen = scenarios.Except(allScen);
if (invalidScen.Count() != 0)
{
Console.WriteLine($"{invalidScen.First()} is an invalid scenario name");
Environment.Exit(-1);
}
}
private static void Main(string[] args)
{
DataWorks.FolderPlotsRemote = DataWorks.FolderPlotsRemoteBase = "Results/";
// technical
Console.CancelKeyPress += InterruptHandler;
// cli
List <string> codesinit = new List <string>();
List <Algorithm> algos = new List <Algorithm>();
List <string> scenarios = new List <string>();
bool doPrecision = true;
bool doRuntime = true;
CLIParse(args, ref codesinit, ref algos, ref scenarios, ref doPrecision, ref doRuntime);
if (!(doPrecision || doRuntime))
{
Console.WriteLine("Both precision and runtime tests are disabled, no results will be produced");
Environment.Exit(-1);
}
if (codesinit == null || codesinit.Count == 0)
{
codesinit = Directory.GetDirectories(DataWorks.FolderData)
.Where(x => !x.EndsWith("_.temp"))
.Select(x => x.Replace(DataWorks.FolderData, ""))
.ToList();
}
string[] codes = codesinit.Distinct().ToArray();
string[] codesLimited = codes
.Where(c => DataWorks.CountMatrixColumns($"{c}/{c}_normal.txt") > 4)
.ToArray();
if (algos == null || algos.Count == 0)
{
algos = AlgoPack.ListAlgorithms.Where(alg => alg.AlgCode != "meanimp" && alg.AlgCode != "linimp").ToList();
}
if (scenarios == null || scenarios.Count == 0)
{
scenarios = EnumMethods.AllExperimentScenarios().Select(EnumMethods.ToLongString).ToList();
}
AlgoPack.GlobalAlgorithmsLocation = "../../../Algorithms/";
// verificaiton that all necessary entries are provided
if (codes == null || codes.Length == 0)
{
throw new InvalidProgramException($"Invalid program state: no datasets found in {DataWorks.FolderData} folder");
}
AlgoPack.ListAlgorithms = algos.ToArray();
algos.Remove(AlgoPack.Tkcm);
algos.Remove(AlgoPack.Spirit);
algos.Remove(AlgoPack.Ssa);
AlgoPack.ListAlgorithmsMulticolumn = algos.ToArray();
AlgoPack.CleanUncollectedResults();
AlgoPack.EnsureFolderStructure(scenarios);
void FullRun(bool enablePrecision, bool enableRuntime)
{
foreach (string code in codes)
{
foreach (ExperimentScenario es in EnumMethods.AllExperimentScenarios().Where(x => scenarios.Contains(x.ToLongString()) || scenarios.Contains(x.ToShortString())))
{
if (es.IsLimited() && !codesLimited.Contains(code))
{
continue;
}
if (enablePrecision)
{
TestRoutines.PrecisionTest(ExperimentType.Recovery, es, code);
}
if (enableRuntime)
{
TestRoutines.RuntimeTest(ExperimentType.Recovery, es, code);
}
}
}
}
FullRun(doPrecision, doRuntime);
//
// multi-run for 1...N runtime tests and averaging the results from them
//
#if false
{
for (int i = 1; i <= 3; i++)
{
DataWorks.FolderPlotsRemote = DataWorks.FolderPlotsRemoteBase + i + "/";
if (!Directory.Exists(DataWorks.FolderPlotsRemote))
{
Directory.CreateDirectory(DataWorks.FolderPlotsRemote);
AlgoPack.EnsureFolderStructure();
}
FullRuntime();
//FullStreaming();
}
DataWorks.FolderPlotsRemote = DataWorks.FolderPlotsRemoteBase;
}
//SingularExperiments.AverageRTRuns(codes, codesLimited, 5);
#endif
#if false
{
string cmptype = "meaninit_vert";
var sw_prec = new StreamWriter(File.Open($"prec_report_{cmptype}.txt", FileMode.Create));
var sw_rt = new StreamWriter(File.Open($"rt_report_{cmptype}.txt", FileMode.Create));
SingularExperiments.MsePerformanceReport(codes, codesLimited, sw_prec.WriteLine, cmptype);
SingularExperiments.SSVIterPerformanceReport(codes, codesLimited, sw_rt.WriteLine, cmptype);
sw_prec.Close();
sw_rt.Close();
}
#endif
//
// time series
//
#if false
{
var data = DataWorks.TimeSeries("BAFU", "*.asc", 3, Utils.Specific.ParseWasserstand, new DateTime(2005, 1, 1), true);
DataWorks.TimeSeriesMerge(data, "BAFU_total.txt");
}
#endif
FinalSequence();
}
private static void Main(string[] args)
{
// technical
Console.CancelKeyPress += InterruptHandler;
string[] codes = null;
string[] codesLimited = null;
List <string> ignoreList = new List <string>();
List <string> scenarios = new List <string>();
Dictionary <string, string> config =
args.Length == 0
? Utils.ReadConfigFile()
: Utils.ReadConfigFile(args[0]);
bool runPrecision = true;
bool runRuntime = true;
foreach (KeyValuePair <string, string> entry in config)
{
switch (entry.Key)
{
case "PlotsFolder":
DataWorks.FolderPlotsRemoteBase = entry.Value.Trim().Replace("~", Environment.GetFolderPath(Environment.SpecialFolder.Personal));
DataWorks.FolderPlotsRemote = DataWorks.FolderPlotsRemoteBase;
break;
case "AlgorithmsLocation":
AlgoPack.GlobalAlgorithmsLocation = entry.Value.Trim().Replace("~", Environment.GetFolderPath(Environment.SpecialFolder.Personal));
break;
case "Datasets":
codes = entry.Value.Split(',').Select(c => c.Trim()).ToArray();
codesLimited = codes
.Where(c => DataWorks.CountMatrixColumns($"{c}/{c}_normal.txt") > 4)
.ToArray();
break;
case "Scenarios":
scenarios.AddRange(entry.Value.Split(',').Select(x => x.Trim().ToLower()));
break;
case "EnableStreaming":
EnumMethods.EnableStreaming = Convert.ToBoolean(entry.Value);
break;
case "EnableContinuous":
EnumMethods.EnableContinuous = Convert.ToBoolean(entry.Value);
break;
case "EnableBatchMid":
EnumMethods.EnableBatchMid = Convert.ToBoolean(entry.Value);
break;
case "EnabledAlgorithms":
ignoreList.AddRange(entry.Value.Split(',').Select(x => x.Trim().ToLower()));
break;
case "DisablePrecision":
runPrecision = !Convert.ToBoolean(entry.Value);
break;
case "DisableRuntime":
runRuntime = !Convert.ToBoolean(entry.Value);
break;
default:
Utils.DelayedWarnings.Enqueue($"Warning: unknown config entry with the key {entry.Key}");
break;
}
}
// verification that all necessary entries are provided
if (DataWorks.FolderPlotsRemoteBase == null)
{
throw new InvalidProgramException("Invalid config file: plots folder has to be supplied (PlotsFolder=)");
}
if (AlgoPack.GlobalAlgorithmsLocation == null)
{
throw new InvalidProgramException("Invalid config file: algorithms folder has to be supplied (AlgorithmsLocation=)");
}
if (codes == null || codes.Length == 0)
{
throw new InvalidProgramException("Invalid config file: datasets are not supplied (Datasets=) or the list is empty");
}
AlgoPack.ListAlgorithms = AlgoPack.ListAlgorithms.Where(x => ignoreList.Contains(x.AlgCode.ToLower())).ToArray();
AlgoPack.ListAlgorithmsStreaming = AlgoPack.ListAlgorithms.Where(x => x.IsStreaming).ToArray();
AlgoPack.ListAlgorithmsMulticolumn = AlgoPack.ListAlgorithms.Where(x => x.IsMulticolumn).ToArray();
AlgoPack.CleanUncollectedResults();
List <ExperimentScenario> activeScenarios = new List <ExperimentScenario>();
foreach (string scen in scenarios)
{
bool found = false;
foreach (ExperimentScenario es in EnumMethods.AllExperimentScenarios())
{
if (scen == es.ToLongString())
{
found = true;
activeScenarios.Add(es);
}
}
if (!found)
{
throw new InvalidProgramException("Invalid config file: list of scenarios contains entries which are not supported by the testing framework");
}
}
void FullRun(bool enablePrecision, bool enableRuntime)
{
foreach (string code in codes)
{
if (EnumMethods.EnableContinuous)
{
AlgoPack.EnsureFolderStructure(ExperimentType.Continuous, activeScenarios.Where(EnumMethods.IsContinuous).Select(es => es.ToLongString()).ToList());
foreach (ExperimentScenario es in activeScenarios.Where(EnumMethods.IsContinuous))
{
if (es.IsLimited() && !codesLimited.Contains(code))
{
continue;
}
if (enablePrecision)
{
TestRoutines.PrecisionTest(ExperimentType.Continuous, es, code);
}
if (enableRuntime)
{
TestRoutines.RuntimeTest(ExperimentType.Continuous, es, code);
}
}
}
if (EnumMethods.EnableBatchMid)
{
AlgoPack.EnsureFolderStructure(ExperimentType.Recovery, activeScenarios.Where(EnumMethods.IsBatchMid).Select(es => es.ToLongString()).ToList());
foreach (ExperimentScenario es in activeScenarios.Where(EnumMethods.IsBatchMid))
{
if (es.IsLimited() && !codesLimited.Contains(code))
{
continue;
}
if (enablePrecision)
{
TestRoutines.PrecisionTest(ExperimentType.Recovery, es, code);
}
if (enableRuntime)
{
TestRoutines.RuntimeTest(ExperimentType.Recovery, es, code);
}
}
}
if (EnumMethods.EnableStreaming)
{
AlgoPack.EnsureFolderStructure(ExperimentType.Streaming, activeScenarios.Where(EnumMethods.IsStreaming).Select(es => es.ToLongString()).ToList());
foreach (ExperimentScenario es in activeScenarios.Where(EnumMethods.IsStreaming))
{
if (es.IsLimited() && !codesLimited.Contains(code))
{
continue;
}
if (enablePrecision)
{
TestRoutines.PrecisionTest(ExperimentType.Streaming, es, code);
}
if (enableRuntime)
{
TestRoutines.RuntimeTest(ExperimentType.Streaming, es, code);
}
}
}
}
}
void FullRuntimeReplot() //service method
{
if (EnumMethods.EnableContinuous)
{
codes.ForEach(c => TestRoutines.RuntimeTestReplot(ExperimentType.Continuous, ExperimentScenario.Missing, c));
codes.ForEach(c => TestRoutines.RuntimeTestReplot(ExperimentType.Continuous, ExperimentScenario.Length, c));
codesLimited.ForEach(c => TestRoutines.RuntimeTestReplot(ExperimentType.Continuous, ExperimentScenario.Columns, c));
}
codes.ForEach(c => TestRoutines.RuntimeTestReplot(ExperimentType.Recovery, ExperimentScenario.Missing, c));
codes.ForEach(c => TestRoutines.RuntimeTestReplot(ExperimentType.Recovery, ExperimentScenario.Length, c));
codesLimited.ForEach(c => TestRoutines.RuntimeTestReplot(ExperimentType.Recovery, ExperimentScenario.Columns, c));
}
FullRun(runPrecision, runRuntime);
FinalSequence();
}