Пример #1
0
        private static void Main(string[] args)
        {
            string[] codes        = null;
            string[] codesLimited = null;

            List <string> ignoreList = new List <string>();

            Dictionary <string, string> config =
                args.Length == 0
                    ? Utils.ReadConfigFile()
                    : Utils.ReadConfigFile(args[0]);

            bool disableTrmf = false;
            bool useBatchCd  = false;

            foreach (KeyValuePair <string, string> entry in config)
            {
                switch (entry.Key)
                {
                case "PlotsFolder":
                    DataWorks.FolderPlotsRemoteBase = entry.Value.Trim().Replace("~", Environment.GetFolderPath(Environment.SpecialFolder.Personal));
                    DataWorks.FolderPlotsRemote     = DataWorks.FolderPlotsRemoteBase;
                    break;

                case "AlgorithmsLocation":
                    AlgoPack.GlobalAlgorithmsLocation = entry.Value.Trim().Replace("~", Environment.GetFolderPath(Environment.SpecialFolder.Personal));
                    break;

                case "Datasets":
                    codes        = entry.Value.Split(',').Select(c => c.Trim()).ToArray();
                    codesLimited = codes
                                   .Where(c => DataWorks.CountMatrixColumns($"{c}/{c}_normal.txt") > 4)
                                   .ToArray();
                    break;

                case "EnableStreaming":
                    EnumMethods.EnableStreaming = Convert.ToBoolean(entry.Value);
                    break;

                case "EnableContinuous":
                    EnumMethods.EnableContinuous = Convert.ToBoolean(entry.Value);
                    break;

                case "EnableMulticolumn":
                    EnumMethods.EnableMulticolumn = Convert.ToBoolean(entry.Value);
                    break;

                case "DisableTrmf":
                    disableTrmf = Convert.ToBoolean(entry.Value);
                    break;

                case "UseBatchCd":
                    useBatchCd = Convert.ToBoolean(entry.Value);
                    break;

                case "IgnoreAlgorithms":
                    ignoreList.AddRange(entry.Value.Split(',').Select(x => x.Trim().ToLower()));
                    break;

                default:
                    Utils.DelayedWarnings.Enqueue($"Warning: unknown config entry with the key {entry.Key}");
                    break;
                }
            }

            // verificaiton that all necessary entries are provided
            if (DataWorks.FolderPlotsRemoteBase == null)
            {
                throw new InvalidProgramException("Invalid config file: plots folder has to be supplied (PlotsFolder=)");
            }

            if (AlgoPack.GlobalAlgorithmsLocation == null)
            {
                throw new InvalidProgramException("Invalid config file: algorithms folder has to be supplied (AlgorithmsLocation=)");
            }

            if (codes == null || codes.Length == 0)
            {
                throw new InvalidProgramException("Invalid config file: datasets are not supplied (Datasets=) or the list is empty");
            }

            Algorithm cdVersion = useBatchCd ? AlgoPack.Cd : AlgoPack.InCd;

            ignoreList = useBatchCd
                ? ignoreList.Select(x => x == "cdrec" ? "cd" : x).ToList()
                : ignoreList.Select(x => x == "cd" ? "cdrec" : x).ToList();

            var listStd = new List <Algorithm> {
                cdVersion
            };

            listStd.AddRange(new[] { AlgoPack.Stmvl, AlgoPack.Tkcm, AlgoPack.Spirit, AlgoPack.Nnmf, AlgoPack.Grouse });
            listStd.AddRange(new[] { AlgoPack.ArImp, AlgoPack.Ssa, AlgoPack.Mrnn, AlgoPack.DynaMMo, AlgoPack.MdIsvd, AlgoPack.SvdImp });// very new batch

            if (!disableTrmf)
            {
                listStd.Add(AlgoPack.Trmf);
            }

            listStd = listStd.Where(x => !ignoreList.Contains(x.AlgCode.ToLower())).ToList();

            AlgoPack.ListAlgorithms = listStd.ToArray();

            listStd.Remove(AlgoPack.Tkcm);
            listStd.Remove(AlgoPack.Spirit);
            listStd.Remove(AlgoPack.Ssa);
            AlgoPack.ListAlgorithmsMulticolumn = listStd.ToArray();

            AlgoPack.ListAlgorithmsStreaming = AlgoPack.ListAlgorithmsStreaming.Where(x => !ignoreList.Contains(x.AlgCode.ToLower())).ToArray();

            AlgoPack.CleanUncollectedResults();
            AlgoPack.EnsureFolderStructure();

            void FullPrecision()
            {
                if (EnumMethods.EnableContinuous)
                {
                    codes.ForEach(c => TestRoutines.PrecisionTest(ExperimentType.Continuous, ExperimentScenario.Missing, c, 1000));
                    codes.ForEach(c => TestRoutines.PrecisionTest(ExperimentType.Continuous, ExperimentScenario.Length, c, 2000));
                    codesLimited.ForEach(c => TestRoutines.PrecisionTest(ExperimentType.Continuous, ExperimentScenario.Columns, c, 1000));
                    codesLimited.ForEach(c => TestRoutines.PrecisionTest(ExperimentType.Continuous, ExperimentScenario.Fullrow, c, 1000));
                }

                codes.ForEach(c => TestRoutines.PrecisionTest(ExperimentType.Recovery, ExperimentScenario.Missing, c, 1000));
                codes.ForEach(c => TestRoutines.PrecisionTest(ExperimentType.Recovery, ExperimentScenario.Length, c, 2000));
                codesLimited.ForEach(c => TestRoutines.PrecisionTest(ExperimentType.Recovery, ExperimentScenario.Columns, c, 1000));
                codesLimited.ForEach(c => TestRoutines.PrecisionTest(ExperimentType.Recovery, ExperimentScenario.Fullrow, c, 1000));
                codes.ForEach(c => TestRoutines.PrecisionTest(ExperimentType.Recovery, ExperimentScenario.BlockSlide, c, 1000));
                codes.ForEach(c => TestRoutines.PrecisionTest(ExperimentType.Recovery, ExperimentScenario.McarElement, c, 1000));
                codes.ForEach(c => TestRoutines.PrecisionTest(ExperimentType.Recovery, ExperimentScenario.McarBlock, c, 1000));
            }

            void FullRuntime()
            {
                if (EnumMethods.EnableContinuous)
                {
                    codes.ForEach(c => TestRoutines.RuntimeTest(ExperimentType.Continuous, ExperimentScenario.Missing, c, 1000));
                    codes.ForEach(c => TestRoutines.RuntimeTest(ExperimentType.Continuous, ExperimentScenario.Length, c, 2000));
                    codesLimited.ForEach(c => TestRoutines.RuntimeTest(ExperimentType.Continuous, ExperimentScenario.Columns, c, 1000));
                    codesLimited.ForEach(c => TestRoutines.RuntimeTest(ExperimentType.Continuous, ExperimentScenario.Fullrow, c, 1000));
                }

                codes.ForEach(c => TestRoutines.RuntimeTest(ExperimentType.Recovery, ExperimentScenario.Missing, c, 1000));
                codes.ForEach(c => TestRoutines.RuntimeTest(ExperimentType.Recovery, ExperimentScenario.Length, c, 2000));
                codesLimited.ForEach(c => TestRoutines.RuntimeTest(ExperimentType.Recovery, ExperimentScenario.Columns, c, 1000));
                codesLimited.ForEach(c => TestRoutines.RuntimeTest(ExperimentType.Recovery, ExperimentScenario.Fullrow, c, 1000));
                codes.ForEach(c => TestRoutines.RuntimeTest(ExperimentType.Recovery, ExperimentScenario.BlockSlide, c, 1000));
                codes.ForEach(c => TestRoutines.RuntimeTest(ExperimentType.Recovery, ExperimentScenario.McarElement, c, 1000));
                codes.ForEach(c => TestRoutines.RuntimeTest(ExperimentType.Recovery, ExperimentScenario.McarBlock, c, 1000));
            }

            void FullRuntimeReplot() //service method
            {
                if (EnumMethods.EnableContinuous)
                {
                    codes.ForEach(c => TestRoutines.RuntimeTestReplot(ExperimentType.Continuous, ExperimentScenario.Missing, c, 1000));
                    codes.ForEach(c => TestRoutines.RuntimeTestReplot(ExperimentType.Continuous, ExperimentScenario.Length, c, 2000));
                    codesLimited.ForEach(c => TestRoutines.RuntimeTestReplot(ExperimentType.Continuous, ExperimentScenario.Columns, c, 1000));
                    codesLimited.ForEach(c => TestRoutines.RuntimeTestReplot(ExperimentType.Continuous, ExperimentScenario.Fullrow, c, 1000));
                }

                codes.ForEach(c => TestRoutines.RuntimeTestReplot(ExperimentType.Recovery, ExperimentScenario.Missing, c, 1000));
                codes.ForEach(c => TestRoutines.RuntimeTestReplot(ExperimentType.Recovery, ExperimentScenario.Length, c, 2000));
                codesLimited.ForEach(c => TestRoutines.RuntimeTestReplot(ExperimentType.Recovery, ExperimentScenario.Columns, c, 1000));
                codesLimited.ForEach(c => TestRoutines.RuntimeTestReplot(ExperimentType.Recovery, ExperimentScenario.Fullrow, c, 1000));
                codes.ForEach(c => TestRoutines.RuntimeTestReplot(ExperimentType.Recovery, ExperimentScenario.BlockSlide, c, 1000));
            }

            void FullStreaming()
            {
                codes.ForEach(c => TestRoutines.RuntimeTest(ExperimentType.Streaming, ExperimentScenario.Missing, c, 1000));
                codes.ForEach(c => TestRoutines.RuntimeTest(ExperimentType.Streaming, ExperimentScenario.Length, c, 2000));
                codesLimited.ForEach(c => TestRoutines.RuntimeTest(ExperimentType.Streaming, ExperimentScenario.Columns, c, 1000));
            }

            void FullMulticolumn()
            {
                codes.ForEach(c => TestRoutines.PrecisionTest(ExperimentType.Recovery, ExperimentScenario.MultiColumnDisjoint, c, 1000));
                codes.ForEach(c => TestRoutines.PrecisionTest(ExperimentType.Recovery, ExperimentScenario.MultiColumnOverlap, c, 1000));

                codes.ForEach(c => TestRoutines.RuntimeTest(ExperimentType.Recovery, ExperimentScenario.MultiColumnDisjoint, c, 1000));
                codes.ForEach(c => TestRoutines.RuntimeTest(ExperimentType.Recovery, ExperimentScenario.MultiColumnOverlap, c, 1000));
            }

            if (EnumMethods.EnableStreaming)
            {
                FullStreaming();
            }

            FullPrecision();

            FullRuntime();

            if (EnumMethods.EnableMulticolumn)
            {
                FullMulticolumn();
            }

            //codes.ForEach(c => TestRoutines.RuntimeTest(ExperimentType.Recovery, ExperimentScenario.MissingMultiColumn, c, 1000));
            //codes.ForEach(c => TestRoutines.RuntimeTest(ExperimentType.Continuous, ExperimentScenario.MissingMultiColumn, c, 2000));

            //codes.ForEach(c => TestRoutines.PrecisionTest(ExperimentType.Recovery, ExperimentScenario.MissingMultiColumn, c, 1000));
            //codes.ForEach(c => TestRoutines.PrecisionTest(ExperimentType.Continuous, ExperimentScenario.MissingMultiColumn, c, 1000));

            //
            // multi-run for 1...N runtime tests and averaging the results from them
            //

            #if false
            {
                for (int i = 1; i <= 5; i++)
                {
                    DataWorks.FolderPlotsRemote = DataWorks.FolderPlotsRemoteBase + i + "/";
                    if (!Directory.Exists(DataWorks.FolderPlotsRemote))
                    {
                        Directory.CreateDirectory(DataWorks.FolderPlotsRemote);
                        AlgoPack.EnsureFolderStructure();
                    }

                    FullRuntime();
                    //FullStreaming();
                }
                DataWorks.FolderPlotsRemote = DataWorks.FolderPlotsRemoteBase;
            }

            //SingularExperiments.AverageRTRuns(codes, codesLimited, 5);
            #endif

            //SingularExperiments.MsePerformanceReport(codes, codesLimited);
            //SingularExperiments.RMSE("~/MVR/CD-RMV/incCD/_data/out/cdmvr200_k3.txt", "~/Downloads/bafu_int.csv", (0, 50, 100));

            #if false
            {
                string cmptype = "meaninit_vert";

                var sw_prec = new StreamWriter(File.Open($"prec_report_{cmptype}.txt", FileMode.Create));
                var sw_rt   = new StreamWriter(File.Open($"rt_report_{cmptype}.txt", FileMode.Create));

                SingularExperiments.MsePerformanceReport(codes, codesLimited, sw_prec.WriteLine, cmptype);
                SingularExperiments.SSVIterPerformanceReport(codes, codesLimited, sw_rt.WriteLine, cmptype);

                sw_prec.Close();
                sw_rt.Close();
            }
            #endif
            //
            // time series
            //

            #if false
            {
                //var data = DataWorks.TimeSeries("BAFU", "*.asc", 3, Utils.Specific.ParseWasserstand, new DateTime(2005, 1, 1), true);
                //DataWorks.TimeSeriesMerge(data, "BAFU_total.txt");
            }
            #endif

            FinalSequence();
        }
Пример #2
0
        private static void Main(string[] args)
        {
            DataWorks.FolderPlotsRemote = DataWorks.FolderPlotsRemoteBase = "Results/";

            // technical
            Console.CancelKeyPress += InterruptHandler;

            // cli
            List <string>    codesinit   = new List <string>();
            List <Algorithm> algos       = new List <Algorithm>();
            List <string>    scenarios   = new List <string>();
            bool             doPrecision = true;
            bool             doRuntime   = true;

            CLIParse(args, ref codesinit, ref algos, ref scenarios, ref doPrecision, ref doRuntime);

            if (!(doPrecision || doRuntime))
            {
                Console.WriteLine("Both precision and runtime tests are disabled, no results will be produced");
                Environment.Exit(-1);
            }

            if (codesinit == null || codesinit.Count == 0)
            {
                codesinit = Directory.GetDirectories(DataWorks.FolderData)
                            .Where(x => !x.EndsWith("_.temp"))
                            .Select(x => x.Replace(DataWorks.FolderData, ""))
                            .ToList();
            }

            string[] codes        = codesinit.Distinct().ToArray();
            string[] codesLimited = codes
                                    .Where(c => DataWorks.CountMatrixColumns($"{c}/{c}_normal.txt") > 4)
                                    .ToArray();

            if (algos == null || algos.Count == 0)
            {
                algos = AlgoPack.ListAlgorithms.Where(alg => alg.AlgCode != "meanimp" && alg.AlgCode != "linimp").ToList();
            }

            if (scenarios == null || scenarios.Count == 0)
            {
                scenarios = EnumMethods.AllExperimentScenarios().Select(EnumMethods.ToLongString).ToList();
            }
            AlgoPack.GlobalAlgorithmsLocation = "../../../Algorithms/";

            // verificaiton that all necessary entries are provided

            if (codes == null || codes.Length == 0)
            {
                throw new InvalidProgramException($"Invalid program state: no datasets found in {DataWorks.FolderData} folder");
            }

            AlgoPack.ListAlgorithms = algos.ToArray();

            algos.Remove(AlgoPack.Tkcm);
            algos.Remove(AlgoPack.Spirit);
            algos.Remove(AlgoPack.Ssa);
            AlgoPack.ListAlgorithmsMulticolumn = algos.ToArray();

            AlgoPack.CleanUncollectedResults();
            AlgoPack.EnsureFolderStructure(scenarios);

            void FullRun(bool enablePrecision, bool enableRuntime)
            {
                foreach (string code in codes)
                {
                    foreach (ExperimentScenario es in EnumMethods.AllExperimentScenarios().Where(x => scenarios.Contains(x.ToLongString()) || scenarios.Contains(x.ToShortString())))
                    {
                        if (es.IsLimited() && !codesLimited.Contains(code))
                        {
                            continue;
                        }
                        if (enablePrecision)
                        {
                            TestRoutines.PrecisionTest(ExperimentType.Recovery, es, code);
                        }
                        if (enableRuntime)
                        {
                            TestRoutines.RuntimeTest(ExperimentType.Recovery, es, code);
                        }
                    }
                }
            }

            FullRun(doPrecision, doRuntime);

            //
            // multi-run for 1...N runtime tests and averaging the results from them
            //

            #if false
            {
                for (int i = 1; i <= 3; i++)
                {
                    DataWorks.FolderPlotsRemote = DataWorks.FolderPlotsRemoteBase + i + "/";
                    if (!Directory.Exists(DataWorks.FolderPlotsRemote))
                    {
                        Directory.CreateDirectory(DataWorks.FolderPlotsRemote);
                        AlgoPack.EnsureFolderStructure();
                    }

                    FullRuntime();
                    //FullStreaming();
                }
                DataWorks.FolderPlotsRemote = DataWorks.FolderPlotsRemoteBase;
            }

            //SingularExperiments.AverageRTRuns(codes, codesLimited, 5);
            #endif

            #if false
            {
                string cmptype = "meaninit_vert";

                var sw_prec = new StreamWriter(File.Open($"prec_report_{cmptype}.txt", FileMode.Create));
                var sw_rt   = new StreamWriter(File.Open($"rt_report_{cmptype}.txt", FileMode.Create));

                SingularExperiments.MsePerformanceReport(codes, codesLimited, sw_prec.WriteLine, cmptype);
                SingularExperiments.SSVIterPerformanceReport(codes, codesLimited, sw_rt.WriteLine, cmptype);

                sw_prec.Close();
                sw_rt.Close();
            }
            #endif
            //
            // time series
            //

            #if false
            {
                var data = DataWorks.TimeSeries("BAFU", "*.asc", 3, Utils.Specific.ParseWasserstand, new DateTime(2005, 1, 1), true);
                DataWorks.TimeSeriesMerge(data, "BAFU_total.txt");
            }
            #endif

            FinalSequence();
        }