예제 #1
0
        /// <summary>
        /// Run the predictor with given args and check if it adds up
        /// </summary>
        protected void Run(RunContext ctx)
        {
            Contracts.Assert(IsActive);
            List <string> args = new List <string>();

            if (ctx.Command != Cmd.Test)
            {
                AddIfNotEmpty(args, ctx.Predictor.Trainer, "tr");
            }
            string dataName = ctx.Command == Cmd.Test ? ctx.Dataset.testFilename : ctx.Dataset.trainFilename;

            AddIfNotEmpty(args, GetDataPath(dataName), "data");
            AddIfNotEmpty(args, 1, "seed");
            //AddIfNotEmpty(args, false, "threads");

            Log("Running '{0}' on '{1}'", ctx.Predictor.Trainer.Kind, ctx.Dataset.name);

            string dir = ctx.BaselineDir;

            if (ctx.Command == Cmd.TrainTest)
            {
                AddIfNotEmpty(args, GetDataPath(ctx.Dataset.testFilename), "test");
            }
            if (ctx.Command == Cmd.TrainTest || ctx.Command == Cmd.Train)
            {
                AddIfNotEmpty(args, GetDataPath(ctx.Dataset.validFilename), "valid");
            }

            // Add in the loader args, and keep a location so we can backtrack and remove it later.
            int    loaderArgIndex = -1;
            string loaderArgs     = GetLoaderTransformSettings(ctx.Dataset);

            if (!string.IsNullOrWhiteSpace(loaderArgs))
            {
                loaderArgIndex = args.Count;
                args.Add(loaderArgs);
            }
            // Add in the dataset transforms. These need to come before the predictor imposed transforms.
            if (ctx.Dataset.mamlExtraSettings != null)
            {
                args.AddRange(ctx.Dataset.mamlExtraSettings);
            }

            // Model file output, used only for train/traintest.
            var modelPath = ctx.Command == Cmd.Train || ctx.Command == Cmd.TrainTest ? ctx.ModelPath() : null;

            AddIfNotEmpty(args, modelPath, "out");

            string basePrefix = ctx.BaselineNamePrefix;

            // Predictions output, for all types of commands except train.
            OutputPath predOutPath = ctx.Command == Cmd.Train ? null : ctx.InitPath(".txt");

            AddIfNotEmpty(args, predOutPath, "dout");

            if (ctx.Predictor.MamlArgs != null)
            {
                args.AddRange(ctx.Predictor.MamlArgs);
            }

            // If CV, do not run the CV in multiple threads.
            if (ctx.Command == Cmd.CV)
            {
                args.Add("threads-");
            }

            if (ctx.ExtraArgs != null)
            {
                foreach (string arg in ctx.ExtraArgs)
                {
                    args.Add(arg);
                }
            }

            AddIfNotEmpty(args, ctx.Predictor.Scorer, "scorer");
            if (ctx.Command != Cmd.Test)
            {
                AddIfNotEmpty(args, ctx.Predictor.Tester, "eval");
            }
            else
            {
                AddIfNotEmpty(args, ctx.ModelOverride.Path, "in");
            }

            string runcmd = string.Join(" ", args.Where(a => !string.IsNullOrWhiteSpace(a)));

            Log("  Running as: {0} {1}", ctx.Command, runcmd);

            int res;

            if (basePrefix == null)
            {
                // Not capturing into a specific log.
                Log("*** Start raw predictor output");
                res = MainForTest(Env, LogWriter, string.Join(" ", ctx.Command, runcmd), ctx.BaselineProgress);
                Log("*** End raw predictor output, return={0}", res);
                return;
            }
            var consOutPath = ctx.StdoutPath();

            TestCore(ctx, ctx.Command.ToString(), runcmd);
            bool matched = consOutPath.CheckEqualityNormalized();

            if (modelPath != null && (ctx.Summary || ctx.SaveAsIni))
            {
                // Save the predictor summary and compare it to baseline.
                string        str   = string.Format("SavePredictorAs in={{{0}}}", modelPath.Path);
                List <string> files = new List <string>();
                if (ctx.Summary)
                {
                    var summaryName = basePrefix + "-summary.txt";
                    files.Add(summaryName);
                    var summaryPath = DeleteOutputPath(dir, summaryName);
                    str += string.Format(" sum={{{0}}}", summaryPath);
                    Log("  Saving summary with: {0}", str);
                }

                if (ctx.SaveAsIni)
                {
                    var iniName = basePrefix + ".ini";
                    files.Add(iniName);
                    var iniPath = DeleteOutputPath(dir, iniName);
                    str += string.Format(" ini={{{0}}}", iniPath);
                    Log("  Saving ini file: {0}", str);
                }

                MainForTest(Env, LogWriter, str);
                files.ForEach(file => CheckEqualityNormalized(dir, file));
            }

            if (ctx.Command == Cmd.Train || ctx.Command == Cmd.Test || ctx.ExpectedToFail)
            {
                return;
            }

            // ResultProcessor output
            if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) // -rp.txt files are not getting generated for Non-Windows Os
            {
                string rpName    = basePrefix + "-rp.txt";
                string rpOutPath = DeleteOutputPath(dir, rpName);

                string[] rpArgs = null;
                if (ctx.Command == Cmd.CV && ctx.ExtraArgs != null && ctx.ExtraArgs.Any(arg => arg.Contains("opf+")))
                {
                    rpArgs = new string[] { "opf+" }
                }
                ;

                // Run result processor on the console output.
                RunResultProcessorTest(new string[] { consOutPath.Path }, rpOutPath, rpArgs);
                CheckEqualityNormalized(dir, rpName);
            }

            // Check the prediction output against its baseline.
            Contracts.Assert(predOutPath != null);
            predOutPath.CheckEquality();

            if (ctx.Command == Cmd.TrainTest)
            {
                // Adjust the args so that we no longer have the loader and transform
                // arguments in there.
                if (loaderArgIndex >= 0)
                {
                    args.RemoveAt(loaderArgIndex);
                }
                bool             foundOut   = false;
                List <int>       toRemove   = new List <int>();
                HashSet <string> removeArgs = new HashSet <string>();
                removeArgs.Add("tr=");
                removeArgs.Add("data=");
                removeArgs.Add("valid=");
                removeArgs.Add("norm=");
                removeArgs.Add("cali=");
                removeArgs.Add("numcali=");
                removeArgs.Add("xf=");
                removeArgs.Add("cache-");
                removeArgs.Add("sf=");

                for (int i = 0; i < args.Count; ++i)
                {
                    if (string.IsNullOrWhiteSpace(args[i]))
                    {
                        continue;
                    }
                    if (removeArgs.Any(x => args[i].StartsWith(x)))
                    {
                        toRemove.Add(i);
                    }
                    if (args[i].StartsWith("out="))
                    {
                        args[i] = string.Format("in={0}", args[i].Substring(4));
                    }
                    if (args[i].StartsWith("test="))
                    {
                        args[i] = string.Format("data={0}", args[i].Substring(5));
                    }
                    foundOut = true;
                }
                Contracts.Assert(foundOut);
                toRemove.Reverse();
                foreach (int i in toRemove)
                {
                    args.RemoveAt(i);
                }
                runcmd = string.Join(" ", args.Where(a => !string.IsNullOrWhiteSpace(a)));

                // Redirect output to the individual log and run the test.
                var        ctx2         = ctx.TestCtx();
                OutputPath consOutPath2 = ctx2.StdoutPath();
                TestCore(ctx2, "Test", runcmd);

                if (CheckTestOutputMatchesTrainTest(consOutPath.Path, consOutPath2.Path, 1))
                {
                    File.Delete(consOutPath2.Path);
                }
                else if (matched)
                {
                    // The TrainTest output matched the baseline, but the SaveLoadTest output did not, so
                    // append some stuff to the .txt output so comparing output to baselines in BeyondCompare
                    // will show the issue.
                    using (var writer = OpenWriter(consOutPath.Path, true))
                    {
                        writer.WriteLine("*** Unit Test Failure! ***");
                        writer.WriteLine("Loaded predictor test results differ! Compare baseline with {0}", consOutPath2.Path);
                        writer.WriteLine("*** Unit Test Failure! ***");
                    }
                }
                // REVIEW: There is nothing analogous to the old predictor output comparison here.
                // The MAML command does not "export" the result of its training programmatically, that would
                // allow us to compare it to the loaded model. To verify that the result of the trained model
                // is the same as its programmatic
            }
        }