public void Can_deploy_to_directory()
{
// setup
var deployDirectory = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "DeploymentDirectory");
if (Directory.Exists(deployDirectory))
Directory.Delete(deployDirectory, true);
Directory.CreateDirectory(deployDirectory);
var writer = new ConsoleWriter();
var runContext = new RunContext();
var runner = new CommandLineRunner(writer,
new PackageProcessor(new ActionExecutor(writer, runContext), writer, new NuGetProcess(writer), runContext),
new NullActionExecutor(),
runContext);
SetDefinitions(new Packages
{
Items = new object[]
{
new Package
{
BasePath = _baseDirectory,
name = "package 1",
NuSpecPath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "TestData/TestNuSpec.nuspec"),
OutputDirectory = _outputDirectory,
Version = "",
Deployments = new PackageDeployments
{
Deployment = new[]
{
new Deployment
{
name = "test deployment",
path = deployDirectory
}
}
}
}
}
});
// act
runner.Run(_rootDirectory, _nugetLocation, _defintionsFileLocation, "", new Dictionary<string, string>());
// assert
Assert.IsTrue(File.Exists(Path.Combine(deployDirectory, "package1.1.2.3.nupkg")));
}
public CustomConvention(RunContext runContext)
{
Classes
.Where(type => type.IsInNamespace(GetType().Namespace))
.NameEndsWith("Tests");
Cases
.Where(method => method.IsVoid())
.Where(method =>
{
var isMarkedExplicit = method.Has<ExplicitAttribute>();
return !isMarkedExplicit || runContext.TargetMember == method;
});
ClassExecution
.CreateInstancePerTestClass();
}
public CustomConvention(RunContext runContext)
{
var customSeed = runContext.Options["seed"].LastOrDefault();
int seed = Environment.TickCount;
if (customSeed != null)
{
seed = Int32.Parse(customSeed);
}
Console.WriteLine("Running shuffled tests using seed: {0}", seed);
Classes
.Where(type => type.IsInNamespace(GetType().Namespace))
.NameEndsWith("Tests");
Methods
.Where(method => method.IsVoid());
ClassExecution.ShuffleCases(new Random(seed));
}
public CustomConvention(RunContext runContext)
{
var desiredCategories = runContext.Options["include"].ToArray();
var shouldRunAll = !desiredCategories.Any();
Classes
.Where(type => type.IsInNamespace(GetType().Namespace))
.NameEndsWith("Tests");
Methods
.Where(method => method.IsVoid())
.Where(method => shouldRunAll || MethodHasAnyDesiredCategory(method, desiredCategories));
if (!shouldRunAll)
{
Console.WriteLine("Categories: " + string.Join(", ", desiredCategories));
Console.WriteLine();
}
}
public ScopeRunner(RunContext ctx)
{
this.context = ctx;
}
public ListedTests(RunContext context) : base(context) { }
public RangeQueryTests(RunContext context) : base(context) { }
public StoredFieldsListTests(RunContext context) : base(context) { }
public void Error_when_proccessing_nuspec_without_global_or_package_version()
{
// setup
var writer = new Mock<IConsoleWriter>();
writer.Setup(x => x.WriteMessage(It.IsAny<string>()));
writer.Setup(x => x.WriteError(It.IsAny<string>()));
var runContext = new RunContext();
var runner = new CommandLineRunner(writer.Object,
new PackageProcessor(new ActionExecutor(writer.Object, runContext), writer.Object, new NuGetProcess(writer.Object), runContext),
new NullActionExecutor(),
runContext);
SetDefinitions(new Packages
{
Items = new object[]
{
new Package
{
BasePath = _baseDirectory,
name = "package 1",
NuSpecPath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "TestData/TestNuSpecWithoutVersion.nuspec"),
OutputDirectory = _outputDirectory,
Version = ""
}
}
});
// act
runner.Run(_rootDirectory, _nugetLocation, _defintionsFileLocation, "", new Dictionary<string, string>());
// assert
writer.Verify(
x =>
x.WriteError("The nuspec for package \"package 1\" expects a version to be supplied but the package didn't define one and also there wasn't a version inputed via the parameter -v."));
}
protected override Task <T> ExecuteInner(RunContext runContext, CancellationToken cancellationToken)
=> runContext.Scheduler.PollForCondition(
this.getConditionState,
this.condition,
cancellationToken);
/// <summary>
/// Run the predictor with given args and check if it adds up
/// </summary>
protected void Run(RunContext ctx, int digitsOfPrecision = DigitsOfPrecision)
{
Contracts.Assert(IsActive);
List <string> args = new List <string>();
if (ctx.Command != Cmd.Test)
{
AddIfNotEmpty(args, ctx.Predictor.Trainer, "tr");
}
string dataName = ctx.Command == Cmd.Test ? ctx.Dataset.testFilename : ctx.Dataset.trainFilename;
AddIfNotEmpty(args, GetDataPath(dataName), "data");
AddIfNotEmpty(args, 1, "seed");
//AddIfNotEmpty(args, false, "threads");
Log("Running '{0}' on '{1}'", ctx.Predictor.Trainer.Kind, ctx.Dataset.name);
string dir = ctx.BaselineDir;
if (ctx.Command == Cmd.TrainTest)
{
AddIfNotEmpty(args, GetDataPath(ctx.Dataset.testFilename), "test");
}
if (ctx.Command == Cmd.TrainTest || ctx.Command == Cmd.Train)
{
AddIfNotEmpty(args, GetDataPath(ctx.Dataset.validFilename), "valid");
}
// Add in the loader args, and keep a location so we can backtrack and remove it later.
int loaderArgIndex = -1;
string loaderArgs = GetLoaderTransformSettings(ctx.Dataset);
if (!string.IsNullOrWhiteSpace(loaderArgs))
{
loaderArgIndex = args.Count;
args.Add(loaderArgs);
}
// Add in the dataset transforms. These need to come before the predictor imposed transforms.
if (ctx.Dataset.mamlExtraSettings != null)
{
args.AddRange(ctx.Dataset.mamlExtraSettings);
}
// Model file output, used only for train/traintest.
var modelPath = ctx.Command == Cmd.Train || ctx.Command == Cmd.TrainTest ? ctx.ModelPath() : null;
AddIfNotEmpty(args, modelPath, "out");
string basePrefix = ctx.BaselineNamePrefix;
// Predictions output, for all types of commands except train.
OutputPath predOutPath = ctx.Command == Cmd.Train ? null : ctx.InitPath(".txt");
AddIfNotEmpty(args, predOutPath, "dout");
if (ctx.Predictor.MamlArgs != null)
{
args.AddRange(ctx.Predictor.MamlArgs);
}
// If CV, do not run the CV in multiple threads.
if (ctx.Command == Cmd.CV)
{
args.Add("threads-");
}
if (ctx.ExtraArgs != null)
{
foreach (string arg in ctx.ExtraArgs)
{
args.Add(arg);
}
}
AddIfNotEmpty(args, ctx.Predictor.Scorer, "scorer");
if (ctx.Command != Cmd.Test)
{
AddIfNotEmpty(args, ctx.Predictor.Tester, "eval");
}
else
{
AddIfNotEmpty(args, ctx.ModelOverride.Path, "in");
}
string runcmd = string.Join(" ", args.Where(a => !string.IsNullOrWhiteSpace(a)));
Log(" Running as: {0} {1}", ctx.Command, runcmd);
int res;
if (basePrefix == null)
{
// Not capturing into a specific log.
Log("*** Start raw predictor output");
res = MainForTest(_env, LogWriter, string.Join(" ", ctx.Command, runcmd), ctx.BaselineProgress);
Log("*** End raw predictor output, return={0}", res);
return;
}
var consOutPath = ctx.StdoutPath();
TestCore(ctx, ctx.Command.ToString(), runcmd, digitsOfPrecision: digitsOfPrecision);
bool matched = consOutPath.CheckEqualityNormalized(digitsOfPrecision);
if (modelPath != null && (ctx.Summary || ctx.SaveAsIni))
{
// Save the predictor summary and compare it to baseline.
string str = string.Format("SavePredictorAs in={{{0}}}", modelPath.Path);
List <string> files = new List <string>();
if (ctx.Summary)
{
var summaryName = basePrefix + "-summary.txt";
files.Add(summaryName);
var summaryPath = DeleteOutputPath(dir, summaryName);
str += string.Format(" sum={{{0}}}", summaryPath);
Log(" Saving summary with: {0}", str);
}
if (ctx.SaveAsIni)
{
var iniName = basePrefix + ".ini";
files.Add(iniName);
var iniPath = DeleteOutputPath(dir, iniName);
str += string.Format(" ini={{{0}}}", iniPath);
Log(" Saving ini file: {0}", str);
}
MainForTest(_env, LogWriter, str);
files.ForEach(file => CheckEqualityNormalized(dir, file, digitsOfPrecision: digitsOfPrecision));
}
if (ctx.Command == Cmd.Train || ctx.Command == Cmd.Test || ctx.ExpectedToFail)
{
return;
}
// ResultProcessor output
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) // -rp.txt files are not getting generated for Non-Windows Os
{
string rpName = basePrefix + "-rp.txt";
string rpOutPath = DeleteOutputPath(dir, rpName);
string[] rpArgs = null;
if (ctx.Command == Cmd.CV && ctx.ExtraArgs != null && ctx.ExtraArgs.Any(arg => arg.Contains("opf+")))
{
rpArgs = new string[] { "opf+" }
}
;
// Run result processor on the console output.
RunResultProcessorTest(new string[] { consOutPath.Path }, rpOutPath, rpArgs);
CheckEqualityNormalized(dir, rpName, digitsOfPrecision: digitsOfPrecision);
}
// Check the prediction output against its baseline.
Contracts.Assert(predOutPath != null);
predOutPath.CheckEquality(digitsOfPrecision: digitsOfPrecision);
if (ctx.Command == Cmd.TrainTest)
{
// Adjust the args so that we no longer have the loader and transform
// arguments in there.
if (loaderArgIndex >= 0)
{
args.RemoveAt(loaderArgIndex);
}
bool foundOut = false;
List <int> toRemove = new List <int>();
HashSet <string> removeArgs = new HashSet <string>();
removeArgs.Add("tr=");
removeArgs.Add("data=");
removeArgs.Add("valid=");
removeArgs.Add("norm=");
removeArgs.Add("cali=");
removeArgs.Add("numcali=");
removeArgs.Add("xf=");
removeArgs.Add("cache-");
removeArgs.Add("sf=");
removeArgs.Add("loader=");
for (int i = 0; i < args.Count; ++i)
{
if (string.IsNullOrWhiteSpace(args[i]))
{
continue;
}
if (removeArgs.Any(x => args[i].StartsWith(x)))
{
toRemove.Add(i);
}
if (args[i].StartsWith("out="))
{
args[i] = string.Format("in={0}", args[i].Substring(4));
}
if (args[i].StartsWith("test="))
{
args[i] = string.Format("data={0}", args[i].Substring(5));
}
foundOut = true;
}
Contracts.Assert(foundOut);
toRemove.Reverse();
foreach (int i in toRemove)
{
args.RemoveAt(i);
}
runcmd = string.Join(" ", args.Where(a => !string.IsNullOrWhiteSpace(a)));
// Redirect output to the individual log and run the test.
var ctx2 = ctx.TestCtx();
OutputPath consOutPath2 = ctx2.StdoutPath();
TestCore(ctx2, "Test", runcmd, digitsOfPrecision: digitsOfPrecision);
if (CheckTestOutputMatchesTrainTest(consOutPath.Path, consOutPath2.Path, 1))
{
File.Delete(consOutPath2.Path);
}
else if (matched)
{
// The TrainTest output matched the baseline, but the SaveLoadTest output did not, so
// append some stuff to the .txt output so comparing output to baselines in BeyondCompare
// will show the issue.
using (var writer = OpenWriter(consOutPath.Path, true))
{
writer.WriteLine("*** Unit Test Failure! ***");
writer.WriteLine("Loaded predictor test results differ! Compare baseline with {0}", consOutPath2.Path);
writer.WriteLine("*** Unit Test Failure! ***");
}
}
// REVIEW: There is nothing analogous to the old predictor output comparison here.
// The MAML command does not "export" the result of its training programmatically, that would
// allow us to compare it to the loaded model. To verify that the result of the trained model
// is the same as its programmatic
}
}
public SegmentsTests(RunContext context) : base(context)
{
}
protected override void InvokeExecutor(LazyExtension <ITestExecutor, ITestExecutorCapabilities> executor, Tuple <Uri, string> executorUriExtensionTuple, RunContext runContext, IFrameworkHandle frameworkHandle)
{
this.InvokeExecutorCallback?.Invoke(executor, executorUriExtensionTuple, runContext, frameworkHandle);
}
protected override IEnumerable <Tuple <Uri, string> > GetExecutorUriExtensionMap(IFrameworkHandle testExecutorFrameworkHandle, RunContext runContext)
{
return(this.GetExecutorUriExtensionMapCallback?.Invoke(testExecutorFrameworkHandle, runContext));
}
/// <summary>
/// Asks the adapter about attaching the debugger to the default test host.
/// </summary>
/// <param name="executor">The executor used to run the tests.</param>
/// <param name="executorUriExtensionTuple">The executor URI.</param>
/// <param name="runContext">The run context.</param>
/// <returns>
/// <see cref="true"/> if must attach the debugger to the default test host,
/// <see cref="false"/> otherwise.
/// </returns>
protected abstract bool ShouldAttachDebuggerToTestHost(
LazyExtension <ITestExecutor, ITestExecutorCapabilities> executor,
Tuple <Uri, string> executorUriExtensionTuple,
RunContext runContext);
private static Var <PredictorModel> ProcessClass(IHostEnvironment env, List <EntryPointNode> macroNodes, int k, string label, Arguments input, EntryPointNode node)
{
Contracts.AssertValue(macroNodes);
// Convert label into T,F based on k.
var labelIndicatorArgs = new LabelIndicatorTransform.Arguments();
labelIndicatorArgs.ClassIndex = k;
labelIndicatorArgs.Column = new[] { new LabelIndicatorTransform.Column()
{
Name = label, Source = label
} };
var inputBindingMap = new Dictionary <string, List <ParameterBinding> >();
var inputMap = new Dictionary <ParameterBinding, VariableBinding>();
var paramBinding = new SimpleParameterBinding(nameof(labelIndicatorArgs.Data));
inputBindingMap.Add(nameof(labelIndicatorArgs.Data), new List <ParameterBinding>()
{
paramBinding
});
inputMap.Add(paramBinding, node.GetInputVariable(nameof(input.TrainingData)));
var outputMap = new Dictionary <string, string>();
var remappedLabelVar = new Var <IDataView>();
outputMap.Add(nameof(CommonOutputs.TransformOutput.OutputData), remappedLabelVar.VarName);
var labelIndicatorNode = EntryPointNode.Create(env, "Transforms.LabelIndicator", labelIndicatorArgs, node.Context,
inputBindingMap, inputMap, outputMap);
macroNodes.Add(labelIndicatorNode);
// Parse the nodes in input.Nodes into a temporary run context.
var subGraphRunContext = new RunContext(env);
var subGraphNodes = EntryPointNode.ValidateNodes(env, subGraphRunContext, input.Nodes);
// Rename all the variables such that they don't conflict with the ones in the outer run context.
var mapping = new Dictionary <string, string>();
bool foundOutput = false;
Var <PredictorModel> predModelVar = null;
foreach (var entryPointNode in subGraphNodes)
{
// Rename variables in input/output maps, and in subgraph context.
entryPointNode.RenameAllVariables(mapping);
foreach (var kvp in mapping)
{
subGraphRunContext.RenameContextVariable(kvp.Key, kvp.Value);
}
// Grab a hold of output model from this subgraph.
if (entryPointNode.GetOutputVariableName("PredictorModel") is string mvn)
{
predModelVar = new Var <PredictorModel> {
VarName = mvn
};
foundOutput = true;
}
// Connect label remapper output to wherever training data was expected within the input graph.
if (entryPointNode.GetInputVariable(nameof(input.TrainingData)) is VariableBinding vb)
{
vb.Rename(remappedLabelVar.VarName);
}
// Change node to use the main context.
entryPointNode.SetContext(node.Context);
}
// Move the variables from the subcontext to the main context.
node.Context.AddContextVariables(subGraphRunContext);
// Make sure we found the output variable for this model.
if (!foundOutput)
{
throw new Exception("Invalid input graph. Does not output predictor model.");
}
// Add training subgraph to our context.
macroNodes.AddRange(subGraphNodes);
return(predModelVar);
}
protected override void Prepare_Impl(RunContext ctx)
{
var actions = ctx.Event.Actions.Where(c => c.Kind == Constants.PushActionName).ToList();
this.Items.AddRange(actions);
}
protected abstract IEnumerable <Tuple <Uri, string> > GetExecutorUriExtensionMap(IFrameworkHandle testExecutorFrameworkHandle, RunContext runContext);
public void Nuspec_overrides_package_and_global_level_versions()
{
// setup
var writer = new ConsoleWriter();
var runContext = new RunContext();
var runner = new CommandLineRunner(writer,
new PackageProcessor(new ActionExecutor(writer, runContext), writer, new NuGetProcess(writer), runContext),
new NullActionExecutor(),
runContext);
SetDefinitions(new Packages
{
Items = new object[]
{
new Package
{
BasePath = _baseDirectory,
name = "package 1",
NuSpecPath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "TestData/TestNuSpec.nuspec"),
OutputDirectory = _outputDirectory,
Version = "4.0.1"
}
}
});
// act
runner.Run(_rootDirectory, _nugetLocation, _defintionsFileLocation, "5.0.1", new Dictionary<string, string>());
// assert
Assert.IsTrue(File.Exists(Path.Combine(_outputDirectory, "package1.1.2.3.nupkg")));
}
protected abstract void InvokeExecutor(LazyExtension <ITestExecutor, ITestExecutorCapabilities> executor, Tuple <Uri, string> executorUriExtensionTuple, RunContext runContext, IFrameworkHandle frameworkHandle);
static int RunStressTestAndGetLostMessageCount( RunContextParam p )
{
TestHelper.ConsoleMonitor.Info().Send( "Prob to sleep: {0}", p.SendingThreadProbToSleep );
int nbThreads = p.PoolThreadCount + p.NewThreadCount;
int criticalErrorCount = 0;
EventHandler<SystemActivityMonitor.LowLevelErrorEventArgs> h = ( sender, e ) => ++criticalErrorCount;
SystemActivityMonitor.OnError += h;
int nbLost = 0;
int maxQueuedCount = 0;
try
{
int nbCalls = 0;
using( RunContext c = new RunContext( p ) )
{
c.RunAndGetPerfTraceCount( ( context, monitor, random ) =>
{
Interlocked.Increment( ref nbCalls );
using( monitor.OpenTrace().Send( "A group..." ) )
{
monitor.Trace().Send( "PerfTrace: {0}", context.IncrementPerfTrace() );
if( p.SendingThreadProbToSleep > random.NextDouble() )
{
int ms = random.Next( 10 );
Thread.Sleep( ms );
}
}
} );
Assert.That( nbCalls, Is.EqualTo( nbThreads * p.LoopCount ) );
nbLost = c.GrandOutput.LostEventCount;
maxQueuedCount = c.GrandOutput.MaxQueuedCount;
}
ActivityMonitor.CriticalErrorCollector.WaitOnErrorFromBackgroundThreadsPending();
}
finally
{
SystemActivityMonitor.OnError -= h;
}
int theoricalTotal = nbThreads * p.LoopCount * 3;
int receivedTotal = CK.Monitoring.GrandOutputHandlers.FakeHandler.TotalHandleCount;
TestHelper.ConsoleMonitor.Info().Send( "Local Test Strategy:{6} - Total should be {0}, Total received = {1}, Binary size = {2}, MaxQueuedCount={3}, Number of lost messages={4}, Number of Critical Errors={5}.",
theoricalTotal,
receivedTotal,
CK.Monitoring.GrandOutputHandlers.FakeHandler.SizeHandled,
maxQueuedCount,
nbLost,
criticalErrorCount,
p.UseLocalTestStrategy );
if( receivedTotal == theoricalTotal )
{
Assert.That( CK.Monitoring.GrandOutputHandlers.FakeHandler.HandlePerfTraceCount, Is.EqualTo( nbThreads * p.LoopCount ) );
}
else
{
Assert.That( criticalErrorCount > 0 );
Assert.That( receivedTotal, Is.EqualTo( theoricalTotal - nbLost ) );
}
return nbLost;
}
public IEnumerable <Tuple <Uri, string> > CallGetExecutorUriExtensionMap(
IFrameworkHandle testExecutorFrameworkHandle, RunContext runContext)
{
return(this.GetExecutorUriExtensionMap(testExecutorFrameworkHandle, runContext));
}
public SanityTests(RunContext context) : base(context) { }
public void CallInvokeExecutor(LazyExtension <ITestExecutor, ITestExecutorCapabilities> executor,
Tuple <Uri, string> executorUriExtensionTuple, RunContext runContext, IFrameworkHandle frameworkHandle)
{
this.InvokeExecutor(executor, executorUriExtensionTuple, runContext, frameworkHandle);
}
public QuerySyntaxTests(RunContext context) : base(context) { }
public static CommonOutputs.MacroOutput <Output> CrossValidate(
IHostEnvironment env,
Arguments input,
EntryPointNode node)
{
env.CheckValue(input, nameof(input));
// This will be the final resulting list of nodes that is returned from the macro.
var subGraphNodes = new List <EntryPointNode>();
//the input transform model
VariableBinding transformModelVarName = null;
if (input.TransformModel != null)
{
transformModelVarName = node.GetInputVariable(nameof(input.TransformModel));
}
// Split the input data into folds.
var splitArgs = new CVSplit.Input();
splitArgs.NumFolds = input.NumFolds;
splitArgs.StratificationColumn = input.StratificationColumn;
var inputBindingMap = new Dictionary <string, List <ParameterBinding> >();
var inputMap = new Dictionary <ParameterBinding, VariableBinding>();
var inputData = node.GetInputVariable(nameof(splitArgs.Data));
ParameterBinding paramBinding = new SimpleParameterBinding(nameof(splitArgs.Data));
inputBindingMap.Add(nameof(splitArgs.Data), new List <ParameterBinding>()
{
paramBinding
});
inputMap.Add(paramBinding, inputData);
var outputMap = new Dictionary <string, string>();
var splitOutputTrainData = new ArrayVar <IDataView>();
var splitOutputTestData = new ArrayVar <IDataView>();
outputMap.Add(nameof(CVSplit.Output.TrainData), splitOutputTrainData.VarName);
outputMap.Add(nameof(CVSplit.Output.TestData), splitOutputTestData.VarName);
var splitNode = EntryPointNode.Create(env, "Models.CrossValidatorDatasetSplitter", splitArgs,
node.Context, inputBindingMap, inputMap, outputMap);
subGraphNodes.Add(splitNode);
var predModelVars = new Var <PredictorModel> [input.NumFolds];
var inputTransformModelVars = new Var <PredictorModel> [input.NumFolds];
var warningsVars = new Var <IDataView> [input.NumFolds];
var overallMetricsVars = new Var <IDataView> [input.NumFolds];
var instanceMetricsVars = new Var <IDataView> [input.NumFolds];
var confusionMatrixVars = new Var <IDataView> [input.NumFolds];
// Instantiate the subgraph for each fold.
for (int k = 0; k < input.NumFolds; k++)
{
// Parse the nodes in input.Nodes into a temporary run context.
var context = new RunContext(env);
var graph = EntryPointNode.ValidateNodes(env, context, input.Nodes);
// Rename all the variables such that they don't conflict with the ones in the outer run context.
var mapping = new Dictionary <string, string>();
foreach (var entryPointNode in graph)
{
entryPointNode.RenameAllVariables(mapping);
}
// Instantiate a TrainTest entry point for this fold.
var args = new TrainTestMacro.Arguments
{
Nodes = new JArray(graph.Select(n => n.ToJson()).ToArray()),
TransformModel = null,
LabelColumn = input.LabelColumn,
GroupColumn = input.GroupColumn,
WeightColumn = input.WeightColumn,
NameColumn = input.NameColumn
};
if (transformModelVarName != null)
{
args.TransformModel = new Var <TransformModel> {
VarName = transformModelVarName.VariableName
}
}
;
args.Inputs.Data = new Var <IDataView>
{
VarName = mapping[input.Inputs.Data.VarName]
};
args.Outputs.PredictorModel = new Var <PredictorModel>
{
VarName = mapping[input.Outputs.PredictorModel.VarName]
};
// Set train/test trainer kind to match.
args.Kind = input.Kind;
// Set the input bindings for the TrainTest entry point.
inputBindingMap = new Dictionary <string, List <ParameterBinding> >();
inputMap = new Dictionary <ParameterBinding, VariableBinding>();
var trainingData = new SimpleParameterBinding(nameof(args.TrainingData));
inputBindingMap.Add(nameof(args.TrainingData), new List <ParameterBinding> {
trainingData
});
inputMap.Add(trainingData, new ArrayIndexVariableBinding(splitOutputTrainData.VarName, k));
var testingData = new SimpleParameterBinding(nameof(args.TestingData));
inputBindingMap.Add(nameof(args.TestingData), new List <ParameterBinding> {
testingData
});
inputMap.Add(testingData, new ArrayIndexVariableBinding(splitOutputTestData.VarName, k));
outputMap = new Dictionary <string, string>();
var transformModelVar = new Var <TransformModel>();
var predModelVar = new Var <PredictorModel>();
outputMap.Add(nameof(TrainTestMacro.Output.PredictorModel), predModelVar.VarName);
predModelVars[k] = predModelVar;
if (transformModelVarName != null && transformModelVarName.VariableName != null)
{
var combineModelsArgs = new ModelOperations.SimplePredictorModelInput();
inputBindingMap = new Dictionary <string, List <ParameterBinding> >();
inputMap = new Dictionary <ParameterBinding, VariableBinding>();
var inputTransformModel = new SimpleVariableBinding(transformModelVarName.VariableName);
var inputPredictorModel = new SimpleVariableBinding(predModelVar.VarName);
paramBinding = new SimpleParameterBinding(nameof(combineModelsArgs.TransformModel));
inputBindingMap.Add(nameof(combineModelsArgs.TransformModel), new List <ParameterBinding>()
{
paramBinding
});
inputMap.Add(paramBinding, inputTransformModel);
paramBinding = new SimpleParameterBinding(nameof(combineModelsArgs.PredictorModel));
inputBindingMap.Add(nameof(combineModelsArgs.PredictorModel), new List <ParameterBinding>()
{
paramBinding
});
inputMap.Add(paramBinding, inputPredictorModel);
outputMap = new Dictionary <string, string>();
var combineNodeOutputPredictorModel = new Var <PredictorModel>();
predModelVars[k] = combineNodeOutputPredictorModel;
outputMap.Add(nameof(ModelOperations.PredictorModelOutput.PredictorModel), combineNodeOutputPredictorModel.VarName);
EntryPointNode combineNode = EntryPointNode.Create(env, "Transforms.TwoHeterogeneousModelCombiner", combineModelsArgs,
node.Context, inputBindingMap, inputMap, outputMap);
subGraphNodes.Add(combineNode);
}
var warningVar = new Var <IDataView>();
outputMap.Add(nameof(TrainTestMacro.Output.Warnings), warningVar.VarName);
warningsVars[k] = warningVar;
var overallMetric = new Var <IDataView>();
outputMap.Add(nameof(TrainTestMacro.Output.OverallMetrics), overallMetric.VarName);
overallMetricsVars[k] = overallMetric;
var instanceMetric = new Var <IDataView>();
outputMap.Add(nameof(TrainTestMacro.Output.PerInstanceMetrics), instanceMetric.VarName);
instanceMetricsVars[k] = instanceMetric;
var confusionMatrix = new Var <IDataView>();
outputMap.Add(nameof(TrainTestMacro.Output.ConfusionMatrix), confusionMatrix.VarName);
confusionMatrixVars[k] = confusionMatrix;
const string trainTestEvaluatorMacroEntryPoint = "Models.TrainTestEvaluator";
subGraphNodes.Add(EntryPointNode.Create(env, trainTestEvaluatorMacroEntryPoint, args, node.Context, inputBindingMap, inputMap, outputMap));
}
// Convert the predictor models to an array of predictor models.
MacroUtils.ConvertIPredictorModelsToArray(env, node.Context, subGraphNodes, predModelVars, node.GetOutputVariableName(nameof(Output.PredictorModel)));
// Convert the warnings, overall, per instance and confusion matrix data views into an array.
var warningsArrayVar = new ArrayVar <IDataView>();
var overallArrayVar = new ArrayVar <IDataView>();
var instanceArrayVar = new ArrayVar <IDataView>();
ArrayVar <IDataView> confusionMatrixArrayVar = null;
MacroUtils.ConvertIdataViewsToArray(env, node.Context, subGraphNodes, warningsVars, warningsArrayVar.VarName);
MacroUtils.ConvertIdataViewsToArray(env, node.Context, subGraphNodes, overallMetricsVars, overallArrayVar.VarName);
MacroUtils.ConvertIdataViewsToArray(env, node.Context, subGraphNodes, instanceMetricsVars, instanceArrayVar.VarName);
if (input.Kind == MacroUtils.TrainerKinds.SignatureBinaryClassifierTrainer ||
input.Kind == MacroUtils.TrainerKinds.SignatureMulticlassClassificationTrainer)
{
confusionMatrixArrayVar = new ArrayVar <IDataView>();
MacroUtils.ConvertIdataViewsToArray(env, node.Context, subGraphNodes, confusionMatrixVars, confusionMatrixArrayVar.VarName);
}
var combineArgs = new CombineMetricsInput();
combineArgs.Kind = input.Kind;
combineArgs.LabelColumn = input.LabelColumn;
combineArgs.WeightColumn = input.WeightColumn;
combineArgs.GroupColumn = input.GroupColumn;
combineArgs.NameColumn = input.NameColumn;
// Set the input bindings for the CombineMetrics entry point.
var combineInputBindingMap = new Dictionary <string, List <ParameterBinding> >();
var combineInputMap = new Dictionary <ParameterBinding, VariableBinding>();
var warningsArray = new SimpleParameterBinding(nameof(combineArgs.Warnings));
combineInputBindingMap.Add(nameof(combineArgs.Warnings), new List <ParameterBinding> {
warningsArray
});
combineInputMap.Add(warningsArray, new SimpleVariableBinding(warningsArrayVar.VarName));
var overallArray = new SimpleParameterBinding(nameof(combineArgs.OverallMetrics));
combineInputBindingMap.Add(nameof(combineArgs.OverallMetrics), new List <ParameterBinding> {
overallArray
});
combineInputMap.Add(overallArray, new SimpleVariableBinding(overallArrayVar.VarName));
var combinePerInstArray = new SimpleParameterBinding(nameof(combineArgs.PerInstanceMetrics));
combineInputBindingMap.Add(nameof(combineArgs.PerInstanceMetrics), new List <ParameterBinding> {
combinePerInstArray
});
combineInputMap.Add(combinePerInstArray, new SimpleVariableBinding(instanceArrayVar.VarName));
if (confusionMatrixArrayVar != null)
{
var combineConfArray = new SimpleParameterBinding(nameof(combineArgs.ConfusionMatrix));
combineInputBindingMap.Add(nameof(combineArgs.ConfusionMatrix), new List <ParameterBinding> {
combineConfArray
});
combineInputMap.Add(combineConfArray, new SimpleVariableBinding(confusionMatrixArrayVar.VarName));
}
var combineOutputMap = new Dictionary <string, string>();
var combineWarningVar = new Var <IDataView>();
combineWarningVar.VarName = node.GetOutputVariableName(nameof(Output.Warnings));
combineOutputMap.Add(nameof(Output.Warnings), combineWarningVar.VarName);
var combineOverallMetric = new Var <IDataView>();
combineOverallMetric.VarName = node.GetOutputVariableName(nameof(Output.OverallMetrics));
combineOutputMap.Add(nameof(Output.OverallMetrics), combineOverallMetric.VarName);
var combineInstanceMetric = new Var <IDataView>();
combineInstanceMetric.VarName = node.GetOutputVariableName(nameof(Output.PerInstanceMetrics));
combineOutputMap.Add(nameof(Output.PerInstanceMetrics), combineInstanceMetric.VarName);
if (confusionMatrixArrayVar != null)
{
var combineConfusionMatrix = new Var <IDataView>();
combineConfusionMatrix.VarName = node.GetOutputVariableName(nameof(Output.ConfusionMatrix));
combineOutputMap.Add(nameof(TrainTestMacro.Output.ConfusionMatrix), combineConfusionMatrix.VarName);
}
var combineMetricsNode = EntryPointNode.Create(env, "Models.CrossValidationResultsCombiner",
combineArgs, node.Context, combineInputBindingMap, combineInputMap, combineOutputMap);
subGraphNodes.Add(combineMetricsNode);
return(new CommonOutputs.MacroOutput <Output>()
{
Nodes = subGraphNodes
});
}
public SupportedFrameworkTests(RunContext context) : base(context) { }
/// <summary> /// Run the action on the publisher /// </summary> /// <param name="publisher"></param> /// <param name="context"></param> protected abstract void OnRun(Publisher publisher, RunContext context);
public DiagnosticsTests(RunContext context) : base(context) { }
public void Run(RunContext context)
{
logger.Trace("Run({0}): {1}", name, GetType().Name);
// Setup scope for any scripting expressions
scope["context"] = context;
scope["Context"] = context;
scope["action"] = this;
scope["Action"] = this;
scope["state"] = parent;
scope["State"] = parent;
scope["StateModel"] = parent.parent;
scope["stateModel"] = parent.parent;
scope["Test"] = parent.parent.parent;
scope["test"] = parent.parent.parent;
scope["self"] = this;
if (when != null)
{
object value = Scripting.EvalExpression(when, scope);
if (!(value is bool))
{
logger.Debug("Run: action '{0}' when return is not boolean, returned: {1}", name, value);
return;
}
if (!(bool)value)
{
logger.Debug("Run: action '{0}' when returned false", name);
return;
}
}
try
{
Publisher publisher = null;
if (this.publisher != null && this.publisher != "Peach.Agent")
{
if (!context.test.publishers.ContainsKey(this.publisher))
{
logger.Debug("Run: Publisher '" + this.publisher + "' not found!");
throw new PeachException("Error, Action '" + name + "' couldn't find publisher named '" + this.publisher + "'.");
}
publisher = context.test.publishers[this.publisher];
}
else
{
publisher = context.test.publishers[0];
}
if (context.controlIteration && context.controlRecordingIteration)
{
logger.Debug("Run: Adding action to controlRecordingActionsExecuted");
context.controlRecordingActionsExecuted.Add(this);
}
else if (context.controlIteration)
{
logger.Debug("Run: Adding action to controlActionsExecuted");
context.controlActionsExecuted.Add(this);
}
started = true;
finished = false;
error = false;
OnStarting();
logger.Debug("ActionType.{0}", GetType().Name.ToString());
RunScript(onStart);
// Save output data
foreach (var item in outputData)
{
parent.parent.SaveData(item.outputName, item.dataModel.Value);
}
OnRun(publisher, context);
// Save input data
foreach (var item in inputData)
{
parent.parent.SaveData(item.inputName, item.dataModel.Value);
}
RunScript(onComplete);
finished = true;
}
catch (ActionChangeStateException)
{
// this is not an error
throw;
}
catch
{
error = true;
throw;
}
finally
{
finished = true;
OnFinished();
}
}
public RelevanceTests(RunContext context) : base(context) { }
protected override void Prepare_Impl(RunContext ctx)
{
this._workflow = ctx.Workflow;
}
public SegmentsTests(RunContext context) : base(context) { }
public ResetDb(RunContext context)
{
_context = context;
}
