public void TestDeterministicSweeperAsyncCancellation() { var random = new Random(42); var env = new MLContext(42); var args = new DeterministicSweeperAsync.Options(); args.BatchSize = 5; args.Relaxation = 1; args.Sweeper = ComponentFactoryUtils.CreateFromFunction( environ => new KdoSweeper(environ, new KdoSweeper.Options() { SweptParameters = new IComponentFactory <INumericValueGenerator>[] { ComponentFactoryUtils.CreateFromFunction( t => new FloatValueGenerator(new FloatParamOptions() { Name = "foo", Min = 1, Max = 5 })), ComponentFactoryUtils.CreateFromFunction( t => new LongValueGenerator(new LongParamOptions() { Name = "bar", Min = 1, Max = 1000, LogBase = true })) } })); var sweeper = new DeterministicSweeperAsync(env, args); int sweeps = 20; var tasks = new List <Task <ParameterSetWithId> >(); int numCompleted = 0; for (int i = 0; i < sweeps; i++) { var task = sweeper.Propose(); if (i < args.BatchSize - args.Relaxation) { Assert.True(task.IsCompleted); sweeper.Update(task.Result.Id, new RunResult(task.Result.ParameterSet, random.NextDouble(), true)); numCompleted++; } else { tasks.Add(task); } } // Cancel after the first barrier and check if the number of registered actions // is indeed 2 * batchSize. sweeper.Cancel(); Task.WaitAll(tasks.ToArray()); foreach (var task in tasks) { if (task.Result != null) { numCompleted++; } } Assert.Equal(args.BatchSize + args.BatchSize, numCompleted); }
public async Task TestNelderMeadSweeperAsync() { var random = new Random(42); var env = new MLContext(42); const int batchSize = 5; const int sweeps = 40; var paramSets = new List <ParameterSet>(); var args = new DeterministicSweeperAsync.Options(); args.BatchSize = batchSize; args.Relaxation = 0; args.Sweeper = ComponentFactoryUtils.CreateFromFunction( environ => { var param = new IComponentFactory <INumericValueGenerator>[] { ComponentFactoryUtils.CreateFromFunction( innerEnviron => new FloatValueGenerator(new FloatParamOptions() { Name = "foo", Min = 1, Max = 5 })), ComponentFactoryUtils.CreateFromFunction( innerEnviron => new LongValueGenerator(new LongParamOptions() { Name = "bar", Min = 1, Max = 1000, LogBase = true })) }; var nelderMeadSweeperArgs = new NelderMeadSweeper.Options() { SweptParameters = param, FirstBatchSweeper = ComponentFactoryUtils.CreateFromFunction <IValueGenerator[], ISweeper>( (firstBatchSweeperEnviron, firstBatchSweeperArgs) => new RandomGridSweeper(environ, new RandomGridSweeper.Options() { SweptParameters = param })) }; return(new NelderMeadSweeper(environ, nelderMeadSweeperArgs)); } ); var sweeper = new DeterministicSweeperAsync(env, args); var mlock = new object(); double[] metrics = new double[sweeps]; for (int i = 0; i < metrics.Length; i++) { metrics[i] = random.NextDouble(); } for (int i = 0; i < sweeps; i++) { var paramWithId = await sweeper.Propose(); if (paramWithId == null) { return; } var result = new RunResult(paramWithId.ParameterSet, metrics[i], true); sweeper.Update(paramWithId.Id, result); lock (mlock) paramSets.Add(paramWithId.ParameterSet); } Assert.True(paramSets.Count <= sweeps); CheckAsyncSweeperResult(paramSets); }
public void TestDeterministicSweeperAsyncParallel() { var random = new Random(42); var env = new MLContext(42); const int batchSize = 5; const int sweeps = 20; var paramSets = new List <ParameterSet>(); var args = new DeterministicSweeperAsync.Options(); args.BatchSize = batchSize; args.Relaxation = batchSize - 2; args.Sweeper = ComponentFactoryUtils.CreateFromFunction( environ => new SmacSweeper(environ, new SmacSweeper.Options() { SweptParameters = new IComponentFactory <INumericValueGenerator>[] { ComponentFactoryUtils.CreateFromFunction( t => new FloatValueGenerator(new FloatParamOptions() { Name = "foo", Min = 1, Max = 5 })), ComponentFactoryUtils.CreateFromFunction( t => new LongValueGenerator(new LongParamOptions() { Name = "bar", Min = 1, Max = 1000, LogBase = true })) } })); var sweeper = new DeterministicSweeperAsync(env, args); var mlock = new object(); var options = new ParallelOptions(); options.MaxDegreeOfParallelism = 4; // Sleep randomly to simulate doing work. int[] sleeps = new int[sweeps]; for (int i = 0; i < sleeps.Length; i++) { sleeps[i] = random.Next(10, 100); } var r = Task.Run(() => Parallel.For(0, sweeps, options, (int i) => { var task = sweeper.Propose(); task.Wait(); Assert.Equal(TaskStatus.RanToCompletion, task.Status); var paramWithId = task.Result; if (paramWithId == null) { return; } Thread.Sleep(sleeps[i]); var result = new RunResult(paramWithId.ParameterSet, 0.42, true); sweeper.Update(paramWithId.Id, result); lock (mlock) paramSets.Add(paramWithId.ParameterSet); })); Assert.True(paramSets.Count <= sweeps); CheckAsyncSweeperResult(paramSets); }
public async Task TestDeterministicSweeperAsync() { var random = new Random(42); var env = new MLContext(42); var args = new DeterministicSweeperAsync.Options(); args.BatchSize = 5; args.Relaxation = args.BatchSize - 1; args.Sweeper = ComponentFactoryUtils.CreateFromFunction( environ => new SmacSweeper(environ, new SmacSweeper.Options() { SweptParameters = new IComponentFactory <INumericValueGenerator>[] { ComponentFactoryUtils.CreateFromFunction( t => new FloatValueGenerator(new FloatParamOptions() { Name = "foo", Min = 1, Max = 5 })), ComponentFactoryUtils.CreateFromFunction( t => new LongValueGenerator(new LongParamOptions() { Name = "bar", Min = 1, Max = 1000, LogBase = true })) } })); var sweeper = new DeterministicSweeperAsync(env, args); // Test single-threaded consumption. int sweeps = 10; var paramSets = new List <ParameterSet>(); for (int i = 0; i < sweeps; i++) { var task = sweeper.Propose(); Assert.True(task.IsCompleted); paramSets.Add(task.CompletedResult().ParameterSet); var result = new RunResult(task.CompletedResult().ParameterSet, random.NextDouble(), true); sweeper.Update(task.CompletedResult().Id, result); } Assert.Equal(sweeps, paramSets.Count); CheckAsyncSweeperResult(paramSets); // Create two batches and test if the 2nd batch is executed after the synchronization barrier is reached. object mlock = new object(); var tasks = new Task <ParameterSetWithId> [sweeps]; args.Relaxation = args.Relaxation - 1; sweeper = new DeterministicSweeperAsync(env, args); paramSets.Clear(); var results = new List <KeyValuePair <int, IRunResult> >(); for (int i = 0; i < args.BatchSize; i++) { var task = sweeper.Propose(); Assert.True(task.IsCompleted); tasks[i] = task; if (task.CompletedResult() == null) { continue; } results.Add(new KeyValuePair <int, IRunResult>(task.CompletedResult().Id, new RunResult(task.CompletedResult().ParameterSet, 0.42, true))); } // Register consumers for the 2nd batch. Those consumers will await until at least one run // in the previous batch has been posted to the sweeper. for (int i = args.BatchSize; i < 2 * args.BatchSize; i++) { var task = sweeper.Propose(); Assert.False(task.IsCompleted); tasks[i] = task; } // Call update to unblock the 2nd batch. foreach (var run in results) { sweeper.Update(run.Key, run.Value); } await Task.WhenAll(tasks); }