public void TestDeterministicSweeperAsyncCancellation()
{
var random = new Random(42);
var env = new MLContext(42);
var args = new DeterministicSweeperAsync.Arguments();
args.BatchSize = 5;
args.Relaxation = 1;
args.Sweeper = ComponentFactoryUtils.CreateFromFunction(
environ => new KdoSweeper(environ,
new KdoSweeper.Arguments()
{
SweptParameters = new IComponentFactory <INumericValueGenerator>[] {
ComponentFactoryUtils.CreateFromFunction(
t => new FloatValueGenerator(new FloatParamArguments()
{
Name = "foo", Min = 1, Max = 5
})),
ComponentFactoryUtils.CreateFromFunction(
t => new LongValueGenerator(new LongParamArguments()
{
Name = "bar", Min = 1, Max = 1000, LogBase = true
}))
}
}));
var sweeper = new DeterministicSweeperAsync(env, args);
int sweeps = 20;
var tasks = new List <Task <ParameterSetWithId> >();
int numCompleted = 0;
for (int i = 0; i < sweeps; i++)
{
var task = sweeper.Propose();
if (i < args.BatchSize - args.Relaxation)
{
Assert.True(task.IsCompleted);
sweeper.Update(task.Result.Id, new RunResult(task.Result.ParameterSet, random.NextDouble(), true));
numCompleted++;
}
else
{
tasks.Add(task);
}
}
// Cancel after the first barrier and check if the number of registered actions
// is indeed 2 * batchSize.
sweeper.Cancel();
Task.WaitAll(tasks.ToArray());
foreach (var task in tasks)
{
if (task.Result != null)
{
numCompleted++;
}
}
Assert.Equal(args.BatchSize + args.BatchSize, numCompleted);
}
public async Task TestNelderMeadSweeperAsync()
{
var random = new Random(42);
var env = new MLContext(42);
const int batchSize = 5;
const int sweeps = 40;
var paramSets = new List <ParameterSet>();
var args = new DeterministicSweeperAsync.Arguments();
args.BatchSize = batchSize;
args.Relaxation = 0;
args.Sweeper = ComponentFactoryUtils.CreateFromFunction(
environ =>
{
var param = new IComponentFactory <INumericValueGenerator>[] {
ComponentFactoryUtils.CreateFromFunction(
innerEnviron => new FloatValueGenerator(new FloatParamArguments()
{
Name = "foo", Min = 1, Max = 5
})),
ComponentFactoryUtils.CreateFromFunction(
innerEnviron => new LongValueGenerator(new LongParamArguments()
{
Name = "bar", Min = 1, Max = 1000, LogBase = true
}))
};
var nelderMeadSweeperArgs = new NelderMeadSweeper.Arguments()
{
SweptParameters = param,
FirstBatchSweeper = ComponentFactoryUtils.CreateFromFunction <IValueGenerator[], ISweeper>(
(firstBatchSweeperEnviron, firstBatchSweeperArgs) =>
new RandomGridSweeper(environ, new RandomGridSweeper.Arguments()
{
SweptParameters = param
}))
};
return(new NelderMeadSweeper(environ, nelderMeadSweeperArgs));
}
);
var sweeper = new DeterministicSweeperAsync(env, args);
var mlock = new object();
double[] metrics = new double[sweeps];
for (int i = 0; i < metrics.Length; i++)
{
metrics[i] = random.NextDouble();
}
for (int i = 0; i < sweeps; i++)
{
var paramWithId = await sweeper.Propose();
if (paramWithId == null)
{
return;
}
var result = new RunResult(paramWithId.ParameterSet, metrics[i], true);
sweeper.Update(paramWithId.Id, result);
lock (mlock)
paramSets.Add(paramWithId.ParameterSet);
}
Assert.True(paramSets.Count <= sweeps);
CheckAsyncSweeperResult(paramSets);
}
public void TestDeterministicSweeperAsyncParallel()
{
var random = new Random(42);
var env = new MLContext(42);
const int batchSize = 5;
const int sweeps = 20;
var paramSets = new List <ParameterSet>();
var args = new DeterministicSweeperAsync.Arguments();
args.BatchSize = batchSize;
args.Relaxation = batchSize - 2;
args.Sweeper = ComponentFactoryUtils.CreateFromFunction(
environ => new SmacSweeper(environ,
new SmacSweeper.Arguments()
{
SweptParameters = new IComponentFactory <INumericValueGenerator>[] {
ComponentFactoryUtils.CreateFromFunction(
t => new FloatValueGenerator(new FloatParamArguments()
{
Name = "foo", Min = 1, Max = 5
})),
ComponentFactoryUtils.CreateFromFunction(
t => new LongValueGenerator(new LongParamArguments()
{
Name = "bar", Min = 1, Max = 1000, LogBase = true
}))
}
}));
var sweeper = new DeterministicSweeperAsync(env, args);
var mlock = new object();
var options = new ParallelOptions();
options.MaxDegreeOfParallelism = 4;
// Sleep randomly to simulate doing work.
int[] sleeps = new int[sweeps];
for (int i = 0; i < sleeps.Length; i++)
{
sleeps[i] = random.Next(10, 100);
}
var r = Parallel.For(0, sweeps, options, (int i) =>
{
var task = sweeper.Propose();
task.Wait();
Assert.Equal(TaskStatus.RanToCompletion, task.Status);
var paramWithId = task.Result;
if (paramWithId == null)
{
return;
}
Thread.Sleep(sleeps[i]);
var result = new RunResult(paramWithId.ParameterSet, 0.42, true);
sweeper.Update(paramWithId.Id, result);
lock (mlock)
paramSets.Add(paramWithId.ParameterSet);
});
Assert.True(paramSets.Count <= sweeps);
CheckAsyncSweeperResult(paramSets);
}
public void TestDeterministicSweeperAsync()
{
var random = new Random(42);
var env = new MLContext(42);
var args = new DeterministicSweeperAsync.Arguments();
args.BatchSize = 5;
args.Relaxation = args.BatchSize - 1;
args.Sweeper = ComponentFactoryUtils.CreateFromFunction(
environ => new SmacSweeper(environ,
new SmacSweeper.Arguments()
{
SweptParameters = new IComponentFactory <INumericValueGenerator>[] {
ComponentFactoryUtils.CreateFromFunction(
t => new FloatValueGenerator(new FloatParamArguments()
{
Name = "foo", Min = 1, Max = 5
})),
ComponentFactoryUtils.CreateFromFunction(
t => new LongValueGenerator(new LongParamArguments()
{
Name = "bar", Min = 1, Max = 1000, LogBase = true
}))
}
}));
var sweeper = new DeterministicSweeperAsync(env, args);
// Test single-threaded consumption.
int sweeps = 10;
var paramSets = new List <ParameterSet>();
for (int i = 0; i < sweeps; i++)
{
var task = sweeper.Propose();
Assert.True(task.IsCompleted);
paramSets.Add(task.Result.ParameterSet);
var result = new RunResult(task.Result.ParameterSet, random.NextDouble(), true);
sweeper.Update(task.Result.Id, result);
}
Assert.Equal(sweeps, paramSets.Count);
CheckAsyncSweeperResult(paramSets);
// Create two batches and test if the 2nd batch is executed after the synchronization barrier is reached.
object mlock = new object();
var tasks = new Task <ParameterSetWithId> [sweeps];
args.Relaxation = args.Relaxation - 1;
sweeper = new DeterministicSweeperAsync(env, args);
paramSets.Clear();
var results = new List <KeyValuePair <int, IRunResult> >();
for (int i = 0; i < args.BatchSize; i++)
{
var task = sweeper.Propose();
Assert.True(task.IsCompleted);
tasks[i] = task;
if (task.Result == null)
{
continue;
}
results.Add(new KeyValuePair <int, IRunResult>(task.Result.Id, new RunResult(task.Result.ParameterSet, 0.42, true)));
}
// Register consumers for the 2nd batch. Those consumers will await until at least one run
// in the previous batch has been posted to the sweeper.
for (int i = args.BatchSize; i < 2 * args.BatchSize; i++)
{
var task = sweeper.Propose();
Assert.False(task.IsCompleted);
tasks[i] = task;
}
// Call update to unblock the 2nd batch.
foreach (var run in results)
{
sweeper.Update(run.Key, run.Value);
}
Task.WaitAll(tasks);
tasks.All(t => t.IsCompleted);
}
