public void TestDeterministicSweeperAsyncCancellation()
        {
            var random = new Random(42);
            var env    = new MLContext(42);
            var args   = new DeterministicSweeperAsync.Options();

            args.BatchSize  = 5;
            args.Relaxation = 1;

            args.Sweeper = ComponentFactoryUtils.CreateFromFunction(
                environ => new KdoSweeper(environ,
                                          new KdoSweeper.Options()
            {
                SweptParameters = new IComponentFactory <INumericValueGenerator>[] {
                    ComponentFactoryUtils.CreateFromFunction(
                        t => new FloatValueGenerator(new FloatParamOptions()
                    {
                        Name = "foo", Min = 1, Max = 5
                    })),
                    ComponentFactoryUtils.CreateFromFunction(
                        t => new LongValueGenerator(new LongParamOptions()
                    {
                        Name = "bar", Min = 1, Max = 1000, LogBase = true
                    }))
                }
            }));

            var sweeper = new DeterministicSweeperAsync(env, args);

            int sweeps       = 20;
            var tasks        = new List <Task <ParameterSetWithId> >();
            int numCompleted = 0;

            for (int i = 0; i < sweeps; i++)
            {
                var task = sweeper.Propose();
                if (i < args.BatchSize - args.Relaxation)
                {
                    Assert.True(task.IsCompleted);
                    sweeper.Update(task.Result.Id, new RunResult(task.Result.ParameterSet, random.NextDouble(), true));
                    numCompleted++;
                }
                else
                {
                    tasks.Add(task);
                }
            }
            // Cancel after the first barrier and check if the number of registered actions
            // is indeed 2 * batchSize.
            sweeper.Cancel();
            Task.WaitAll(tasks.ToArray());
            foreach (var task in tasks)
            {
                if (task.Result != null)
                {
                    numCompleted++;
                }
            }
            Assert.Equal(args.BatchSize + args.BatchSize, numCompleted);
        }
예제 #2
0
        public async Task TestNelderMeadSweeperAsync()
        {
            var       random    = new Random(42);
            var       env       = new MLContext(42);
            const int batchSize = 5;
            const int sweeps    = 40;
            var       paramSets = new List <ParameterSet>();
            var       args      = new DeterministicSweeperAsync.Options();

            args.BatchSize  = batchSize;
            args.Relaxation = 0;

            args.Sweeper = ComponentFactoryUtils.CreateFromFunction(
                environ =>
            {
                var param = new IComponentFactory <INumericValueGenerator>[] {
                    ComponentFactoryUtils.CreateFromFunction(
                        innerEnviron => new FloatValueGenerator(new FloatParamOptions()
                    {
                        Name = "foo", Min = 1, Max = 5
                    })),
                    ComponentFactoryUtils.CreateFromFunction(
                        innerEnviron => new LongValueGenerator(new LongParamOptions()
                    {
                        Name = "bar", Min = 1, Max = 1000, LogBase = true
                    }))
                };

                var nelderMeadSweeperArgs = new NelderMeadSweeper.Options()
                {
                    SweptParameters   = param,
                    FirstBatchSweeper = ComponentFactoryUtils.CreateFromFunction <IValueGenerator[], ISweeper>(
                        (firstBatchSweeperEnviron, firstBatchSweeperArgs) =>
                        new RandomGridSweeper(environ, new RandomGridSweeper.Options()
                    {
                        SweptParameters = param
                    }))
                };

                return(new NelderMeadSweeper(environ, nelderMeadSweeperArgs));
            }
                );

            var sweeper = new DeterministicSweeperAsync(env, args);
            var mlock   = new object();

            double[] metrics = new double[sweeps];
            for (int i = 0; i < metrics.Length; i++)
            {
                metrics[i] = random.NextDouble();
            }

            for (int i = 0; i < sweeps; i++)
            {
                var paramWithId = await sweeper.Propose();

                if (paramWithId == null)
                {
                    return;
                }
                var result = new RunResult(paramWithId.ParameterSet, metrics[i], true);
                sweeper.Update(paramWithId.Id, result);
                lock (mlock)
                    paramSets.Add(paramWithId.ParameterSet);
            }
            Assert.True(paramSets.Count <= sweeps);
            CheckAsyncSweeperResult(paramSets);
        }
예제 #3
0
        public void TestDeterministicSweeperAsyncParallel()
        {
            var       random    = new Random(42);
            var       env       = new MLContext(42);
            const int batchSize = 5;
            const int sweeps    = 20;
            var       paramSets = new List <ParameterSet>();
            var       args      = new DeterministicSweeperAsync.Options();

            args.BatchSize  = batchSize;
            args.Relaxation = batchSize - 2;

            args.Sweeper = ComponentFactoryUtils.CreateFromFunction(
                environ => new SmacSweeper(environ,
                                           new SmacSweeper.Options()
            {
                SweptParameters = new IComponentFactory <INumericValueGenerator>[] {
                    ComponentFactoryUtils.CreateFromFunction(
                        t => new FloatValueGenerator(new FloatParamOptions()
                    {
                        Name = "foo", Min = 1, Max = 5
                    })),
                    ComponentFactoryUtils.CreateFromFunction(
                        t => new LongValueGenerator(new LongParamOptions()
                    {
                        Name = "bar", Min = 1, Max = 1000, LogBase = true
                    }))
                }
            }));

            var sweeper = new DeterministicSweeperAsync(env, args);

            var mlock   = new object();
            var options = new ParallelOptions();

            options.MaxDegreeOfParallelism = 4;

            // Sleep randomly to simulate doing work.
            int[] sleeps = new int[sweeps];
            for (int i = 0; i < sleeps.Length; i++)
            {
                sleeps[i] = random.Next(10, 100);
            }
            var r = Task.Run(() => Parallel.For(0, sweeps, options, (int i) =>
            {
                var task = sweeper.Propose();
                task.Wait();
                Assert.Equal(TaskStatus.RanToCompletion, task.Status);
                var paramWithId = task.Result;
                if (paramWithId == null)
                {
                    return;
                }
                Thread.Sleep(sleeps[i]);
                var result = new RunResult(paramWithId.ParameterSet, 0.42, true);
                sweeper.Update(paramWithId.Id, result);
                lock (mlock)
                    paramSets.Add(paramWithId.ParameterSet);
            }));

            Assert.True(paramSets.Count <= sweeps);
            CheckAsyncSweeperResult(paramSets);
        }
예제 #4
0
        public async Task TestDeterministicSweeperAsync()
        {
            var random = new Random(42);
            var env    = new MLContext(42);
            var args   = new DeterministicSweeperAsync.Options();

            args.BatchSize  = 5;
            args.Relaxation = args.BatchSize - 1;

            args.Sweeper = ComponentFactoryUtils.CreateFromFunction(
                environ => new SmacSweeper(environ,
                                           new SmacSweeper.Options()
            {
                SweptParameters = new IComponentFactory <INumericValueGenerator>[] {
                    ComponentFactoryUtils.CreateFromFunction(
                        t => new FloatValueGenerator(new FloatParamOptions()
                    {
                        Name = "foo", Min = 1, Max = 5
                    })),
                    ComponentFactoryUtils.CreateFromFunction(
                        t => new LongValueGenerator(new LongParamOptions()
                    {
                        Name = "bar", Min = 1, Max = 1000, LogBase = true
                    }))
                }
            }));

            var sweeper = new DeterministicSweeperAsync(env, args);

            // Test single-threaded consumption.
            int sweeps    = 10;
            var paramSets = new List <ParameterSet>();

            for (int i = 0; i < sweeps; i++)
            {
                var task = sweeper.Propose();
                Assert.True(task.IsCompleted);
                paramSets.Add(task.CompletedResult().ParameterSet);
                var result = new RunResult(task.CompletedResult().ParameterSet, random.NextDouble(), true);
                sweeper.Update(task.CompletedResult().Id, result);
            }
            Assert.Equal(sweeps, paramSets.Count);
            CheckAsyncSweeperResult(paramSets);

            // Create two batches and test if the 2nd batch is executed after the synchronization barrier is reached.
            object mlock = new object();
            var    tasks = new Task <ParameterSetWithId> [sweeps];

            args.Relaxation = args.Relaxation - 1;
            sweeper         = new DeterministicSweeperAsync(env, args);
            paramSets.Clear();
            var results = new List <KeyValuePair <int, IRunResult> >();

            for (int i = 0; i < args.BatchSize; i++)
            {
                var task = sweeper.Propose();
                Assert.True(task.IsCompleted);
                tasks[i] = task;
                if (task.CompletedResult() == null)
                {
                    continue;
                }
                results.Add(new KeyValuePair <int, IRunResult>(task.CompletedResult().Id, new RunResult(task.CompletedResult().ParameterSet, 0.42, true)));
            }
            // Register consumers for the 2nd batch. Those consumers will await until at least one run
            // in the previous batch has been posted to the sweeper.
            for (int i = args.BatchSize; i < 2 * args.BatchSize; i++)
            {
                var task = sweeper.Propose();
                Assert.False(task.IsCompleted);
                tasks[i] = task;
            }
            // Call update to unblock the 2nd batch.
            foreach (var run in results)
            {
                sweeper.Update(run.Key, run.Value);
            }

            await Task.WhenAll(tasks);
        }