public static void Clock() { float epsilon = .2f; string uniqueKey = "clock"; int numFeatures = 1000; int numIter = 1000; int numWarmup = 100; int numInteractions = 1; uint numActions = 10; double timeInit = 0, timeChoose = 0, timeSerializedLog = 0; System.Diagnostics.Stopwatch watch = new System.Diagnostics.Stopwatch(); for (int iter = 0; iter < numIter + numWarmup; iter++) { watch.Restart(); StringRecorder <SimpleContext> recorder = new StringRecorder <SimpleContext>(); StringPolicy policy = new StringPolicy(); MwtExplorer <SimpleContext> mwt = new MwtExplorer <SimpleContext>("mwt", recorder); EpsilonGreedyExplorer <SimpleContext> explorer = new EpsilonGreedyExplorer <SimpleContext>(policy, epsilon, numActions); timeInit += (iter < numWarmup) ? 0 : watch.Elapsed.TotalMilliseconds; Feature[] f = new Feature[numFeatures]; for (int i = 0; i < numFeatures; i++) { f[i].Id = (uint)i + 1; f[i].Value = 0.5f; } watch.Restart(); SimpleContext context = new SimpleContext(f); for (int i = 0; i < numInteractions; i++) { mwt.ChooseAction(explorer, uniqueKey, context); } timeChoose += (iter < numWarmup) ? 0 : watch.Elapsed.TotalMilliseconds; watch.Restart(); string interactions = recorder.GetRecording(); timeSerializedLog += (iter < numWarmup) ? 0 : watch.Elapsed.TotalMilliseconds; for (int i = 0; i < numInteractions; i++) { mwt.ChooseAction(explorer, uniqueKey, context); } } Console.WriteLine("--- PER ITERATION ---"); Console.WriteLine("# iterations: {0}, # interactions: {1}, # context features {2}", numIter, numInteractions, numFeatures); Console.WriteLine("Init: {0} micro", timeInit * 1000 / numIter); Console.WriteLine("Choose Action: {0} micro", timeChoose * 1000 / (numIter * numInteractions)); Console.WriteLine("Get Serialized Log: {0} micro", timeSerializedLog * 1000 / numIter); Console.WriteLine("--- TOTAL TIME: {0} micro", (timeInit + timeChoose + timeSerializedLog) * 1000); }
static void ExploreTauFirst <TContext> ( string appId, int policyType, JToken configPolicy, int tau, int numActions, string[] experimentalUnitIdList, TContext[] contextList, string outputFile ) { var recorder = new StringRecorder <TContext>(); bool isVariableActionContext = typeof(IVariableActionContext).IsAssignableFrom(typeof(TContext)); switch (policyType) { case 0: // fixed policy { var policyAction = configPolicy["Action"].Value <uint>(); var policy = new TestPolicy <TContext> { ActionToChoose = policyAction }; var explorer = new TauFirstExplorer(tau); var mwt = isVariableActionContext ? MwtExplorer.Create(appId, new VariableActionProvider <TContext>(), recorder, explorer, policy) : MwtExplorer.Create(appId, numActions, recorder, explorer, policy); for (int i = 0; i < experimentalUnitIdList.Length; i++) { int numActionsVariable = isVariableActionContext ? ((IVariableActionContext)contextList[i]).GetNumberOfActions() : int.MaxValue; mwt.ChooseAction(experimentalUnitIdList[i], contextList[i]); } File.AppendAllText(outputFile, recorder.GetRecording()); break; } } }
static void ExploreGeneric <TContext> ( string appId, int policyType, JToken configPolicy, int numActions, string[] experimentalUnitIdList, TContext[] contextList, string outputFile ) { var recorder = new StringRecorder <TContext>(); bool isVariableActionContext = typeof(IVariableActionContext).IsAssignableFrom(typeof(TContext)); switch (policyType) { case 0: // fixed all-equal scorer { var scorerScore = configPolicy["Score"].Value <int>(); var scorer = new TestScorer <TContext>(scorerScore, numActions); var explorer = new GenericExplorer(); var mwt = isVariableActionContext ? MwtExplorer.Create(appId, new VariableActionProvider <TContext>(), recorder, explorer, scorer) : MwtExplorer.Create(appId, numActions, recorder, explorer, scorer); for (int i = 0; i < experimentalUnitIdList.Length; i++) { int numActionsVariable = isVariableActionContext ? ((IVariableActionContext)contextList[i]).GetNumberOfActions() : int.MaxValue; mwt.ChooseAction(experimentalUnitIdList[i], contextList[i]); } File.AppendAllText(outputFile, recorder.GetRecording()); break; } case 1: // integer-progression scorer { var scorerStartScore = configPolicy["Start"].Value <int>(); var scorer = new TestScorer <TContext>(scorerStartScore, numActions, uniform: false); var explorer = new GenericExplorer(); var mwt = isVariableActionContext ? MwtExplorer.Create(appId, new VariableActionProvider <TContext>(), recorder, explorer, scorer) : MwtExplorer.Create(appId, numActions, recorder, explorer, scorer); for (int i = 0; i < experimentalUnitIdList.Length; i++) { int numActionsVariable = isVariableActionContext ? ((IVariableActionContext)contextList[i]).GetNumberOfActions() : int.MaxValue; mwt.ChooseAction(experimentalUnitIdList[i], contextList[i]); } File.AppendAllText(outputFile, recorder.GetRecording()); break; } } }
public static void Run() { string exploration_type = "greedy"; if (exploration_type == "greedy") { // Initialize Epsilon-Greedy explore algorithm using built-in StringRecorder and SimpleContext types // Creates a recorder of built-in StringRecorder type for string serialization StringRecorder <SimpleContext> recorder = new StringRecorder <SimpleContext>(); int numActions = 10; float epsilon = 0.2f; // Creates an Epsilon-Greedy explorer using the specified settings var explorer = new EpsilonGreedyExplorer(epsilon); // Creates an MwtExplorer instance using the recorder above // Creates a policy that interacts with SimpleContext type var mwtt = MwtExplorer.Create("mwt", numActions, recorder, explorer, new StringPolicy()); // Creates a context of built-in SimpleContext type SimpleContext context = new SimpleContext(new float[] { .5f, 1.3f, -.5f }); // Performs exploration by passing an instance of the Epsilon-Greedy exploration algorithm into MwtExplorer // using a sample string to uniquely identify this event string uniqueKey = "eventid"; int action = mwtt.ChooseAction(uniqueKey, context); Console.WriteLine(recorder.GetRecording()); return; } else if (exploration_type == "tau-first") { // Initialize Tau-First explore algorithm using custom Recorder, Policy & Context types MyRecorder recorder = new MyRecorder(); int numActions = 10; int tau = 0; //MwtExplorer<MyContext> mwtt = new MwtExplorer<MyContext>("mwt", recorder); var mwtt = MwtExplorer.Create("mwt", numActions, recorder, new TauFirstExplorer(tau), new MyPolicy()); int action = mwtt.ChooseAction("key", new MyContext()); Console.WriteLine(String.Join(",", recorder.GetAllInteractions().Select(it => it.Action))); return; } else if (exploration_type == "bootstrap") { // TODO: add support for bootstrap //// Initialize Bootstrap explore algorithm using custom Recorder, Policy & Context types //MyRecorder recorder = new MyRecorder(); ////MwtExplorer<MyContext> mwtt = new MwtExplorer<MyContext>("mwt", recorder); //uint numActions = 10; //uint numbags = 2; //MyPolicy[] policies = new MyPolicy[numbags]; //for (int i = 0; i < numbags; i++) //{ // policies[i] = new MyPolicy(i * 2); //} //var mwtt = MwtExplorer.Create("mwt", recorder, new BootstrapExplorer(numActions)); //uint action = mwtt.ChooseAction(new BootstrapExplorer<MyContext>(policies, numActions), "key", new MyContext()); //Console.WriteLine(String.Join(",", recorder.GetAllInteractions().Select(it => it.Action))); return; } else if (exploration_type == "softmax") { // TODO: add support for softmax //// Initialize Softmax explore algorithm using custom Recorder, Scorer & Context types //MyRecorder recorder = new MyRecorder(); //MwtExplorer<MyContext> mwtt = new MwtExplorer<MyContext>("mwt", recorder); //uint numActions = 10; //float lambda = 0.5f; //MyScorer scorer = new MyScorer(numActions); //uint action = mwtt.ChooseAction(new SoftmaxExplorer<MyContext>(scorer, lambda, numActions), "key", new MyContext()); //Console.WriteLine(String.Join(",", recorder.GetAllInteractions().Select(it => it.Action))); return; } else if (exploration_type == "generic") { // TODO: add support for generic //// Initialize Generic explore algorithm using custom Recorder, Scorer & Context types //MyRecorder recorder = new MyRecorder(); //MwtExplorer<MyContext> mwtt = new MwtExplorer<MyContext>("mwt", recorder); //uint numActions = 10; //MyScorer scorer = new MyScorer(numActions); //uint action = mwtt.ChooseAction(new GenericExplorer<MyContext>(scorer, numActions), "key", new MyContext()); //Console.WriteLine(String.Join(",", recorder.GetAllInteractions().Select(it => it.Action))); return; } else { //add error here } }
public static void Run() { string exploration_type = "greedy"; if (exploration_type == "greedy") { // Initialize Epsilon-Greedy explore algorithm using built-in StringRecorder and SimpleContext types StringRecorder <SimpleContext> recorder = new StringRecorder <SimpleContext>(); MwtExplorer <SimpleContext> mwtt = new MwtExplorer <SimpleContext>("mwt", recorder); uint numActions = 10; float epsilon = 0.2f; StringPolicy policy = new StringPolicy(); SimpleContext context = new SimpleContext(new Feature[] { new Feature() { Id = 1, Value = 0.5f }, new Feature() { Id = 4, Value = 1.3f }, new Feature() { Id = 9, Value = -0.5f }, }); uint action = mwtt.ChooseAction(new EpsilonGreedyExplorer <SimpleContext>(policy, epsilon, numActions), "key", context); Console.WriteLine(recorder.GetRecording()); return; } else if (exploration_type == "tau-first") { // Initialize Tau-First explore algorithm using custom Recorder, Policy & Context types MyRecorder recorder = new MyRecorder(); MwtExplorer <MyContext> mwtt = new MwtExplorer <MyContext>("mwt", recorder); uint numActions = 10; uint tau = 0; MyPolicy policy = new MyPolicy(); uint action = mwtt.ChooseAction(new TauFirstExplorer <MyContext>(policy, tau, numActions), "key", new MyContext()); Console.WriteLine(String.Join(",", recorder.GetData())); return; } else if (exploration_type == "bagging") { // Initialize Bagging explore algorithm using custom Recorder, Policy & Context types MyRecorder recorder = new MyRecorder(); MwtExplorer <MyContext> mwtt = new MwtExplorer <MyContext>("mwt", recorder); uint numActions = 10; uint numbags = 2; MyPolicy[] policies = new MyPolicy[numbags]; for (int i = 0; i < numbags; i++) { policies[i] = new MyPolicy(i * 2); } uint action = mwtt.ChooseAction(new BaggingExplorer <MyContext>(policies, numbags, numActions), "key", new MyContext()); Console.WriteLine(String.Join(",", recorder.GetData())); return; } else if (exploration_type == "softmax") { // Initialize Softmax explore algorithm using custom Recorder, Scorer & Context types MyRecorder recorder = new MyRecorder(); MwtExplorer <MyContext> mwtt = new MwtExplorer <MyContext>("mwt", recorder); uint numActions = 10; float lambda = 0.5f; MyScorer scorer = new MyScorer(numActions); uint action = mwtt.ChooseAction(new SoftmaxExplorer <MyContext>(scorer, lambda, numActions), "key", new MyContext()); Console.WriteLine(String.Join(",", recorder.GetData())); return; } else if (exploration_type == "generic") { // Initialize Generic explore algorithm using custom Recorder, Scorer & Context types MyRecorder recorder = new MyRecorder(); MwtExplorer <MyContext> mwtt = new MwtExplorer <MyContext>("mwt", recorder); uint numActions = 10; MyScorer scorer = new MyScorer(numActions); uint action = mwtt.ChooseAction(new GenericExplorer <MyContext>(scorer, numActions), "key", new MyContext()); Console.WriteLine(String.Join(",", recorder.GetData())); return; } else { //add error here } }
public static void Run() { string exploration_type = "greedy"; if (exploration_type == "greedy") { // Initialize Epsilon-Greedy explore algorithm using built-in StringRecorder and SimpleContext types // Creates a recorder of built-in StringRecorder type for string serialization StringRecorder <SimpleContext> recorder = new StringRecorder <SimpleContext>(); // Creates an MwtExplorer instance using the recorder above MwtExplorer <SimpleContext> mwtt = new MwtExplorer <SimpleContext>("mwt", recorder); // Creates a policy that interacts with SimpleContext type StringPolicy policy = new StringPolicy(); uint numActions = 10; float epsilon = 0.2f; // Creates an Epsilon-Greedy explorer using the specified settings EpsilonGreedyExplorer <SimpleContext> explorer = new EpsilonGreedyExplorer <SimpleContext>(policy, epsilon, numActions); // Creates a context of built-in SimpleContext type SimpleContext context = new SimpleContext(new Feature[] { new Feature() { Id = 1, Value = 0.5f }, new Feature() { Id = 4, Value = 1.3f }, new Feature() { Id = 9, Value = -0.5f }, }); // Performs exploration by passing an instance of the Epsilon-Greedy exploration algorithm into MwtExplorer // using a sample string to uniquely identify this event string uniqueKey = "eventid"; uint action = mwtt.ChooseAction(explorer, uniqueKey, context); Console.WriteLine(recorder.GetRecording()); return; } else if (exploration_type == "tau-first") { // Initialize Tau-First explore algorithm using custom Recorder, Policy & Context types MyRecorder recorder = new MyRecorder(); MwtExplorer <MyContext> mwtt = new MwtExplorer <MyContext>("mwt", recorder); uint numActions = 10; uint tau = 0; MyPolicy policy = new MyPolicy(); uint action = mwtt.ChooseAction(new TauFirstExplorer <MyContext>(policy, tau, numActions), "key", new MyContext()); Console.WriteLine(String.Join(",", recorder.GetAllInteractions().Select(it => it.Action))); return; } else if (exploration_type == "bootstrap") { // Initialize Bootstrap explore algorithm using custom Recorder, Policy & Context types MyRecorder recorder = new MyRecorder(); MwtExplorer <MyContext> mwtt = new MwtExplorer <MyContext>("mwt", recorder); uint numActions = 10; uint numbags = 2; MyPolicy[] policies = new MyPolicy[numbags]; for (int i = 0; i < numbags; i++) { policies[i] = new MyPolicy(i * 2); } uint action = mwtt.ChooseAction(new BootstrapExplorer <MyContext>(policies, numActions), "key", new MyContext()); Console.WriteLine(String.Join(",", recorder.GetAllInteractions().Select(it => it.Action))); return; } else if (exploration_type == "softmax") { // Initialize Softmax explore algorithm using custom Recorder, Scorer & Context types MyRecorder recorder = new MyRecorder(); MwtExplorer <MyContext> mwtt = new MwtExplorer <MyContext>("mwt", recorder); uint numActions = 10; float lambda = 0.5f; MyScorer scorer = new MyScorer(numActions); uint action = mwtt.ChooseAction(new SoftmaxExplorer <MyContext>(scorer, lambda, numActions), "key", new MyContext()); Console.WriteLine(String.Join(",", recorder.GetAllInteractions().Select(it => it.Action))); return; } else if (exploration_type == "generic") { // Initialize Generic explore algorithm using custom Recorder, Scorer & Context types MyRecorder recorder = new MyRecorder(); MwtExplorer <MyContext> mwtt = new MwtExplorer <MyContext>("mwt", recorder); uint numActions = 10; MyScorer scorer = new MyScorer(numActions); uint action = mwtt.ChooseAction(new GenericExplorer <MyContext>(scorer, numActions), "key", new MyContext()); Console.WriteLine(String.Join(",", recorder.GetAllInteractions().Select(it => it.Action))); return; } else { //add error here } }