private static void EpsilonGreedyWithContext <TContext>(uint numActions, TContext testContext, TestPolicy <TContext> policy, IExplorer <TContext> explorer)
where TContext : TestContext
{
string uniqueKey = "ManagedTestId";
TestRecorder <TContext> recorder = new TestRecorder <TContext>();
MwtExplorer <TContext> mwtt = new MwtExplorer <TContext>("mwt", recorder);
testContext.Id = 100;
uint expectedAction = policy.ChooseAction(testContext);
uint chosenAction = mwtt.ChooseAction(explorer, uniqueKey, testContext);
Assert.AreEqual(expectedAction, chosenAction);
chosenAction = mwtt.ChooseAction(explorer, uniqueKey, testContext);
Assert.AreEqual(expectedAction, chosenAction);
var interactions = recorder.GetAllInteractions();
Assert.AreEqual(2, interactions.Count);
Assert.AreEqual(testContext.Id, interactions[0].Context.Id);
// Verify that policy action is chosen all the time
explorer.EnableExplore(false);
for (int i = 0; i < 1000; i++)
{
chosenAction = mwtt.ChooseAction(explorer, uniqueKey, testContext);
Assert.AreEqual(expectedAction, chosenAction);
}
}
public void EpsilonGreedy()
{
uint numActions = 10;
float epsilon = 0f;
string uniqueKey = "ManagedTestId";
TestRecorder <TestContext> recorder = new TestRecorder <TestContext>();
TestPolicy policy = new TestPolicy();
MwtExplorer <TestContext> mwtt = new MwtExplorer <TestContext>("mwt", recorder);
TestContext testContext = new TestContext();
testContext.Id = 100;
var explorer = new EpsilonGreedyExplorer <TestContext>(policy, epsilon, numActions);
uint expectedAction = policy.ChooseAction(testContext);
uint chosenAction = mwtt.ChooseAction(explorer, uniqueKey, testContext);
Assert.AreEqual(expectedAction, chosenAction);
chosenAction = mwtt.ChooseAction(explorer, uniqueKey, testContext);
Assert.AreEqual(expectedAction, chosenAction);
var interactions = recorder.GetAllInteractions();
Assert.AreEqual(2, interactions.Count);
Assert.AreEqual(testContext.Id, interactions[0].Context.Id);
}
public static void Clock()
{
float epsilon = .2f;
string uniqueKey = "clock";
int numFeatures = 1000;
int numIter = 1000;
int numWarmup = 100;
int numInteractions = 1;
uint numActions = 10;
double timeInit = 0, timeChoose = 0, timeSerializedLog = 0;
System.Diagnostics.Stopwatch watch = new System.Diagnostics.Stopwatch();
for (int iter = 0; iter < numIter + numWarmup; iter++)
{
watch.Restart();
StringRecorder <SimpleContext> recorder = new StringRecorder <SimpleContext>();
StringPolicy policy = new StringPolicy();
MwtExplorer <SimpleContext> mwt = new MwtExplorer <SimpleContext>("mwt", recorder);
EpsilonGreedyExplorer <SimpleContext> explorer = new EpsilonGreedyExplorer <SimpleContext>(policy, epsilon, numActions);
timeInit += (iter < numWarmup) ? 0 : watch.Elapsed.TotalMilliseconds;
Feature[] f = new Feature[numFeatures];
for (int i = 0; i < numFeatures; i++)
{
f[i].Id = (uint)i + 1;
f[i].Value = 0.5f;
}
watch.Restart();
SimpleContext context = new SimpleContext(f);
for (int i = 0; i < numInteractions; i++)
{
mwt.ChooseAction(explorer, uniqueKey, context);
}
timeChoose += (iter < numWarmup) ? 0 : watch.Elapsed.TotalMilliseconds;
watch.Restart();
string interactions = recorder.GetRecording();
timeSerializedLog += (iter < numWarmup) ? 0 : watch.Elapsed.TotalMilliseconds;
for (int i = 0; i < numInteractions; i++)
{
mwt.ChooseAction(explorer, uniqueKey, context);
}
}
Console.WriteLine("--- PER ITERATION ---");
Console.WriteLine("# iterations: {0}, # interactions: {1}, # context features {2}", numIter, numInteractions, numFeatures);
Console.WriteLine("Init: {0} micro", timeInit * 1000 / numIter);
Console.WriteLine("Choose Action: {0} micro", timeChoose * 1000 / (numIter * numInteractions));
Console.WriteLine("Get Serialized Log: {0} micro", timeSerializedLog * 1000 / numIter);
Console.WriteLine("--- TOTAL TIME: {0} micro", (timeInit + timeChoose + timeSerializedLog) * 1000);
}
private static void SoftmaxWithContext <TContext>(uint numActions, IExplorer <TContext> explorer, TContext[] contexts)
where TContext : TestContext
{
var recorder = new TestRecorder <TContext>();
var mwtt = new MwtExplorer <TContext>("mwt", recorder);
uint[] actions = new uint[numActions];
Random rand = new Random();
for (uint i = 0; i < contexts.Length; i++)
{
uint chosenAction = mwtt.ChooseAction(explorer, rand.NextDouble().ToString(), contexts[i]);
actions[chosenAction - 1]++; // action id is one-based
}
for (uint i = 0; i < numActions; i++)
{
Assert.IsTrue(actions[i] > 0);
}
var interactions = recorder.GetAllInteractions();
Assert.AreEqual(contexts.Length, interactions.Count);
for (int i = 0; i < contexts.Length; i++)
{
Assert.AreEqual(i, interactions[i].Context.Id);
}
}
public void UsageBadVariableActionContext()
{
int numExceptionsCaught = 0;
int numExceptionsExpected = 5;
var tryCatchArgumentException = (Action <Action>)((action) => {
try
{
action();
}
catch (ArgumentException ex)
{
if (ex.ParamName.ToLower() == "ctx")
{
numExceptionsCaught++;
}
}
});
tryCatchArgumentException(() => {
var mwt = new MwtExplorer <TestContext>("test", new TestRecorder <TestContext>());
var policy = new TestPolicy <TestContext>();
var explorer = new EpsilonGreedyExplorer <TestContext>(policy, 0.2f);
mwt.ChooseAction(explorer, "key", new TestContext());
});
tryCatchArgumentException(() =>
{
var mwt = new MwtExplorer <TestContext>("test", new TestRecorder <TestContext>());
var policy = new TestPolicy <TestContext>();
var explorer = new TauFirstExplorer <TestContext>(policy, 10);
mwt.ChooseAction(explorer, "key", new TestContext());
});
tryCatchArgumentException(() =>
{
var mwt = new MwtExplorer <TestContext>("test", new TestRecorder <TestContext>());
var policies = new TestPolicy <TestContext> [2];
for (int i = 0; i < 2; i++)
{
policies[i] = new TestPolicy <TestContext>(i * 2);
}
var explorer = new BootstrapExplorer <TestContext>(policies);
mwt.ChooseAction(explorer, "key", new TestContext());
});
tryCatchArgumentException(() =>
{
var mwt = new MwtExplorer <TestContext>("test", new TestRecorder <TestContext>());
var scorer = new TestScorer <TestContext>(10);
var explorer = new SoftmaxExplorer <TestContext>(scorer, 0.5f);
mwt.ChooseAction(explorer, "key", new TestContext());
});
tryCatchArgumentException(() =>
{
var mwt = new MwtExplorer <TestContext>("test", new TestRecorder <TestContext>());
var scorer = new TestScorer <TestContext>(10);
var explorer = new GenericExplorer <TestContext>(scorer);
mwt.ChooseAction(explorer, "key", new TestContext());
});
Assert.AreEqual(numExceptionsExpected, numExceptionsCaught);
}
public void TauFirst()
{
uint numActions = 10;
uint tau = 0;
string uniqueKey = "ManagedTestId";
TestRecorder <TestContext> recorder = new TestRecorder <TestContext>();
TestPolicy policy = new TestPolicy();
MwtExplorer <TestContext> mwtt = new MwtExplorer <TestContext>("mwt", recorder);
TestContext testContext = new TestContext()
{
Id = 100
};
var explorer = new TauFirstExplorer <TestContext>(policy, tau, numActions);
uint expectedAction = policy.ChooseAction(testContext);
uint chosenAction = mwtt.ChooseAction(explorer, uniqueKey, testContext);
Assert.AreEqual(expectedAction, chosenAction);
var interactions = recorder.GetAllInteractions();
Assert.AreEqual(0, interactions.Count);
}
private void EndToEnd(MwtExplorer <SimpleContext> mwtt, IExplorer <SimpleContext> explorer, TestRecorder <SimpleContext> recorder)
{
uint numActions = 10;
Random rand = new Random();
List <float> rewards = new List <float>();
for (int i = 0; i < 1000; i++)
{
Feature[] f = new Feature[rand.Next(800, 1201)];
for (int j = 0; j < f.Length; j++)
{
f[j].Id = (uint)(j + 1);
f[j].Value = (float)rand.NextDouble();
}
SimpleContext c = new SimpleContext(f);
mwtt.ChooseAction(explorer, i.ToString(), c);
rewards.Add((float)rand.NextDouble());
}
var testInteractions = recorder.GetAllInteractions();
Interaction[] partialInteractions = new Interaction[testInteractions.Count];
for (int i = 0; i < testInteractions.Count; i++)
{
partialInteractions[i] = new Interaction()
{
ApplicationContext = new OldSimpleContext(testInteractions[i].Context.GetFeatures(), null),
ChosenAction = testInteractions[i].Action,
Probability = testInteractions[i].Probability,
Id = testInteractions[i].UniqueKey
};
}
MwtRewardReporter mrr = new MwtRewardReporter(partialInteractions);
for (int i = 0; i < partialInteractions.Length; i++)
{
Assert.AreEqual(true, mrr.ReportReward(partialInteractions[i].GetId(), rewards[i]));
}
Interaction[] completeInteractions = mrr.GetAllInteractions();
MwtOptimizer mop = new MwtOptimizer(completeInteractions, numActions);
string modelFile = "model";
mop.OptimizePolicyVWCSOAA(modelFile);
Assert.IsTrue(System.IO.File.Exists(modelFile));
float evaluatedValue = mop.EvaluatePolicyVWCSOAA(modelFile);
Assert.IsFalse(float.IsNaN(evaluatedValue));
System.IO.File.Delete(modelFile);
}
public void Bootstrap()
{
uint numActions = 10;
uint numbags = 2;
string uniqueKey = "ManagedTestId";
TestRecorder <TestContext> recorder = new TestRecorder <TestContext>();
TestPolicy[] policies = new TestPolicy[numbags];
for (int i = 0; i < numbags; i++)
{
policies[i] = new TestPolicy(i * 2);
}
TestContext testContext1 = new TestContext()
{
Id = 99
};
TestContext testContext2 = new TestContext()
{
Id = 100
};
MwtExplorer <TestContext> mwtt = new MwtExplorer <TestContext>("mwt", recorder);
var explorer = new BootstrapExplorer <TestContext>(policies, numActions);
uint expectedAction = policies[0].ChooseAction(testContext1);
uint chosenAction = mwtt.ChooseAction(explorer, uniqueKey, testContext1);
Assert.AreEqual(expectedAction, chosenAction);
chosenAction = mwtt.ChooseAction(explorer, uniqueKey, testContext2);
Assert.AreEqual(expectedAction, chosenAction);
var interactions = recorder.GetAllInteractions();
Assert.AreEqual(2, interactions.Count);
Assert.AreEqual(testContext1.Id, interactions[0].Context.Id);
Assert.AreEqual(testContext2.Id, interactions[1].Context.Id);
}
private static void GenericWithContext <TContext>(uint numActions, TContext testContext, IExplorer <TContext> explorer)
where TContext : TestContext
{
string uniqueKey = "ManagedTestId";
var recorder = new TestRecorder <TContext>();
var mwtt = new MwtExplorer <TContext>("mwt", recorder);
uint chosenAction = mwtt.ChooseAction(explorer, uniqueKey, testContext);
var interactions = recorder.GetAllInteractions();
Assert.AreEqual(1, interactions.Count);
Assert.AreEqual(testContext.Id, interactions[0].Context.Id);
}
public void SoftmaxScores()
{
uint numActions = 10;
float lambda = 0.5f;
TestRecorder <TestContext> recorder = new TestRecorder <TestContext>();
TestScorer <TestContext> scorer = new TestScorer <TestContext>(numActions, uniform: false);
MwtExplorer <TestContext> mwtt = new MwtExplorer <TestContext>("mwt", recorder);
var explorer = new SoftmaxExplorer <TestContext>(scorer, lambda, numActions);
Random rand = new Random();
mwtt.ChooseAction(explorer, rand.NextDouble().ToString(), new TestContext()
{
Id = 100
});
mwtt.ChooseAction(explorer, rand.NextDouble().ToString(), new TestContext()
{
Id = 101
});
mwtt.ChooseAction(explorer, rand.NextDouble().ToString(), new TestContext()
{
Id = 102
});
var interactions = recorder.GetAllInteractions();
Assert.AreEqual(3, interactions.Count);
for (int i = 0; i < interactions.Count; i++)
{
// Scores are not equal therefore probabilities should not be uniform
Assert.AreNotEqual(interactions[i].Probability, 1.0f / numActions);
Assert.AreEqual(100 + i, interactions[i].Context.Id);
}
}
public void Softmax()
{
uint numActions = 10;
float lambda = 0.5f;
uint numActionsCover = 100;
float C = 5;
TestRecorder <TestContext> recorder = new TestRecorder <TestContext>();
TestScorer <TestContext> scorer = new TestScorer <TestContext>(numActions);
MwtExplorer <TestContext> mwtt = new MwtExplorer <TestContext>("mwt", recorder);
var explorer = new SoftmaxExplorer <TestContext>(scorer, lambda, numActions);
uint numDecisions = (uint)(numActions * Math.Log(numActions * 1.0) + Math.Log(numActionsCover * 1.0 / numActions) * C * numActions);
uint[] actions = new uint[numActions];
Random rand = new Random();
for (uint i = 0; i < numDecisions; i++)
{
uint chosenAction = mwtt.ChooseAction(explorer, rand.NextDouble().ToString(), new TestContext()
{
Id = (int)i
});
actions[chosenAction - 1]++; // action id is one-based
}
for (uint i = 0; i < numActions; i++)
{
Assert.IsTrue(actions[i] > 0);
}
var interactions = recorder.GetAllInteractions();
Assert.AreEqual(numDecisions, (uint)interactions.Count);
for (int i = 0; i < numDecisions; i++)
{
Assert.AreEqual(i, interactions[i].Context.Id);
}
}
public void Generic()
{
uint numActions = 10;
string uniqueKey = "ManagedTestId";
TestRecorder <TestContext> recorder = new TestRecorder <TestContext>();
TestScorer <TestContext> scorer = new TestScorer <TestContext>(numActions);
MwtExplorer <TestContext> mwtt = new MwtExplorer <TestContext>("mwt", recorder);
var explorer = new GenericExplorer <TestContext>(scorer, numActions);
TestContext testContext = new TestContext()
{
Id = 100
};
uint chosenAction = mwtt.ChooseAction(explorer, uniqueKey, testContext);
var interactions = recorder.GetAllInteractions();
Assert.AreEqual(1, interactions.Count);
Assert.AreEqual(testContext.Id, interactions[0].Context.Id);
}
public static void Run()
{
string exploration_type = "greedy";
if (exploration_type == "greedy")
{
// Initialize Epsilon-Greedy explore algorithm using built-in StringRecorder and SimpleContext types
// Creates a recorder of built-in StringRecorder type for string serialization
StringRecorder <SimpleContext> recorder = new StringRecorder <SimpleContext>();
// Creates an MwtExplorer instance using the recorder above
MwtExplorer <SimpleContext> mwtt = new MwtExplorer <SimpleContext>("mwt", recorder);
// Creates a policy that interacts with SimpleContext type
StringPolicy policy = new StringPolicy();
uint numActions = 10;
float epsilon = 0.2f;
// Creates an Epsilon-Greedy explorer using the specified settings
EpsilonGreedyExplorer <SimpleContext> explorer = new EpsilonGreedyExplorer <SimpleContext>(policy, epsilon, numActions);
// Creates a context of built-in SimpleContext type
SimpleContext context = new SimpleContext(new Feature[] {
new Feature()
{
Id = 1, Value = 0.5f
},
new Feature()
{
Id = 4, Value = 1.3f
},
new Feature()
{
Id = 9, Value = -0.5f
},
});
// Performs exploration by passing an instance of the Epsilon-Greedy exploration algorithm into MwtExplorer
// using a sample string to uniquely identify this event
string uniqueKey = "eventid";
uint action = mwtt.ChooseAction(explorer, uniqueKey, context);
Console.WriteLine(recorder.GetRecording());
return;
}
else if (exploration_type == "tau-first")
{
// Initialize Tau-First explore algorithm using custom Recorder, Policy & Context types
MyRecorder recorder = new MyRecorder();
MwtExplorer <MyContext> mwtt = new MwtExplorer <MyContext>("mwt", recorder);
uint numActions = 10;
uint tau = 0;
MyPolicy policy = new MyPolicy();
uint action = mwtt.ChooseAction(new TauFirstExplorer <MyContext>(policy, tau, numActions), "key", new MyContext());
Console.WriteLine(String.Join(",", recorder.GetAllInteractions().Select(it => it.Action)));
return;
}
else if (exploration_type == "bootstrap")
{
// Initialize Bootstrap explore algorithm using custom Recorder, Policy & Context types
MyRecorder recorder = new MyRecorder();
MwtExplorer <MyContext> mwtt = new MwtExplorer <MyContext>("mwt", recorder);
uint numActions = 10;
uint numbags = 2;
MyPolicy[] policies = new MyPolicy[numbags];
for (int i = 0; i < numbags; i++)
{
policies[i] = new MyPolicy(i * 2);
}
uint action = mwtt.ChooseAction(new BootstrapExplorer <MyContext>(policies, numActions), "key", new MyContext());
Console.WriteLine(String.Join(",", recorder.GetAllInteractions().Select(it => it.Action)));
return;
}
else if (exploration_type == "softmax")
{
// Initialize Softmax explore algorithm using custom Recorder, Scorer & Context types
MyRecorder recorder = new MyRecorder();
MwtExplorer <MyContext> mwtt = new MwtExplorer <MyContext>("mwt", recorder);
uint numActions = 10;
float lambda = 0.5f;
MyScorer scorer = new MyScorer(numActions);
uint action = mwtt.ChooseAction(new SoftmaxExplorer <MyContext>(scorer, lambda, numActions), "key", new MyContext());
Console.WriteLine(String.Join(",", recorder.GetAllInteractions().Select(it => it.Action)));
return;
}
else if (exploration_type == "generic")
{
// Initialize Generic explore algorithm using custom Recorder, Scorer & Context types
MyRecorder recorder = new MyRecorder();
MwtExplorer <MyContext> mwtt = new MwtExplorer <MyContext>("mwt", recorder);
uint numActions = 10;
MyScorer scorer = new MyScorer(numActions);
uint action = mwtt.ChooseAction(new GenericExplorer <MyContext>(scorer, numActions), "key", new MyContext());
Console.WriteLine(String.Join(",", recorder.GetAllInteractions().Select(it => it.Action)));
return;
}
else
{ //add error here
}
}
public void SoftmaxScores()
{
uint numActions = 10;
float lambda = 0.5f;
var recorder = new TestRecorder <TestContext>();
var scorer = new TestScorer <TestContext>(numActions, uniform: false);
var mwtt = new MwtExplorer <TestContext>("mwt", recorder);
var explorer = new SoftmaxExplorer <TestContext>(scorer, lambda, numActions);
Random rand = new Random();
mwtt.ChooseAction(explorer, rand.NextDouble().ToString(), new TestContext()
{
Id = 100
});
mwtt.ChooseAction(explorer, rand.NextDouble().ToString(), new TestContext()
{
Id = 101
});
mwtt.ChooseAction(explorer, rand.NextDouble().ToString(), new TestContext()
{
Id = 102
});
var interactions = recorder.GetAllInteractions();
Assert.AreEqual(3, interactions.Count);
for (int i = 0; i < interactions.Count; i++)
{
// Scores are not equal therefore probabilities should not be uniform
Assert.AreNotEqual(interactions[i].Probability, 1.0f / numActions);
Assert.AreEqual(100 + i, interactions[i].Context.Id);
}
// Verify that policy action is chosen all the time
TestContext context = new TestContext {
Id = 100
};
List <float> scores = scorer.ScoreActions(context);
float maxScore = 0;
uint highestScoreAction = 0;
for (int i = 0; i < scores.Count; i++)
{
if (maxScore < scores[i])
{
maxScore = scores[i];
highestScoreAction = (uint)i + 1;
}
}
explorer.EnableExplore(false);
for (int i = 0; i < 1000; i++)
{
uint chosenAction = mwtt.ChooseAction(explorer, rand.NextDouble().ToString(), new TestContext()
{
Id = (int)i
});
Assert.AreEqual(highestScoreAction, chosenAction);
}
}
public static void Run()
{
string interactionFile = "serialized.txt";
MwtLogger logger = new MwtLogger(interactionFile);
MwtExplorer mwt = new MwtExplorer("test", logger);
uint numActions = 10;
float epsilon = 0.2f;
uint tau = 0;
uint bags = 2;
float lambda = 0.5f;
int policyParams = 1003;
CustomParams customParams = new CustomParams()
{
Value1 = policyParams, Value2 = policyParams + 1
};
/*** Initialize Epsilon-Greedy explore algorithm using a default policy function that accepts parameters ***/
mwt.InitializeEpsilonGreedy <int>(epsilon, new StatefulPolicyDelegate <int>(SampleStatefulPolicyFunc), policyParams, numActions);
/*** Initialize Epsilon-Greedy explore algorithm using a stateless default policy function ***/
//mwt.InitializeEpsilonGreedy(epsilon, new StatelessPolicyDelegate(SampleStatelessPolicyFunc), numActions);
/*** Initialize Tau-First explore algorithm using a default policy function that accepts parameters ***/
//mwt.InitializeTauFirst<CustomParams>(tau, new StatefulPolicyDelegate<CustomParams>(SampleStatefulPolicyFunc), customParams, numActions);
/*** Initialize Tau-First explore algorithm using a stateless default policy function ***/
//mwt.InitializeTauFirst(tau, new StatelessPolicyDelegate(SampleStatelessPolicyFunc), numActions);
/*** Initialize Bagging explore algorithm using a default policy function that accepts parameters ***/
//StatefulPolicyDelegate<int>[] funcs =
//{
// new StatefulPolicyDelegate<int>(SampleStatefulPolicyFunc),
// new StatefulPolicyDelegate<int>(SampleStatefulPolicyFunc2)
//};
//int[] parameters = { policyParams, policyParams };
//mwt.InitializeBagging<int>(bags, funcs, parameters, numActions);
/*** Initialize Bagging explore algorithm using a stateless default policy function ***/
//StatelessPolicyDelegate[] funcs =
//{
// new StatelessPolicyDelegate(SampleStatelessPolicyFunc),
// new StatelessPolicyDelegate(SampleStatelessPolicyFunc2)
//};
//mwt.InitializeBagging(bags, funcs, numActions);
/*** Initialize Softmax explore algorithm using a default policy function that accepts parameters ***/
//mwt.InitializeSoftmax<int>(lambda, new StatefulScorerDelegate<int>(SampleStatefulScorerFunc), policyParams, numActions);
/*** Initialize Softmax explore algorithm using a stateless default policy function ***/
//mwt.InitializeSoftmax(lambda, new StatelessScorerDelegate(SampleStatelessScorerFunc), numActions);
FEATURE[] f = new FEATURE[2];
f[0].X = 0.5f;
f[0].Index = 1;
f[1].X = 0.9f;
f[1].Index = 2;
string otherContext = "Some other context data that might be helpful to log";
CONTEXT context = new CONTEXT(f, otherContext);
UInt32 chosenAction = mwt.ChooseAction(context, "myId");
INTERACTION[] interactions = mwt.GetAllInteractions();
mwt.Unintialize();
MwtRewardReporter mrr = new MwtRewardReporter(interactions);
string joinKey = "myId";
float reward = 0.5f;
if (!mrr.ReportReward(joinKey, reward))
{
throw new Exception();
}
MwtOptimizer mot = new MwtOptimizer(interactions, numActions);
float eval1 = mot.EvaluatePolicy(new StatefulPolicyDelegate <int>(SampleStatefulPolicyFunc), policyParams);
mot.OptimizePolicyVWCSOAA("model_file");
float eval2 = mot.EvaluatePolicyVWCSOAA("model_file");
Console.WriteLine(chosenAction);
Console.WriteLine(interactions);
logger.Flush();
// Create a new logger to read back interaction data
logger = new MwtLogger(interactionFile);
INTERACTION[] inters = logger.GetAllInteractions();
// Load and save reward data to file
string rewardFile = "rewards.txt";
RewardStore rewardStore = new RewardStore(rewardFile);
rewardStore.Add(new float[2] {
1.0f, 0.4f
});
rewardStore.Flush();
// Read back reward data
rewardStore = new RewardStore(rewardFile);
float[] rewards = rewardStore.GetAllRewards();
}
public static void Clock()
{
float epsilon = .2f;
int policyParams = 1003;
string uniqueKey = "clock";
int numFeatures = 1000;
int numIter = 1000;
int numWarmup = 100;
int numInteractions = 1;
uint numActions = 10;
string otherContext = null;
double timeInit = 0, timeChoose = 0, timeSerializedLog = 0, timeTypedLog = 0;
System.Diagnostics.Stopwatch watch = new System.Diagnostics.Stopwatch();
for (int iter = 0; iter < numIter + numWarmup; iter++)
{
watch.Restart();
MwtExplorer mwt = new MwtExplorer("test");
mwt.InitializeEpsilonGreedy <int>(epsilon, new StatefulPolicyDelegate <int>(SampleStatefulPolicyFunc), policyParams, numActions);
timeInit += (iter < numWarmup) ? 0 : watch.Elapsed.TotalMilliseconds;
FEATURE[] f = new FEATURE[numFeatures];
for (int i = 0; i < numFeatures; i++)
{
f[i].Index = (uint)i + 1;
f[i].X = 0.5f;
}
watch.Restart();
CONTEXT context = new CONTEXT(f, otherContext);
for (int i = 0; i < numInteractions; i++)
{
mwt.ChooseAction(context, uniqueKey);
}
timeChoose += (iter < numWarmup) ? 0 : watch.Elapsed.TotalMilliseconds;
watch.Restart();
string interactions = mwt.GetAllInteractionsAsString();
timeSerializedLog += (iter < numWarmup) ? 0 : watch.Elapsed.TotalMilliseconds;
for (int i = 0; i < numInteractions; i++)
{
mwt.ChooseAction(context, uniqueKey);
}
watch.Restart();
mwt.GetAllInteractions();
timeTypedLog += (iter < numWarmup) ? 0 : watch.Elapsed.TotalMilliseconds;
}
Console.WriteLine("--- PER ITERATION ---");
Console.WriteLine("# iterations: {0}, # interactions: {1}, # context features {2}", numIter, numInteractions, numFeatures);
Console.WriteLine("Init: {0} micro", timeInit * 1000 / numIter);
Console.WriteLine("Choose Action: {0} micro", timeChoose * 1000 / (numIter * numInteractions));
Console.WriteLine("Get Serialized Log: {0} micro", timeSerializedLog * 1000 / numIter);
Console.WriteLine("Get Typed Log: {0} micro", timeTypedLog * 1000 / numIter);
Console.WriteLine("--- TOTAL TIME: {0} micro", (timeInit + timeChoose + timeSerializedLog + timeTypedLog) * 1000);
}
public static void Run()
{
string exploration_type = "greedy";
if (exploration_type == "greedy")
{
// Initialize Epsilon-Greedy explore algorithm using built-in StringRecorder and SimpleContext types
StringRecorder <SimpleContext> recorder = new StringRecorder <SimpleContext>();
MwtExplorer <SimpleContext> mwtt = new MwtExplorer <SimpleContext>("mwt", recorder);
uint numActions = 10;
float epsilon = 0.2f;
StringPolicy policy = new StringPolicy();
SimpleContext context = new SimpleContext(new Feature[] {
new Feature()
{
Id = 1, Value = 0.5f
},
new Feature()
{
Id = 4, Value = 1.3f
},
new Feature()
{
Id = 9, Value = -0.5f
},
});
uint action = mwtt.ChooseAction(new EpsilonGreedyExplorer <SimpleContext>(policy, epsilon, numActions), "key", context);
Console.WriteLine(recorder.GetRecording());
return;
}
else if (exploration_type == "tau-first")
{
// Initialize Tau-First explore algorithm using custom Recorder, Policy & Context types
MyRecorder recorder = new MyRecorder();
MwtExplorer <MyContext> mwtt = new MwtExplorer <MyContext>("mwt", recorder);
uint numActions = 10;
uint tau = 0;
MyPolicy policy = new MyPolicy();
uint action = mwtt.ChooseAction(new TauFirstExplorer <MyContext>(policy, tau, numActions), "key", new MyContext());
Console.WriteLine(String.Join(",", recorder.GetData()));
return;
}
else if (exploration_type == "bagging")
{
// Initialize Bagging explore algorithm using custom Recorder, Policy & Context types
MyRecorder recorder = new MyRecorder();
MwtExplorer <MyContext> mwtt = new MwtExplorer <MyContext>("mwt", recorder);
uint numActions = 10;
uint numbags = 2;
MyPolicy[] policies = new MyPolicy[numbags];
for (int i = 0; i < numbags; i++)
{
policies[i] = new MyPolicy(i * 2);
}
uint action = mwtt.ChooseAction(new BaggingExplorer <MyContext>(policies, numbags, numActions), "key", new MyContext());
Console.WriteLine(String.Join(",", recorder.GetData()));
return;
}
else if (exploration_type == "softmax")
{
// Initialize Softmax explore algorithm using custom Recorder, Scorer & Context types
MyRecorder recorder = new MyRecorder();
MwtExplorer <MyContext> mwtt = new MwtExplorer <MyContext>("mwt", recorder);
uint numActions = 10;
float lambda = 0.5f;
MyScorer scorer = new MyScorer(numActions);
uint action = mwtt.ChooseAction(new SoftmaxExplorer <MyContext>(scorer, lambda, numActions), "key", new MyContext());
Console.WriteLine(String.Join(",", recorder.GetData()));
return;
}
else if (exploration_type == "generic")
{
// Initialize Generic explore algorithm using custom Recorder, Scorer & Context types
MyRecorder recorder = new MyRecorder();
MwtExplorer <MyContext> mwtt = new MwtExplorer <MyContext>("mwt", recorder);
uint numActions = 10;
MyScorer scorer = new MyScorer(numActions);
uint action = mwtt.ChooseAction(new GenericExplorer <MyContext>(scorer, numActions), "key", new MyContext());
Console.WriteLine(String.Join(",", recorder.GetData()));
return;
}
else
{ //add error here
}
}