public void TestRcv1ModelUpdateFromStream()
{
joinServer.Reset();
int numActions = 10;
int numFeatures = 1024;
var dsConfig = new DecisionServiceConfiguration<TestRcv1Context>(
authorizationToken: MockCommandCenter.AuthorizationToken,
explorer: new EpsilonGreedyExplorer<TestRcv1Context>(new TestRcv1ContextPolicy(), epsilon: 0.5f, numActions: (uint)numActions))
{
JoinServerType = JoinServerType.CustomSolution,
LoggingServiceAddress = MockJoinServer.MockJoinServerAddress,
PollingForModelPeriod = TimeSpan.MinValue,
PollingForSettingsPeriod = TimeSpan.MinValue
};
var ds = new DecisionService<TestRcv1Context>(dsConfig);
string uniqueKey = "eventid";
string modelFile = "test_vw_adf{0}.model";
for (int i = 1; i <= 100; i++)
{
Random rg = new Random(i);
if (i % 50 == 1)
{
int modelIndex = i / 50;
string currentModelFile = string.Format(modelFile, modelIndex);
byte[] modelContent = commandCenter.GetCBModelBlobContent(numExamples: 3 + modelIndex, numFeatures: numFeatures, numActions: numActions);
var modelStream = new MemoryStream(modelContent);
ds.UpdatePolicy(new VWPolicy<TestRcv1Context>(modelStream));
}
var context = TestRcv1Context.CreateRandom(numActions, numFeatures, rg);
DateTime timeStamp = DateTime.UtcNow;
uint action = ds.ChooseAction(new UniqueEventID { Key = uniqueKey }, context);
// verify the actions are in the expected range
Assert.IsTrue(action >= 1 && action <= numActions);
ds.ReportReward(i / 100f, new UniqueEventID { Key = uniqueKey });
}
ds.Flush();
Assert.AreEqual(200, joinServer.EventBatchList.Sum(b => b.ExperimentalUnitFragments.Count));
}
public void TestADFModelUpdateFromFile()
{
joinServer.Reset();
var dsConfig = new DecisionServiceConfiguration<TestADFContextWithFeatures, TestADFFeatures>(
authorizationToken: MockCommandCenter.AuthorizationToken,
explorer: new EpsilonGreedyExplorer<TestADFContextWithFeatures>(new TestADFWithFeaturesPolicy(), epsilon: 0.5f))
{
JoinServerType = JoinServerType.CustomSolution,
LoggingServiceAddress = MockJoinServer.MockJoinServerAddress,
PollingForModelPeriod = TimeSpan.MinValue,
PollingForSettingsPeriod = TimeSpan.MinValue
};
var ds = new DecisionService<TestADFContextWithFeatures, TestADFFeatures>(dsConfig);
string uniqueKey = "eventid";
string modelFile = "test_vw_adf{0}.model";
var actualModelFiles = new List<string>();
for (int i = 1; i <= 100; i++)
{
Random rg = new Random(i);
if (i % 50 == 1)
{
int modelIndex = i / 50;
string currentModelFile = string.Format(modelFile, modelIndex);
byte[] modelContent = commandCenter.GetCBADFModelBlobContent(numExamples: 3 + modelIndex, numFeatureVectors: 4 + modelIndex);
System.IO.File.WriteAllBytes(currentModelFile, modelContent);
ds.UpdatePolicy(new VWPolicy<TestADFContextWithFeatures, TestADFFeatures>(GetFeaturesFromContext, currentModelFile));
actualModelFiles.Add(currentModelFile);
}
int numActions = rg.Next(5, 20);
var context = TestADFContextWithFeatures.CreateRandom(numActions, rg);
uint[] action = ds.ChooseAction(new UniqueEventID { Key = uniqueKey }, context, (uint)context.ActionDependentFeatures.Count);
Assert.AreEqual(numActions, action.Length);
// verify all unique actions in the list
Assert.AreEqual(action.Length, action.Distinct().Count());
// verify the actions are in the expected range
Assert.AreEqual((numActions * (numActions + 1)) / 2, action.Sum(a => a));
ds.ReportReward(i / 100f, new UniqueEventID { Key = uniqueKey });
}
ds.Flush();
Assert.AreEqual(200, joinServer.EventBatchList.Sum(b => b.ExperimentalUnitFragments.Count));
foreach (string actualModelFile in actualModelFiles)
{
System.IO.File.Delete(actualModelFile);
}
}
public static void SampleCodeUsingActionDependentFeatures()
{
// Create configuration for the decision service
var serviceConfig = new DecisionServiceConfiguration<ADFContext, ADFFeatures>(
authorizationToken: "",
explorer: new EpsilonGreedyExplorer<ADFContext>(new ADFPolicy(), epsilon: 0.8f))
{
PollingForModelPeriod = TimeSpan.MinValue,
PollingForSettingsPeriod = TimeSpan.MinValue
};
var service = new DecisionService<ADFContext, ADFFeatures>(serviceConfig);
string uniqueKey = "eventid";
var rg = new Random(uniqueKey.GetHashCode());
var vwPolicy = new VWPolicy<ADFContext, ADFFeatures>(GetFeaturesFromContext);
for (int i = 1; i < 100; i++)
{
if (i == 30)
{
string vwModelFile = TrainNewVWModelWithRandomData(numExamples: 5, numActions: 10);
vwPolicy = new VWPolicy<ADFContext, ADFFeatures>(GetFeaturesFromContext, vwModelFile);
// Alternatively, VWPolicy can also be loaded from an IO stream:
// var vwModelStream = new MemoryStream(File.ReadAllBytes(vwModelFile));
// vwPolicy = new VWPolicy<ADFContext, ADFFeatures>(GetFeaturesFromContext, vwModelStream);
// Manually updates decision service with a new policy for consuming VW models.
service.UpdatePolicy(vwPolicy);
}
if (i == 60)
{
string vwModelFile = TrainNewVWModelWithRandomData(numExamples: 6, numActions: 8);
// Evolves the existing VWPolicy with a new model
vwPolicy.ModelUpdate(vwModelFile);
}
int numActions = rg.Next(5, 10);
uint[] action = service.ChooseAction(new UniqueEventID { Key = uniqueKey }, ADFContext.CreateRandom(numActions, rg), (uint)numActions);
service.ReportReward(i / 100f, new UniqueEventID { Key = uniqueKey });
}
service.Flush();
}
