public void TestSettingsBlobPolling()
{
joinServer.Reset();
commandCenter.CreateBlobs(createSettingsBlob: true, createModelBlob: false, vwArgs: "--cb_explore 2");
var dsConfig = new DecisionServiceConfiguration(MockCommandCenter.SettingsBlobUri);
dsConfig.JoinServerType = JoinServerType.CustomSolution;
dsConfig.LoggingServiceAddress = MockJoinServer.MockJoinServerAddress;
dsConfig.PollingForSettingsPeriod = TimeSpan.FromMilliseconds(500);
using (var ds = DecisionService
.Create <TestContext>(dsConfig)
// TODO .WithTopSlotEpsilonGreedy(.2f)
.ExploitUntilModelReady(new ConstantPolicy <TestContext>()))
{
byte[] settingsBytes = null;
dsConfig.SettingsPollSuccessCallback = data => settingsBytes = data;
for (int i = 0; i < 50 && settingsBytes == null; i++)
{
Thread.Sleep(100);
if (i % 2 == 0)
{
// change the settings blob's etag frequently to make sure polling detects it
commandCenter.CreateBlobs(createSettingsBlob: true, createModelBlob: false, vwArgs: "abc", initialExplorationEpsilon: 0.4f);
}
}
Assert.IsNotNull(settingsBytes);
Assert.IsTrue(Enumerable.SequenceEqual(settingsBytes, commandCenter.GetSettingsBlobContent("abc", 0.4f)));
}
}
public void TestSingleActionDSUploadSingleEvent()
{
joinServer.Reset();
string uniqueKey = "test interaction";
commandCenter.CreateBlobs(createSettingsBlob: true, createModelBlob: false, vwArgs: "--cb_explore_adf --epsilon 0.5");
var dsConfig = new DecisionServiceConfiguration(MockCommandCenter.SettingsBlobUri);
dsConfig.JoinServerType = JoinServerType.CustomSolution;
dsConfig.LoggingServiceAddress = MockJoinServer.MockJoinServerAddress;
int chosenAction;
using (var ds = DecisionService
.Create <TestContext>(dsConfig)
.ExploitUntilModelReady(new ConstantPolicy <TestContext>()))
{
chosenAction = ds.ChooseAction(uniqueKey, new TestContext());
}
Assert.AreEqual(1, joinServer.RequestCount);
Assert.AreEqual(1, joinServer.EventBatchList.Count);
Assert.AreEqual(1, joinServer.EventBatchList[0].ExperimentalUnitFragments.Count);
Assert.AreEqual(uniqueKey, joinServer.EventBatchList[0].ExperimentalUnitFragments[0].Id);
Assert.IsTrue(joinServer.EventBatchList[0].ExperimentalUnitFragments[0].Value.ToLower().Contains("\"a\":[" + chosenAction + ","));
}
public void TestDevModeSettingsAndExampleLog()
{
joinServer.Reset();
var testTraceListener = new TestTraceListener();
Trace.Listeners.Add(testTraceListener);
string vwArgs = "--cb_explore_adf --epsilon 0.5";
commandCenter.CreateBlobs(createSettingsBlob: true, createModelBlob: false, vwArgs: vwArgs);
var dsConfig = new DecisionServiceConfiguration(MockCommandCenter.SettingsBlobUri)
{
JoinServerType = JoinServerType.CustomSolution,
LoggingServiceAddress = MockJoinServer.MockJoinServerAddress,
PollingForModelPeriod = TimeSpan.MinValue,
PollingForSettingsPeriod = TimeSpan.MinValue,
DevelopmentMode = true
};
int numInteractionEvents = 25;
var eventIdList = new List <string>();
using (var ds = DecisionService
.Create <TestADFContextWithFeatures>(dsConfig)
// .With<TestADFContextWithFeatures, TestADFFeatures>(context => context.ActionDependentFeatures)
// TODO .WithTopSlotEpsilonGreedy(.5f)
.ExploitUntilModelReady(new ConstantPolicy <TestADFContextWithFeatures>(ctx => ctx.ActionDependentFeatures.Count)))
{
byte[] modelContent = commandCenter.GetCBADFModelBlobContent(numExamples: 5, numFeatureVectors: 10, vwDefaultArgs: vwArgs);
using (var modelStream = new MemoryStream(modelContent))
{
ds.UpdateModel(modelStream);
}
for (int i = 1; i <= numInteractionEvents; i++)
{
var interId = "inter" + i;
var obserId = "obser" + i;
Random rg = new Random(i);
int numActions = rg.Next(5, 20);
var context = TestADFContextWithFeatures.CreateRandom(numActions, rg);
int[] action = ds.ChooseRanking(interId, context);
ds.ReportReward(i / 100f, obserId);
eventIdList.Add(interId);
eventIdList.Add(obserId);
}
}
// Number of batches must be exactly equal to number of events uploaded in development mode
// and each batch must contain exactly one event
Assert.AreEqual(numInteractionEvents * 2, joinServer.EventBatchList.Count);
Assert.AreEqual(numInteractionEvents * 2, joinServer.EventBatchList.Sum(ebl => ebl.ExperimentalUnitFragments.Count));
var eblList = joinServer.EventBatchList.Select(ebl => ebl.ExperimentalUnitFragments[0].Id).OrderBy(id => id);
Assert.IsTrue(eblList.SequenceEqual(eventIdList.OrderBy(id => id)));
// Trace messages must contain context information
Assert.AreEqual(numInteractionEvents, testTraceListener.Messages.Count(m => m.Contains("Example Context")));
}
public void TestMultiActionDSUploadSingleEvent()
{
joinServer.Reset();
string uniqueKey = "test interaction";
var dsConfig = new DecisionServiceConfiguration(MockCommandCenter.SettingsBlobUri);
dsConfig.JoinServerType = JoinServerType.CustomSolution;
dsConfig.LoggingServiceAddress = MockJoinServer.MockJoinServerAddress;
int[] chosenActions;
using (var ds = DecisionService.Create <TestContext>(dsConfig)
//.WithTopSlotEpsilonGreedy(.2f)
.ExploitUntilModelReady(new ConstantPolicy <TestContext>()))
{
chosenActions = ds.ChooseRanking(uniqueKey, new TestContext());
}
Assert.AreEqual(1, joinServer.RequestCount);
Assert.AreEqual(1, joinServer.EventBatchList.Count);
Assert.AreEqual(1, joinServer.EventBatchList[0].ExperimentalUnitFragments.Count);
Assert.AreEqual(uniqueKey, joinServer.EventBatchList[0].ExperimentalUnitFragments[0].Id);
Assert.IsTrue(joinServer.EventBatchList[0].ExperimentalUnitFragments[0].Value.ToLower().Contains("\"a\":[" + string.Join(",", chosenActions) + "],"));
}
public async Task TestSingleActionDSUploadMultipleEvents()
{
joinServer.Reset();
string uniqueKey = "test interaction";
var dsConfig = new DecisionServiceConfiguration(MockCommandCenter.SettingsBlobUri);
dsConfig.JoinServerType = JoinServerType.CustomSolution;
dsConfig.LoggingServiceAddress = MockJoinServer.MockJoinServerAddress;
using (var ds = DecisionService
.Create <TestContext>(dsConfig)
.ExploitUntilModelReady(new ConstantPolicy <TestContext>()))
{
int chosenAction1 = await ds.ChooseActionAsync(uniqueKey, new TestContext());
int chosenAction2 = await ds.ChooseActionAsync(uniqueKey, new TestContext());
ds.ReportReward(1.0f, uniqueKey);
ds.ReportOutcome(JsonConvert.SerializeObject(new { value = "test outcome" }), uniqueKey);
}
Assert.AreEqual(4, joinServer.EventBatchList.Sum(batch => batch.ExperimentalUnitFragments.Count));
}
public void TestRcv1ModelUpdateFromStream()
{
joinServer.Reset();
int numActions = 10;
int numFeatures = 1024;
string vwArgs = "--cb_explore 10 --epsilon 0.5";
commandCenter.CreateBlobs(createSettingsBlob: true, createModelBlob: false, vwArgs: vwArgs);
var dsConfig = new DecisionServiceConfiguration(MockCommandCenter.SettingsBlobUri)
//explorer: new EpsilonGreedyExplorer<TestRcv1Context>(new ConstantPolicy<TestRcv1Context>(ctx => ctx.ActionDependentFeatures.Count), epsilon: 0.5f, numActions: (int)numActions))
{
JoinServerType = JoinServerType.CustomSolution,
LoggingServiceAddress = MockJoinServer.MockJoinServerAddress,
PollingForModelPeriod = TimeSpan.MinValue,
PollingForSettingsPeriod = TimeSpan.MinValue
};
using (var ds = DecisionService
.Create <TestRcv1Context>(dsConfig, TypeInspector.Default)
// TODOD: .WithEpsilonGreedy(.5f)
.ExploitUntilModelReady(new ConstantPolicy <TestRcv1Context>(ctx => ctx.Features.Count)))
{
string uniqueKey = "eventid";
for (int i = 1; i <= 100; i++)
{
Random rg = new Random(i);
if (i % 50 == 1)
{
int modelIndex = i / 50;
byte[] modelContent = commandCenter.GetCBModelBlobContent(numExamples: 3 + modelIndex, numFeatures: numFeatures, numActions: numActions, vwArgs: vwArgs);
using (var modelStream = new MemoryStream(modelContent))
{
ds.UpdateModel(modelStream);
}
}
var context = TestRcv1Context.CreateRandom(numActions, numFeatures, rg);
DateTime timeStamp = DateTime.UtcNow;
int action = ds.ChooseAction(uniqueKey, context);
// verify the actions are in the expected range
Assert.IsTrue(action >= 1 && action <= numActions);
ds.ReportReward(i / 100f, uniqueKey);
}
}
Assert.AreEqual(200, joinServer.EventBatchList.Sum(b => b.ExperimentalUnitFragments.Count));
}
public void TestADFModelUpdateFromStream()
{
joinServer.Reset();
string vwArgs = "--cb_explore_adf --epsilon 0.5";
commandCenter.CreateBlobs(createSettingsBlob: true, createModelBlob: false, vwArgs: vwArgs);
var dsConfig = new DecisionServiceConfiguration(MockCommandCenter.SettingsBlobUri)
{
JoinServerType = JoinServerType.CustomSolution,
LoggingServiceAddress = MockJoinServer.MockJoinServerAddress,
PollingForModelPeriod = TimeSpan.MinValue,
PollingForSettingsPeriod = TimeSpan.MinValue
};
using (var ds = DecisionService.Create <TestADFContextWithFeatures>(dsConfig)
.ExploitUntilModelReady(new ConstantPolicy <TestADFContextWithFeatures>(ctx => ctx.ActionDependentFeatures.Count)))
{
string uniqueKey = "eventid";
for (int i = 1; i <= 100; i++)
{
Random rg = new Random(i);
if (i % 50 == 1)
{
int modelIndex = i / 50;
byte[] modelContent = commandCenter.GetCBADFModelBlobContent(numExamples: 3 + modelIndex, numFeatureVectors: 4 + modelIndex, vwDefaultArgs: vwArgs);
using (var modelStream = new MemoryStream(modelContent))
{
ds.UpdateModel(modelStream);
}
}
int numActions = rg.Next(5, 20);
var context = TestADFContextWithFeatures.CreateRandom(numActions, rg);
int[] action = ds.ChooseRanking(uniqueKey, context);
Assert.AreEqual(numActions, action.Length);
// verify all unique actions in the list
Assert.AreEqual(action.Length, action.Distinct().Count());
// verify the actions are in the expected range
Assert.AreEqual((numActions * (numActions + 1)) / 2, action.Sum(a => a));
ds.ReportReward(i / 100f, uniqueKey);
}
}
Assert.AreEqual(200, joinServer.EventBatchList.Sum(b => b.ExperimentalUnitFragments.Count));
}
public void TestMultiActionDSUploadSelective()
{
joinServer.Reset();
string uniqueKey = "test interaction";
var dsConfig = new DecisionServiceConfiguration(MockCommandCenter.SettingsBlobUri);
dsConfig.JoinServerType = JoinServerType.CustomSolution;
dsConfig.LoggingServiceAddress = MockJoinServer.MockJoinServerAddress;
dsConfig.InteractionUploadConfiguration = new BatchingConfiguration();
dsConfig.InteractionUploadConfiguration.MaxDuration = TimeSpan.FromMinutes(10); // allow enough time for queue to buffer events
dsConfig.InteractionUploadConfiguration.MaxDegreeOfSerializationParallelism = 1; // single-threaded for easy verification
int numEvents = 100;
// Set queue capacity to same number of events so selective dropping starts at 50% full
dsConfig.InteractionUploadConfiguration.MaxUploadQueueCapacity = numEvents;
dsConfig.InteractionUploadConfiguration.DroppingPolicy = new DroppingPolicy
{
MaxQueueLevelBeforeDrop = .5f,
// when threshold is reached, drop half of the events
ProbabilityOfDrop = .5f
};
using (var ds = DecisionService.Create <TestContext>(dsConfig)
//.WithTopSlotEpsilonGreedy(.2f)
.ExploitUntilModelReady(new ConstantPolicy <TestContext>()))
{
for (int i = 0; i < numEvents; i++)
{
int[] chosenAction1 = ds.ChooseRanking(uniqueKey, new TestContext());
}
}
// Some events must have been dropped so the total count cannot be same as original
Assert.IsTrue(joinServer.EventBatchList.Sum(batch => batch.ExperimentalUnitFragments.Count) < numEvents);
// Get number of events that have been downsampled, i.e. selected with probability q
int numSampledEvents = joinServer.EventBatchList
.SelectMany(batch => batch.ExperimentalUnitFragments)
.Where(e => e.Value.Contains("\"pdrop\":0.5"))
.Count();
Assert.IsTrue(numSampledEvents > 0);
// half of the events are selected with probability 0.5, so this should definitely be less than half the total events
Assert.IsTrue(numSampledEvents < numEvents / 2);
}
public void TestSingleActionOfflineModeCustomLogger()
{
var dsConfig = new DecisionServiceConfiguration("")
{
OfflineMode = true, OfflineApplicationID = ""
};
var metaData = new ApplicationClientMetadata
{
TrainArguments = "--cb_explore_adf --epsilon 0.3",
InitialExplorationEpsilon = 1f
};
var recorder = new TestLogger();
int numChooseAction = 100;
using (var ds = DecisionService.Create <TestContext>(dsConfig, metaData: metaData)
.WithRecorder(recorder)
.ExploitUntilModelReady(new ConstantPolicy <TestContext>()))
{
for (int i = 0; i < numChooseAction; i++)
{
ds.ChooseAction(i.ToString(), new TestContext());
}
Assert.AreEqual(numChooseAction, recorder.NumRecord);
int numReward = 200;
for (int i = 0; i < numReward; i++)
{
ds.ReportReward(i, i.ToString());
}
Assert.AreEqual(numReward, recorder.NumReward);
int numOutcome = 300;
for (int i = 0; i < numOutcome; i++)
{
ds.ReportOutcome(i.ToString(), i.ToString());
}
Assert.AreEqual(numOutcome, recorder.NumOutcome);
}
Assert.AreEqual(0, recorder.NumRecord);
Assert.AreEqual(0, recorder.NumReward);
Assert.AreEqual(0, recorder.NumOutcome);
}
public async Task TestNoModelFoundForImmediateDownload()
{
// create mock blobs for settings and models
this.commandCenter.CreateBlobs(createSettingsBlob: true, createModelBlob: false, vwArgs: "--cb_explore_adf --epsilon 0.2");
var serviceConfig = new DecisionServiceConfiguration(MockCommandCenter.SettingsBlobUri)
{
JoinServerType = JoinServerType.CustomSolution,
LoggingServiceAddress = MockJoinServer.MockJoinServerAddress,
};
using (var service = DecisionService.Create <TestADFContextWithFeatures>(serviceConfig))
{
await service.DownloadModelAndUpdate(new System.Threading.CancellationToken());
}
}
public void TestMultiActionOnlineModeCustomLogger()
{
joinServer.Reset();
var dsConfig = new DecisionServiceConfiguration(MockCommandCenter.SettingsBlobUri);
var recorder = new TestLogger();
dsConfig.PollingForModelPeriod = TimeSpan.MinValue;
dsConfig.PollingForSettingsPeriod = TimeSpan.MinValue;
dsConfig.JoinServerType = JoinServerType.CustomSolution;
int numChooseAction = 100;
using (var ds = DecisionService.Create <TestContext>(dsConfig, JsonTypeInspector.Default)
//.WithTopSlotEpsilonGreedy(.2f)
.WithRecorder(recorder)
.ExploitUntilModelReady(new ConstantPolicy <TestContext>()))
{
for (int i = 0; i < numChooseAction; i++)
{
ds.ChooseAction(i.ToString(), new TestContext());
}
Assert.AreEqual(numChooseAction, recorder.NumRecord);
int numReward = 200;
for (int i = 0; i < numReward; i++)
{
ds.ReportReward(i, i.ToString());
}
Assert.AreEqual(numReward, recorder.NumReward);
int numOutcome = 300;
for (int i = 0; i < numOutcome; i++)
{
ds.ReportOutcome(i.ToString(), i.ToString());
}
Assert.AreEqual(numOutcome, recorder.NumOutcome);
}
Assert.AreEqual(0, recorder.NumRecord);
Assert.AreEqual(0, recorder.NumReward);
Assert.AreEqual(0, recorder.NumOutcome);
}
public void TestSingleActionOnlineModeCustomLogger()
{
var dsConfig = new DecisionServiceConfiguration(MockCommandCenter.SettingsBlobUri)
{
JoinServerType = JoinServerType.CustomSolution
};
var recorder = new TestLogger();
int numChooseAction = 100;
using (var ds = DecisionService
.Create <TestContext>(dsConfig)
// TODO: .WithEpsilonGreedy(.2f)
.WithRecorder(recorder)
.ExploitUntilModelReady(new ConstantPolicy <TestContext>()))
{
for (int i = 0; i < numChooseAction; i++)
{
ds.ChooseAction(i.ToString(), new TestContext());
}
Assert.AreEqual(numChooseAction, recorder.NumRecord);
int numReward = 200;
for (int i = 0; i < numReward; i++)
{
ds.ReportReward(i, i.ToString());
}
Assert.AreEqual(numReward, recorder.NumReward);
int numOutcome = 300;
for (int i = 0; i < numOutcome; i++)
{
ds.ReportOutcome(i.ToString(), i.ToString());
}
Assert.AreEqual(numOutcome, recorder.NumOutcome);
}
Assert.AreEqual(0, recorder.NumRecord);
Assert.AreEqual(0, recorder.NumReward);
Assert.AreEqual(0, recorder.NumOutcome);
}
public void TestModelImmediateDownload()
{
// create mock blobs for settings and models
this.commandCenter.CreateBlobs(createSettingsBlob: true, createModelBlob: true, vwArgs: "--cb_explore_adf --epsilon 0.2");
var serviceConfig = new DecisionServiceConfiguration(MockCommandCenter.SettingsBlobUri)
{
JoinServerType = JoinServerType.CustomSolution,
LoggingServiceAddress = MockJoinServer.MockJoinServerAddress,
};
using (var service = DecisionService.Create <TestADFContextWithFeatures>(serviceConfig))
{
// download model right away
service.DownloadModelAndUpdate(new System.Threading.CancellationToken()).Wait();
}
// No exception if everything works
}
public void TestSingleActionOfflineModeArgument()
{
var dsConfig = new DecisionServiceConfiguration("")
{
OfflineMode = true, OfflineApplicationID = ""
};
try
{
var metaData = new ApplicationClientMetadata
{
TrainArguments = "--cb_explore_adf --epsilon 0.3",
InitialExplorationEpsilon = 1f
};
using (var ds = DecisionService.Create <TestContext>(dsConfig, metaData: metaData))
{ }
}
catch (ArgumentException ex)
{
Assert.AreEqual("Recorder", ex.ParamName);
}
}
public void TestMultiActionDSUploadMultipleEvents()
{
joinServer.Reset();
string uniqueKey = "test interaction";
var dsConfig = new DecisionServiceConfiguration(MockCommandCenter.SettingsBlobUri);
dsConfig.JoinServerType = JoinServerType.CustomSolution;
dsConfig.LoggingServiceAddress = MockJoinServer.MockJoinServerAddress;
using (var ds = DecisionService.Create <TestContext>(dsConfig)
//.WithTopSlotEpsilonGreedy(.2f)
.ExploitUntilModelReady(new ConstantPolicy <TestContext>()))
{
int[] chosenAction1 = ds.ChooseRanking(uniqueKey, new TestContext());
int[] chosenAction2 = ds.ChooseRanking(uniqueKey, new TestContext());
ds.ReportReward(1.0f, uniqueKey);
ds.ReportOutcome(new { value = "test outcome" }, uniqueKey);
}
Assert.AreEqual(4, joinServer.EventBatchList.Sum(batch => batch.ExperimentalUnitFragments.Count));
}
public void TestADFExplorationResult()
{
joinServer.Reset();
commandCenter.CreateBlobs(createSettingsBlob: true, createModelBlob: false, vwArgs: "--cb_explore_adf --epsilon 0.5");
var dsConfig = new DecisionServiceConfiguration(MockCommandCenter.SettingsBlobUri)
{
PollingForModelPeriod = TimeSpan.MinValue,
PollingForSettingsPeriod = TimeSpan.MinValue,
JoinServerType = JoinServerType.CustomSolution,
LoggingServiceAddress = MockJoinServer.MockJoinServerAddress
};
using (var ds = DecisionService.Create <TestADFContext>(dsConfig)
.ExploitUntilModelReady(new ConstantPolicy <TestADFContext>(ctx => ctx.ActionDependentFeatures.Count)))
{
string uniqueKey = "eventid";
for (int i = 1; i <= 100; i++)
{
var adfContext = new TestADFContext(i);
int[] action = ds.ChooseRanking(uniqueKey, adfContext);
Assert.AreEqual(i, action.Length);
// verify all unique actions in the list
Assert.AreEqual(action.Length, action.Distinct().Count());
// verify the actions are in the expected range
Assert.AreEqual((i * (i + 1)) / 2, action.Sum(a => a));
ds.ReportReward(i / 100f, uniqueKey);
}
}
Assert.AreEqual(200, joinServer.EventBatchList.Sum(b => b.ExperimentalUnitFragments.Count));
}
public void TestSingleActionDSThreadSafeUpload()
{
joinServer.Reset();
string uniqueKey = "test interaction";
commandCenter.CreateBlobs(createSettingsBlob: true, createModelBlob: false, vwArgs: "--cb_explore_adf --epsilon 0.2");
var createObservation = (Func <int, string>)((i) => { return(string.Format("00000", i)); });
var dsConfig = new DecisionServiceConfiguration(MockCommandCenter.SettingsBlobUri);
dsConfig.JoinServerType = JoinServerType.CustomSolution;
dsConfig.LoggingServiceAddress = MockJoinServer.MockJoinServerAddress;
int numEvents = 1000;
var chosenActions = new ConcurrentBag <int>();
using (var ds = DecisionService
.Create <TestContext>(dsConfig)
.ExploitUntilModelReady(new ConstantPolicy <TestContext>()))
{
Parallel.For(0, numEvents, new ParallelOptions {
MaxDegreeOfParallelism = Environment.ProcessorCount * 2
}, (i) =>
{
chosenActions.Add(ds.ChooseAction(uniqueKey, new TestContext()));
ds.ReportOutcome(new { value = createObservation((int)i) }, uniqueKey);
});
}
List <PartialDecisionServiceMessage> batchList = this.joinServer.EventBatchList;
int numActualEvents = batchList.Sum(b => b.ExperimentalUnitFragments.Count);
Assert.AreEqual(numEvents * 2, numActualEvents);
List <string> uniqueKeys = batchList
.SelectMany(b => b.ExperimentalUnitFragments.Select(f => f.Id))
.Distinct()
.ToList();
Assert.AreEqual(1, uniqueKeys.Count);
Assert.AreEqual(uniqueKey, uniqueKeys[0]);
var completeFragments = batchList
.SelectMany(b => b.ExperimentalUnitFragments
.Select(f => JsonConvert.DeserializeObject <MultiActionCompleteExperimentalUnitFragment>(f.Value)));
// Test actual interactions received
List <MultiActionCompleteExperimentalUnitFragment> interactions = completeFragments
.Where(f => f.Value == null)
.OrderBy(f => f.Actions[0])
.ToList();
// Test values of the interactions
Assert.AreEqual(numEvents, interactions.Count);
var chosenActionList = chosenActions.OrderBy(a => a).ToList();
for (int i = 0; i < interactions.Count; i++)
{
Assert.AreEqual((int)chosenActionList[i], interactions[i].Actions[0]);
}
// Test actual observations received
List <MultiActionCompleteExperimentalUnitFragment> observations = completeFragments
.Where(f => f.Value != null)
.OrderBy(f => f.Value)
.ToList();
// Test values of the observations
Assert.AreEqual(numEvents, observations.Count);
for (int i = 0; i < observations.Count; i++)
{
Assert.AreEqual(JsonConvert.SerializeObject(new { value = createObservation(i) }), observations[i].Value);
}
}
/// <summary>
/// Sample code simulating a news recommendation scenario. In this simple example,
/// the rendering server has to pick 1 out of 4 news topics to show to users (e.g. as featured article).
/// </summary>
/// <remarks>
/// NOTE: For this sample to work, the proper settings must be set at deployment time:
/// Vowpal Wabbit Switches = --cb_explore_adf --epsilon 0.2 --cb_type dr
/// </remarks>
public static async Task NewsRecommendation()
{
if (String.IsNullOrWhiteSpace(SettingsBlobUri))
{
Console.WriteLine("Please specify a valid settings URL.");
return;
}
// Create configuration for the decision service
var serviceConfig = new DecisionServiceConfiguration(settingsBlobUri: SettingsBlobUri);
// Enable development mode to easily debug / diagnose data flow and system properties
// This should be turned off in production deployment
serviceConfig.DevelopmentMode = true;
using (var service = DecisionService.Create <UserContextADF>(serviceConfig))
{
var random = new Random();
int user = 0; // user id
int maxDecisionHistory = 100; // max number of past decisions to record
var correctDecisions = new Queue <int>(); // keeps track of past decisions
Console.WriteLine("Press Ctrl + C at any time to cancel the process.");
// Each topic has its own set of features
var topicFeatures = Enumerable.Range(1, 10)
.Select(_ => new TopicFeature
{
Features = Enumerable.Range(1, 10).Select(f => (float)random.NextDouble()).ToArray()
})
.ToArray();
while (true)
{
user++;
string uniqueKey = Guid.NewGuid().ToString();
var userContext = new UserContextADF
{
Gender = random.NextDouble() > 0.5 ? "male" : "female",
TopicFeatures = topicFeatures.Take(random.Next(5, topicFeatures.Length)).ToArray()
};
int[] topicRanking = await service.ChooseRankingAsync(uniqueKey, userContext);
int topicId = topicRanking[0];
// Display the news topic chosen by exploration process.
Console.WriteLine("Topic {0} was chosen for user {1}.", topicId, user + 1);
// Report {0,1} reward as a simple float.
// In a real scenario, one could associated a reward of 1 if user
// clicks on the article and 0 otherwise.
float reward = 0;
if (userContext.Gender == "male" && topicId == 3)
{
reward = 1;
}
else if (userContext.Gender == "female" && topicId == 8)
{
reward = 1;
}
service.ReportReward(reward, uniqueKey);
correctDecisions.Enqueue((int)reward);
if (correctDecisions.Count == maxDecisionHistory)
{
correctDecisions.Dequeue();
}
if (user % 50 == 0)
{
Console.WriteLine("Correct decisions out of last {0} interactions: {1}", maxDecisionHistory, correctDecisions.Sum());
}
System.Threading.Thread.Sleep(TimeSpan.FromMilliseconds(50));
}
}
}
private float UploadFoodContextData(string settingsBlobUri, bool firstPass)
{
var serviceConfig = new DecisionServiceConfiguration(settingsBlobUri);
if (firstPass)
{
serviceConfig.PollingForModelPeriod = TimeSpan.MinValue;
this.OnlineTrainerReset();
}
using (var service = DecisionService.Create <FoodContext>(serviceConfig))
{
if (!firstPass)
{
Thread.Sleep(10000);
}
string uniqueKey = "scratch-key-gal";
string[] locations = { "HealthyTown", "LessHealthyTown" };
var rg = new Random(uniqueKey.GetHashCode());
int numActions = 3; // ["Hamburger deal 1", "Hamburger deal 2" (better), "Salad deal"]
var csv = new StringBuilder();
int counterCorrect = 0;
int counterTotal = 0;
var header = "Location,Action,Reward";
csv.AppendLine(header);
// number of iterations
for (int i = 0; i < 10000 * locations.Length; i++)
{
// randomly select a location
int iL = rg.Next(0, locations.Length);
string location = locations[iL];
DateTime timeStamp = DateTime.UtcNow;
string key = uniqueKey + Guid.NewGuid().ToString();
FoodContext currentContext = new FoodContext();
currentContext.UserLocation = location;
currentContext.Actions = Enumerable.Range(1, numActions).ToArray();
int[] action = service.ChooseRanking(key, currentContext);
counterTotal += 1;
// We expect healthy town to get salad and unhealthy town to get the second burger (action 2)
if (location.Equals("HealthyTown") && action[0] == 3)
{
counterCorrect += 1;
}
else if (location.Equals("LessHealthyTown") && action[0] == 2)
{
counterCorrect += 1;
}
var csvLocation = location;
var csvAction = action[0].ToString();
float reward = 0;
double currentRand = rg.NextDouble();
if (location.Equals("HealthyTown"))
{
// for healthy town, buy burger 1 with probability 0.1, burger 2 with probability 0.15, salad with probability 0.6
if ((action[0] == 1 && currentRand < 0.1) ||
(action[0] == 2 && currentRand < 0.15) ||
(action[0] == 3 && currentRand < 0.6))
{
reward = 10;
}
}
else
{
// for unhealthy town, buy burger 1 with probability 0.4, burger 2 with probability 0.6, salad with probability 0.2
if ((action[0] == 1 && currentRand < 0.4) ||
(action[0] == 2 && currentRand < 0.6) ||
(action[0] == 3 && currentRand < 0.2))
{
reward = 10;
}
}
service.ReportReward(reward, key);
var newLine = string.Format("{0},{1},{2}", csvLocation, csvAction, "0");
csv.AppendLine(newLine);
System.Threading.Thread.Sleep(1);
}
return((float)counterCorrect / counterTotal);
}
}
private static async Task MainAsync()
{
// Requires Visual C++ 2015 Runtime installed
// - https://www.microsoft.com/en-us/download/details.aspx?id=48145
// Requires platform to be x64
// 1. Sign in on https://ds.microsoft.com
// 2. Create new app
// a. Select "Custom App"
// b. Supply Azure storage account
// c. Empty Action Endpoint field - this disables action id based featurization
// d. Empty Action Set Endpoint field
// e. Set Provision FrontEnd field to "false" - avoids HTTP scoring provisioning
// f. Set Vowpal Wabbit arguments to --cb_explore_adf --epsilon 0.2 -q DT -q LT
// f. Click create
// 3. Wait for creation and hit edit
// a. Copy Client Library URL and use in the sample code below
// 4. nuget install Microsoft.Research.MultiWorldTesting.ClientLibrary
// a. at least version 2.0.0.6
var config = new DecisionServiceConfiguration(settingsBlobUri: "<<INSERT URL FROM STEP 3A")
{
ModelPollFailureCallback = ex => Console.WriteLine(ex.Message),
SettingsPollFailureCallback = ex => Console.WriteLine(ex.Message)
};
// you should keep this object around (e.g. static variable) as it maintains the background process
// to download new models and upload logged events
using (var ds = DecisionService.Create <DecisionContext>(config, typeInspector: JsonTypeInspector.Default))
{
// If you add the timestamp calculating reward latencies becomes very easy on our end
// In general you can use existing request ids, just be careful as they're used to seed the random
// generator for exploration.
// Also they're used to join rank and reward calls, so they must be unique within the experimental unit duration
var eventId = EventIdUtil.AddTimestamp(Guid.NewGuid().ToString("n"), EventIdUtil.BeginningOfTime);
var context = new DecisionContext
{
Demographics = new DemographicNamespace
{
Gender = "female",
},
Location = new LocationNamespace
{
Country = "USA",
City = "New York"
},
Actions = new[]
{
new ActionDependentFeatures
{
Topic = new TopicNamespace
{
Category = "Fashion"
}
},
new ActionDependentFeatures
{
Topic = new TopicNamespace
{
Category = "Technology"
}
}
}
};
// get the decision
var ranking = await ds.ChooseRankingAsync(
uniqueKey : eventId,
context : context);
// report the reward if you observed desired behavior on first item in ranking
// e.g. the first item returned by ChooseRankingAsync got clicked.
ds.ReportReward(reward: 1f, uniqueKey: eventId);
}
}
public static async Task Sample()
{
// 1. Sign in on https://ds.microsoft.com
// 2. Create new app
// a. Select “Custom App”
// b. Supply Azure storage account
// c. Empty Action Endpoint field - this disables action id based featurization
// d. Empty Action Set Endpoint field
// e. Set Provision FrontEnd field to “false” - avoids HTTP scoring provisioning
// f. Set Vowpal Wabbit arguments to --cb_explore_adf --epsilon 0.2 -q gc -q gC -q dC
// f. Click create
// 3. Wait for creation and hit edit
// a. Copy Client Library URL and use in the sample code below
// 4. nuget install Microsoft.Research.MultiWorldTesting.ClientLibrary
// a. at least version 2.0.0.6
var config = new DecisionServiceConfiguration(settingsBlobUri: "<<INSERT URL FROM STEP 3A")
{
ModelPollFailureCallback = ex => Console.WriteLine(ex.Message),
SettingsPollFailureCallback = ex => Console.WriteLine(ex.Message)
};
// you should keep this object around (e.g. static variable) as it maintains the background process
// to download new models and upload logged events
var ds = DecisionService.Create <MyDecisionContext>(config, typeInspector: JsonTypeInspector.Default);
// Some sample data
var context = new MyDecisionContext
{
Demographics = new MyDecisionContextDemographics
{
Age = 15,
AgeGroup = "Teenager",
Gender = "male"
},
Geo = new MyDecisionContextGeo
{
City = "New York",
Country = "USA"
},
Actions = new[]
{
new MyDecisionContextAction
{
Categories = new MyDecisionContextActionCategorization {
Categories = "Fashion OldSchool"
},
Celebrities = new Dictionary <string, float>
{
{ "Markus", 0.9f },
{ "Bob", 0.1f }
}
},
new MyDecisionContextAction
{
Categories = new MyDecisionContextActionCategorization {
Categories = "Tech Microsoft"
},
Celebrities = new Dictionary <string, float>
{
{ "Bill", 0.7f },
{ "Bob", 0.2f }
}
}
}
};
for (int i = 0; i < 5; i++)
{
// this is used as both correlation event id to link the ranking request to the observerd outcome
// as well as random seed for the pseudo random generator
var eventId = Guid.NewGuid().ToString("n");
// get the decision
var ranking = await ds.ChooseRankingAsync(
uniqueKey : eventId,
context : context);
// report the reward if you observed desired behavior on first item in ranking
// e.g. the first item returned by ChooseRankingAsync got clicked.
ds.ReportReward(reward: 1f, uniqueKey: eventId);
}
Console.WriteLine("done");
Thread.Sleep(TimeSpan.FromHours(1));
Console.WriteLine();
}
/// <summary>
/// Sample code simulating a news recommendation scenario. In this simple example,
/// the rendering server has to pick 1 out of 10 news topics to show to users (e.g. as featured article).
/// </summary>
/// <remarks>
/// NOTE: For this sample to work, the proper settings must be set at deployment time:
/// Number Of Actions = 10
/// Vowpal Wabbit Switches = --cb_explore 10 --epsilon 0.2 --cb_type dr
/// </remarks>
public static void NewsRecommendation()
{
if (String.IsNullOrWhiteSpace(SettingsBlobUri))
{
Console.WriteLine("Please specify a valid settings URL.");
return;
}
int numTopics = 10; // number of different topic choices to show
// Create configuration for the decision service.
var serviceConfig = new DecisionServiceConfiguration(settingsBlobUri: SettingsBlobUri);
// Enable development mode to easily debug / diagnose data flow and system properties
// This should be turned off in production deployment
serviceConfig.DevelopmentMode = true;
// Create the main service object with above configurations.
// Specify the exploration algorithm to use, here we will use Epsilon-Greedy.
// For more details about this and other algorithms, refer to the MWT onboarding whitepaper.
using (var service = DecisionService.Create <UserContext>(serviceConfig))
{
// Create a default policy which is used before a machine-learned model.
// This policy is used in epsilon-greedy exploration using the initial
// exploration epsilon specified at deployment time or in the Management Center.
var defaultPolicy = new NewsDisplayPolicy(numTopics);
var random = new Random();
int user = 0;
Console.WriteLine("Press Ctrl + C at any time to cancel the process.");
int maxDecisionHistory = 100;
var correctDecisions = new Queue <int>();
while (true)
{
user++;
// Generate a random GUID id for each user.
var uniqueKey = Guid.NewGuid().ToString();
// Generate random feature vector for each user.
var userContext = new UserContext()
{
Gender = random.NextDouble() > 0.5 ? "male" : "female"
};
for (int f = 1; f <= 10; f++)
{
userContext.Features.Add(f.ToString(), (float)random.NextDouble());
}
// Perform exploration given user features.
int topicId = service.ChooseAction(uniqueKey, userContext, defaultPolicy);
// Display the news topic chosen by exploration process.
Console.WriteLine("Topic {0} was chosen for user {1}.", topicId, user + 1);
// Report {0,1} reward as a simple float.
// In a real scenario, one could associated a reward of 1 if user
// clicks on the article and 0 otherwise.
float reward = 0;
if (userContext.Gender == "male" && topicId == 3)
{
reward = 1;
}
else if (userContext.Gender == "female" && topicId == 8)
{
reward = 1;
}
service.ReportReward(reward, uniqueKey);
correctDecisions.Enqueue((int)reward);
if (correctDecisions.Count == maxDecisionHistory)
{
correctDecisions.Dequeue();
}
if (user % 50 == 0)
{
Console.WriteLine("Correct decisions out of last {0} interactions: {1}", maxDecisionHistory, correctDecisions.Sum());
}
System.Threading.Thread.Sleep(TimeSpan.FromMilliseconds(50));
}
}
}
