public void IndexAsync()
{
AsyncManager.OutstandingOperations.Increment();
var explorer = new EpsilonGreedyExplorer <string>(new MartPolicy(), epsilon: .2f, numActions: 10);
var serviceConfig = new DecisionServiceConfiguration <string>(
authorizationToken: "c01ff675-5710-4814-a961-d03d2d6bce65",
explorer: explorer);
var service = new DecisionService <string>(serviceConfig);
var rand = new Random();
for (int i = 0; i < 10; i++)
{
int context = rand.Next(100);
string uniqueKey = i.ToString();
service.ChooseAction(uniqueKey, context.ToString());
service.ReportReward((float)(context % 2), uniqueKey);
}
service.Flush();
AsyncManager.OutstandingOperations.Decrement();
}
public void TestSingleActionDSUploadSingleEvent()
{
joinServer.Reset();
string uniqueKey = "test interaction";
commandCenter.CreateBlobs(createSettingsBlob: true, createModelBlob: false, vwArgs: "--cb_explore_adf --epsilon 0.5");
var dsConfig = new DecisionServiceConfiguration(MockCommandCenter.SettingsBlobUri);
dsConfig.JoinServerType = JoinServerType.CustomSolution;
dsConfig.LoggingServiceAddress = MockJoinServer.MockJoinServerAddress;
int chosenAction;
using (var ds = DecisionService
.Create <TestContext>(dsConfig)
.ExploitUntilModelReady(new ConstantPolicy <TestContext>()))
{
chosenAction = ds.ChooseAction(uniqueKey, new TestContext());
}
Assert.AreEqual(1, joinServer.RequestCount);
Assert.AreEqual(1, joinServer.EventBatchList.Count);
Assert.AreEqual(1, joinServer.EventBatchList[0].ExperimentalUnitFragments.Count);
Assert.AreEqual(uniqueKey, joinServer.EventBatchList[0].ExperimentalUnitFragments[0].Id);
Assert.IsTrue(joinServer.EventBatchList[0].ExperimentalUnitFragments[0].Value.ToLower().Contains("\"a\":[" + chosenAction + ","));
}
public void TestMultiActionDSUploadSingleEvent()
{
joinServer.Reset();
string uniqueKey = "test interaction";
var dsConfig = new DecisionServiceConfiguration(MockCommandCenter.SettingsBlobUri);
dsConfig.JoinServerType = JoinServerType.CustomSolution;
dsConfig.LoggingServiceAddress = MockJoinServer.MockJoinServerAddress;
int[] chosenActions;
using (var ds = DecisionService.Create <TestContext>(dsConfig)
//.WithTopSlotEpsilonGreedy(.2f)
.ExploitUntilModelReady(new ConstantPolicy <TestContext>()))
{
chosenActions = ds.ChooseRanking(uniqueKey, new TestContext());
}
Assert.AreEqual(1, joinServer.RequestCount);
Assert.AreEqual(1, joinServer.EventBatchList.Count);
Assert.AreEqual(1, joinServer.EventBatchList[0].ExperimentalUnitFragments.Count);
Assert.AreEqual(uniqueKey, joinServer.EventBatchList[0].ExperimentalUnitFragments[0].Id);
Assert.IsTrue(joinServer.EventBatchList[0].ExperimentalUnitFragments[0].Value.ToLower().Contains("\"a\":[" + string.Join(",", chosenActions) + "],"));
}
public void TestBatchingByTime()
{
var serviceConfig = new DecisionServiceConfiguration<TestContext>("",
new EpsilonGreedyExplorer<TestContext>(new TestPolicy(), epsilon: 0.2f, numActions: 10))
{
BatchConfig = new BatchingConfiguration()
{
MaxDuration = TimeSpan.FromMilliseconds(100),
MaxEventCount = 2,
MaxBufferSizeInBytes = 10 * 1024 * 1024
}
};
var service = new DecisionService<TestContext>(serviceConfig);
string uniqueKey = "eventid";
service.ChooseAction(uniqueKey, new TestContext());
System.Threading.Thread.Sleep(200);
service.ChooseAction(uniqueKey, new TestContext());
var batch = JsonConvert.DeserializeObject<EventBatch>(File.ReadAllText(batchOutputFile), new EventJsonConverter());
Assert.AreEqual(1, batch.Events.Count);
}
public async Task TestSingleActionDSUploadMultipleEvents()
{
joinServer.Reset();
string uniqueKey = "test interaction";
var dsConfig = new DecisionServiceConfiguration(MockCommandCenter.SettingsBlobUri);
dsConfig.JoinServerType = JoinServerType.CustomSolution;
dsConfig.LoggingServiceAddress = MockJoinServer.MockJoinServerAddress;
using (var ds = DecisionService
.Create <TestContext>(dsConfig)
.ExploitUntilModelReady(new ConstantPolicy <TestContext>()))
{
int chosenAction1 = await ds.ChooseActionAsync(uniqueKey, new TestContext());
int chosenAction2 = await ds.ChooseActionAsync(uniqueKey, new TestContext());
ds.ReportReward(1.0f, uniqueKey);
ds.ReportOutcome(JsonConvert.SerializeObject(new { value = "test outcome" }), uniqueKey);
}
Assert.AreEqual(4, joinServer.EventBatchList.Sum(batch => batch.ExperimentalUnitFragments.Count));
}
public void TestDevModeSettingsAndExampleLog()
{
joinServer.Reset();
var testTraceListener = new TestTraceListener();
Trace.Listeners.Add(testTraceListener);
string vwArgs = "--cb_explore_adf --epsilon 0.5";
commandCenter.CreateBlobs(createSettingsBlob: true, createModelBlob: false, vwArgs: vwArgs);
var dsConfig = new DecisionServiceConfiguration(MockCommandCenter.SettingsBlobUri)
{
JoinServerType = JoinServerType.CustomSolution,
LoggingServiceAddress = MockJoinServer.MockJoinServerAddress,
PollingForModelPeriod = TimeSpan.MinValue,
PollingForSettingsPeriod = TimeSpan.MinValue,
DevelopmentMode = true
};
int numInteractionEvents = 25;
var eventIdList = new List <string>();
using (var ds = DecisionService
.Create <TestADFContextWithFeatures>(dsConfig)
// .With<TestADFContextWithFeatures, TestADFFeatures>(context => context.ActionDependentFeatures)
// TODO .WithTopSlotEpsilonGreedy(.5f)
.ExploitUntilModelReady(new ConstantPolicy <TestADFContextWithFeatures>(ctx => ctx.ActionDependentFeatures.Count)))
{
byte[] modelContent = commandCenter.GetCBADFModelBlobContent(numExamples: 5, numFeatureVectors: 10, vwDefaultArgs: vwArgs);
using (var modelStream = new MemoryStream(modelContent))
{
ds.UpdateModel(modelStream);
}
for (int i = 1; i <= numInteractionEvents; i++)
{
var interId = "inter" + i;
var obserId = "obser" + i;
Random rg = new Random(i);
int numActions = rg.Next(5, 20);
var context = TestADFContextWithFeatures.CreateRandom(numActions, rg);
int[] action = ds.ChooseRanking(interId, context);
ds.ReportReward(i / 100f, obserId);
eventIdList.Add(interId);
eventIdList.Add(obserId);
}
}
// Number of batches must be exactly equal to number of events uploaded in development mode
// and each batch must contain exactly one event
Assert.AreEqual(numInteractionEvents * 2, joinServer.EventBatchList.Count);
Assert.AreEqual(numInteractionEvents * 2, joinServer.EventBatchList.Sum(ebl => ebl.ExperimentalUnitFragments.Count));
var eblList = joinServer.EventBatchList.Select(ebl => ebl.ExperimentalUnitFragments[0].Id).OrderBy(id => id);
Assert.IsTrue(eblList.SequenceEqual(eventIdList.OrderBy(id => id)));
// Trace messages must contain context information
Assert.AreEqual(numInteractionEvents, testTraceListener.Messages.Count(m => m.Contains("Example Context")));
}
public static void Create(uint numActions, string modelOutputDir, int policyAction)
{
if (Explorer == null)
{
Explorer = new EpsilonGreedyExplorer <TContext>(new MartPolicy <TContext>(policyAction), LoadSettings().Epsilon, numActions);
}
if (Configuration == null)
{
Configuration = new DecisionServiceConfiguration <TContext>(appToken, Explorer)
{
BlobOutputDir = modelOutputDir,
BatchConfig = new BatchingConfiguration
{
MaxDuration = TimeSpan.FromSeconds(2),
MaxBufferSizeInBytes = 1024,
MaxEventCount = 100,
MaxUploadQueueCapacity = 4,
UploadRetryPolicy = BatchUploadRetryPolicy.Retry
}
};
}
if (Service == null)
{
Service = new DecisionService <TContext>(Configuration);
}
if (!File.Exists(settingsFile))
{
File.WriteAllText(settingsFile, JsonConvert.SerializeObject(new DecisionServiceSettings()));
}
}
public static void SampleCodeUsingASAJoinServer()
{
// Create configuration for the decision service
var serviceConfig = new DecisionServiceConfiguration<ExpandedContext, ExpandedActionDependentFeatures>(
authorizationToken: "sample-code",
explorer: new EpsilonGreedyExplorer<ExpandedContext>(new ExpandedPolicy(), epsilon: 0.8f))
{
EventHubConnectionString = "",
EventHubInputName = "",
GetContextFeaturesFunc = ExpandedContext.GetFeaturesFromContext
};
var service = new DecisionService<ExpandedContext, ExpandedActionDependentFeatures>(serviceConfig);
//string uniqueKey = "sample-asa-client-";
string uniqueKey = "scratch-key-";
var rg = new Random(uniqueKey.GetHashCode());
for (int i = 1; i < 20; i++)
{
int numActions = rg.Next(5, 10);
DateTime timeStamp = DateTime.UtcNow;
string key = uniqueKey + Guid.NewGuid().ToString();
uint[] action = service.ChooseAction(new UniqueEventID { Key = key, TimeStamp = timeStamp }, ExpandedContext.CreateRandom(numActions, rg), (uint)numActions);
service.ReportReward(i / 100f, new UniqueEventID { Key = key, TimeStamp = timeStamp });
System.Threading.Thread.Sleep(1);
}
service.Flush();
}
public void TestSettingsBlobPolling()
{
joinServer.Reset();
commandCenter.CreateBlobs(createSettingsBlob: true, createModelBlob: false, vwArgs: "--cb_explore 2");
var dsConfig = new DecisionServiceConfiguration(MockCommandCenter.SettingsBlobUri);
dsConfig.JoinServerType = JoinServerType.CustomSolution;
dsConfig.LoggingServiceAddress = MockJoinServer.MockJoinServerAddress;
dsConfig.PollingForSettingsPeriod = TimeSpan.FromMilliseconds(500);
using (var ds = DecisionService
.Create <TestContext>(dsConfig)
// TODO .WithTopSlotEpsilonGreedy(.2f)
.ExploitUntilModelReady(new ConstantPolicy <TestContext>()))
{
byte[] settingsBytes = null;
dsConfig.SettingsPollSuccessCallback = data => settingsBytes = data;
for (int i = 0; i < 50 && settingsBytes == null; i++)
{
Thread.Sleep(100);
if (i % 2 == 0)
{
// change the settings blob's etag frequently to make sure polling detects it
commandCenter.CreateBlobs(createSettingsBlob: true, createModelBlob: false, vwArgs: "abc", initialExplorationEpsilon: 0.4f);
}
}
Assert.IsNotNull(settingsBytes);
Assert.IsTrue(Enumerable.SequenceEqual(settingsBytes, commandCenter.GetSettingsBlobContent("abc", 0.4f)));
}
}
public void TestRcv1ModelUpdateFromStream()
{
joinServer.Reset();
int numActions = 10;
int numFeatures = 1024;
string vwArgs = "--cb_explore 10 --epsilon 0.5";
commandCenter.CreateBlobs(createSettingsBlob: true, createModelBlob: false, vwArgs: vwArgs);
var dsConfig = new DecisionServiceConfiguration(MockCommandCenter.SettingsBlobUri)
//explorer: new EpsilonGreedyExplorer<TestRcv1Context>(new ConstantPolicy<TestRcv1Context>(ctx => ctx.ActionDependentFeatures.Count), epsilon: 0.5f, numActions: (int)numActions))
{
JoinServerType = JoinServerType.CustomSolution,
LoggingServiceAddress = MockJoinServer.MockJoinServerAddress,
PollingForModelPeriod = TimeSpan.MinValue,
PollingForSettingsPeriod = TimeSpan.MinValue
};
using (var ds = DecisionService
.Create <TestRcv1Context>(dsConfig, TypeInspector.Default)
// TODOD: .WithEpsilonGreedy(.5f)
.ExploitUntilModelReady(new ConstantPolicy <TestRcv1Context>(ctx => ctx.Features.Count)))
{
string uniqueKey = "eventid";
for (int i = 1; i <= 100; i++)
{
Random rg = new Random(i);
if (i % 50 == 1)
{
int modelIndex = i / 50;
byte[] modelContent = commandCenter.GetCBModelBlobContent(numExamples: 3 + modelIndex, numFeatures: numFeatures, numActions: numActions, vwArgs: vwArgs);
using (var modelStream = new MemoryStream(modelContent))
{
ds.UpdateModel(modelStream);
}
}
var context = TestRcv1Context.CreateRandom(numActions, numFeatures, rg);
DateTime timeStamp = DateTime.UtcNow;
int action = ds.ChooseAction(uniqueKey, context);
// verify the actions are in the expected range
Assert.IsTrue(action >= 1 && action <= numActions);
ds.ReportReward(i / 100f, uniqueKey);
}
}
Assert.AreEqual(200, joinServer.EventBatchList.Sum(b => b.ExperimentalUnitFragments.Count));
}
public void TestADFModelUpdateFromStream()
{
joinServer.Reset();
string vwArgs = "--cb_explore_adf --epsilon 0.5";
commandCenter.CreateBlobs(createSettingsBlob: true, createModelBlob: false, vwArgs: vwArgs);
var dsConfig = new DecisionServiceConfiguration(MockCommandCenter.SettingsBlobUri)
{
JoinServerType = JoinServerType.CustomSolution,
LoggingServiceAddress = MockJoinServer.MockJoinServerAddress,
PollingForModelPeriod = TimeSpan.MinValue,
PollingForSettingsPeriod = TimeSpan.MinValue
};
using (var ds = DecisionService.Create <TestADFContextWithFeatures>(dsConfig)
.ExploitUntilModelReady(new ConstantPolicy <TestADFContextWithFeatures>(ctx => ctx.ActionDependentFeatures.Count)))
{
string uniqueKey = "eventid";
for (int i = 1; i <= 100; i++)
{
Random rg = new Random(i);
if (i % 50 == 1)
{
int modelIndex = i / 50;
byte[] modelContent = commandCenter.GetCBADFModelBlobContent(numExamples: 3 + modelIndex, numFeatureVectors: 4 + modelIndex, vwDefaultArgs: vwArgs);
using (var modelStream = new MemoryStream(modelContent))
{
ds.UpdateModel(modelStream);
}
}
int numActions = rg.Next(5, 20);
var context = TestADFContextWithFeatures.CreateRandom(numActions, rg);
int[] action = ds.ChooseRanking(uniqueKey, context);
Assert.AreEqual(numActions, action.Length);
// verify all unique actions in the list
Assert.AreEqual(action.Length, action.Distinct().Count());
// verify the actions are in the expected range
Assert.AreEqual((numActions * (numActions + 1)) / 2, action.Sum(a => a));
ds.ReportReward(i / 100f, uniqueKey);
}
}
Assert.AreEqual(200, joinServer.EventBatchList.Sum(b => b.ExperimentalUnitFragments.Count));
}
public void TestMultiActionDSUploadSelective()
{
joinServer.Reset();
string uniqueKey = "test interaction";
var dsConfig = new DecisionServiceConfiguration(MockCommandCenter.SettingsBlobUri);
dsConfig.JoinServerType = JoinServerType.CustomSolution;
dsConfig.LoggingServiceAddress = MockJoinServer.MockJoinServerAddress;
dsConfig.InteractionUploadConfiguration = new BatchingConfiguration();
dsConfig.InteractionUploadConfiguration.MaxDuration = TimeSpan.FromMinutes(10); // allow enough time for queue to buffer events
dsConfig.InteractionUploadConfiguration.MaxDegreeOfSerializationParallelism = 1; // single-threaded for easy verification
int numEvents = 100;
// Set queue capacity to same number of events so selective dropping starts at 50% full
dsConfig.InteractionUploadConfiguration.MaxUploadQueueCapacity = numEvents;
dsConfig.InteractionUploadConfiguration.DroppingPolicy = new DroppingPolicy
{
MaxQueueLevelBeforeDrop = .5f,
// when threshold is reached, drop half of the events
ProbabilityOfDrop = .5f
};
using (var ds = DecisionService.Create <TestContext>(dsConfig)
//.WithTopSlotEpsilonGreedy(.2f)
.ExploitUntilModelReady(new ConstantPolicy <TestContext>()))
{
for (int i = 0; i < numEvents; i++)
{
int[] chosenAction1 = ds.ChooseRanking(uniqueKey, new TestContext());
}
}
// Some events must have been dropped so the total count cannot be same as original
Assert.IsTrue(joinServer.EventBatchList.Sum(batch => batch.ExperimentalUnitFragments.Count) < numEvents);
// Get number of events that have been downsampled, i.e. selected with probability q
int numSampledEvents = joinServer.EventBatchList
.SelectMany(batch => batch.ExperimentalUnitFragments)
.Where(e => e.Value.Contains("\"pdrop\":0.5"))
.Count();
Assert.IsTrue(numSampledEvents > 0);
// half of the events are selected with probability 0.5, so this should definitely be less than half the total events
Assert.IsTrue(numSampledEvents < numEvents / 2);
}
public void TestSingleActionOfflineModeCustomLogger()
{
var dsConfig = new DecisionServiceConfiguration("")
{
OfflineMode = true, OfflineApplicationID = ""
};
var metaData = new ApplicationClientMetadata
{
TrainArguments = "--cb_explore_adf --epsilon 0.3",
InitialExplorationEpsilon = 1f
};
var recorder = new TestLogger();
int numChooseAction = 100;
using (var ds = DecisionService.Create <TestContext>(dsConfig, metaData: metaData)
.WithRecorder(recorder)
.ExploitUntilModelReady(new ConstantPolicy <TestContext>()))
{
for (int i = 0; i < numChooseAction; i++)
{
ds.ChooseAction(i.ToString(), new TestContext());
}
Assert.AreEqual(numChooseAction, recorder.NumRecord);
int numReward = 200;
for (int i = 0; i < numReward; i++)
{
ds.ReportReward(i, i.ToString());
}
Assert.AreEqual(numReward, recorder.NumReward);
int numOutcome = 300;
for (int i = 0; i < numOutcome; i++)
{
ds.ReportOutcome(i.ToString(), i.ToString());
}
Assert.AreEqual(numOutcome, recorder.NumOutcome);
}
Assert.AreEqual(0, recorder.NumRecord);
Assert.AreEqual(0, recorder.NumReward);
Assert.AreEqual(0, recorder.NumOutcome);
}
public async Task TestNoModelFoundForImmediateDownload()
{
// create mock blobs for settings and models
this.commandCenter.CreateBlobs(createSettingsBlob: true, createModelBlob: false, vwArgs: "--cb_explore_adf --epsilon 0.2");
var serviceConfig = new DecisionServiceConfiguration(MockCommandCenter.SettingsBlobUri)
{
JoinServerType = JoinServerType.CustomSolution,
LoggingServiceAddress = MockJoinServer.MockJoinServerAddress,
};
using (var service = DecisionService.Create <TestADFContextWithFeatures>(serviceConfig))
{
await service.DownloadModelAndUpdate(new System.Threading.CancellationToken());
}
}
public void TestMultiActionOnlineModeCustomLogger()
{
joinServer.Reset();
var dsConfig = new DecisionServiceConfiguration(MockCommandCenter.SettingsBlobUri);
var recorder = new TestLogger();
dsConfig.PollingForModelPeriod = TimeSpan.MinValue;
dsConfig.PollingForSettingsPeriod = TimeSpan.MinValue;
dsConfig.JoinServerType = JoinServerType.CustomSolution;
int numChooseAction = 100;
using (var ds = DecisionService.Create <TestContext>(dsConfig, JsonTypeInspector.Default)
//.WithTopSlotEpsilonGreedy(.2f)
.WithRecorder(recorder)
.ExploitUntilModelReady(new ConstantPolicy <TestContext>()))
{
for (int i = 0; i < numChooseAction; i++)
{
ds.ChooseAction(i.ToString(), new TestContext());
}
Assert.AreEqual(numChooseAction, recorder.NumRecord);
int numReward = 200;
for (int i = 0; i < numReward; i++)
{
ds.ReportReward(i, i.ToString());
}
Assert.AreEqual(numReward, recorder.NumReward);
int numOutcome = 300;
for (int i = 0; i < numOutcome; i++)
{
ds.ReportOutcome(i.ToString(), i.ToString());
}
Assert.AreEqual(numOutcome, recorder.NumOutcome);
}
Assert.AreEqual(0, recorder.NumRecord);
Assert.AreEqual(0, recorder.NumReward);
Assert.AreEqual(0, recorder.NumOutcome);
}
public void TestSingleActionOnlineModeCustomLogger()
{
var dsConfig = new DecisionServiceConfiguration(MockCommandCenter.SettingsBlobUri)
{
JoinServerType = JoinServerType.CustomSolution
};
var recorder = new TestLogger();
int numChooseAction = 100;
using (var ds = DecisionService
.Create <TestContext>(dsConfig)
// TODO: .WithEpsilonGreedy(.2f)
.WithRecorder(recorder)
.ExploitUntilModelReady(new ConstantPolicy <TestContext>()))
{
for (int i = 0; i < numChooseAction; i++)
{
ds.ChooseAction(i.ToString(), new TestContext());
}
Assert.AreEqual(numChooseAction, recorder.NumRecord);
int numReward = 200;
for (int i = 0; i < numReward; i++)
{
ds.ReportReward(i, i.ToString());
}
Assert.AreEqual(numReward, recorder.NumReward);
int numOutcome = 300;
for (int i = 0; i < numOutcome; i++)
{
ds.ReportOutcome(i.ToString(), i.ToString());
}
Assert.AreEqual(numOutcome, recorder.NumOutcome);
}
Assert.AreEqual(0, recorder.NumRecord);
Assert.AreEqual(0, recorder.NumReward);
Assert.AreEqual(0, recorder.NumOutcome);
}
public void TestModelImmediateDownload()
{
// create mock blobs for settings and models
this.commandCenter.CreateBlobs(createSettingsBlob: true, createModelBlob: true, vwArgs: "--cb_explore_adf --epsilon 0.2");
var serviceConfig = new DecisionServiceConfiguration(MockCommandCenter.SettingsBlobUri)
{
JoinServerType = JoinServerType.CustomSolution,
LoggingServiceAddress = MockJoinServer.MockJoinServerAddress,
};
using (var service = DecisionService.Create <TestADFContextWithFeatures>(serviceConfig))
{
// download model right away
service.DownloadModelAndUpdate(new System.Threading.CancellationToken()).Wait();
}
// No exception if everything works
}
public async Task InitialFullExplorationTest()
{
var recorder = new MyRecorder();
using (var model = new MemoryStream())
{
using (var vw = new VowpalWabbit("--cb_explore_adf --epsilon 0.3"))
{
vw.Learn(new[] { "1:-3:0.2 | b:2" });
vw.ID = "123";
vw.SaveModel(model);
}
var config = new DecisionServiceConfiguration("")
{
OfflineMode = true, OfflineApplicationID = "", DevelopmentMode = true
};
var metaData = new ApplicationClientMetadata
{
TrainArguments = "--cb_explore_adf --epsilon 0.3",
InitialExplorationEpsilon = 1f
};
using (var ds = DecisionService.CreateJson(config, metaData: metaData).WithRecorder(recorder))
{
var decision = await ds.ChooseRankingAsync("abc", "{\"a\":1,\"_multi\":[{\"b\":2}]}");
// since there's not a model loaded why should get 100% exploration
// Assert.AreEqual(1f, recorder.LastExplorerState.Probability);
model.Position = 0;
ds.UpdateModel(model);
decision = await ds.ChooseRankingAsync("abc", "{\"a\":1,\"_multi\":[{\"b\":2}, {\"b\":3}]}");
// Assert.AreNotEqual(1f, recorder.LastExplorerState.Probability);
var vwState = recorder.LastMapperState as VWState;
Assert.IsNotNull(vwState);
Assert.AreEqual("123", vwState.ModelId);
}
}
}
public void TestSingleActionOfflineModeArgument()
{
var dsConfig = new DecisionServiceConfiguration("")
{
OfflineMode = true, OfflineApplicationID = ""
};
try
{
var metaData = new ApplicationClientMetadata
{
TrainArguments = "--cb_explore_adf --epsilon 0.3",
InitialExplorationEpsilon = 1f
};
using (var ds = DecisionService.Create <TestContext>(dsConfig, metaData: metaData))
{ }
}
catch (ArgumentException ex)
{
Assert.AreEqual("Recorder", ex.ParamName);
}
}
public void TestMultiActionDSUploadMultipleEvents()
{
joinServer.Reset();
string uniqueKey = "test interaction";
var dsConfig = new DecisionServiceConfiguration(MockCommandCenter.SettingsBlobUri);
dsConfig.JoinServerType = JoinServerType.CustomSolution;
dsConfig.LoggingServiceAddress = MockJoinServer.MockJoinServerAddress;
using (var ds = DecisionService.Create <TestContext>(dsConfig)
//.WithTopSlotEpsilonGreedy(.2f)
.ExploitUntilModelReady(new ConstantPolicy <TestContext>()))
{
int[] chosenAction1 = ds.ChooseRanking(uniqueKey, new TestContext());
int[] chosenAction2 = ds.ChooseRanking(uniqueKey, new TestContext());
ds.ReportReward(1.0f, uniqueKey);
ds.ReportOutcome(new { value = "test outcome" }, uniqueKey);
}
Assert.AreEqual(4, joinServer.EventBatchList.Sum(batch => batch.ExperimentalUnitFragments.Count));
}
public static void Initialize(string settingsUrl)
{
var telemetry = new TelemetryClient();
var config = new DecisionServiceConfiguration(settingsUrl)
{
InteractionUploadConfiguration = new BatchingConfiguration
{
MaxDuration = TimeSpan.FromSeconds(2),
UploadRetryPolicy = BatchUploadRetryPolicy.ExponentialRetry
},
ModelPollFailureCallback = e => telemetry.TrackException(e, new Dictionary <string, string> {
{ "Pool failure", "model" }
}),
SettingsPollFailureCallback = e => telemetry.TrackException(e, new Dictionary <string, string> {
{ "Pool failure", "settings" }
}),
// AzureStorageConnectionString = ConfigurationManager.AppSettings[ApplicationMetadataStore.AKConnectionString]
};
client = DecisionService.CreateJson(config);
}
public void TestADFExplorationResult()
{
joinServer.Reset();
var dsConfig = new DecisionServiceConfiguration<TestADFContext, DummyADFType>(
authorizationToken: MockCommandCenter.AuthorizationToken,
explorer: new EpsilonGreedyExplorer<TestADFContext>(new TestADFPolicy(), epsilon: 0.5f))
{
PollingForModelPeriod = TimeSpan.MinValue,
PollingForSettingsPeriod = TimeSpan.MinValue,
JoinServerType = JoinServerType.CustomSolution,
LoggingServiceAddress = MockJoinServer.MockJoinServerAddress
};
var ds = new DecisionService<TestADFContext, DummyADFType>(dsConfig);
string uniqueKey = "eventid";
for (int i = 1; i <= 100; i++)
{
var adfContext = new TestADFContext(i);
uint[] action = ds.ChooseAction(new UniqueEventID { Key = uniqueKey }, adfContext, (uint)adfContext.ActionDependentFeatures.Count);
Assert.AreEqual(i, action.Length);
// verify all unique actions in the list
Assert.AreEqual(action.Length, action.Distinct().Count());
// verify the actions are in the expected range
Assert.AreEqual((i * (i + 1)) / 2, action.Sum(a => a));
ds.ReportReward(i / 100f, new UniqueEventID { Key = uniqueKey });
}
ds.Flush();
Assert.AreEqual(200, joinServer.EventBatchList.Sum(b => b.ExperimentalUnitFragments.Count));
}
public static DecisionServiceClient <string> AddOrGetExisting(Action <byte[]> modelSuccessNotifier)
{
return(DecisionServiceStaticClient.AddOrGetExisting("single", _ =>
{
var telemetry = new TelemetryClient();
string azureStorageConnectionString = ConfigurationManager.AppSettings[ApplicationMetadataStore.AKConnectionString];
var storageAccount = CloudStorageAccount.Parse(azureStorageConnectionString);
var blobClient = storageAccount.CreateCloudBlobClient();
var settingsBlobContainer = blobClient.GetContainerReference(ApplicationBlobConstants.SettingsContainerName);
var clientSettingsBlob = settingsBlobContainer.GetBlockBlobReference(ApplicationBlobConstants.LatestClientSettingsBlobName);
//var settingsUrl = clientSettingsBlob.StorageUri.PrimaryUri.ToString();
var settingsUrl = APIUtil.GetSettingsUrl();
telemetry.TrackEvent($"DecisionServiceClient created: '{settingsUrl}'");
var config = new DecisionServiceConfiguration(settingsUrl)
{
InteractionUploadConfiguration = new BatchingConfiguration
{
MaxDuration = TimeSpan.FromSeconds(2),
UploadRetryPolicy = BatchUploadRetryPolicy.ExponentialRetry
},
ModelPollSuccessCallback = modelSuccessNotifier,
ModelPollFailureCallback = e => telemetry.TrackException(e, new Dictionary <string, string> {
{ "Pool failure", "model" }
}),
SettingsPollFailureCallback = e => telemetry.TrackException(e, new Dictionary <string, string> {
{ "Pool failure", "settings" }
}),
AzureStorageConnectionString = ConfigurationManager.AppSettings[ApplicationMetadataStore.AKConnectionString]
};
return DecisionService.CreateJson(config);
}));
}
public static DecisionServiceClient<string> AddOrGetExisting(Action<byte[]> modelSuccessNotifier)
{
return DecisionServiceStaticClient.AddOrGetExisting("single", _ =>
{
var telemetry = new TelemetryClient();
string azureStorageConnectionString = ConfigurationManager.AppSettings[ApplicationMetadataStore.AKConnectionString];
var storageAccount = CloudStorageAccount.Parse(azureStorageConnectionString);
var blobClient = storageAccount.CreateCloudBlobClient();
var settingsBlobContainer = blobClient.GetContainerReference(ApplicationBlobConstants.SettingsContainerName);
var clientSettingsBlob = settingsBlobContainer.GetBlockBlobReference(ApplicationBlobConstants.LatestClientSettingsBlobName);
//var settingsUrl = clientSettingsBlob.StorageUri.PrimaryUri.ToString();
var settingsUrl = APIUtil.GetSettingsUrl();
telemetry.TrackEvent($"DecisionServiceClient created: '{settingsUrl}'");
var config = new DecisionServiceConfiguration(settingsUrl)
{
InteractionUploadConfiguration = new BatchingConfiguration
{
// TODO: these are not production ready configurations. do we need to move those to C&C as well?
MaxBufferSizeInBytes = 1,
MaxDuration = TimeSpan.FromSeconds(1),
MaxEventCount = 1,
MaxUploadQueueCapacity = 1,
UploadRetryPolicy = BatchUploadRetryPolicy.ExponentialRetry
},
ModelPollSuccessCallback = modelSuccessNotifier,
ModelPollFailureCallback = e => telemetry.TrackException(e, new Dictionary<string, string> { { "Pool failure", "model" } }),
SettingsPollFailureCallback = e => telemetry.TrackException(e, new Dictionary<string, string> { { "Pool failure", "settings" } }),
AzureStorageConnectionString = ConfigurationManager.AppSettings[ApplicationMetadataStore.AKConnectionString]
};
return DecisionService.CreateJson(config);
});
}
public void TestADFExplorationResult()
{
joinServer.Reset();
commandCenter.CreateBlobs(createSettingsBlob: true, createModelBlob: false, vwArgs: "--cb_explore_adf --epsilon 0.5");
var dsConfig = new DecisionServiceConfiguration(MockCommandCenter.SettingsBlobUri)
{
PollingForModelPeriod = TimeSpan.MinValue,
PollingForSettingsPeriod = TimeSpan.MinValue,
JoinServerType = JoinServerType.CustomSolution,
LoggingServiceAddress = MockJoinServer.MockJoinServerAddress
};
using (var ds = DecisionService.Create <TestADFContext>(dsConfig)
.ExploitUntilModelReady(new ConstantPolicy <TestADFContext>(ctx => ctx.ActionDependentFeatures.Count)))
{
string uniqueKey = "eventid";
for (int i = 1; i <= 100; i++)
{
var adfContext = new TestADFContext(i);
int[] action = ds.ChooseRanking(uniqueKey, adfContext);
Assert.AreEqual(i, action.Length);
// verify all unique actions in the list
Assert.AreEqual(action.Length, action.Distinct().Count());
// verify the actions are in the expected range
Assert.AreEqual((i * (i + 1)) / 2, action.Sum(a => a));
ds.ReportReward(i / 100f, uniqueKey);
}
}
Assert.AreEqual(200, joinServer.EventBatchList.Sum(b => b.ExperimentalUnitFragments.Count));
}
public void TestBatchingByCount()
{
var serviceConfig = new DecisionServiceConfiguration<TestContext>("",
new EpsilonGreedyExplorer<TestContext>(new TestPolicy(), epsilon: 0.2f, numActions: 10))
{
BatchConfig = new BatchingConfiguration()
{
MaxDuration = TimeSpan.FromDays(30),
MaxEventCount = 2,
MaxBufferSizeInBytes = 10 * 1024 * 1024
}
};
var service = new DecisionService<TestContext>(serviceConfig);
string uniqueKey = "eventid";
service.ChooseAction(uniqueKey, new TestContext());
service.ReportOutcome("my json outcome", uniqueKey);
service.ReportReward(0.5f, uniqueKey);
var batch = JsonConvert.DeserializeObject<EventBatch>(File.ReadAllText(batchOutputFile), new EventJsonConverter());
Assert.AreEqual(2, batch.Events.Count);
}
public void TestRcv1ModelUpdateFromStream()
{
joinServer.Reset();
int numActions = 10;
int numFeatures = 1024;
var dsConfig = new DecisionServiceConfiguration<TestRcv1Context>(
authorizationToken: MockCommandCenter.AuthorizationToken,
explorer: new EpsilonGreedyExplorer<TestRcv1Context>(new TestRcv1ContextPolicy(), epsilon: 0.5f, numActions: (uint)numActions))
{
JoinServerType = JoinServerType.CustomSolution,
LoggingServiceAddress = MockJoinServer.MockJoinServerAddress,
PollingForModelPeriod = TimeSpan.MinValue,
PollingForSettingsPeriod = TimeSpan.MinValue
};
var ds = new DecisionService<TestRcv1Context>(dsConfig);
string uniqueKey = "eventid";
string modelFile = "test_vw_adf{0}.model";
for (int i = 1; i <= 100; i++)
{
Random rg = new Random(i);
if (i % 50 == 1)
{
int modelIndex = i / 50;
string currentModelFile = string.Format(modelFile, modelIndex);
byte[] modelContent = commandCenter.GetCBModelBlobContent(numExamples: 3 + modelIndex, numFeatures: numFeatures, numActions: numActions);
var modelStream = new MemoryStream(modelContent);
ds.UpdatePolicy(new VWPolicy<TestRcv1Context>(modelStream));
}
var context = TestRcv1Context.CreateRandom(numActions, numFeatures, rg);
DateTime timeStamp = DateTime.UtcNow;
uint action = ds.ChooseAction(new UniqueEventID { Key = uniqueKey }, context);
// verify the actions are in the expected range
Assert.IsTrue(action >= 1 && action <= numActions);
ds.ReportReward(i / 100f, new UniqueEventID { Key = uniqueKey });
}
ds.Flush();
Assert.AreEqual(200, joinServer.EventBatchList.Sum(b => b.ExperimentalUnitFragments.Count));
}
public void SimplePolicyHttpTest()
{
var deployment = new ProvisioningUtil().Deploy();
wc = new WebClient();
wc.Headers.Add("auth", deployment.ManagementPassword);
deployment.ConfigureDecisionService("--cb_explore 4 --epsilon 1", initialExplorationEpsilon: 1, isExplorationEnabled: true);
Thread.Sleep(TimeSpan.FromSeconds(5));
// 4 Actions
// why does this need to be different from default?
var config = new DecisionServiceConfiguration(deployment.SettingsUrl)
{
InteractionUploadConfiguration = new BatchingConfiguration
{
MaxEventCount = 64
},
ObservationUploadConfiguration = new BatchingConfiguration
{
MaxEventCount = 64
}
};
config.InteractionUploadConfiguration.ErrorHandler += JoinServiceBatchConfiguration_ErrorHandler;
config.InteractionUploadConfiguration.SuccessHandler += JoinServiceBatchConfiguration_SuccessHandler;
this.features = new string[] { "a", "b", "c", "d" };
this.freq = new Dictionary <string, int>();
this.rnd = new Random(123);
// reset the model
deployment.OnlineTrainerReset();
Console.WriteLine("Waiting after reset...");
Thread.Sleep(TimeSpan.FromSeconds(2));
Console.WriteLine("Exploration");
var expectedEvents = 0;
for (int i = 0; i < 1000; i++)
{
int featureIndex = i % features.Length;
expectedEvents += SendEvents(deployment, wc, featureIndex);
}
// Thread.Sleep(500);
// TODO: flush doesn't work
// Assert.AreEqual(expectedEvents, this.eventCount);
// 4 actions times 4 feature values
Assert.AreEqual(4 * 4, freq.Keys.Count);
var total = freq.Values.Sum();
foreach (var k in freq.Keys.OrderBy(k => k))
{
var f = freq[k] / (float)total;
Assert.IsTrue(f < 0.08); // 1/(4*4) = 0.0625
Console.WriteLine("{0} | {1}", k, f);
}
freq.Clear();
deployment.ConfigureDecisionService("--cb_explore 4 --epsilon 0", initialExplorationEpsilon: 1, isExplorationEnabled: false);
Thread.Sleep(TimeSpan.FromSeconds(5));
// check here to make sure model was updated
Console.WriteLine("Exploitation");
expectedEvents = 0;
for (int i = 0; i < 1000; i++)
{
var featureIndex = i % features.Length;
expectedEvents += SendEvents(deployment, wc, featureIndex, false);
}
total = freq.Values.Sum();
foreach (var k in freq.Keys.OrderBy(k => k))
{
var f = freq[k] / (float)total;
Assert.AreEqual(0.25f, f, 0.1);
Console.WriteLine("{0} | {1}", k, f);
}
}
public void TestSingleActionDSThreadSafeUpload()
{
joinServer.Reset();
string uniqueKey = "test interaction";
commandCenter.CreateBlobs(createSettingsBlob: true, createModelBlob: false, vwArgs: "--cb_explore_adf --epsilon 0.2");
var createObservation = (Func <int, string>)((i) => { return(string.Format("00000", i)); });
var dsConfig = new DecisionServiceConfiguration(MockCommandCenter.SettingsBlobUri);
dsConfig.JoinServerType = JoinServerType.CustomSolution;
dsConfig.LoggingServiceAddress = MockJoinServer.MockJoinServerAddress;
int numEvents = 1000;
var chosenActions = new ConcurrentBag <int>();
using (var ds = DecisionService
.Create <TestContext>(dsConfig)
.ExploitUntilModelReady(new ConstantPolicy <TestContext>()))
{
Parallel.For(0, numEvents, new ParallelOptions {
MaxDegreeOfParallelism = Environment.ProcessorCount * 2
}, (i) =>
{
chosenActions.Add(ds.ChooseAction(uniqueKey, new TestContext()));
ds.ReportOutcome(new { value = createObservation((int)i) }, uniqueKey);
});
}
List <PartialDecisionServiceMessage> batchList = this.joinServer.EventBatchList;
int numActualEvents = batchList.Sum(b => b.ExperimentalUnitFragments.Count);
Assert.AreEqual(numEvents * 2, numActualEvents);
List <string> uniqueKeys = batchList
.SelectMany(b => b.ExperimentalUnitFragments.Select(f => f.Id))
.Distinct()
.ToList();
Assert.AreEqual(1, uniqueKeys.Count);
Assert.AreEqual(uniqueKey, uniqueKeys[0]);
var completeFragments = batchList
.SelectMany(b => b.ExperimentalUnitFragments
.Select(f => JsonConvert.DeserializeObject <MultiActionCompleteExperimentalUnitFragment>(f.Value)));
// Test actual interactions received
List <MultiActionCompleteExperimentalUnitFragment> interactions = completeFragments
.Where(f => f.Value == null)
.OrderBy(f => f.Actions[0])
.ToList();
// Test values of the interactions
Assert.AreEqual(numEvents, interactions.Count);
var chosenActionList = chosenActions.OrderBy(a => a).ToList();
for (int i = 0; i < interactions.Count; i++)
{
Assert.AreEqual((int)chosenActionList[i], interactions[i].Actions[0]);
}
// Test actual observations received
List <MultiActionCompleteExperimentalUnitFragment> observations = completeFragments
.Where(f => f.Value != null)
.OrderBy(f => f.Value)
.ToList();
// Test values of the observations
Assert.AreEqual(numEvents, observations.Count);
for (int i = 0; i < observations.Count; i++)
{
Assert.AreEqual(JsonConvert.SerializeObject(new { value = createObservation(i) }), observations[i].Value);
}
}
/// <summary>
/// Sample code simulating a news recommendation scenario. In this simple example,
/// the rendering server has to pick 1 out of 4 news topics to show to users (e.g. as featured article).
/// </summary>
/// <remarks>
/// NOTE: For this sample to work, the proper settings must be set at deployment time:
/// Vowpal Wabbit Switches = --cb_explore_adf --epsilon 0.2 --cb_type dr
/// </remarks>
public static async Task NewsRecommendation()
{
if (String.IsNullOrWhiteSpace(SettingsBlobUri))
{
Console.WriteLine("Please specify a valid settings URL.");
return;
}
// Create configuration for the decision service
var serviceConfig = new DecisionServiceConfiguration(settingsBlobUri: SettingsBlobUri);
// Enable development mode to easily debug / diagnose data flow and system properties
// This should be turned off in production deployment
serviceConfig.DevelopmentMode = true;
using (var service = DecisionService.Create <UserContextADF>(serviceConfig))
{
var random = new Random();
int user = 0; // user id
int maxDecisionHistory = 100; // max number of past decisions to record
var correctDecisions = new Queue <int>(); // keeps track of past decisions
Console.WriteLine("Press Ctrl + C at any time to cancel the process.");
// Each topic has its own set of features
var topicFeatures = Enumerable.Range(1, 10)
.Select(_ => new TopicFeature
{
Features = Enumerable.Range(1, 10).Select(f => (float)random.NextDouble()).ToArray()
})
.ToArray();
while (true)
{
user++;
string uniqueKey = Guid.NewGuid().ToString();
var userContext = new UserContextADF
{
Gender = random.NextDouble() > 0.5 ? "male" : "female",
TopicFeatures = topicFeatures.Take(random.Next(5, topicFeatures.Length)).ToArray()
};
int[] topicRanking = await service.ChooseRankingAsync(uniqueKey, userContext);
int topicId = topicRanking[0];
// Display the news topic chosen by exploration process.
Console.WriteLine("Topic {0} was chosen for user {1}.", topicId, user + 1);
// Report {0,1} reward as a simple float.
// In a real scenario, one could associated a reward of 1 if user
// clicks on the article and 0 otherwise.
float reward = 0;
if (userContext.Gender == "male" && topicId == 3)
{
reward = 1;
}
else if (userContext.Gender == "female" && topicId == 8)
{
reward = 1;
}
service.ReportReward(reward, uniqueKey);
correctDecisions.Enqueue((int)reward);
if (correctDecisions.Count == maxDecisionHistory)
{
correctDecisions.Dequeue();
}
if (user % 50 == 0)
{
Console.WriteLine("Correct decisions out of last {0} interactions: {1}", maxDecisionHistory, correctDecisions.Sum());
}
System.Threading.Thread.Sleep(TimeSpan.FromMilliseconds(50));
}
}
}
public async Task SimplePolicyTest(DecisionServiceDeployment deployment)
{
deployment.OnlineTrainerWaitForStartup();
deployment.ConfigureDecisionService("--cb_explore 4 --epsilon 0", initialExplorationEpsilon: 1, isExplorationEnabled: true);
// 4 Actions
// why does this need to be different from default?
var config = new DecisionServiceConfiguration(deployment.SettingsUrl)
{
InteractionUploadConfiguration = new BatchingConfiguration
{
MaxEventCount = 64
},
ObservationUploadConfiguration = new BatchingConfiguration
{
MaxEventCount = 64
},
PollingForModelPeriod = TimeSpan.FromMinutes(5)
};
config.InteractionUploadConfiguration.ErrorHandler += JoinServiceBatchConfiguration_ErrorHandler;
config.InteractionUploadConfiguration.SuccessHandler += JoinServiceBatchConfiguration_SuccessHandler;
this.features = new string[] { "a", "b", "c", "d" };
this.freq = new Dictionary <string, int>();
this.rnd = new Random(123);
deployment.OnlineTrainerReset();
{
var expectedEvents = 0;
using (var client = Microsoft.Research.MultiWorldTesting.ClientLibrary.DecisionService.Create <MyContext>(config))
{
for (int i = 0; i < 50; i++)
{
expectedEvents += SendEvents(client, 128);
// Thread.Sleep(500);
}
}
// TODO: flush doesn't work
// Assert.AreEqual(expectedEvents, this.eventCount);
}
// 4 actions times 4 feature values
Assert.AreEqual(4 * 4, freq.Keys.Count);
Console.WriteLine("Exploration");
var total = freq.Values.Sum();
foreach (var k in freq.Keys.OrderBy(k => k))
{
var f = freq[k] / (float)total;
Assert.IsTrue(f < 0.08);
Console.WriteLine("{0} | {1}", k, f);
}
freq.Clear();
await Task.Delay(TimeSpan.FromMinutes(2));
// TODO: update eps: 0
using (var client = Microsoft.Research.MultiWorldTesting.ClientLibrary.DecisionService.Create <MyContext>(config))
{
int i;
for (i = 0; i < 120; i++)
{
try
{
client.DownloadModelAndUpdate(new System.Threading.CancellationToken()).Wait();
break;
}
catch (Exception)
{
await Task.Delay(TimeSpan.FromSeconds(1));
}
}
Assert.IsTrue(i < 30, "Unable to download model");
for (i = 0; i < 1024; i++)
{
var key = Guid.NewGuid().ToString();
var featureIndex = i % features.Length;
var action = client.ChooseAction(key, new MyContext {
Feature = features[featureIndex]
});
var stat = string.Format("'{0}' '{1}' ", features[featureIndex], action);
int count;
if (freq.TryGetValue(stat, out count))
{
freq[stat]++;
}
else
{
freq.Add(stat, count);
}
}
}
Console.WriteLine("Exploitation");
total = freq.Values.Sum();
foreach (var k in freq.Keys.OrderBy(k => k))
{
var f = freq[k] / (float)total;
Assert.AreEqual(0.25f, f, 0.1);
Console.WriteLine("{0} | {1}", k, f);
}
}
private float UploadFoodContextData(string settingsBlobUri, bool firstPass)
{
var serviceConfig = new DecisionServiceConfiguration(settingsBlobUri);
if (firstPass)
{
serviceConfig.PollingForModelPeriod = TimeSpan.MinValue;
this.OnlineTrainerReset();
}
using (var service = DecisionService.Create <FoodContext>(serviceConfig))
{
if (!firstPass)
{
Thread.Sleep(10000);
}
string uniqueKey = "scratch-key-gal";
string[] locations = { "HealthyTown", "LessHealthyTown" };
var rg = new Random(uniqueKey.GetHashCode());
int numActions = 3; // ["Hamburger deal 1", "Hamburger deal 2" (better), "Salad deal"]
var csv = new StringBuilder();
int counterCorrect = 0;
int counterTotal = 0;
var header = "Location,Action,Reward";
csv.AppendLine(header);
// number of iterations
for (int i = 0; i < 10000 * locations.Length; i++)
{
// randomly select a location
int iL = rg.Next(0, locations.Length);
string location = locations[iL];
DateTime timeStamp = DateTime.UtcNow;
string key = uniqueKey + Guid.NewGuid().ToString();
FoodContext currentContext = new FoodContext();
currentContext.UserLocation = location;
currentContext.Actions = Enumerable.Range(1, numActions).ToArray();
int[] action = service.ChooseRanking(key, currentContext);
counterTotal += 1;
// We expect healthy town to get salad and unhealthy town to get the second burger (action 2)
if (location.Equals("HealthyTown") && action[0] == 3)
{
counterCorrect += 1;
}
else if (location.Equals("LessHealthyTown") && action[0] == 2)
{
counterCorrect += 1;
}
var csvLocation = location;
var csvAction = action[0].ToString();
float reward = 0;
double currentRand = rg.NextDouble();
if (location.Equals("HealthyTown"))
{
// for healthy town, buy burger 1 with probability 0.1, burger 2 with probability 0.15, salad with probability 0.6
if ((action[0] == 1 && currentRand < 0.1) ||
(action[0] == 2 && currentRand < 0.15) ||
(action[0] == 3 && currentRand < 0.6))
{
reward = 10;
}
}
else
{
// for unhealthy town, buy burger 1 with probability 0.4, burger 2 with probability 0.6, salad with probability 0.2
if ((action[0] == 1 && currentRand < 0.4) ||
(action[0] == 2 && currentRand < 0.6) ||
(action[0] == 3 && currentRand < 0.2))
{
reward = 10;
}
}
service.ReportReward(reward, key);
var newLine = string.Format("{0},{1},{2}", csvLocation, csvAction, "0");
csv.AppendLine(newLine);
System.Threading.Thread.Sleep(1);
}
return((float)counterCorrect / counterTotal);
}
}
/// <summary>
/// Sample code simulating a news recommendation scenario. In this simple example,
/// the rendering server has to pick 1 out of 10 news topics to show to users (e.g. as featured article).
/// </summary>
/// <remarks>
/// NOTE: For this sample to work, the proper settings must be set at deployment time:
/// Number Of Actions = 10
/// Vowpal Wabbit Switches = --cb_explore 10 --epsilon 0.2 --cb_type dr
/// </remarks>
public static void NewsRecommendation()
{
if (String.IsNullOrWhiteSpace(SettingsBlobUri))
{
Console.WriteLine("Please specify a valid settings URL.");
return;
}
int numTopics = 10; // number of different topic choices to show
// Create configuration for the decision service.
var serviceConfig = new DecisionServiceConfiguration(settingsBlobUri: SettingsBlobUri);
// Enable development mode to easily debug / diagnose data flow and system properties
// This should be turned off in production deployment
serviceConfig.DevelopmentMode = true;
// Create the main service object with above configurations.
// Specify the exploration algorithm to use, here we will use Epsilon-Greedy.
// For more details about this and other algorithms, refer to the MWT onboarding whitepaper.
using (var service = DecisionService.Create <UserContext>(serviceConfig))
{
// Create a default policy which is used before a machine-learned model.
// This policy is used in epsilon-greedy exploration using the initial
// exploration epsilon specified at deployment time or in the Management Center.
var defaultPolicy = new NewsDisplayPolicy(numTopics);
var random = new Random();
int user = 0;
Console.WriteLine("Press Ctrl + C at any time to cancel the process.");
int maxDecisionHistory = 100;
var correctDecisions = new Queue <int>();
while (true)
{
user++;
// Generate a random GUID id for each user.
var uniqueKey = Guid.NewGuid().ToString();
// Generate random feature vector for each user.
var userContext = new UserContext()
{
Gender = random.NextDouble() > 0.5 ? "male" : "female"
};
for (int f = 1; f <= 10; f++)
{
userContext.Features.Add(f.ToString(), (float)random.NextDouble());
}
// Perform exploration given user features.
int topicId = service.ChooseAction(uniqueKey, userContext, defaultPolicy);
// Display the news topic chosen by exploration process.
Console.WriteLine("Topic {0} was chosen for user {1}.", topicId, user + 1);
// Report {0,1} reward as a simple float.
// In a real scenario, one could associated a reward of 1 if user
// clicks on the article and 0 otherwise.
float reward = 0;
if (userContext.Gender == "male" && topicId == 3)
{
reward = 1;
}
else if (userContext.Gender == "female" && topicId == 8)
{
reward = 1;
}
service.ReportReward(reward, uniqueKey);
correctDecisions.Enqueue((int)reward);
if (correctDecisions.Count == maxDecisionHistory)
{
correctDecisions.Dequeue();
}
if (user % 50 == 0)
{
Console.WriteLine("Correct decisions out of last {0} interactions: {1}", maxDecisionHistory, correctDecisions.Sum());
}
System.Threading.Thread.Sleep(TimeSpan.FromMilliseconds(50));
}
}
}
public static async Task Sample()
{
// 1. Sign in on https://ds.microsoft.com
// 2. Create new app
// a. Select “Custom App”
// b. Supply Azure storage account
// c. Empty Action Endpoint field - this disables action id based featurization
// d. Empty Action Set Endpoint field
// e. Set Provision FrontEnd field to “false” - avoids HTTP scoring provisioning
// f. Set Vowpal Wabbit arguments to --cb_explore_adf --epsilon 0.2 -q gc -q gC -q dC
// f. Click create
// 3. Wait for creation and hit edit
// a. Copy Client Library URL and use in the sample code below
// 4. nuget install Microsoft.Research.MultiWorldTesting.ClientLibrary
// a. at least version 2.0.0.6
var config = new DecisionServiceConfiguration(settingsBlobUri: "<<INSERT URL FROM STEP 3A")
{
ModelPollFailureCallback = ex => Console.WriteLine(ex.Message),
SettingsPollFailureCallback = ex => Console.WriteLine(ex.Message)
};
// you should keep this object around (e.g. static variable) as it maintains the background process
// to download new models and upload logged events
var ds = DecisionService.Create <MyDecisionContext>(config, typeInspector: JsonTypeInspector.Default);
// Some sample data
var context = new MyDecisionContext
{
Demographics = new MyDecisionContextDemographics
{
Age = 15,
AgeGroup = "Teenager",
Gender = "male"
},
Geo = new MyDecisionContextGeo
{
City = "New York",
Country = "USA"
},
Actions = new[]
{
new MyDecisionContextAction
{
Categories = new MyDecisionContextActionCategorization {
Categories = "Fashion OldSchool"
},
Celebrities = new Dictionary <string, float>
{
{ "Markus", 0.9f },
{ "Bob", 0.1f }
}
},
new MyDecisionContextAction
{
Categories = new MyDecisionContextActionCategorization {
Categories = "Tech Microsoft"
},
Celebrities = new Dictionary <string, float>
{
{ "Bill", 0.7f },
{ "Bob", 0.2f }
}
}
}
};
for (int i = 0; i < 5; i++)
{
// this is used as both correlation event id to link the ranking request to the observerd outcome
// as well as random seed for the pseudo random generator
var eventId = Guid.NewGuid().ToString("n");
// get the decision
var ranking = await ds.ChooseRankingAsync(
uniqueKey : eventId,
context : context);
// report the reward if you observed desired behavior on first item in ranking
// e.g. the first item returned by ChooseRankingAsync got clicked.
ds.ReportReward(reward: 1f, uniqueKey: eventId);
}
Console.WriteLine("done");
Thread.Sleep(TimeSpan.FromHours(1));
Console.WriteLine();
}
private static async Task MainAsync()
{
// Requires Visual C++ 2015 Runtime installed
// - https://www.microsoft.com/en-us/download/details.aspx?id=48145
// Requires platform to be x64
// 1. Sign in on https://ds.microsoft.com
// 2. Create new app
// a. Select "Custom App"
// b. Supply Azure storage account
// c. Empty Action Endpoint field - this disables action id based featurization
// d. Empty Action Set Endpoint field
// e. Set Provision FrontEnd field to "false" - avoids HTTP scoring provisioning
// f. Set Vowpal Wabbit arguments to --cb_explore_adf --epsilon 0.2 -q DT -q LT
// f. Click create
// 3. Wait for creation and hit edit
// a. Copy Client Library URL and use in the sample code below
// 4. nuget install Microsoft.Research.MultiWorldTesting.ClientLibrary
// a. at least version 2.0.0.6
var config = new DecisionServiceConfiguration(settingsBlobUri: "<<INSERT URL FROM STEP 3A")
{
ModelPollFailureCallback = ex => Console.WriteLine(ex.Message),
SettingsPollFailureCallback = ex => Console.WriteLine(ex.Message)
};
// you should keep this object around (e.g. static variable) as it maintains the background process
// to download new models and upload logged events
using (var ds = DecisionService.Create <DecisionContext>(config, typeInspector: JsonTypeInspector.Default))
{
// If you add the timestamp calculating reward latencies becomes very easy on our end
// In general you can use existing request ids, just be careful as they're used to seed the random
// generator for exploration.
// Also they're used to join rank and reward calls, so they must be unique within the experimental unit duration
var eventId = EventIdUtil.AddTimestamp(Guid.NewGuid().ToString("n"), EventIdUtil.BeginningOfTime);
var context = new DecisionContext
{
Demographics = new DemographicNamespace
{
Gender = "female",
},
Location = new LocationNamespace
{
Country = "USA",
City = "New York"
},
Actions = new[]
{
new ActionDependentFeatures
{
Topic = new TopicNamespace
{
Category = "Fashion"
}
},
new ActionDependentFeatures
{
Topic = new TopicNamespace
{
Category = "Technology"
}
}
}
};
// get the decision
var ranking = await ds.ChooseRankingAsync(
uniqueKey : eventId,
context : context);
// report the reward if you observed desired behavior on first item in ranking
// e.g. the first item returned by ChooseRankingAsync got clicked.
ds.ReportReward(reward: 1f, uniqueKey: eventId);
}
}
public void TestADFModelUpdateFromFile()
{
joinServer.Reset();
var dsConfig = new DecisionServiceConfiguration<TestADFContextWithFeatures, TestADFFeatures>(
authorizationToken: MockCommandCenter.AuthorizationToken,
explorer: new EpsilonGreedyExplorer<TestADFContextWithFeatures>(new TestADFWithFeaturesPolicy(), epsilon: 0.5f))
{
JoinServerType = JoinServerType.CustomSolution,
LoggingServiceAddress = MockJoinServer.MockJoinServerAddress,
PollingForModelPeriod = TimeSpan.MinValue,
PollingForSettingsPeriod = TimeSpan.MinValue
};
var ds = new DecisionService<TestADFContextWithFeatures, TestADFFeatures>(dsConfig);
string uniqueKey = "eventid";
string modelFile = "test_vw_adf{0}.model";
var actualModelFiles = new List<string>();
for (int i = 1; i <= 100; i++)
{
Random rg = new Random(i);
if (i % 50 == 1)
{
int modelIndex = i / 50;
string currentModelFile = string.Format(modelFile, modelIndex);
byte[] modelContent = commandCenter.GetCBADFModelBlobContent(numExamples: 3 + modelIndex, numFeatureVectors: 4 + modelIndex);
System.IO.File.WriteAllBytes(currentModelFile, modelContent);
ds.UpdatePolicy(new VWPolicy<TestADFContextWithFeatures, TestADFFeatures>(GetFeaturesFromContext, currentModelFile));
actualModelFiles.Add(currentModelFile);
}
int numActions = rg.Next(5, 20);
var context = TestADFContextWithFeatures.CreateRandom(numActions, rg);
uint[] action = ds.ChooseAction(new UniqueEventID { Key = uniqueKey }, context, (uint)context.ActionDependentFeatures.Count);
Assert.AreEqual(numActions, action.Length);
// verify all unique actions in the list
Assert.AreEqual(action.Length, action.Distinct().Count());
// verify the actions are in the expected range
Assert.AreEqual((numActions * (numActions + 1)) / 2, action.Sum(a => a));
ds.ReportReward(i / 100f, new UniqueEventID { Key = uniqueKey });
}
ds.Flush();
Assert.AreEqual(200, joinServer.EventBatchList.Sum(b => b.ExperimentalUnitFragments.Count));
foreach (string actualModelFile in actualModelFiles)
{
System.IO.File.Delete(actualModelFile);
}
}
public static void SampleCodeUsingActionDependentFeatures()
{
// Create configuration for the decision service
var serviceConfig = new DecisionServiceConfiguration<ADFContext, ADFFeatures>(
authorizationToken: "",
explorer: new EpsilonGreedyExplorer<ADFContext>(new ADFPolicy(), epsilon: 0.8f))
{
PollingForModelPeriod = TimeSpan.MinValue,
PollingForSettingsPeriod = TimeSpan.MinValue
};
var service = new DecisionService<ADFContext, ADFFeatures>(serviceConfig);
string uniqueKey = "eventid";
var rg = new Random(uniqueKey.GetHashCode());
var vwPolicy = new VWPolicy<ADFContext, ADFFeatures>(GetFeaturesFromContext);
for (int i = 1; i < 100; i++)
{
if (i == 30)
{
string vwModelFile = TrainNewVWModelWithRandomData(numExamples: 5, numActions: 10);
vwPolicy = new VWPolicy<ADFContext, ADFFeatures>(GetFeaturesFromContext, vwModelFile);
// Alternatively, VWPolicy can also be loaded from an IO stream:
// var vwModelStream = new MemoryStream(File.ReadAllBytes(vwModelFile));
// vwPolicy = new VWPolicy<ADFContext, ADFFeatures>(GetFeaturesFromContext, vwModelStream);
// Manually updates decision service with a new policy for consuming VW models.
service.UpdatePolicy(vwPolicy);
}
if (i == 60)
{
string vwModelFile = TrainNewVWModelWithRandomData(numExamples: 6, numActions: 8);
// Evolves the existing VWPolicy with a new model
vwPolicy.ModelUpdate(vwModelFile);
}
int numActions = rg.Next(5, 10);
uint[] action = service.ChooseAction(new UniqueEventID { Key = uniqueKey }, ADFContext.CreateRandom(numActions, rg), (uint)numActions);
service.ReportReward(i / 100f, new UniqueEventID { Key = uniqueKey });
}
service.Flush();
}
