public void TestSettingsBlobPolling() { joinServer.Reset(); commandCenter.CreateBlobs(createSettingsBlob: true, createModelBlob: false, vwArgs: "--cb_explore 2"); var dsConfig = new DecisionServiceConfiguration(MockCommandCenter.SettingsBlobUri); dsConfig.JoinServerType = JoinServerType.CustomSolution; dsConfig.LoggingServiceAddress = MockJoinServer.MockJoinServerAddress; dsConfig.PollingForSettingsPeriod = TimeSpan.FromMilliseconds(500); using (var ds = DecisionService .Create <TestContext>(dsConfig) // TODO .WithTopSlotEpsilonGreedy(.2f) .ExploitUntilModelReady(new ConstantPolicy <TestContext>())) { byte[] settingsBytes = null; dsConfig.SettingsPollSuccessCallback = data => settingsBytes = data; for (int i = 0; i < 50 && settingsBytes == null; i++) { Thread.Sleep(100); if (i % 2 == 0) { // change the settings blob's etag frequently to make sure polling detects it commandCenter.CreateBlobs(createSettingsBlob: true, createModelBlob: false, vwArgs: "abc", initialExplorationEpsilon: 0.4f); } } Assert.IsNotNull(settingsBytes); Assert.IsTrue(Enumerable.SequenceEqual(settingsBytes, commandCenter.GetSettingsBlobContent("abc", 0.4f))); } }
public void TestSingleActionDSUploadSingleEvent() { joinServer.Reset(); string uniqueKey = "test interaction"; commandCenter.CreateBlobs(createSettingsBlob: true, createModelBlob: false, vwArgs: "--cb_explore_adf --epsilon 0.5"); var dsConfig = new DecisionServiceConfiguration(MockCommandCenter.SettingsBlobUri); dsConfig.JoinServerType = JoinServerType.CustomSolution; dsConfig.LoggingServiceAddress = MockJoinServer.MockJoinServerAddress; int chosenAction; using (var ds = DecisionService .Create <TestContext>(dsConfig) .ExploitUntilModelReady(new ConstantPolicy <TestContext>())) { chosenAction = ds.ChooseAction(uniqueKey, new TestContext()); } Assert.AreEqual(1, joinServer.RequestCount); Assert.AreEqual(1, joinServer.EventBatchList.Count); Assert.AreEqual(1, joinServer.EventBatchList[0].ExperimentalUnitFragments.Count); Assert.AreEqual(uniqueKey, joinServer.EventBatchList[0].ExperimentalUnitFragments[0].Id); Assert.IsTrue(joinServer.EventBatchList[0].ExperimentalUnitFragments[0].Value.ToLower().Contains("\"a\":[" + chosenAction + ",")); }
public void TestDevModeSettingsAndExampleLog() { joinServer.Reset(); var testTraceListener = new TestTraceListener(); Trace.Listeners.Add(testTraceListener); string vwArgs = "--cb_explore_adf --epsilon 0.5"; commandCenter.CreateBlobs(createSettingsBlob: true, createModelBlob: false, vwArgs: vwArgs); var dsConfig = new DecisionServiceConfiguration(MockCommandCenter.SettingsBlobUri) { JoinServerType = JoinServerType.CustomSolution, LoggingServiceAddress = MockJoinServer.MockJoinServerAddress, PollingForModelPeriod = TimeSpan.MinValue, PollingForSettingsPeriod = TimeSpan.MinValue, DevelopmentMode = true }; int numInteractionEvents = 25; var eventIdList = new List <string>(); using (var ds = DecisionService .Create <TestADFContextWithFeatures>(dsConfig) // .With<TestADFContextWithFeatures, TestADFFeatures>(context => context.ActionDependentFeatures) // TODO .WithTopSlotEpsilonGreedy(.5f) .ExploitUntilModelReady(new ConstantPolicy <TestADFContextWithFeatures>(ctx => ctx.ActionDependentFeatures.Count))) { byte[] modelContent = commandCenter.GetCBADFModelBlobContent(numExamples: 5, numFeatureVectors: 10, vwDefaultArgs: vwArgs); using (var modelStream = new MemoryStream(modelContent)) { ds.UpdateModel(modelStream); } for (int i = 1; i <= numInteractionEvents; i++) { var interId = "inter" + i; var obserId = "obser" + i; Random rg = new Random(i); int numActions = rg.Next(5, 20); var context = TestADFContextWithFeatures.CreateRandom(numActions, rg); int[] action = ds.ChooseRanking(interId, context); ds.ReportReward(i / 100f, obserId); eventIdList.Add(interId); eventIdList.Add(obserId); } } // Number of batches must be exactly equal to number of events uploaded in development mode // and each batch must contain exactly one event Assert.AreEqual(numInteractionEvents * 2, joinServer.EventBatchList.Count); Assert.AreEqual(numInteractionEvents * 2, joinServer.EventBatchList.Sum(ebl => ebl.ExperimentalUnitFragments.Count)); var eblList = joinServer.EventBatchList.Select(ebl => ebl.ExperimentalUnitFragments[0].Id).OrderBy(id => id); Assert.IsTrue(eblList.SequenceEqual(eventIdList.OrderBy(id => id))); // Trace messages must contain context information Assert.AreEqual(numInteractionEvents, testTraceListener.Messages.Count(m => m.Contains("Example Context"))); }
public void TestMultiActionDSUploadSingleEvent() { joinServer.Reset(); string uniqueKey = "test interaction"; var dsConfig = new DecisionServiceConfiguration(MockCommandCenter.SettingsBlobUri); dsConfig.JoinServerType = JoinServerType.CustomSolution; dsConfig.LoggingServiceAddress = MockJoinServer.MockJoinServerAddress; int[] chosenActions; using (var ds = DecisionService.Create <TestContext>(dsConfig) //.WithTopSlotEpsilonGreedy(.2f) .ExploitUntilModelReady(new ConstantPolicy <TestContext>())) { chosenActions = ds.ChooseRanking(uniqueKey, new TestContext()); } Assert.AreEqual(1, joinServer.RequestCount); Assert.AreEqual(1, joinServer.EventBatchList.Count); Assert.AreEqual(1, joinServer.EventBatchList[0].ExperimentalUnitFragments.Count); Assert.AreEqual(uniqueKey, joinServer.EventBatchList[0].ExperimentalUnitFragments[0].Id); Assert.IsTrue(joinServer.EventBatchList[0].ExperimentalUnitFragments[0].Value.ToLower().Contains("\"a\":[" + string.Join(",", chosenActions) + "],")); }
public async Task TestSingleActionDSUploadMultipleEvents() { joinServer.Reset(); string uniqueKey = "test interaction"; var dsConfig = new DecisionServiceConfiguration(MockCommandCenter.SettingsBlobUri); dsConfig.JoinServerType = JoinServerType.CustomSolution; dsConfig.LoggingServiceAddress = MockJoinServer.MockJoinServerAddress; using (var ds = DecisionService .Create <TestContext>(dsConfig) .ExploitUntilModelReady(new ConstantPolicy <TestContext>())) { int chosenAction1 = await ds.ChooseActionAsync(uniqueKey, new TestContext()); int chosenAction2 = await ds.ChooseActionAsync(uniqueKey, new TestContext()); ds.ReportReward(1.0f, uniqueKey); ds.ReportOutcome(JsonConvert.SerializeObject(new { value = "test outcome" }), uniqueKey); } Assert.AreEqual(4, joinServer.EventBatchList.Sum(batch => batch.ExperimentalUnitFragments.Count)); }
public void TestRcv1ModelUpdateFromStream() { joinServer.Reset(); int numActions = 10; int numFeatures = 1024; string vwArgs = "--cb_explore 10 --epsilon 0.5"; commandCenter.CreateBlobs(createSettingsBlob: true, createModelBlob: false, vwArgs: vwArgs); var dsConfig = new DecisionServiceConfiguration(MockCommandCenter.SettingsBlobUri) //explorer: new EpsilonGreedyExplorer<TestRcv1Context>(new ConstantPolicy<TestRcv1Context>(ctx => ctx.ActionDependentFeatures.Count), epsilon: 0.5f, numActions: (int)numActions)) { JoinServerType = JoinServerType.CustomSolution, LoggingServiceAddress = MockJoinServer.MockJoinServerAddress, PollingForModelPeriod = TimeSpan.MinValue, PollingForSettingsPeriod = TimeSpan.MinValue }; using (var ds = DecisionService .Create <TestRcv1Context>(dsConfig, TypeInspector.Default) // TODOD: .WithEpsilonGreedy(.5f) .ExploitUntilModelReady(new ConstantPolicy <TestRcv1Context>(ctx => ctx.Features.Count))) { string uniqueKey = "eventid"; for (int i = 1; i <= 100; i++) { Random rg = new Random(i); if (i % 50 == 1) { int modelIndex = i / 50; byte[] modelContent = commandCenter.GetCBModelBlobContent(numExamples: 3 + modelIndex, numFeatures: numFeatures, numActions: numActions, vwArgs: vwArgs); using (var modelStream = new MemoryStream(modelContent)) { ds.UpdateModel(modelStream); } } var context = TestRcv1Context.CreateRandom(numActions, numFeatures, rg); DateTime timeStamp = DateTime.UtcNow; int action = ds.ChooseAction(uniqueKey, context); // verify the actions are in the expected range Assert.IsTrue(action >= 1 && action <= numActions); ds.ReportReward(i / 100f, uniqueKey); } } Assert.AreEqual(200, joinServer.EventBatchList.Sum(b => b.ExperimentalUnitFragments.Count)); }
public void TestADFModelUpdateFromStream() { joinServer.Reset(); string vwArgs = "--cb_explore_adf --epsilon 0.5"; commandCenter.CreateBlobs(createSettingsBlob: true, createModelBlob: false, vwArgs: vwArgs); var dsConfig = new DecisionServiceConfiguration(MockCommandCenter.SettingsBlobUri) { JoinServerType = JoinServerType.CustomSolution, LoggingServiceAddress = MockJoinServer.MockJoinServerAddress, PollingForModelPeriod = TimeSpan.MinValue, PollingForSettingsPeriod = TimeSpan.MinValue }; using (var ds = DecisionService.Create <TestADFContextWithFeatures>(dsConfig) .ExploitUntilModelReady(new ConstantPolicy <TestADFContextWithFeatures>(ctx => ctx.ActionDependentFeatures.Count))) { string uniqueKey = "eventid"; for (int i = 1; i <= 100; i++) { Random rg = new Random(i); if (i % 50 == 1) { int modelIndex = i / 50; byte[] modelContent = commandCenter.GetCBADFModelBlobContent(numExamples: 3 + modelIndex, numFeatureVectors: 4 + modelIndex, vwDefaultArgs: vwArgs); using (var modelStream = new MemoryStream(modelContent)) { ds.UpdateModel(modelStream); } } int numActions = rg.Next(5, 20); var context = TestADFContextWithFeatures.CreateRandom(numActions, rg); int[] action = ds.ChooseRanking(uniqueKey, context); Assert.AreEqual(numActions, action.Length); // verify all unique actions in the list Assert.AreEqual(action.Length, action.Distinct().Count()); // verify the actions are in the expected range Assert.AreEqual((numActions * (numActions + 1)) / 2, action.Sum(a => a)); ds.ReportReward(i / 100f, uniqueKey); } } Assert.AreEqual(200, joinServer.EventBatchList.Sum(b => b.ExperimentalUnitFragments.Count)); }
public void TestMultiActionDSUploadSelective() { joinServer.Reset(); string uniqueKey = "test interaction"; var dsConfig = new DecisionServiceConfiguration(MockCommandCenter.SettingsBlobUri); dsConfig.JoinServerType = JoinServerType.CustomSolution; dsConfig.LoggingServiceAddress = MockJoinServer.MockJoinServerAddress; dsConfig.InteractionUploadConfiguration = new BatchingConfiguration(); dsConfig.InteractionUploadConfiguration.MaxDuration = TimeSpan.FromMinutes(10); // allow enough time for queue to buffer events dsConfig.InteractionUploadConfiguration.MaxDegreeOfSerializationParallelism = 1; // single-threaded for easy verification int numEvents = 100; // Set queue capacity to same number of events so selective dropping starts at 50% full dsConfig.InteractionUploadConfiguration.MaxUploadQueueCapacity = numEvents; dsConfig.InteractionUploadConfiguration.DroppingPolicy = new DroppingPolicy { MaxQueueLevelBeforeDrop = .5f, // when threshold is reached, drop half of the events ProbabilityOfDrop = .5f }; using (var ds = DecisionService.Create <TestContext>(dsConfig) //.WithTopSlotEpsilonGreedy(.2f) .ExploitUntilModelReady(new ConstantPolicy <TestContext>())) { for (int i = 0; i < numEvents; i++) { int[] chosenAction1 = ds.ChooseRanking(uniqueKey, new TestContext()); } } // Some events must have been dropped so the total count cannot be same as original Assert.IsTrue(joinServer.EventBatchList.Sum(batch => batch.ExperimentalUnitFragments.Count) < numEvents); // Get number of events that have been downsampled, i.e. selected with probability q int numSampledEvents = joinServer.EventBatchList .SelectMany(batch => batch.ExperimentalUnitFragments) .Where(e => e.Value.Contains("\"pdrop\":0.5")) .Count(); Assert.IsTrue(numSampledEvents > 0); // half of the events are selected with probability 0.5, so this should definitely be less than half the total events Assert.IsTrue(numSampledEvents < numEvents / 2); }
public void TestSingleActionOfflineModeCustomLogger() { var dsConfig = new DecisionServiceConfiguration("") { OfflineMode = true, OfflineApplicationID = "" }; var metaData = new ApplicationClientMetadata { TrainArguments = "--cb_explore_adf --epsilon 0.3", InitialExplorationEpsilon = 1f }; var recorder = new TestLogger(); int numChooseAction = 100; using (var ds = DecisionService.Create <TestContext>(dsConfig, metaData: metaData) .WithRecorder(recorder) .ExploitUntilModelReady(new ConstantPolicy <TestContext>())) { for (int i = 0; i < numChooseAction; i++) { ds.ChooseAction(i.ToString(), new TestContext()); } Assert.AreEqual(numChooseAction, recorder.NumRecord); int numReward = 200; for (int i = 0; i < numReward; i++) { ds.ReportReward(i, i.ToString()); } Assert.AreEqual(numReward, recorder.NumReward); int numOutcome = 300; for (int i = 0; i < numOutcome; i++) { ds.ReportOutcome(i.ToString(), i.ToString()); } Assert.AreEqual(numOutcome, recorder.NumOutcome); } Assert.AreEqual(0, recorder.NumRecord); Assert.AreEqual(0, recorder.NumReward); Assert.AreEqual(0, recorder.NumOutcome); }
public async Task TestNoModelFoundForImmediateDownload() { // create mock blobs for settings and models this.commandCenter.CreateBlobs(createSettingsBlob: true, createModelBlob: false, vwArgs: "--cb_explore_adf --epsilon 0.2"); var serviceConfig = new DecisionServiceConfiguration(MockCommandCenter.SettingsBlobUri) { JoinServerType = JoinServerType.CustomSolution, LoggingServiceAddress = MockJoinServer.MockJoinServerAddress, }; using (var service = DecisionService.Create <TestADFContextWithFeatures>(serviceConfig)) { await service.DownloadModelAndUpdate(new System.Threading.CancellationToken()); } }
public void TestMultiActionOnlineModeCustomLogger() { joinServer.Reset(); var dsConfig = new DecisionServiceConfiguration(MockCommandCenter.SettingsBlobUri); var recorder = new TestLogger(); dsConfig.PollingForModelPeriod = TimeSpan.MinValue; dsConfig.PollingForSettingsPeriod = TimeSpan.MinValue; dsConfig.JoinServerType = JoinServerType.CustomSolution; int numChooseAction = 100; using (var ds = DecisionService.Create <TestContext>(dsConfig, JsonTypeInspector.Default) //.WithTopSlotEpsilonGreedy(.2f) .WithRecorder(recorder) .ExploitUntilModelReady(new ConstantPolicy <TestContext>())) { for (int i = 0; i < numChooseAction; i++) { ds.ChooseAction(i.ToString(), new TestContext()); } Assert.AreEqual(numChooseAction, recorder.NumRecord); int numReward = 200; for (int i = 0; i < numReward; i++) { ds.ReportReward(i, i.ToString()); } Assert.AreEqual(numReward, recorder.NumReward); int numOutcome = 300; for (int i = 0; i < numOutcome; i++) { ds.ReportOutcome(i.ToString(), i.ToString()); } Assert.AreEqual(numOutcome, recorder.NumOutcome); } Assert.AreEqual(0, recorder.NumRecord); Assert.AreEqual(0, recorder.NumReward); Assert.AreEqual(0, recorder.NumOutcome); }
public void TestSingleActionOnlineModeCustomLogger() { var dsConfig = new DecisionServiceConfiguration(MockCommandCenter.SettingsBlobUri) { JoinServerType = JoinServerType.CustomSolution }; var recorder = new TestLogger(); int numChooseAction = 100; using (var ds = DecisionService .Create <TestContext>(dsConfig) // TODO: .WithEpsilonGreedy(.2f) .WithRecorder(recorder) .ExploitUntilModelReady(new ConstantPolicy <TestContext>())) { for (int i = 0; i < numChooseAction; i++) { ds.ChooseAction(i.ToString(), new TestContext()); } Assert.AreEqual(numChooseAction, recorder.NumRecord); int numReward = 200; for (int i = 0; i < numReward; i++) { ds.ReportReward(i, i.ToString()); } Assert.AreEqual(numReward, recorder.NumReward); int numOutcome = 300; for (int i = 0; i < numOutcome; i++) { ds.ReportOutcome(i.ToString(), i.ToString()); } Assert.AreEqual(numOutcome, recorder.NumOutcome); } Assert.AreEqual(0, recorder.NumRecord); Assert.AreEqual(0, recorder.NumReward); Assert.AreEqual(0, recorder.NumOutcome); }
public void TestModelImmediateDownload() { // create mock blobs for settings and models this.commandCenter.CreateBlobs(createSettingsBlob: true, createModelBlob: true, vwArgs: "--cb_explore_adf --epsilon 0.2"); var serviceConfig = new DecisionServiceConfiguration(MockCommandCenter.SettingsBlobUri) { JoinServerType = JoinServerType.CustomSolution, LoggingServiceAddress = MockJoinServer.MockJoinServerAddress, }; using (var service = DecisionService.Create <TestADFContextWithFeatures>(serviceConfig)) { // download model right away service.DownloadModelAndUpdate(new System.Threading.CancellationToken()).Wait(); } // No exception if everything works }
public void TestSingleActionOfflineModeArgument() { var dsConfig = new DecisionServiceConfiguration("") { OfflineMode = true, OfflineApplicationID = "" }; try { var metaData = new ApplicationClientMetadata { TrainArguments = "--cb_explore_adf --epsilon 0.3", InitialExplorationEpsilon = 1f }; using (var ds = DecisionService.Create <TestContext>(dsConfig, metaData: metaData)) { } } catch (ArgumentException ex) { Assert.AreEqual("Recorder", ex.ParamName); } }
public void TestMultiActionDSUploadMultipleEvents() { joinServer.Reset(); string uniqueKey = "test interaction"; var dsConfig = new DecisionServiceConfiguration(MockCommandCenter.SettingsBlobUri); dsConfig.JoinServerType = JoinServerType.CustomSolution; dsConfig.LoggingServiceAddress = MockJoinServer.MockJoinServerAddress; using (var ds = DecisionService.Create <TestContext>(dsConfig) //.WithTopSlotEpsilonGreedy(.2f) .ExploitUntilModelReady(new ConstantPolicy <TestContext>())) { int[] chosenAction1 = ds.ChooseRanking(uniqueKey, new TestContext()); int[] chosenAction2 = ds.ChooseRanking(uniqueKey, new TestContext()); ds.ReportReward(1.0f, uniqueKey); ds.ReportOutcome(new { value = "test outcome" }, uniqueKey); } Assert.AreEqual(4, joinServer.EventBatchList.Sum(batch => batch.ExperimentalUnitFragments.Count)); }
public void TestADFExplorationResult() { joinServer.Reset(); commandCenter.CreateBlobs(createSettingsBlob: true, createModelBlob: false, vwArgs: "--cb_explore_adf --epsilon 0.5"); var dsConfig = new DecisionServiceConfiguration(MockCommandCenter.SettingsBlobUri) { PollingForModelPeriod = TimeSpan.MinValue, PollingForSettingsPeriod = TimeSpan.MinValue, JoinServerType = JoinServerType.CustomSolution, LoggingServiceAddress = MockJoinServer.MockJoinServerAddress }; using (var ds = DecisionService.Create <TestADFContext>(dsConfig) .ExploitUntilModelReady(new ConstantPolicy <TestADFContext>(ctx => ctx.ActionDependentFeatures.Count))) { string uniqueKey = "eventid"; for (int i = 1; i <= 100; i++) { var adfContext = new TestADFContext(i); int[] action = ds.ChooseRanking(uniqueKey, adfContext); Assert.AreEqual(i, action.Length); // verify all unique actions in the list Assert.AreEqual(action.Length, action.Distinct().Count()); // verify the actions are in the expected range Assert.AreEqual((i * (i + 1)) / 2, action.Sum(a => a)); ds.ReportReward(i / 100f, uniqueKey); } } Assert.AreEqual(200, joinServer.EventBatchList.Sum(b => b.ExperimentalUnitFragments.Count)); }
public void TestSingleActionDSThreadSafeUpload() { joinServer.Reset(); string uniqueKey = "test interaction"; commandCenter.CreateBlobs(createSettingsBlob: true, createModelBlob: false, vwArgs: "--cb_explore_adf --epsilon 0.2"); var createObservation = (Func <int, string>)((i) => { return(string.Format("00000", i)); }); var dsConfig = new DecisionServiceConfiguration(MockCommandCenter.SettingsBlobUri); dsConfig.JoinServerType = JoinServerType.CustomSolution; dsConfig.LoggingServiceAddress = MockJoinServer.MockJoinServerAddress; int numEvents = 1000; var chosenActions = new ConcurrentBag <int>(); using (var ds = DecisionService .Create <TestContext>(dsConfig) .ExploitUntilModelReady(new ConstantPolicy <TestContext>())) { Parallel.For(0, numEvents, new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount * 2 }, (i) => { chosenActions.Add(ds.ChooseAction(uniqueKey, new TestContext())); ds.ReportOutcome(new { value = createObservation((int)i) }, uniqueKey); }); } List <PartialDecisionServiceMessage> batchList = this.joinServer.EventBatchList; int numActualEvents = batchList.Sum(b => b.ExperimentalUnitFragments.Count); Assert.AreEqual(numEvents * 2, numActualEvents); List <string> uniqueKeys = batchList .SelectMany(b => b.ExperimentalUnitFragments.Select(f => f.Id)) .Distinct() .ToList(); Assert.AreEqual(1, uniqueKeys.Count); Assert.AreEqual(uniqueKey, uniqueKeys[0]); var completeFragments = batchList .SelectMany(b => b.ExperimentalUnitFragments .Select(f => JsonConvert.DeserializeObject <MultiActionCompleteExperimentalUnitFragment>(f.Value))); // Test actual interactions received List <MultiActionCompleteExperimentalUnitFragment> interactions = completeFragments .Where(f => f.Value == null) .OrderBy(f => f.Actions[0]) .ToList(); // Test values of the interactions Assert.AreEqual(numEvents, interactions.Count); var chosenActionList = chosenActions.OrderBy(a => a).ToList(); for (int i = 0; i < interactions.Count; i++) { Assert.AreEqual((int)chosenActionList[i], interactions[i].Actions[0]); } // Test actual observations received List <MultiActionCompleteExperimentalUnitFragment> observations = completeFragments .Where(f => f.Value != null) .OrderBy(f => f.Value) .ToList(); // Test values of the observations Assert.AreEqual(numEvents, observations.Count); for (int i = 0; i < observations.Count; i++) { Assert.AreEqual(JsonConvert.SerializeObject(new { value = createObservation(i) }), observations[i].Value); } }
/// <summary> /// Sample code simulating a news recommendation scenario. In this simple example, /// the rendering server has to pick 1 out of 4 news topics to show to users (e.g. as featured article). /// </summary> /// <remarks> /// NOTE: For this sample to work, the proper settings must be set at deployment time: /// Vowpal Wabbit Switches = --cb_explore_adf --epsilon 0.2 --cb_type dr /// </remarks> public static async Task NewsRecommendation() { if (String.IsNullOrWhiteSpace(SettingsBlobUri)) { Console.WriteLine("Please specify a valid settings URL."); return; } // Create configuration for the decision service var serviceConfig = new DecisionServiceConfiguration(settingsBlobUri: SettingsBlobUri); // Enable development mode to easily debug / diagnose data flow and system properties // This should be turned off in production deployment serviceConfig.DevelopmentMode = true; using (var service = DecisionService.Create <UserContextADF>(serviceConfig)) { var random = new Random(); int user = 0; // user id int maxDecisionHistory = 100; // max number of past decisions to record var correctDecisions = new Queue <int>(); // keeps track of past decisions Console.WriteLine("Press Ctrl + C at any time to cancel the process."); // Each topic has its own set of features var topicFeatures = Enumerable.Range(1, 10) .Select(_ => new TopicFeature { Features = Enumerable.Range(1, 10).Select(f => (float)random.NextDouble()).ToArray() }) .ToArray(); while (true) { user++; string uniqueKey = Guid.NewGuid().ToString(); var userContext = new UserContextADF { Gender = random.NextDouble() > 0.5 ? "male" : "female", TopicFeatures = topicFeatures.Take(random.Next(5, topicFeatures.Length)).ToArray() }; int[] topicRanking = await service.ChooseRankingAsync(uniqueKey, userContext); int topicId = topicRanking[0]; // Display the news topic chosen by exploration process. Console.WriteLine("Topic {0} was chosen for user {1}.", topicId, user + 1); // Report {0,1} reward as a simple float. // In a real scenario, one could associated a reward of 1 if user // clicks on the article and 0 otherwise. float reward = 0; if (userContext.Gender == "male" && topicId == 3) { reward = 1; } else if (userContext.Gender == "female" && topicId == 8) { reward = 1; } service.ReportReward(reward, uniqueKey); correctDecisions.Enqueue((int)reward); if (correctDecisions.Count == maxDecisionHistory) { correctDecisions.Dequeue(); } if (user % 50 == 0) { Console.WriteLine("Correct decisions out of last {0} interactions: {1}", maxDecisionHistory, correctDecisions.Sum()); } System.Threading.Thread.Sleep(TimeSpan.FromMilliseconds(50)); } } }
private float UploadFoodContextData(string settingsBlobUri, bool firstPass) { var serviceConfig = new DecisionServiceConfiguration(settingsBlobUri); if (firstPass) { serviceConfig.PollingForModelPeriod = TimeSpan.MinValue; this.OnlineTrainerReset(); } using (var service = DecisionService.Create <FoodContext>(serviceConfig)) { if (!firstPass) { Thread.Sleep(10000); } string uniqueKey = "scratch-key-gal"; string[] locations = { "HealthyTown", "LessHealthyTown" }; var rg = new Random(uniqueKey.GetHashCode()); int numActions = 3; // ["Hamburger deal 1", "Hamburger deal 2" (better), "Salad deal"] var csv = new StringBuilder(); int counterCorrect = 0; int counterTotal = 0; var header = "Location,Action,Reward"; csv.AppendLine(header); // number of iterations for (int i = 0; i < 10000 * locations.Length; i++) { // randomly select a location int iL = rg.Next(0, locations.Length); string location = locations[iL]; DateTime timeStamp = DateTime.UtcNow; string key = uniqueKey + Guid.NewGuid().ToString(); FoodContext currentContext = new FoodContext(); currentContext.UserLocation = location; currentContext.Actions = Enumerable.Range(1, numActions).ToArray(); int[] action = service.ChooseRanking(key, currentContext); counterTotal += 1; // We expect healthy town to get salad and unhealthy town to get the second burger (action 2) if (location.Equals("HealthyTown") && action[0] == 3) { counterCorrect += 1; } else if (location.Equals("LessHealthyTown") && action[0] == 2) { counterCorrect += 1; } var csvLocation = location; var csvAction = action[0].ToString(); float reward = 0; double currentRand = rg.NextDouble(); if (location.Equals("HealthyTown")) { // for healthy town, buy burger 1 with probability 0.1, burger 2 with probability 0.15, salad with probability 0.6 if ((action[0] == 1 && currentRand < 0.1) || (action[0] == 2 && currentRand < 0.15) || (action[0] == 3 && currentRand < 0.6)) { reward = 10; } } else { // for unhealthy town, buy burger 1 with probability 0.4, burger 2 with probability 0.6, salad with probability 0.2 if ((action[0] == 1 && currentRand < 0.4) || (action[0] == 2 && currentRand < 0.6) || (action[0] == 3 && currentRand < 0.2)) { reward = 10; } } service.ReportReward(reward, key); var newLine = string.Format("{0},{1},{2}", csvLocation, csvAction, "0"); csv.AppendLine(newLine); System.Threading.Thread.Sleep(1); } return((float)counterCorrect / counterTotal); } }
private static async Task MainAsync() { // Requires Visual C++ 2015 Runtime installed // - https://www.microsoft.com/en-us/download/details.aspx?id=48145 // Requires platform to be x64 // 1. Sign in on https://ds.microsoft.com // 2. Create new app // a. Select "Custom App" // b. Supply Azure storage account // c. Empty Action Endpoint field - this disables action id based featurization // d. Empty Action Set Endpoint field // e. Set Provision FrontEnd field to "false" - avoids HTTP scoring provisioning // f. Set Vowpal Wabbit arguments to --cb_explore_adf --epsilon 0.2 -q DT -q LT // f. Click create // 3. Wait for creation and hit edit // a. Copy Client Library URL and use in the sample code below // 4. nuget install Microsoft.Research.MultiWorldTesting.ClientLibrary // a. at least version 2.0.0.6 var config = new DecisionServiceConfiguration(settingsBlobUri: "<<INSERT URL FROM STEP 3A") { ModelPollFailureCallback = ex => Console.WriteLine(ex.Message), SettingsPollFailureCallback = ex => Console.WriteLine(ex.Message) }; // you should keep this object around (e.g. static variable) as it maintains the background process // to download new models and upload logged events using (var ds = DecisionService.Create <DecisionContext>(config, typeInspector: JsonTypeInspector.Default)) { // If you add the timestamp calculating reward latencies becomes very easy on our end // In general you can use existing request ids, just be careful as they're used to seed the random // generator for exploration. // Also they're used to join rank and reward calls, so they must be unique within the experimental unit duration var eventId = EventIdUtil.AddTimestamp(Guid.NewGuid().ToString("n"), EventIdUtil.BeginningOfTime); var context = new DecisionContext { Demographics = new DemographicNamespace { Gender = "female", }, Location = new LocationNamespace { Country = "USA", City = "New York" }, Actions = new[] { new ActionDependentFeatures { Topic = new TopicNamespace { Category = "Fashion" } }, new ActionDependentFeatures { Topic = new TopicNamespace { Category = "Technology" } } } }; // get the decision var ranking = await ds.ChooseRankingAsync( uniqueKey : eventId, context : context); // report the reward if you observed desired behavior on first item in ranking // e.g. the first item returned by ChooseRankingAsync got clicked. ds.ReportReward(reward: 1f, uniqueKey: eventId); } }
public static async Task Sample() { // 1. Sign in on https://ds.microsoft.com // 2. Create new app // a. Select “Custom App” // b. Supply Azure storage account // c. Empty Action Endpoint field - this disables action id based featurization // d. Empty Action Set Endpoint field // e. Set Provision FrontEnd field to “false” - avoids HTTP scoring provisioning // f. Set Vowpal Wabbit arguments to --cb_explore_adf --epsilon 0.2 -q gc -q gC -q dC // f. Click create // 3. Wait for creation and hit edit // a. Copy Client Library URL and use in the sample code below // 4. nuget install Microsoft.Research.MultiWorldTesting.ClientLibrary // a. at least version 2.0.0.6 var config = new DecisionServiceConfiguration(settingsBlobUri: "<<INSERT URL FROM STEP 3A") { ModelPollFailureCallback = ex => Console.WriteLine(ex.Message), SettingsPollFailureCallback = ex => Console.WriteLine(ex.Message) }; // you should keep this object around (e.g. static variable) as it maintains the background process // to download new models and upload logged events var ds = DecisionService.Create <MyDecisionContext>(config, typeInspector: JsonTypeInspector.Default); // Some sample data var context = new MyDecisionContext { Demographics = new MyDecisionContextDemographics { Age = 15, AgeGroup = "Teenager", Gender = "male" }, Geo = new MyDecisionContextGeo { City = "New York", Country = "USA" }, Actions = new[] { new MyDecisionContextAction { Categories = new MyDecisionContextActionCategorization { Categories = "Fashion OldSchool" }, Celebrities = new Dictionary <string, float> { { "Markus", 0.9f }, { "Bob", 0.1f } } }, new MyDecisionContextAction { Categories = new MyDecisionContextActionCategorization { Categories = "Tech Microsoft" }, Celebrities = new Dictionary <string, float> { { "Bill", 0.7f }, { "Bob", 0.2f } } } } }; for (int i = 0; i < 5; i++) { // this is used as both correlation event id to link the ranking request to the observerd outcome // as well as random seed for the pseudo random generator var eventId = Guid.NewGuid().ToString("n"); // get the decision var ranking = await ds.ChooseRankingAsync( uniqueKey : eventId, context : context); // report the reward if you observed desired behavior on first item in ranking // e.g. the first item returned by ChooseRankingAsync got clicked. ds.ReportReward(reward: 1f, uniqueKey: eventId); } Console.WriteLine("done"); Thread.Sleep(TimeSpan.FromHours(1)); Console.WriteLine(); }
/// <summary> /// Sample code simulating a news recommendation scenario. In this simple example, /// the rendering server has to pick 1 out of 10 news topics to show to users (e.g. as featured article). /// </summary> /// <remarks> /// NOTE: For this sample to work, the proper settings must be set at deployment time: /// Number Of Actions = 10 /// Vowpal Wabbit Switches = --cb_explore 10 --epsilon 0.2 --cb_type dr /// </remarks> public static void NewsRecommendation() { if (String.IsNullOrWhiteSpace(SettingsBlobUri)) { Console.WriteLine("Please specify a valid settings URL."); return; } int numTopics = 10; // number of different topic choices to show // Create configuration for the decision service. var serviceConfig = new DecisionServiceConfiguration(settingsBlobUri: SettingsBlobUri); // Enable development mode to easily debug / diagnose data flow and system properties // This should be turned off in production deployment serviceConfig.DevelopmentMode = true; // Create the main service object with above configurations. // Specify the exploration algorithm to use, here we will use Epsilon-Greedy. // For more details about this and other algorithms, refer to the MWT onboarding whitepaper. using (var service = DecisionService.Create <UserContext>(serviceConfig)) { // Create a default policy which is used before a machine-learned model. // This policy is used in epsilon-greedy exploration using the initial // exploration epsilon specified at deployment time or in the Management Center. var defaultPolicy = new NewsDisplayPolicy(numTopics); var random = new Random(); int user = 0; Console.WriteLine("Press Ctrl + C at any time to cancel the process."); int maxDecisionHistory = 100; var correctDecisions = new Queue <int>(); while (true) { user++; // Generate a random GUID id for each user. var uniqueKey = Guid.NewGuid().ToString(); // Generate random feature vector for each user. var userContext = new UserContext() { Gender = random.NextDouble() > 0.5 ? "male" : "female" }; for (int f = 1; f <= 10; f++) { userContext.Features.Add(f.ToString(), (float)random.NextDouble()); } // Perform exploration given user features. int topicId = service.ChooseAction(uniqueKey, userContext, defaultPolicy); // Display the news topic chosen by exploration process. Console.WriteLine("Topic {0} was chosen for user {1}.", topicId, user + 1); // Report {0,1} reward as a simple float. // In a real scenario, one could associated a reward of 1 if user // clicks on the article and 0 otherwise. float reward = 0; if (userContext.Gender == "male" && topicId == 3) { reward = 1; } else if (userContext.Gender == "female" && topicId == 8) { reward = 1; } service.ReportReward(reward, uniqueKey); correctDecisions.Enqueue((int)reward); if (correctDecisions.Count == maxDecisionHistory) { correctDecisions.Dequeue(); } if (user % 50 == 0) { Console.WriteLine("Correct decisions out of last {0} interactions: {1}", maxDecisionHistory, correctDecisions.Sum()); } System.Threading.Thread.Sleep(TimeSpan.FromMilliseconds(50)); } } }