public async Task EndToEnd_ADF() { var util = new ProvisioningUtil(); util.DeleteExistingResourceGroupsMatchingPrefix(); var deployment = util.Deploy(); new EndToEndOnlineTrainerTest().E2ERankerStochasticRewards(deployment); }
public void ProvisionOnlyTest() { var deployment = new ProvisioningUtil().Deploy(); Assert.IsNotNull(deployment.ManagementCenterUrl); Assert.IsNotNull(deployment.ManagementPassword); Assert.IsNotNull(deployment.OnlineTrainerUrl); Assert.IsNotNull(deployment.OnlineTrainerToken); Assert.IsNotNull(deployment.WebServiceToken); Assert.IsNotNull(deployment.SettingsUrl); }
public async Task EndToEnd_Simple() { var util = new ProvisioningUtil(); util.DeleteExistingResourceGroupsMatchingPrefix(); var deployment = util.Deploy(); // var deployment = new DecisionServiceDeployment(@" // { // ""management Center URL"": { // ""type"": ""String"", // ""value"": ""https://mc-l6bzp222nonnslkr2h7lahyo2k.azurewebsites.net"" // }, // ""management Center Password"": { // ""type"": ""String"", // ""value"": ""t26uhzni7gf3g"" // }, // ""client Library URL"": { // ""type"": ""String"", // ""value"": ""https://storagel6bzp222nonns.blob.core.windows.net/mwt-settings/client?sv=2015-12-11&sr=b&sig=%2BaC%2FQAQdceavnQzGac7d8NwQDpIqCmPWKrT4sNrIASs%3D&st=2016-08-01T22%3A39%3A42Z&se=2017-08-01T22%3A40%3A42Z&sp=r"" // }, // ""web Service Token"": { // ""type"": ""String"", // ""value"": ""3pmfcjtevpcns"" // }, // ""online Trainer Token"": { // ""type"": ""String"", // ""value"": ""5a6cgj6753342"" // }, // ""online Trainer URL"": { // ""type"": ""String"", // ""value"": ""http://trainer-l6bzp222nonnslkr2h7lahyo2k.cloudapp.net"" // } //}"); await new SimplePolicyTestClass().SimplePolicyTest(deployment); }
public void APITestDriveTest() { var deployment = new ProvisioningUtil().Deploy(); var locs = new[] { "Seattle", "New York" }; var genders = new[] { "Male", "Female" }; var industries = new[] { "Tech", "Law" }; var ages = new[] { "Young", "Old" }; var rnd = new Random(123); var confusionMatrix = new int[2, 2]; using (var wc = new WebClient()) { wc.Headers.Add("auth: " + deployment.WebServiceToken); var urlPolicy = $"{deployment.ManagementCenterUrl}/API/Policy"; var urlReward = $"{deployment.ManagementCenterUrl}/API/Reward"; for (int i = 0; i < 32 * 1024; i++) { var index = rnd.Next(2); // mapping var expectedAction = index; if (rnd.NextDouble() < .2) { expectedAction++; } expectedAction = (expectedAction % 2) + 1; var jsonContext = JsonConvert.SerializeObject(new { Location = locs[index] }); var response = wc.UploadData(urlPolicy, "POST", Encoding.UTF8.GetBytes(jsonContext)); var decision = JsonConvert.DeserializeObject <PolicyDecision>(Encoding.UTF8.GetString(response)); if (decision.Action == expectedAction) { wc.UploadData($"{urlReward}/?eventId={decision.EventId}", "POST", Encoding.UTF8.GetBytes("1")); } confusionMatrix[index, decision.Action - 1]++; Thread.Sleep(100); if (i % 5 == 0) { Trace.WriteLine($"Model {decision.ModelTime}"); for (int j = 0; j < 2; j++) { Trace.WriteLine($"Location {locs[j],-9}: {confusionMatrix[j, 0],-4} {confusionMatrix[j, 1],-4}"); } // float sum = confusionMatrix.OfType<int>().Sum(); for (int j = 0; j < 2; j++) { var a1 = confusionMatrix[j, 0]; var a2 = confusionMatrix[j, 1]; float sum = a1 + a2; Trace.WriteLine($"Location {locs[j],-9}: {a1/sum:0.00} {a2 / sum:0.00}"); } Trace.WriteLine(""); } } } }
public void SimplePolicyHttpTest() { var deployment = new ProvisioningUtil().Deploy(); wc = new WebClient(); wc.Headers.Add("auth", deployment.ManagementPassword); deployment.ConfigureDecisionService("--cb_explore 4 --epsilon 1", initialExplorationEpsilon: 1, isExplorationEnabled: true); Thread.Sleep(TimeSpan.FromSeconds(5)); // 4 Actions // why does this need to be different from default? var config = new DecisionServiceConfiguration(deployment.SettingsUrl) { InteractionUploadConfiguration = new BatchingConfiguration { MaxEventCount = 64 }, ObservationUploadConfiguration = new BatchingConfiguration { MaxEventCount = 64 } }; config.InteractionUploadConfiguration.ErrorHandler += JoinServiceBatchConfiguration_ErrorHandler; config.InteractionUploadConfiguration.SuccessHandler += JoinServiceBatchConfiguration_SuccessHandler; this.features = new string[] { "a", "b", "c", "d" }; this.freq = new Dictionary <string, int>(); this.rnd = new Random(123); // reset the model deployment.OnlineTrainerReset(); Console.WriteLine("Waiting after reset..."); Thread.Sleep(TimeSpan.FromSeconds(2)); Console.WriteLine("Exploration"); var expectedEvents = 0; for (int i = 0; i < 1000; i++) { int featureIndex = i % features.Length; expectedEvents += SendEvents(deployment, wc, featureIndex); } // Thread.Sleep(500); // TODO: flush doesn't work // Assert.AreEqual(expectedEvents, this.eventCount); // 4 actions times 4 feature values Assert.AreEqual(4 * 4, freq.Keys.Count); var total = freq.Values.Sum(); foreach (var k in freq.Keys.OrderBy(k => k)) { var f = freq[k] / (float)total; Assert.IsTrue(f < 0.08); // 1/(4*4) = 0.0625 Console.WriteLine("{0} | {1}", k, f); } freq.Clear(); deployment.ConfigureDecisionService("--cb_explore 4 --epsilon 0", initialExplorationEpsilon: 1, isExplorationEnabled: false); Thread.Sleep(TimeSpan.FromSeconds(5)); // check here to make sure model was updated Console.WriteLine("Exploitation"); expectedEvents = 0; for (int i = 0; i < 1000; i++) { var featureIndex = i % features.Length; expectedEvents += SendEvents(deployment, wc, featureIndex, false); } total = freq.Values.Sum(); foreach (var k in freq.Keys.OrderBy(k => k)) { var f = freq[k] / (float)total; Assert.AreEqual(0.25f, f, 0.1); Console.WriteLine("{0} | {1}", k, f); } }