public async Task VerifyStartEarlyProcessingForTwoSupportedCommandHistory()
{
var expectedTelemetryCount = 2;
var(azPredictor, telemetryClient) = CreateTestObjects(throwException: false, expectedTelemetryCount + 1);
IReadOnlyList <string> history = new List <string>()
{
"Get-AzContext",
"New-AzVM -Name hello -Location WestUS",
};
azPredictor.StartEarlyProcessing(AzPredictorTelemetryTests.AzPredictorClient, history);
var maskedCommands = new List <string>()
{
"Get-AzContext",
"New-AzVM -Location *** -Name ***",
};
await telemetryClient.HistoryTaskCompletionSource.Task;
Assert.Equal(maskedCommands[1], telemetryClient.HistoryData.Command);
Assert.Equal(AzPredictorTelemetryTests.AzPredictorClient, telemetryClient.HistoryData.ClientId);
await telemetryClient.RequestPredictionTaskCompletionSource.Task;
Assert.Equal(maskedCommands, telemetryClient.RequestPredictionData.Commands);
Assert.Equal(AzPredictorTelemetryTests.AzPredictorClient, telemetryClient.RequestPredictionData.ClientId);
VerifyTelemetryRecordCount(expectedTelemetryCount, telemetryClient);
Assert.EndsWith("CommandHistory", telemetryClient.RecordedTelemetry[0].EventName);
Assert.Equal(AzPredictorTelemetryTests.AzPredictorClient, telemetryClient.RecordedTelemetry[0].Properties["ClientId"]);
Assert.Equal(maskedCommands[1], telemetryClient.RecordedTelemetry[0].Properties["History"]);
Assert.EndsWith("RequestPrediction", telemetryClient.RecordedTelemetry[1].EventName);
Assert.Equal(AzPredictorTelemetryTests.AzPredictorClient, telemetryClient.RecordedTelemetry[1].Properties["ClientId"]);
Assert.Equal(string.Join("\n", maskedCommands), telemetryClient.RecordedTelemetry[1].Properties["Command"]);
// The correlation id are changed in OnHistory.
AzPredictorTelemetryTests.EnsureDifferentCorrelationId(telemetryClient.RequestPredictionData, telemetryClient.HistoryData);
// SetssionId is not changed.
AzPredictorTelemetryTests.EnsureSameSessionId(telemetryClient.RequestPredictionData, telemetryClient.HistoryData);
Assert.Null(telemetryClient.GetSuggestionData);
Assert.Null(telemetryClient.SuggestionDisplayedData);
Assert.Null(telemetryClient.SuggestionAcceptedData);
Assert.Null(telemetryClient.ParameterMapData);
}
public async Task VerifyStartEarlyProcessingForOneSupportedCommandWithoutParameter()
{
var expectedTelemetryCount = 2;
var(azPredictor, telemetryClient) = CreateTestObjects(throwException: false, expectedTelemetryCount + 1);
var testCaseClientId = "TestCase";
// There is only one command.
IReadOnlyList <string> history = new List <string>()
{
"Get-AzContext",
};
azPredictor.StartEarlyProcessing(testCaseClientId, history);
await telemetryClient.HistoryTaskCompletionSource.Task;
Assert.Equal(history[0], telemetryClient.HistoryData.Command);
Assert.Equal(testCaseClientId, telemetryClient.HistoryData.ClientId);
await telemetryClient.RequestPredictionTaskCompletionSource.Task;
Assert.Equal(new List <string>()
{
AzPredictorConstants.CommandPlaceholder, history[0]
}, telemetryClient.RequestPredictionData.Commands);
Assert.Equal(testCaseClientId, telemetryClient.RequestPredictionData.ClientId);
VerifyTelemetryRecordCount(expectedTelemetryCount, telemetryClient);
Assert.EndsWith("CommandHistory", telemetryClient.RecordedTelemetry[0].EventName);
Assert.Equal(testCaseClientId, telemetryClient.RecordedTelemetry[0].Properties["ClientId"]);
Assert.Equal(history[0], telemetryClient.RecordedTelemetry[0].Properties["History"]);
Assert.EndsWith("RequestPrediction", telemetryClient.RecordedTelemetry[1].EventName);
Assert.Equal(testCaseClientId, telemetryClient.RecordedTelemetry[1].Properties["ClientId"]);
Assert.Equal($"{AzPredictorConstants.CommandPlaceholder}\n{history[0]}", telemetryClient.RecordedTelemetry[1].Properties["Command"]);
// The correlation id are changed in OnHistory.
AzPredictorTelemetryTests.EnsureDifferentCorrelationId(telemetryClient.RequestPredictionData, telemetryClient.HistoryData);
// SetssionId is not changed.
AzPredictorTelemetryTests.EnsureSameSessionId(telemetryClient.RequestPredictionData, telemetryClient.HistoryData);
Assert.Null(telemetryClient.GetSuggestionData);
Assert.Null(telemetryClient.SuggestionDisplayedData);
Assert.Null(telemetryClient.SuggestionAcceptedData);
Assert.Null(telemetryClient.ParameterMapData);
}
public void VerifySuggestionSessionIdChanged()
{
var expectedTelemetryCount = 2;
var(azPredictor, telemetryClient) = CreateTestObjects(throwException: false, expectedTelemetryCount + 1);
var predictionContext = PredictionContext.Create("New-AzResourceGroup -Name 'ResourceGroup01' -Location 'Central US' -WhatIf");
var firstSuggestionPackage = azPredictor.GetSuggestion(AzPredictorTelemetryTests.AzPredictorClient, predictionContext, CancellationToken.None);
var firstGetSuggestionData = telemetryClient.GetSuggestionData;
var secondSuggestionPackage = azPredictor.GetSuggestion(AzPredictorTelemetryTests.AzPredictorClient, predictionContext, CancellationToken.None);
var secondGetSuggestionData = telemetryClient.GetSuggestionData;
Assert.NotEqual(secondSuggestionPackage.Session, firstSuggestionPackage.Session);
AzPredictorTelemetryTests.EnsureSameCorrelationId(secondGetSuggestionData, firstGetSuggestionData);
AzPredictorTelemetryTests.EnsureSameSessionId(secondGetSuggestionData, firstGetSuggestionData);
VerifyTelemetryRecordCount(expectedTelemetryCount, telemetryClient);
}
public async Task VerifyStartEarlyProcessingException()
{
var expectedTelemetryCount = 2;
var(azPredictor, telemetryClient) = CreateTestObjects(throwException: true, expectedTelemetryCount + 1);
var history = new List <string>()
{
"New-AzVM -Name hello -Location WestUS",
};
var maskedCommand = "New-AzVM -Location *** -Name ***";
azPredictor.StartEarlyProcessing(AzPredictorTelemetryTests.AzPredictorClient, history);
await telemetryClient.HistoryTaskCompletionSource.Task;
Assert.Equal(maskedCommand, telemetryClient.HistoryData.Command);
Assert.Equal(AzPredictorTelemetryTests.AzPredictorClient, telemetryClient.HistoryData.ClientId);
await telemetryClient.RequestPredictionTaskCompletionSource.Task;
Assert.IsType <MockTestException>(telemetryClient.RequestPredictionData.Exception);
Assert.Equal(AzPredictorTelemetryTests.AzPredictorClient, telemetryClient.RequestPredictionData.ClientId);
VerifyTelemetryRecordCount(expectedTelemetryCount, telemetryClient);
Assert.EndsWith("CommandHistory", telemetryClient.RecordedTelemetry[0].EventName);
Assert.Equal(AzPredictorTelemetryTests.AzPredictorClient, telemetryClient.RecordedTelemetry[0].Properties["ClientId"]);
Assert.Equal(maskedCommand, telemetryClient.RecordedTelemetry[0].Properties["History"]);
Assert.EndsWith("RequestPrediction", telemetryClient.RecordedTelemetry[1].EventName);
Assert.Equal(AzPredictorTelemetryTests.AzPredictorClient, telemetryClient.RecordedTelemetry[1].Properties["ClientId"]);
Assert.Equal($"{AzPredictorConstants.CommandPlaceholder}\n{maskedCommand}", telemetryClient.RecordedTelemetry[1].Properties["Command"]);
Assert.StartsWith($"Type: {typeof(MockTestException)}\nStack Trace: ", telemetryClient.RecordedTelemetry[1].Properties["Exception"]);
// The correlation id are changed in OnHistory.
AzPredictorTelemetryTests.EnsureDifferentCorrelationId(telemetryClient.RequestPredictionData, telemetryClient.HistoryData);
// SetssionId is not changed.
AzPredictorTelemetryTests.EnsureSameSessionId(telemetryClient.RequestPredictionData, telemetryClient.HistoryData);
}
public async Task VerifyStartEarlyProcessingForSupportedAndUnsupportedCommands()
{
var expectedTelemetryCount = 2;
var(azPredictor, telemetryClient) = CreateTestObjects(throwException: false, expectedTelemetryCount + 1);
var history = new List <string>()
{
"New-AzVM -Name hello -Location WestUS",
"git status",
};
azPredictor.StartEarlyProcessing(AzPredictorTelemetryTests.AzPredictorClient, history);
var maskedCommands = new List <string>()
{
AzPredictorConstants.CommandPlaceholder,
"New-AzVM -Location *** -Name ***",
};
await telemetryClient.HistoryTaskCompletionSource.Task;
Assert.Equal(maskedCommands[0], telemetryClient.HistoryData.Command);
Assert.Equal(AzPredictorTelemetryTests.AzPredictorClient, telemetryClient.HistoryData.ClientId);
await telemetryClient.RequestPredictionTaskCompletionSource.Task;
Assert.Equal(maskedCommands, telemetryClient.RequestPredictionData.Commands);
Assert.Equal(AzPredictorTelemetryTests.AzPredictorClient, telemetryClient.RequestPredictionData.ClientId);
// The correlation id are changed in OnHistory.
AzPredictorTelemetryTests.EnsureDifferentCorrelationId(telemetryClient.RequestPredictionData, telemetryClient.HistoryData);
// SetssionId is not changed.
AzPredictorTelemetryTests.EnsureSameSessionId(telemetryClient.RequestPredictionData, telemetryClient.HistoryData);
VerifyTelemetryRecordCount(expectedTelemetryCount, telemetryClient);
}
/// <summary>
/// Verify that GetSuggestion, SuggestionDisplayed, and SessionAccepted all have the same suggestion session id.
/// </summary>
//[Fact]
private void VerifySameSuggestionSessionId()
{
var expectedTelemetryCount = 1;
var(azPredictor, telemetryClient) = CreateTestObjects(throwException: false, expectedTelemetryCount + 1);
var predictionContext = PredictionContext.Create("New-AzResourceGroup -Name 'ResourceGroup01' -Location 'Central US' -WhatIf");
var suggestionPackage = azPredictor.GetSuggestion(AzPredictorTelemetryTests.AzPredictorClient, predictionContext, CancellationToken.None);
Assert.Equal(AzPredictorTelemetryTests.AzPredictorClient, telemetryClient.GetSuggestionData.ClientId);
Assert.Equal(suggestionPackage.Session.Value, telemetryClient.GetSuggestionData.SuggestionSessionId);
Assert.NotNull(telemetryClient.GetSuggestionData.Suggestion);
Assert.NotNull(telemetryClient.GetSuggestionData.UserInput);
VerifyTelemetryRecordCount(expectedTelemetryCount, telemetryClient);
Assert.EndsWith("GetSuggestion", telemetryClient.RecordedTelemetry[0].EventName);
Assert.Equal(AzPredictorTelemetryTests.AzPredictorClient, telemetryClient.RecordedTelemetry[0].Properties["ClientId"]);
Assert.Equal(suggestionPackage.Session.Value.ToString(CultureInfo.InvariantCulture), telemetryClient.RecordedTelemetry[0].Properties["SuggestionSessionId"]);
Assert.Equal("New-AzResourceGroup -Location *** -Name *** -WhatIf ***", telemetryClient.RecordedTelemetry[0].Properties["UserInut"]);
Assert.Equal("", telemetryClient.RecordedTelemetry[0].Properties["Suggestion"]);
var displayCountOrIndex = 3;
telemetryClient.ResetWaitingTasks();
telemetryClient.ExceptedTelemetryRecordCount = expectedTelemetryCount + 1;
azPredictor.OnSuggestionDisplayed(AzPredictorTelemetryTests.AzPredictorClient, suggestionPackage.Session.Value, displayCountOrIndex);
VerifyTelemetryRecordCount(expectedTelemetryCount, telemetryClient);
Assert.EndsWith("DisplaySuggestion", telemetryClient.RecordedTelemetry[0].EventName);
Assert.Equal(suggestionPackage.Session.Value.ToString(CultureInfo.InvariantCulture), telemetryClient.RecordedTelemetry[0].Properties["SuggestionSessionId"]);
Assert.Equal(AzPredictorTelemetryTests.AzPredictorClient, telemetryClient.SuggestionDisplayedData.ClientId);
Assert.Equal("ListView", telemetryClient.RecordedTelemetry[0].Properties["SuggestionDisplayMode"]);
Assert.Equal(displayCountOrIndex.ToString(CultureInfo.InvariantCulture), telemetryClient.RecordedTelemetry[0].Properties["SuggestionCount"]);
Assert.False(telemetryClient.RecordedTelemetry[0].Properties.ContainsKey("SuggestionIndex"));
var acceptedSuggestion = "SuggestionAccepted";
telemetryClient.ResetWaitingTasks();
telemetryClient.ExceptedTelemetryRecordCount = expectedTelemetryCount + 1;
azPredictor.OnSuggestionAccepted(AzPredictorTelemetryTests.AzPredictorClient, suggestionPackage.Session.Value, acceptedSuggestion);
VerifyTelemetryRecordCount(expectedTelemetryCount, telemetryClient);
Assert.EndsWith("AcceptSuggestion", telemetryClient.RecordedTelemetry[0].EventName);
Assert.Equal(AzPredictorTelemetryTests.AzPredictorClient, telemetryClient.RecordedTelemetry[0].Properties["ClientId"]);
Assert.Equal(suggestionPackage.Session.Value.ToString(CultureInfo.InvariantCulture), telemetryClient.RecordedTelemetry[0].Properties["SuggestionSessionId"]);
Assert.Equal(acceptedSuggestion, telemetryClient.RecordedTelemetry[0].Properties["AccepedSuggestion"]);
Assert.Equal(suggestionPackage.Session.Value, telemetryClient.SuggestionDisplayedData.SuggestionSessionId);
Assert.Equal(displayCountOrIndex, telemetryClient.SuggestionDisplayedData.SuggestionCountOrIndex);
Assert.Equal(suggestionPackage.Session.Value, telemetryClient.SuggestionAcceptedData.SuggestionSessionId);
Assert.Equal(acceptedSuggestion, telemetryClient.SuggestionAcceptedData.Suggestion);
AzPredictorTelemetryTests.EnsureSameCorrelationId(telemetryClient.GetSuggestionData, telemetryClient.SuggestionDisplayedData);
AzPredictorTelemetryTests.EnsureSameCorrelationId(telemetryClient.GetSuggestionData, telemetryClient.SuggestionAcceptedData);
AzPredictorTelemetryTests.EnsureSameSessionId(telemetryClient.GetSuggestionData, telemetryClient.SuggestionDisplayedData);
AzPredictorTelemetryTests.EnsureSameSessionId(telemetryClient.GetSuggestionData, telemetryClient.SuggestionAcceptedData);
Assert.Null(telemetryClient.HistoryData);
Assert.Null(telemetryClient.RequestPredictionData);
}
public async Task VerifyStartEarlyProcessingForUnsupportedCommandAfterSupportedOnes()
{
var expectedTelemetryCount = 2;
var(azPredictor, telemetryClient) = CreateTestObjects(throwException: false, expectedTelemetryCount + 1);
var history = new List <string>()
{
"New-AzResourceGroup -Name:resourceGroup01",
"New-AzVM -Name hello -Location WestUS"
};
var maskedCommands = new List <string>()
{
"New-AzResourceGroup -Name:***",
"New-AzVM -Location *** -Name ***"
};
azPredictor.StartEarlyProcessing(AzPredictorTelemetryTests.AzPredictorClient, history);
await telemetryClient.HistoryTaskCompletionSource.Task;
Assert.Equal(maskedCommands[1], telemetryClient.HistoryData.Command);
Assert.Equal(AzPredictorTelemetryTests.AzPredictorClient, telemetryClient.HistoryData.ClientId);
await telemetryClient.RequestPredictionTaskCompletionSource.Task;
Assert.Equal(maskedCommands, telemetryClient.RequestPredictionData.Commands);
Assert.Equal(AzPredictorTelemetryTests.AzPredictorClient, telemetryClient.RequestPredictionData.ClientId);
VerifyTelemetryRecordCount(expectedTelemetryCount, telemetryClient);
Assert.EndsWith("CommandHistory", telemetryClient.RecordedTelemetry[0].EventName);
Assert.Equal(AzPredictorTelemetryTests.AzPredictorClient, telemetryClient.RecordedTelemetry[0].Properties["ClientId"]);
Assert.Equal(maskedCommands[1], telemetryClient.RecordedTelemetry[0].Properties["History"]);
Assert.EndsWith("RequestPrediction", telemetryClient.RecordedTelemetry[1].EventName);
Assert.Equal(AzPredictorTelemetryTests.AzPredictorClient, telemetryClient.RecordedTelemetry[1].Properties["ClientId"]);
Assert.Equal(string.Join("\n", maskedCommands), telemetryClient.RecordedTelemetry[1].Properties["Command"]);
var firstHistoryData = telemetryClient.HistoryData;
var firstRequestPredictionData = telemetryClient.RequestPredictionData;
telemetryClient.ResetWaitingTasks();
expectedTelemetryCount = 1;
telemetryClient.ExceptedTelemetryRecordCount = expectedTelemetryCount + 1;
history.Add("git status");
azPredictor.StartEarlyProcessing(AzPredictorTelemetryTests.AzPredictorClient, history);
await telemetryClient.HistoryTaskCompletionSource.Task;
Assert.Equal(AzPredictorConstants.CommandPlaceholder, telemetryClient.HistoryData.Command);
Assert.Equal(AzPredictorTelemetryTests.AzPredictorClient, telemetryClient.HistoryData.ClientId);
// Don't need to await on telemetryClient.RequestPredictioinTask, because "git" isn't a supported command and RequestPredictionsAsync isn't called.
// The commands to request predictions are the same as previous request.
Assert.Equal(new List <string>()
{
"New-AzResourceGroup -Name:***", "New-AzVM -Location *** -Name ***"
}, telemetryClient.RequestPredictionData.Commands);
Assert.Equal(AzPredictorTelemetryTests.AzPredictorClient, telemetryClient.RequestPredictionData.ClientId);
VerifyTelemetryRecordCount(expectedTelemetryCount, telemetryClient);
Assert.EndsWith("CommandHistory", telemetryClient.RecordedTelemetry[0].EventName);
Assert.Equal(AzPredictorTelemetryTests.AzPredictorClient, telemetryClient.RecordedTelemetry[0].Properties["ClientId"]);
Assert.Equal(AzPredictorConstants.CommandPlaceholder, telemetryClient.RecordedTelemetry[0].Properties["History"]);
var secondHistoryData = telemetryClient.HistoryData;
// Make sure that the RequestPrediction event can be correlated to the right History event.
AzPredictorTelemetryTests.EnsureDifferentCorrelationId(firstHistoryData, firstRequestPredictionData);
AzPredictorTelemetryTests.EnsureSameCorrelationId(firstRequestPredictionData, secondHistoryData);
AzPredictorTelemetryTests.EnsureSameSessionId(firstHistoryData, firstRequestPredictionData);
AzPredictorTelemetryTests.EnsureSameSessionId(firstRequestPredictionData, secondHistoryData);
telemetryClient.ResetWaitingTasks();
expectedTelemetryCount = 1;
telemetryClient.ExceptedTelemetryRecordCount = expectedTelemetryCount + 1;
history.Add(@"$a='NewResourceName'");
azPredictor.StartEarlyProcessing(AzPredictorTelemetryTests.AzPredictorClient, history);
await telemetryClient.HistoryTaskCompletionSource.Task;
Assert.Equal(AzPredictorConstants.CommandPlaceholder, telemetryClient.HistoryData.Command);
Assert.Equal(AzPredictorTelemetryTests.AzPredictorClient, telemetryClient.HistoryData.ClientId);
// Don't need to await on telemetryClient.RequestPredictioinTask, because assignment isn't a supported command and RequestPredictionsAsync isn't called.
// The commands to request predictions are the same as previous request.
Assert.Equal(new List <string>()
{
"New-AzResourceGroup -Name:***", "New-AzVM -Location *** -Name ***"
}, telemetryClient.RequestPredictionData.Commands);
Assert.Equal(AzPredictorTelemetryTests.AzPredictorClient, telemetryClient.RequestPredictionData.ClientId);
VerifyTelemetryRecordCount(expectedTelemetryCount, telemetryClient);
Assert.EndsWith("CommandHistory", telemetryClient.RecordedTelemetry[0].EventName);
Assert.Equal(AzPredictorTelemetryTests.AzPredictorClient, telemetryClient.RecordedTelemetry[0].Properties["ClientId"]);
Assert.Equal(AzPredictorConstants.CommandPlaceholder, telemetryClient.RecordedTelemetry[0].Properties["History"]);
// There is no new request prediction. The correlation id isn't changed.
AzPredictorTelemetryTests.EnsureSameCorrelationId(telemetryClient.RequestPredictionData, telemetryClient.HistoryData);
// SetssionId is not changed.
AzPredictorTelemetryTests.EnsureSameSessionId(telemetryClient.RequestPredictionData, telemetryClient.HistoryData);
telemetryClient.ResetWaitingTasks();
expectedTelemetryCount = 2;
telemetryClient.ExceptedTelemetryRecordCount = expectedTelemetryCount + 1;
history.Add("Get-AzResourceGroup -Name:ResourceGroup01");
azPredictor.StartEarlyProcessing(AzPredictorTelemetryTests.AzPredictorClient, history);
maskedCommands = new List <string>()
{
"New-AzVM -Location *** -Name ***",
"Get-AzResourceGroup -Name:***",
};
await telemetryClient.HistoryTaskCompletionSource.Task;
Assert.Equal(maskedCommands[1], telemetryClient.HistoryData.Command);
Assert.Equal(AzPredictorTelemetryTests.AzPredictorClient, telemetryClient.HistoryData.ClientId);
await telemetryClient.RequestPredictionTaskCompletionSource.Task;
Assert.Equal(maskedCommands, telemetryClient.RequestPredictionData.Commands);
Assert.Equal(AzPredictorTelemetryTests.AzPredictorClient, telemetryClient.RequestPredictionData.ClientId);
VerifyTelemetryRecordCount(expectedTelemetryCount, telemetryClient);
Assert.EndsWith("CommandHistory", telemetryClient.RecordedTelemetry[0].EventName);
Assert.Equal(AzPredictorTelemetryTests.AzPredictorClient, telemetryClient.RecordedTelemetry[0].Properties["ClientId"]);
Assert.Equal(maskedCommands[1], telemetryClient.RecordedTelemetry[0].Properties["History"]);
Assert.EndsWith("RequestPrediction", telemetryClient.RecordedTelemetry[1].EventName);
Assert.Equal(AzPredictorTelemetryTests.AzPredictorClient, telemetryClient.RecordedTelemetry[1].Properties["ClientId"]);
Assert.Equal(string.Join("\n", maskedCommands), telemetryClient.RecordedTelemetry[1].Properties["Command"]);
// The correlation id are changed in OnHistory.
AzPredictorTelemetryTests.EnsureDifferentCorrelationId(telemetryClient.RequestPredictionData, telemetryClient.HistoryData);
// SetssionId is not changed.
AzPredictorTelemetryTests.EnsureSameSessionId(telemetryClient.RequestPredictionData, telemetryClient.HistoryData);
}