/// <summary>
/// Iterate over command elements to extract local parameter values.
///
/// Store these values by a key
/// consisting of the suffix of the command + the parameter name. There are some exceptions, e.g.
/// credential, location, where the parameter name itself is the key.
///
/// For example, New-AzResourceGroup -Name Hello -Location 'EastUS' will store into local parameters:
/// ResourceGroupName => Hello
/// Location => 'EastUS'
/// </summary>
/// <param name="command">The command ast elements</param>
private void ExtractLocalParameters(System.Collections.ObjectModel.ReadOnlyCollection <CommandElementAst> command)
{
// Azure PowerShell command is in the form of {Verb}-Az{Noun}, e.g. New-AzResource.
// We need to extract the noun to construct the parameter name.
var commandName = command.FirstOrDefault()?.ToString();
var commandNoun = ParameterValuePredictor.GetAzCommandNoun(commandName).ToLower();
if (commandNoun == null)
{
return;
}
for (int i = 2; i < command.Count; i += 2)
{
if (command[i - 1] is CommandParameterAst parameterAst && command[i] is StringConstantExpressionAst)
{
var parameterName = command[i - 1].ToString().ToLower().Trim('-');
if (_command_param_to_resource_map.ContainsKey(commandNoun))
{
if (_command_param_to_resource_map[commandNoun].ContainsKey(parameterName))
{
var key = _command_param_to_resource_map[commandNoun][parameterName];
var parameterValue = command[i].ToString();
this._localParameterValues.AddOrUpdate(key, parameterValue, (k, v) => parameterValue);
}
}
}
}
}
/// <summary>
/// Creates a new instance of <see cref="AzPredictorService"/>.
/// </summary>
/// <param name="serviceUri">The URI of the Aladdin service.</param>
/// <param name="telemetryClient">The telemetry client.</param>
/// <param name="azContext">The Az context which this module runs in.</param>
public AzPredictorService(string serviceUri, ITelemetryClient telemetryClient, IAzContext azContext)
{
Validation.CheckArgument(!string.IsNullOrWhiteSpace(serviceUri), $"{nameof(serviceUri)} cannot be null or whitespace.");
Validation.CheckArgument(telemetryClient, $"{nameof(telemetryClient)} cannot be null.");
Validation.CheckArgument(azContext, $"{nameof(azContext)} cannot be null.");
_parameterValuePredictor = new ParameterValuePredictor(telemetryClient);
_commandsEndpoint = $"{serviceUri}{AzPredictorConstants.CommandsEndpoint}?clientType={AzPredictorService.ClientType}&context.versionNumber={azContext.AzVersion}";
_predictionsEndpoint = serviceUri + AzPredictorConstants.PredictionsEndpoint;
_telemetryClient = telemetryClient;
_azContext = azContext;
_client = new HttpClient();
RequestAllPredictiveCommands();
}
/// <summary>
/// Predictor must be initialized with a list of string suggestions.
/// </summary>
/// <param name="modelPredictions">List of suggestions from the model, sorted by frequency (most to least)</param>
/// <param name="parameterValuePredictor">Provide the prediction to the parameter values.</param>
public Predictor(IList <string> modelPredictions, ParameterValuePredictor parameterValuePredictor)
{
this._parameterValuePredictor = parameterValuePredictor;
this._predictions = new List <Prediction>();
foreach (var predictionTextRaw in modelPredictions ?? Enumerable.Empty <string>())
{
var predictionText = EscapePredictionText(predictionTextRaw);
Ast ast = Parser.ParseInput(predictionText, out Token[] tokens, out _);
var commandAst = (ast.Find((ast) => ast is CommandAst, searchNestedScriptBlocks: false) as CommandAst);
if (commandAst?.CommandElements[0] is StringConstantExpressionAst commandName)
{
var parameterSet = new ParameterSet(commandAst);
this._predictions.Add(new Prediction(commandName.Value, parameterSet));
}
}
}
/// <summary>
/// Creates a new instance of <see cref="CommandLinePredictor"/>.
/// </summary>
/// <param name="modelPredictions">List of suggestions from the model, sorted by frequency (most to least).</param>
/// <param name="parameterValuePredictor">Provide the prediction to the parameter values.</param>
public CommandLinePredictor(IList <PredictiveCommand> modelPredictions, ParameterValuePredictor parameterValuePredictor)
{
Validation.CheckArgument(modelPredictions, $"{nameof(modelPredictions)} cannot be null.");
_parameterValuePredictor = parameterValuePredictor;
var commnadLines = new List <CommandLine>();
foreach (var predictiveCommand in modelPredictions ?? Enumerable.Empty <PredictiveCommand>())
{
var predictionText = CommandLineUtilities.EscapePredictionText(predictiveCommand.Command);
Ast ast = Parser.ParseInput(predictionText, out Token[] tokens, out _);
var commandAst = (ast.Find((ast) => ast is CommandAst, searchNestedScriptBlocks: false) as CommandAst);
if (commandAst?.CommandElements[0] is StringConstantExpressionAst commandName)
{
var parameterSet = new ParameterSet(commandAst);
this._commandLinePredictions.Add(new CommandLine(commandName.Value, predictiveCommand.Description, parameterSet));
}
}
}
/// <summary>
/// Creates a new instance of <see cref="CommandLinePredictor"/>.
/// </summary>
/// <param name="modelPredictions">List of suggestions from the model, sorted by frequency (most to least).</param>
/// <param name="parameterValuePredictor">Provide the prediction to the parameter values.</param>
/// <param name="telemetryClient">The telemetry client.</param>
/// <param name="azContext">The current PowerShell conext.</param>
public CommandLinePredictor(IList <PredictiveCommand> modelPredictions, ParameterValuePredictor parameterValuePredictor, ITelemetryClient telemetryClient, IAzContext azContext = null)
{
Validation.CheckArgument(modelPredictions, $"{nameof(modelPredictions)} cannot be null.");
_telemetryClient = telemetryClient;
_parameterValuePredictor = parameterValuePredictor;
var commnadLines = new List <CommandLine>();
if (modelPredictions != null)
{
for (var i = 0; i < modelPredictions.Count; ++i)
{
try
{
this._commandLinePredictions.Add(new CommandLine(modelPredictions[i], azContext));
}
catch (Exception e)
{
_telemetryClient?.OnParseCommandLineFailure(new CommandLineParsingTelemetryData(modelPredictions[i].Command, e));
}
}
}
}
/// <summary>
/// Iterate over command elements to extract local parameter values.
///
/// Store these values by a key
/// consisting of the suffix of the command + the parameter name. There are some exceptions, e.g.
/// credential, location, where the parameter name itself is the key.
///
/// For example, New-AzResourceGroup -Name Hello -Location 'EastUS' will store into local parameters:
/// ResourceGroupName => Hello
/// Location => 'EastUS'
/// </summary>
/// <param name="command">The command ast element</param>
/// <remarks>
/// This doesn't support positional parameter.
/// </remarks>
private void ExtractLocalParameters(CommandAst command)
{
// Azure PowerShell command is in the form of {Verb}-Az{Noun}, e.g. New-AzResource.
// We need to extract the noun to construct the parameter name.
var commandName = command.GetCommandName();
var commandNoun = ParameterValuePredictor.GetAzCommandNoun(commandName)?.ToLower();
if (commandNoun == null)
{
return;
}
Dictionary <string, string> commandNounMap = null;
_command_param_to_resource_map?.TryGetValue(commandNoun, out commandNounMap);
for (int i = 1; i < command.CommandElements.Count;)
{
if (command.CommandElements[i] is CommandParameterAst parameterAst)
{
var parameterName = parameterAst.ParameterName.ToLower();
string parameterValue = null;
// In the form of "-Name:Value"
if (parameterAst.Argument != null)
{
parameterValue = parameterAst.Argument.ToString();
++i;
}
else if (i + 1 < command.CommandElements.Count)
{
// We don't support positional parameter.
// The next element is either
// 1. The value of this parameter name.
// 2. This parameter is a switch parameter which doesn't have a value. The next element is a parameter.
var nextElement = command.CommandElements[i + 1];
if (nextElement is CommandParameterAst)
{
++i;
continue;
}
parameterValue = command.CommandElements[i + 1].ToString();
i += 2;
}
var parameterKey = parameterName;
if (commandNounMap != null)
{
if (commandNounMap.TryGetValue(parameterName, out var mappedValue))
{
parameterKey = mappedValue;
}
}
_localParameterValues.AddOrUpdate(parameterKey, parameterValue, (k, v) => parameterValue);
}
}
}
/// <summary>
/// Returns suggestions given the user input.
/// </summary>
/// <param name="inputCommandName">The command name extracted from the user input.</param>
/// <param name="inputParameterSet">The parameter set extracted from the user input.</param>
/// <param name="rawUserInput">The string format of the command line from user input.</param>
/// <param name="presentCommands">Commands already present. Contents may be added to this collection.</param>
/// <param name="suggestionCount">The number of suggestions to return.</param>
/// <param name="maxAllowedCommandDuplicate">The maximum amount of the same commands in the list of predictions.</param>
/// <param name="cancellationToken">The cancellation token</param>
/// <returns>The collections of suggestions.</returns>
public CommandLineSuggestion GetSuggestion(string inputCommandName,
ParameterSet inputParameterSet,
string rawUserInput,
IDictionary <string, int> presentCommands,
int suggestionCount,
int maxAllowedCommandDuplicate,
CancellationToken cancellationToken)
{
Validation.CheckArgument(!string.IsNullOrWhiteSpace(inputCommandName), $"{nameof(inputCommandName)} cannot be null or whitespace.");
Validation.CheckArgument(inputParameterSet, $"{nameof(inputParameterSet)} cannot be null.");
Validation.CheckArgument(!string.IsNullOrWhiteSpace(rawUserInput), $"{nameof(rawUserInput)} cannot be null or whitespace.");
Validation.CheckArgument(presentCommands, $"{nameof(presentCommands)} cannot be null.");
Validation.CheckArgument <ArgumentOutOfRangeException>(suggestionCount > 0, $"{nameof(suggestionCount)} must be larger than 0.");
Validation.CheckArgument <ArgumentOutOfRangeException>(maxAllowedCommandDuplicate > 0, $"{nameof(maxAllowedCommandDuplicate)} must be larger than 0.");
const int commandCollectionCapacity = 10;
CommandLineSuggestion result = new();
var duplicateResults = new Dictionary <string, DuplicateResult>(commandCollectionCapacity, StringComparer.OrdinalIgnoreCase);
var isCommandNameComplete = (inputParameterSet.Parameters.Count > 0) || rawUserInput.EndsWith(' ');
Func <string, bool> commandNameQuery = (command) => command.Equals(inputCommandName, StringComparison.OrdinalIgnoreCase);
if (!isCommandNameComplete)
{
commandNameQuery = (command) => command.StartsWith(inputCommandName, StringComparison.OrdinalIgnoreCase);
}
// Try to find the matching command and arrange the parameters in the order of the input.
//
// Predictions should be flexible, e.g. if "Command -Name N -Location L" is a possibility,
// then "Command -Location L -Name N" should also be possible.
//
// resultBuilder and usedParams are used to store the information to construct the result.
// We want to avoid too much heap allocation for the performance purpose.
const int parameterCollectionCapacity = 10;
var resultBuilder = new StringBuilder();
var usedParams = new HashSet <int>(parameterCollectionCapacity);
for (var i = 0; i < _commandLinePredictions.Count && result.Count < suggestionCount; ++i)
{
if (commandNameQuery(_commandLinePredictions[i].Name))
{
cancellationToken.ThrowIfCancellationRequested();
resultBuilder.Clear();
resultBuilder.Append(_commandLinePredictions[i].Name);
usedParams.Clear();
string commandNoun = ParameterValuePredictor.GetCommandNoun(_commandLinePredictions[i].Name)?.ToLower();
if (DoesPredictionParameterSetMatchInput(resultBuilder, inputParameterSet, commandNoun, _commandLinePredictions[i].ParameterSet, usedParams))
{
PredictRestOfParameters(resultBuilder, commandNoun, _commandLinePredictions[i].ParameterSet.Parameters, usedParams);
if (resultBuilder.Length <= rawUserInput.Length)
{
// We don't add anything to to the raw user input. So skip this.
continue;
}
var prediction = resultBuilder.ToString();
if (!presentCommands.ContainsKey(_commandLinePredictions[i].Name))
{
result.AddSuggestion(new PredictiveSuggestion(prediction, _commandLinePredictions[i].Description), _commandLinePredictions[i].SourceText);
presentCommands.Add(_commandLinePredictions[i].Name, 1);
}
else if (presentCommands[_commandLinePredictions[i].Name] < maxAllowedCommandDuplicate)
{
result.AddSuggestion(new PredictiveSuggestion(prediction, _commandLinePredictions[i].Description), _commandLinePredictions[i].SourceText);
presentCommands[_commandLinePredictions[i].Name] += 1;
}
else
{
_ = duplicateResults.TryAdd(prediction, new DuplicateResult(_commandLinePredictions[i].SourceText, _commandLinePredictions[i].Description));
}
}
}
}
var resultCount = result.Count;
if ((resultCount < suggestionCount) && (duplicateResults.Count > 0))
{
foreach (var temp in duplicateResults.Take(suggestionCount - resultCount))
{
result.AddSuggestion(new PredictiveSuggestion(temp.Key, temp.Value.Description), temp.Value.Source);
}
}
return(result);
}
/// <summary>
/// Iterate over command elements to extract local parameter values.
///
/// Store these values by a key
/// consisting of the suffix of the command + the parameter name. There are some exceptions, e.g.
/// credential, location, where the parameter name itself is the key.
///
/// For example, New-AzResourceGroup -Name Hello -Location 'EastUS' will store into local parameters:
/// ResourceGroupName => Hello
/// Location => 'EastUS'
/// </summary>
/// <param name="command">The command ast element</param>
/// <remarks>
/// This doesn't support positional parameter.
/// </remarks>
private void ExtractLocalParameters(CommandAst command)
{
// Azure PowerShell command is in the form of {Verb}-Az{Noun}, e.g. New-AzResource.
// We need to extract the noun to construct the parameter name.
var commandName = command.GetCommandName();
var commandNoun = ParameterValuePredictor.GetCommandNoun(commandName)?.ToLower();
if (commandNoun == null)
{
return;
}
Dictionary <string, string> commandNounMap = null;
_commandParamToResourceMap?.TryGetValue(commandNoun, out commandNounMap);
bool isParameterUpdated = false;
var parameterSet = new ParameterSet(command, _azContext);
for (var i = 0; i < parameterSet.Parameters.Count; ++i)
{
var parameterName = parameterSet.Parameters[i].Name.ToLower();
var parameterValue = parameterSet.Parameters[i].Value;
if (string.IsNullOrWhiteSpace(parameterValue) || string.IsNullOrWhiteSpace(parameterName))
{
continue;
}
var parameterKey = parameterName;
var mappedValue = parameterKey;
if (commandNounMap?.TryGetValue(parameterName, out mappedValue) == true)
{
parameterKey = mappedValue;
}
this._localParameterValues.AddOrUpdate(parameterKey, parameterValue, (k, v) => parameterValue);
isParameterUpdated = true;
}
if (isParameterUpdated)
{
_cancellationTokenSource?.Cancel();
_cancellationTokenSource = new CancellationTokenSource();
var cancellationToken = _cancellationTokenSource.Token;
Task.Run(() =>
{
String localParameterValuesJson = JsonSerializer.Serialize <ConcurrentDictionary <string, string> >(_localParameterValues, JsonUtilities.DefaultSerializerOptions);
_mutex.WaitOne();
cancellationToken.ThrowIfCancellationRequested();
try
{
System.IO.File.WriteAllText(_paramValueHistoryFilePath, localParameterValuesJson);
}
finally
{
_mutex.ReleaseMutex();
}
}, cancellationToken);
}
}
/// <summary>
/// Iterate over command elements to extract local parameter values.
///
/// Store these values by a key
/// consisting of the suffix of the command + the parameter name. There are some exceptions, e.g.
/// credential, location, where the parameter name itself is the key.
///
/// For example, New-AzResourceGroup -Name Hello -Location 'EastUS' will store into local parameters:
/// ResourceGroupName => Hello
/// Location => 'EastUS'
/// </summary>
/// <param name="command">The command ast element</param>
/// <remarks>
/// This doesn't support positional parameter.
/// </remarks>
private void ExtractLocalParameters(CommandAst command)
{
// Azure PowerShell command is in the form of {Verb}-Az{Noun}, e.g. New-AzResource.
// We need to extract the noun to construct the parameter name.
var commandName = command.GetCommandName();
var commandNoun = ParameterValuePredictor.GetAzCommandNoun(commandName)?.ToLower();
if (commandNoun == null)
{
return;
}
Dictionary <string, string> commandNounMap = null;
_commandParamToResourceMap?.TryGetValue(commandNoun, out commandNounMap);
// Don't need to check the last command element. If it's a switch parameter, we don't have a value to store.
// If it's a value, it should be counted in the previous command element (at Count - 2).
for (int i = 1; i < command.CommandElements.Count - 1;)
{
if (command.CommandElements[i] is CommandParameterAst parameterAst)
{
var parameterName = parameterAst.ParameterName.ToLower();
string parameterValue = null;
// In the form of "-Name:Value"
if (parameterAst.Argument != null)
{
parameterValue = parameterAst.Argument.ToString();
++i;
}
else if (i + 1 < command.CommandElements.Count)
{
// We don't support positional parameter.
// The next element is either
// 1. The value of this parameter name.
// 2. This parameter is a switch parameter which doesn't have a value. The next element is a parameter.
var nextElement = command.CommandElements[i + 1];
if (nextElement is CommandParameterAst)
{
++i;
continue;
}
parameterValue = command.CommandElements[i + 1].ToString();
i += 2;
}
var parameterKey = parameterName;
if (commandNounMap != null)
{
if (commandNounMap.TryGetValue(parameterName, out var mappedValue))
{
parameterKey = mappedValue;
}
}
_cancellationTokenSource?.Cancel();
_cancellationTokenSource = new CancellationTokenSource();
Task.Run(() =>
{
this._localParameterValues.AddOrUpdate(parameterKey, parameterValue, (k, v) => parameterValue);
if (_cancellationTokenSource.IsCancellationRequested)
{
throw new OperationCanceledException();
}
String localParameterValuesJson = JsonSerializer.Serialize <ConcurrentDictionary <string, string> >(_localParameterValues, JsonUtilities.DefaultSerializerOptions);
_mutex.WaitOne();
try
{
System.IO.File.WriteAllText(_paramValueHistoryFilePath, localParameterValuesJson);
}
finally
{
_mutex.ReleaseMutex();
}
});
}
}
}
