public static CommandTreeTokenizerResult Tokenize(IEnumerable <string> args)
{
var tokens = new List <CommandTreeToken>();
var position = 0;
var previousReader = default(TextBuffer);
var context = new CommandTreeTokenizerContext();
foreach (var arg in args)
{
var start = position;
var reader = new TextBuffer(previousReader, arg);
// Parse the token.
position = ParseToken(context, reader, position, start, tokens);
context.FlushRemaining();
previousReader = reader;
}
previousReader?.Dispose();
return(new CommandTreeTokenizerResult(
new CommandTreeTokenStream(tokens),
context.Remaining));
}
private static CommandTreeToken ScanQuotedString(CommandTreeTokenizerContext context, TextBuffer reader)
{
var position = reader.Position;
context.FlushRemaining();
reader.Consume('\"');
var builder = new StringBuilder();
var terminated = false;
while (!reader.ReachedEnd)
{
var character = reader.Peek();
if (character == '\"')
{
terminated = true;
reader.Read();
break;
}
builder.Append(reader.Read());
}
if (!terminated)
{
var unterminatedQuote = builder.ToString();
var token = new CommandTreeToken(CommandTreeToken.Kind.String, position, unterminatedQuote, $"\"{unterminatedQuote}");
throw CommandParseException.UnterminatedQuote(reader.Original, token);
}
var quotedString = builder.ToString();
// Add to the context
context.AddRemaining(quotedString);
return(new CommandTreeToken(
CommandTreeToken.Kind.String,
position, quotedString,
quotedString));
}
private static int ParseToken(CommandTreeTokenizerContext context, TextBuffer reader, int position, int start, List <CommandTreeToken> tokens)
{
while (reader.Peek() != -1)
{
if (reader.ReachedEnd)
{
position += reader.Position - start;
break;
}
var character = reader.Peek();
// Eat whitespace
if (char.IsWhiteSpace(character))
{
reader.Consume();
continue;
}
if (character == '-')
{
// Option
tokens.AddRange(ScanOptions(context, reader));
}
else
{
// Command or argument
tokens.Add(ScanString(context, reader));
}
// Flush remaining tokens
context.FlushRemaining();
}
return(position);
}