public static CommandTreeTokenizerResult Tokenize(IEnumerable <string> args) { var tokens = new List <CommandTreeToken>(); var position = 0; var previousReader = default(TextBuffer); var context = new CommandTreeTokenizerContext(); foreach (var arg in args) { var start = position; var reader = new TextBuffer(previousReader, arg); // Parse the token. position = ParseToken(context, reader, position, start, tokens); context.FlushRemaining(); previousReader = reader; } previousReader?.Dispose(); return(new CommandTreeTokenizerResult( new CommandTreeTokenStream(tokens), context.Remaining)); }
private static CommandTreeToken ScanQuotedString(CommandTreeTokenizerContext context, TextBuffer reader) { var position = reader.Position; context.FlushRemaining(); reader.Consume('\"'); var builder = new StringBuilder(); var terminated = false; while (!reader.ReachedEnd) { var character = reader.Peek(); if (character == '\"') { terminated = true; reader.Read(); break; } builder.Append(reader.Read()); } if (!terminated) { var unterminatedQuote = builder.ToString(); var token = new CommandTreeToken(CommandTreeToken.Kind.String, position, unterminatedQuote, $"\"{unterminatedQuote}"); throw CommandParseException.UnterminatedQuote(reader.Original, token); } var quotedString = builder.ToString(); // Add to the context context.AddRemaining(quotedString); return(new CommandTreeToken( CommandTreeToken.Kind.String, position, quotedString, quotedString)); }
private static int ParseToken(CommandTreeTokenizerContext context, TextBuffer reader, int position, int start, List <CommandTreeToken> tokens) { while (reader.Peek() != -1) { if (reader.ReachedEnd) { position += reader.Position - start; break; } var character = reader.Peek(); // Eat whitespace if (char.IsWhiteSpace(character)) { reader.Consume(); continue; } if (character == '-') { // Option tokens.AddRange(ScanOptions(context, reader)); } else { // Command or argument tokens.Add(ScanString(context, reader)); } // Flush remaining tokens context.FlushRemaining(); } return(position); }