private void ScanStackedOptions( ElementsCursor <SyntaxToken> cursor, List <SyntaxError> errors, SyntaxToken nameMarker, SyntaxToken name, IReadOnlyList <string> commandName, List <ArgumentSyntax> arguments) { var builder = new SyntaxTokenBuilder(); var optionsSpan = name.Span; string optionNames = name.StringValue; int optionsCount = optionNames.Length; GetShortOptionName(errors, builder, optionsSpan, optionNames, 0); var shortName = builder.Build(); var option = new OptionSyntax(nameMarker, shortName, valueMarker: null, value: null); arguments.Add(option); nameMarker = null; for (int i = 1; i < optionsCount - 1; i++) { GetShortOptionName(errors, builder, optionsSpan, optionNames, i); shortName = builder.Build(); option = new OptionSyntax(nameMarker, shortName, valueMarker: null, value: null); arguments.Add(option); } GetShortOptionName(errors, builder, optionsSpan, optionNames, optionsCount - 1); if (name.HasTrailingTrivia) { builder.TrailingTrivia = name.TrailingTrivia; } name = builder.Build(); option = ScanOption(cursor, errors, nameMarker, name, commandName); arguments.Add(option); }
private ValueSyntax ScanValue(ElementsCursor <SyntaxToken> cursor, List <SyntaxError> errors) { var token = cursor.Peek(); cursor.MoveNext(); if (token.Kind == SyntaxKind.QuotedLiteralToken || !HasMoreTokens(cursor)) { if (!HasSpaceAfterOrEnd(token, cursor.Peek(1))) { // Quoted literals do not merge with any other literal // even if they don't have spaces between them. var builder = new SyntaxTokenBuilder(token); builder.TrailingTrivia = _syntaxFactory.WhitespaceTriviaAfter(token, length: 0, missing: true); token = builder.Build(); errors.Add(_syntaxFactory.MissingWhitespaceError(token.TrailingTrivia.Span)); } if (token.Kind == SyntaxKind.QuotedLiteralToken && !HasClosingQuote(token)) { var span = token.Span; errors.Add(_syntaxFactory.MissingClosingQuoteError( new TextSpan(span.Source, span.End, 0))); } return(new ValueSyntax(token)); } return(ScanMultiTokenValue(cursor, errors, token)); }
private static void ValidateArgumentEnd(ElementsCursor <char> cursor) { if (cursor.IsAtTheEnd()) { throw new FormatException($"Missing closing bracket at symbol {cursor.Position}."); } }
private string ScanArgumentFormat(string messageTemplate, ElementsCursor <char> cursor) { if (!Skip(cursor, ':')) { return(null); } int start = cursor.Position; int length = 0; while (true) { ValidateArgumentEnd(cursor); char c = cursor.Peek(); if (c == '}') { if (length == 0) { throw new FormatException($"Missing argument format at symbol {cursor.Position}."); } string format = messageTemplate.Substring(start, length); return(format); } else { length++; cursor.MoveNext(); } } }
private SyntaxToken ScanEndOfInput( ElementsCursor <SyntaxToken> cursor, List <SyntaxError> errors) { var token = cursor.Peek(); if (token.Kind == SyntaxKind.EndOfInputToken) { return(token); } var builder = new SyntaxTokenBuilder(); var span = token.FullSpan; _syntaxFactory.EndOfInputToken(builder, span.Source, span.Start, missing: true); errors.Add(_syntaxFactory.MissingEndOfInputError(builder.Span)); if (!span.IsEmpty) { int start = span.Start; int length = cursor.TerminalElement.FullSpan.End - start; builder.TrailingTrivia = _syntaxFactory.UndefinedTrivia(span.Source, start, length); errors.Add(_syntaxFactory.UnrecognizedInputError(builder.TrailingTrivia.Span)); } return(builder.Build()); }
private OperandSyntax ScanOperand( ElementsCursor <SyntaxToken> cursor, List <SyntaxError> errors, int operandIndex) { var valueToken = ScanValue(cursor, errors); return(new OperandSyntax(operandIndex, valueToken)); }
private static bool Skip(ElementsCursor <char> cursor, char value) { if (cursor.Peek() == value) { cursor.MoveNext(); return(true); } return(false); }
public void MoveNext(int[] elements, int delta, int expected) { // Arrange var cursor = new ElementsCursor <int>(elements, 0); // Act cursor.MoveNext(delta); // Assert Assert.Equal(expected, cursor.Position); }
public void PeekOrGetTerminal(int[] elements, int delta, int expected) { // Arrange var cursor = new ElementsCursor <int>(elements, 0); // Act int actual = cursor.Peek(delta); // Assert Assert.Equal(expected, actual); }
public MessageTemplate Parse(string messageTemplate) { if (messageTemplate == null) { throw new ArgumentNullException(nameof(messageTemplate)); } if (messageTemplate.Length == 0) { return(new MessageTemplate(Array.Empty <MessageTemplate.Token>())); } var chars = new CharsReadOnlyList(messageTemplate); var cursor = new ElementsCursor <char>(chars, EndOfInput); var tokens = new List <MessageTemplate.Token>(); while (!cursor.IsAtTheEnd()) { switch (cursor.Peek()) { case '{': if (cursor.Peek(1) == '{') { // Escaped opening bracket tokens.Add(ScanTextToken(cursor)); } else { // Argument tokens.Add(ScanArgumentToken(messageTemplate, cursor)); } break; case '}': if (cursor.Peek(1) == '}') { // Escaped closing bracket tokens.Add(ScanTextToken(cursor)); } else { throw new FormatException($"Unescaped closing bracket at symbol {cursor.Position}."); } break; default: tokens.Add(ScanTextToken(cursor)); break; } } return(new MessageTemplate(tokens.ToArray())); }
public void BeAtTheEnd(int[] elements) { // Arrange var cursor = new ElementsCursor <int>(elements, 0); // Assert for (int i = 0; i < elements.Length; i++) { Assert.True(!cursor.IsAtTheEnd()); cursor.MoveNext(); } Assert.True(cursor.IsAtTheEnd()); }
private ArgumentsSectionSyntax ScanArgumentsSection( ElementsCursor <SyntaxToken> cursor, List <SyntaxError> errors, CancellationToken cancellation, IReadOnlyList <string> commandName) { var arguments = new List <ArgumentSyntax>(); bool operandsOnly = false; int operandIndex = 0; while (HasMoreTokens(cursor) && !cancellation.IsCancellationRequested) { var token = cursor.Peek(); if (!operandsOnly && IsEndOfOptions(token, cursor.Peek(1))) { cursor.MoveNext(); var endOfOptions = new EndOfOptionsSyntax(token); arguments.Add(endOfOptions); operandsOnly = true; } else if (!operandsOnly && IsOptionName(token, cursor.Peek(1))) { var nameMarker = token; var name = cursor.Peek(1); cursor.MoveNext(2); if (IsStackedOption(nameMarker, name)) { ScanStackedOptions(cursor, errors, nameMarker, name, commandName, arguments); } else { var option = ScanOption(cursor, errors, nameMarker, name, commandName); arguments.Add(option); } } else { var operand = ScanOperand(cursor, errors, operandIndex); arguments.Add(operand); operandIndex++; } } cancellation.ThrowIfCancellationRequested(); if (arguments.Count == 0) { return(null); } return(new ArgumentsSectionSyntax(arguments)); }
private MessageTemplate.TextToken ScanTextToken(ElementsCursor <char> cursor) { var text = new StringBuilder(128); while (!cursor.IsAtTheEnd()) { char c = cursor.Peek(); if (c == '{') { if (cursor.Peek(1) == '{') { // Escaped opening bracket text.Append(c); cursor.MoveNext(2); } else { // Beginning of an argument break; } } else if (c == '}') { if (cursor.Peek(1) == '}') { // Escaped closing bracket text.Append(c); cursor.MoveNext(2); } else { // Unescaped closing bracket // Error handled in the top method break; } } else { text.Append(c); cursor.MoveNext(); } } return(new MessageTemplate.TextToken(text.ToString())); }
public CommandLineSyntax Parse(string commandLine, CancellationToken cancellation = default) { if (commandLine == null) { throw new ArgumentNullException(nameof(commandLine)); } var tokens = _lexer.Lex(commandLine, cancellation).ToArray(); var terminalElement = GetTerminalElement(tokens, commandLine); var cursor = new ElementsCursor <SyntaxToken>(tokens, terminalElement); var errors = new List <SyntaxError>(); var commandNameSyntax = ScanCommandName(cursor, errors, cancellation, out IReadOnlyList <string> commandName); var argumentsSectionSyntax = ScanArgumentsSection(cursor, errors, cancellation, commandName); var endOfInputToken = ScanEndOfInput(cursor, errors); cancellation.ThrowIfCancellationRequested(); return(new CommandLineSyntax(commandNameSyntax, argumentsSectionSyntax, endOfInputToken, errors)); }
private void LexSyntaxToken( SyntaxTokenBuilder builder, string input, ElementsCursor <char> cursor) { switch (cursor.Peek()) { case EndOfInput: _syntaxFactory.EndOfInputToken(builder, input, cursor.Position); break; case '-': char next = cursor.Peek(1); if (next == '-') { _syntaxFactory.DashDashToken(builder, input, cursor.Position); cursor.MoveNext(2); } else if (Char.IsDigit(next)) { // Possibly a negative number. ScanLiteralOrIdentifierToken(builder, input, cursor); } else { _syntaxFactory.DashToken(builder, input, cursor.Position); cursor.MoveNext(); } break; case '=': _syntaxFactory.EqualsToken(builder, input, cursor.Position); cursor.MoveNext(); break; case ':': _syntaxFactory.ColonToken(builder, input, cursor.Position); cursor.MoveNext(); break; default: ScanLiteralOrIdentifierToken(builder, input, cursor); break; } }
private CommandNameSyntax ScanCommandName( ElementsCursor <SyntaxToken> cursor, List <SyntaxError> errors, CancellationToken cancellation, out IReadOnlyList <string> commandName) { var nameParts = new List <SyntaxToken>(); var command = new List <string>(); while (HasMoreTokens(cursor) && !cancellation.IsCancellationRequested) { var token = cursor.Peek(); if (!IsIdentifier(token)) { break; } string subcommand = token.StringValue; if (!_semanticModel.HasSubCommand(command, subcommand)) { break; } if (!HasSpaceAfterOrEnd(token, cursor.Peek(1))) { var builder = new SyntaxTokenBuilder(token); builder.TrailingTrivia = _syntaxFactory.WhitespaceTriviaAfter(token, length: 0, missing: true); token = builder.Build(); errors.Add(_syntaxFactory.MissingWhitespaceError(token.TrailingTrivia.Span)); } command.Add(subcommand); nameParts.Add(token); cursor.MoveNext(); } cancellation.ThrowIfCancellationRequested(); commandName = command; if (nameParts.Count == 0) { return(null); } return(new CommandNameSyntax(nameParts)); }
private SyntaxTrivia ScanWhitespaceTrivia(string input, ElementsCursor <char> cursor) { int start = cursor.Position; while (!cursor.IsAtTheEnd()) { var c = cursor.Peek(); if (!Char.IsWhiteSpace(c)) { break; } cursor.MoveNext(); } if (cursor.Position == start) { return(null); } return(_syntaxFactory.WhitespaceTrivia(input, start, cursor.Position - start)); }
private int ScanArgumentAlignment(string messageTemplate, ElementsCursor <char> cursor) { if (!Skip(cursor, ',')) { return(0); } int start = cursor.Position; int length = 0; while (true) { ValidateArgumentEnd(cursor); char c = cursor.Peek(); if (c == '}' || c == ':') { if (length == 0) { throw new FormatException($"Missing argument alignment at symbol {cursor.Position}."); } string alignment = messageTemplate.Substring(start, length); try { return(Int32.Parse(alignment)); } catch (Exception ex) { throw new FormatException($"Invalid argument alignment at symbol {start}.", ex); } } else { length++; cursor.MoveNext(); } } }
public IEnumerable <SyntaxToken> Lex(string commandLine, CancellationToken cancellation = default) { if (commandLine == null) { throw new ArgumentNullException(nameof(commandLine)); } cancellation.ThrowIfCancellationRequested(); var chars = new CharsReadOnlyList(commandLine); var cursor = new ElementsCursor <char>(chars, EndOfInput); var builder = new SyntaxTokenBuilder(); do { builder.LeadingTrivia = LexSyntaxTrivia(commandLine, cursor); LexSyntaxToken(builder, commandLine, cursor); builder.TrailingTrivia = LexSyntaxTrivia(commandLine, cursor); cancellation.ThrowIfCancellationRequested(); yield return(builder.Build()); } while (builder.Kind != SyntaxKind.EndOfInputToken); }
private ValueSyntax ScanMultiTokenValue( ElementsCursor <SyntaxToken> cursor, List <SyntaxError> errors, SyntaxToken first) { var tokens = new List <SyntaxToken>(); var token = first; tokens.Add(token); while (HasMoreTokens(cursor)) { var next = cursor.Peek(); if (HasSpaceAfterOrEnd(token, next)) { break; } if (next.Kind == SyntaxKind.QuotedLiteralToken) { // Quoted literals do not merge with any other literal // even if they don't have spaces between them. var builder = new SyntaxTokenBuilder(token); builder.TrailingTrivia = _syntaxFactory.WhitespaceTriviaAfter(token, length: 0, missing: true); tokens[tokens.Count - 1] = builder.Build(); errors.Add(_syntaxFactory.MissingWhitespaceError(builder.TrailingTrivia.Span)); break; } token = next; tokens.Add(token); cursor.MoveNext(); } return(new ValueSyntax(tokens)); }
private OptionSyntax ScanOption(ElementsCursor <SyntaxToken> cursor, List <SyntaxError> errors, SyntaxToken nameMarker, SyntaxToken name, IReadOnlyList <string> commandName) { SyntaxToken valueMarker = null; ValueSyntax value = null; if (!HasMoreTokens(cursor)) { return(new OptionSyntax(nameMarker, name, valueMarker, value)); } var token = cursor.Peek(); if (IsOptionValueMarker(token)) { valueMarker = token; cursor.MoveNext(); token = cursor.Peek(); } else { if (!HasSpaceAfterOrEnd(name, token)) { var builder = new SyntaxTokenBuilder(name); builder.TrailingTrivia = _syntaxFactory.WhitespaceTriviaAfter(name, length: 0, missing: true); name = builder.Build(); errors.Add(_syntaxFactory.MissingWhitespaceError(name.TrailingTrivia.Span)); } _semanticModel.TryGetOptionType(commandName, name.StringValue, out Type valueType); if (IsFlag(valueType)) { // Flag can have a value only if a value marker is specified // because it have value 'true' by default. return(new OptionSyntax(nameMarker, name, valueMarker, value)); } } bool missingValue = valueMarker != null && (!HasMoreTokens(cursor) || (valueMarker.HasTrailingTrivia && IsOptionNameOrEndOfOptions(token, cursor.Peek(1)))); if (missingValue) { SyntaxToken valueToken = null; var builder = new SyntaxTokenBuilder(); var span = valueMarker.Span; _syntaxFactory.LiteralToken(builder, span.Source, span.End, 0, null, missing: true); if (valueMarker.HasTrailingTrivia) { // Place trailing trivia after missing option value builder.TrailingTrivia = valueMarker.TrailingTrivia; valueToken = builder.Build(); // and remove it from value marker. builder.InitializeWith(valueMarker); builder.TrailingTrivia = null; valueMarker = builder.Build(); } else { valueToken = builder.Build(); } errors.Add(_syntaxFactory.MissingOptionValueError(valueToken.Span)); value = new ValueSyntax(valueToken); return(new OptionSyntax(nameMarker, name, valueMarker, value)); } bool isValue = (valueMarker != null && !valueMarker.HasTrailingTrivia) || !IsOptionNameOrEndOfOptions(token, cursor.Peek(1)); if (isValue) { value = ScanValue(cursor, errors); } return(new OptionSyntax(nameMarker, name, valueMarker, value)); }
private SyntaxTrivia LexSyntaxTrivia(string input, ElementsCursor <char> cursor) { return(ScanWhitespaceTrivia(input, cursor)); }
private void ScanLiteralOrIdentifierToken( SyntaxTokenBuilder builder, string input, ElementsCursor <char> cursor) { int start = cursor.Position; char?quote = null; var stringValue = new StringBuilder(128); char first = cursor.Peek(); bool isIdentifier = Char.IsLetter(first); if (first == '"' || first == '\'') { quote = first; isIdentifier = false; cursor.MoveNext(); } while (!cursor.IsAtTheEnd()) { char c = cursor.Peek(); if (Char.IsWhiteSpace(c)) { if (!quote.HasValue) { break; } stringValue.Append(c); cursor.MoveNext(); } else if (c == '=' || c == ':') { if (isIdentifier) { break; } stringValue.Append(c); cursor.MoveNext(); } else if (c == '"' || c == '\'') { // Start of another quoted literal. if (!quote.HasValue) { break; } if (c == quote) { char next = cursor.Peek(1); if (next == c) { // Escaped quote sequence. stringValue.Append(c); cursor.MoveNext(2); } else { // Closing quote symbol. cursor.MoveNext(); break; } } else { // Another quote symbol. stringValue.Append(c); cursor.MoveNext(); } } else { if (isIdentifier && !IsIdentifierSymbol(c)) { isIdentifier = false; } stringValue.Append(c); cursor.MoveNext(); } } // We don't know for sure that this is a pure identifier without a context // but we definitely can identify a pure literal. if (isIdentifier) { _syntaxFactory.IdentifierOrLiteralToken(builder, input, start, cursor.Position - start, stringValue.ToString()); } else { _syntaxFactory.LiteralToken(builder, input, start, cursor.Position - start, stringValue.ToString(), quoted: quote.HasValue); } }
private bool HasMoreTokens(ElementsCursor <SyntaxToken> cursor) { return(!cursor.IsAtTheEnd() && cursor.Peek().Kind != cursor.TerminalElement.Kind); }
private MessageTemplate.ArgumentToken ScanArgumentToken(string messageTemplate, ElementsCursor <char> cursor) { Skip(cursor, '{'); var index = ScanArgumentIndex(messageTemplate, cursor); var alignment = ScanArgumentAlignment(messageTemplate, cursor); var format = ScanArgumentFormat(messageTemplate, cursor); if (!Skip(cursor, '}')) { throw new FormatException($"Missing closing bracket at symbol {cursor.Position}."); } return(new MessageTemplate.ArgumentToken(index, alignment, format)); }