private void ScanStackedOptions(
ElementsCursor <SyntaxToken> cursor, List <SyntaxError> errors,
SyntaxToken nameMarker, SyntaxToken name, IReadOnlyList <string> commandName, List <ArgumentSyntax> arguments)
{
var builder = new SyntaxTokenBuilder();
var optionsSpan = name.Span;
string optionNames = name.StringValue;
int optionsCount = optionNames.Length;
GetShortOptionName(errors, builder, optionsSpan, optionNames, 0);
var shortName = builder.Build();
var option = new OptionSyntax(nameMarker, shortName, valueMarker: null, value: null);
arguments.Add(option);
nameMarker = null;
for (int i = 1; i < optionsCount - 1; i++)
{
GetShortOptionName(errors, builder, optionsSpan, optionNames, i);
shortName = builder.Build();
option = new OptionSyntax(nameMarker, shortName, valueMarker: null, value: null);
arguments.Add(option);
}
GetShortOptionName(errors, builder, optionsSpan, optionNames, optionsCount - 1);
if (name.HasTrailingTrivia)
{
builder.TrailingTrivia = name.TrailingTrivia;
}
name = builder.Build();
option = ScanOption(cursor, errors, nameMarker, name, commandName);
arguments.Add(option);
}
private SyntaxToken ScanEndOfInput(
ElementsCursor <SyntaxToken> cursor, List <SyntaxError> errors)
{
var token = cursor.Peek();
if (token.Kind == SyntaxKind.EndOfInputToken)
{
return(token);
}
var builder = new SyntaxTokenBuilder();
var span = token.FullSpan;
_syntaxFactory.EndOfInputToken(builder, span.Source, span.Start, missing: true);
errors.Add(_syntaxFactory.MissingEndOfInputError(builder.Span));
if (!span.IsEmpty)
{
int start = span.Start;
int length = cursor.TerminalElement.FullSpan.End - start;
builder.TrailingTrivia = _syntaxFactory.UndefinedTrivia(span.Source, start, length);
errors.Add(_syntaxFactory.UnrecognizedInputError(builder.TrailingTrivia.Span));
}
return(builder.Build());
}
private ValueSyntax ScanValue(ElementsCursor <SyntaxToken> cursor, List <SyntaxError> errors)
{
var token = cursor.Peek();
cursor.MoveNext();
if (token.Kind == SyntaxKind.QuotedLiteralToken || !HasMoreTokens(cursor))
{
if (!HasSpaceAfterOrEnd(token, cursor.Peek(1)))
{
// Quoted literals do not merge with any other literal
// even if they don't have spaces between them.
var builder = new SyntaxTokenBuilder(token);
builder.TrailingTrivia = _syntaxFactory.WhitespaceTriviaAfter(token, length: 0, missing: true);
token = builder.Build();
errors.Add(_syntaxFactory.MissingWhitespaceError(token.TrailingTrivia.Span));
}
if (token.Kind == SyntaxKind.QuotedLiteralToken &&
!HasClosingQuote(token))
{
var span = token.Span;
errors.Add(_syntaxFactory.MissingClosingQuoteError(
new TextSpan(span.Source, span.End, 0)));
}
return(new ValueSyntax(token));
}
return(ScanMultiTokenValue(cursor, errors, token));
}
public void LiteralToken(SyntaxTokenBuilder builder,
string input, int position, int length, string stringValue,
bool quoted = false, bool missing = false)
{
builder.InitializeWith(quoted ? SyntaxKind.QuotedLiteralToken : SyntaxKind.LiteralToken,
new TextSpan(input, position, length), stringValue, missing);
}
public void IdentifierToken(SyntaxTokenBuilder builder,
string input, int position, int length, string stringValue,
bool missing = false)
{
builder.InitializeWith(SyntaxKind.IdentifierToken,
new TextSpan(input, position, length), stringValue, missing);
}
private SyntaxToken GetTerminalElement(SyntaxToken[] tokens, string commandLine)
{
var terminalElement = tokens.LastOrDefault();
if (terminalElement == null || terminalElement.Kind != SyntaxKind.EndOfInputToken)
{
var builder = new SyntaxTokenBuilder();
_syntaxFactory.EndOfInputToken(builder,
commandLine, terminalElement?.FullSpan.End ?? 0, missing: true);
terminalElement = builder.Build();
}
return(terminalElement);
}
private void GetShortOptionName(
List <SyntaxError> errors, SyntaxTokenBuilder builder,
TextSpan optionsSpan, string optionNames, int optionIndex)
{
char optionName = optionNames[optionIndex];
if (!Char.IsLetter(optionName))
{
errors.Add(_syntaxFactory.InvalidOptionNameError(
new TextSpan(optionsSpan.Source, optionsSpan.Start + optionIndex, 1)));
}
_syntaxFactory.IdentifierToken(builder,
optionsSpan.Source, optionsSpan.Start + optionIndex, 1, optionName.ToString());
}
private void LexSyntaxToken(
SyntaxTokenBuilder builder, string input, ElementsCursor <char> cursor)
{
switch (cursor.Peek())
{
case EndOfInput:
_syntaxFactory.EndOfInputToken(builder, input, cursor.Position);
break;
case '-':
char next = cursor.Peek(1);
if (next == '-')
{
_syntaxFactory.DashDashToken(builder, input, cursor.Position);
cursor.MoveNext(2);
}
else if (Char.IsDigit(next))
{
// Possibly a negative number.
ScanLiteralOrIdentifierToken(builder, input, cursor);
}
else
{
_syntaxFactory.DashToken(builder, input, cursor.Position);
cursor.MoveNext();
}
break;
case '=':
_syntaxFactory.EqualsToken(builder, input, cursor.Position);
cursor.MoveNext();
break;
case ':':
_syntaxFactory.ColonToken(builder, input, cursor.Position);
cursor.MoveNext();
break;
default:
ScanLiteralOrIdentifierToken(builder, input, cursor);
break;
}
}
private CommandNameSyntax ScanCommandName(
ElementsCursor <SyntaxToken> cursor, List <SyntaxError> errors, CancellationToken cancellation,
out IReadOnlyList <string> commandName)
{
var nameParts = new List <SyntaxToken>();
var command = new List <string>();
while (HasMoreTokens(cursor) && !cancellation.IsCancellationRequested)
{
var token = cursor.Peek();
if (!IsIdentifier(token))
{
break;
}
string subcommand = token.StringValue;
if (!_semanticModel.HasSubCommand(command, subcommand))
{
break;
}
if (!HasSpaceAfterOrEnd(token, cursor.Peek(1)))
{
var builder = new SyntaxTokenBuilder(token);
builder.TrailingTrivia = _syntaxFactory.WhitespaceTriviaAfter(token, length: 0, missing: true);
token = builder.Build();
errors.Add(_syntaxFactory.MissingWhitespaceError(token.TrailingTrivia.Span));
}
command.Add(subcommand);
nameParts.Add(token);
cursor.MoveNext();
}
cancellation.ThrowIfCancellationRequested();
commandName = command;
if (nameParts.Count == 0)
{
return(null);
}
return(new CommandNameSyntax(nameParts));
}
public IEnumerable <SyntaxToken> Lex(string commandLine, CancellationToken cancellation = default)
{
if (commandLine == null)
{
throw new ArgumentNullException(nameof(commandLine));
}
cancellation.ThrowIfCancellationRequested();
var chars = new CharsReadOnlyList(commandLine);
var cursor = new ElementsCursor <char>(chars, EndOfInput);
var builder = new SyntaxTokenBuilder();
do
{
builder.LeadingTrivia = LexSyntaxTrivia(commandLine, cursor);
LexSyntaxToken(builder, commandLine, cursor);
builder.TrailingTrivia = LexSyntaxTrivia(commandLine, cursor);
cancellation.ThrowIfCancellationRequested();
yield return(builder.Build());
} while (builder.Kind != SyntaxKind.EndOfInputToken);
}
private ValueSyntax ScanMultiTokenValue(
ElementsCursor <SyntaxToken> cursor, List <SyntaxError> errors,
SyntaxToken first)
{
var tokens = new List <SyntaxToken>();
var token = first;
tokens.Add(token);
while (HasMoreTokens(cursor))
{
var next = cursor.Peek();
if (HasSpaceAfterOrEnd(token, next))
{
break;
}
if (next.Kind == SyntaxKind.QuotedLiteralToken)
{
// Quoted literals do not merge with any other literal
// even if they don't have spaces between them.
var builder = new SyntaxTokenBuilder(token);
builder.TrailingTrivia = _syntaxFactory.WhitespaceTriviaAfter(token, length: 0, missing: true);
tokens[tokens.Count - 1] = builder.Build();
errors.Add(_syntaxFactory.MissingWhitespaceError(builder.TrailingTrivia.Span));
break;
}
token = next;
tokens.Add(token);
cursor.MoveNext();
}
return(new ValueSyntax(tokens));
}
private void ScanLiteralOrIdentifierToken(
SyntaxTokenBuilder builder, string input, ElementsCursor <char> cursor)
{
int start = cursor.Position;
char?quote = null;
var stringValue = new StringBuilder(128);
char first = cursor.Peek();
bool isIdentifier = Char.IsLetter(first);
if (first == '"' || first == '\'')
{
quote = first;
isIdentifier = false;
cursor.MoveNext();
}
while (!cursor.IsAtTheEnd())
{
char c = cursor.Peek();
if (Char.IsWhiteSpace(c))
{
if (!quote.HasValue)
{
break;
}
stringValue.Append(c);
cursor.MoveNext();
}
else if (c == '=' || c == ':')
{
if (isIdentifier)
{
break;
}
stringValue.Append(c);
cursor.MoveNext();
}
else if (c == '"' || c == '\'')
{
// Start of another quoted literal.
if (!quote.HasValue)
{
break;
}
if (c == quote)
{
char next = cursor.Peek(1);
if (next == c)
{
// Escaped quote sequence.
stringValue.Append(c);
cursor.MoveNext(2);
}
else
{
// Closing quote symbol.
cursor.MoveNext();
break;
}
}
else
{
// Another quote symbol.
stringValue.Append(c);
cursor.MoveNext();
}
}
else
{
if (isIdentifier && !IsIdentifierSymbol(c))
{
isIdentifier = false;
}
stringValue.Append(c);
cursor.MoveNext();
}
}
// We don't know for sure that this is a pure identifier without a context
// but we definitely can identify a pure literal.
if (isIdentifier)
{
_syntaxFactory.IdentifierOrLiteralToken(builder, input,
start, cursor.Position - start, stringValue.ToString());
}
else
{
_syntaxFactory.LiteralToken(builder, input,
start, cursor.Position - start, stringValue.ToString(), quoted: quote.HasValue);
}
}
private OptionSyntax ScanOption(ElementsCursor <SyntaxToken> cursor, List <SyntaxError> errors,
SyntaxToken nameMarker, SyntaxToken name, IReadOnlyList <string> commandName)
{
SyntaxToken valueMarker = null;
ValueSyntax value = null;
if (!HasMoreTokens(cursor))
{
return(new OptionSyntax(nameMarker, name, valueMarker, value));
}
var token = cursor.Peek();
if (IsOptionValueMarker(token))
{
valueMarker = token;
cursor.MoveNext();
token = cursor.Peek();
}
else
{
if (!HasSpaceAfterOrEnd(name, token))
{
var builder = new SyntaxTokenBuilder(name);
builder.TrailingTrivia = _syntaxFactory.WhitespaceTriviaAfter(name, length: 0, missing: true);
name = builder.Build();
errors.Add(_syntaxFactory.MissingWhitespaceError(name.TrailingTrivia.Span));
}
_semanticModel.TryGetOptionType(commandName, name.StringValue, out Type valueType);
if (IsFlag(valueType))
{
// Flag can have a value only if a value marker is specified
// because it have value 'true' by default.
return(new OptionSyntax(nameMarker, name, valueMarker, value));
}
}
bool missingValue = valueMarker != null &&
(!HasMoreTokens(cursor) || (valueMarker.HasTrailingTrivia && IsOptionNameOrEndOfOptions(token, cursor.Peek(1))));
if (missingValue)
{
SyntaxToken valueToken = null;
var builder = new SyntaxTokenBuilder();
var span = valueMarker.Span;
_syntaxFactory.LiteralToken(builder, span.Source, span.End, 0, null, missing: true);
if (valueMarker.HasTrailingTrivia)
{
// Place trailing trivia after missing option value
builder.TrailingTrivia = valueMarker.TrailingTrivia;
valueToken = builder.Build();
// and remove it from value marker.
builder.InitializeWith(valueMarker);
builder.TrailingTrivia = null;
valueMarker = builder.Build();
}
else
{
valueToken = builder.Build();
}
errors.Add(_syntaxFactory.MissingOptionValueError(valueToken.Span));
value = new ValueSyntax(valueToken);
return(new OptionSyntax(nameMarker, name, valueMarker, value));
}
bool isValue = (valueMarker != null && !valueMarker.HasTrailingTrivia) ||
!IsOptionNameOrEndOfOptions(token, cursor.Peek(1));
if (isValue)
{
value = ScanValue(cursor, errors);
}
return(new OptionSyntax(nameMarker, name, valueMarker, value));
}
public void ColonToken(SyntaxTokenBuilder builder,
string input, int position, bool missing = false)
{
builder.InitializeWith(SyntaxKind.ColonToken,
new TextSpan(input, position, 1), ":", missing);
}
public void DashDashToken(SyntaxTokenBuilder builder,
string input, int position, bool missing = false)
{
builder.InitializeWith(SyntaxKind.DashDashToken,
new TextSpan(input, position, 2), "--", missing);
}
public void EndOfInputToken(SyntaxTokenBuilder builder,
string input, int position, bool missing = false)
{
builder.InitializeWith(SyntaxKind.EndOfInputToken,
new TextSpan(input, position, 0), "<eoi>", missing);
}
