public void Test_3_5_4_Keywords_B()
{
// An unadorned identifier is an identifier which is not immediately preceded by a '#'.
// If a ':' followed by an '=' immediately follows an unadorned identifier,
// with no intervening white space, then the token is to be parsed as an
// identifier followed by an assignmentOperator not as an keyword followed by an '='.
Scanner lexer = this.GetLexer("isGood1Time:=");
// Identifier token
object obj = lexer.GetToken();
Assert.IsInstanceOfType(obj, typeof(IdentifierToken));
Assert.IsNotInstanceOfType(obj, typeof(KeywordToken));
IdentifierToken token = (IdentifierToken)obj;
Assert.IsTrue(token.IsValid);
Assert.IsNull(token.ScanError);
Assert.AreEqual(0, token.StartPosition.Position);
Assert.AreEqual(10, token.StopPosition.Position);
Assert.AreEqual("isGood1Time", token.Value);
// Assignment token
obj = lexer.GetToken();
Assert.IsInstanceOfType(obj, typeof(AssignmentOperatorToken));
AssignmentOperatorToken assign = (AssignmentOperatorToken)obj;
Assert.IsTrue(assign.IsValid);
Assert.IsNull(assign.ScanError);
Assert.AreEqual(11, assign.StartPosition.Position);
Assert.AreEqual(12, assign.StopPosition.Position);
// Should be the last one
obj = lexer.GetToken();
Assert.IsInstanceOfType(obj, typeof(EofToken));
}
public void Test_3_5_5_Operators_E()
{
Scanner lexer = this.GetLexer(":=");
object obj = lexer.GetToken();
Assert.IsInstanceOfType(obj, typeof(AssignmentOperatorToken));
AssignmentOperatorToken token = (AssignmentOperatorToken)obj;
Assert.IsTrue(token.IsValid);
Assert.IsNull(token.ScanError);
Assert.AreEqual(0, token.StartPosition.Position);
Assert.AreEqual(1, token.StopPosition.Position);
// Should be the last one
obj = lexer.GetToken();
Assert.IsInstanceOfType(obj, typeof(EofToken));
}
protected virtual AssignmentNode ParseAssignment(SemanticNode parent, IdentifierToken identifier, AssignmentOperatorToken assignmentOperator)
{
// PARSE: <assignment> ::= <assignment target> assignmentOperator <expression>
// <assignment target> := identifier
AssignmentNode result = new AssignmentNode(parent, identifier, assignmentOperator);
Token token = this.GetNextTokenxx(Preference.NegativeSign);
ExpressionNode expression = this.ParseExpression(result, token);
if (expression == null)
{
this.ReportParserError(result, SemanticErrors.MissingExpression, token);
}
else
{
result.SetContents(expression);
}
return(result);
}
public IEnumerable <IToken> Tokenize(SafeStreamReader source)
{
uint lineNumber = 1;
uint lexemeStartPositionInLine = 1;
uint absolutePosition = 1;
var maybeCurrentChar = Option <int> .None;
var currentLexemeBuffer = new StringBuilder();
var maybeToken = Option <IToken> .None;;
while ((maybeCurrentChar = source.Read()).IsSome)
{
var currentChar = maybeCurrentChar.Value();
maybeToken = Option <IToken> .None;
switch (currentChar)
{
case var c when string.IsNullOrWhiteSpace(char.ConvertFromUtf32(c)):
// if a whitespace was encountered - strip it
// and yield whatever in the buffer to the output
maybeToken = FlushBuffer(
currentLexemeBuffer,
ref absolutePosition,
lineNumber,
ref lexemeStartPositionInLine
);
if (maybeToken.IsSome)
{
yield return(maybeToken.ValueUnsafe());
}
switch (c)
{
case '\r':
yield return(source.Read()
.Some <IToken>(cn =>
cn == '\n' ?
(IToken) new NewLineSymbolToken(
absolutePosition,
lineNumber,
lexemeStartPositionInLine
) :
(IToken) new UnrecognizedToken(
$"\r{cn}",
absolutePosition,
lineNumber,
lexemeStartPositionInLine
)
)
.None(new UnrecognizedToken(
$"\r",
absolutePosition,
lineNumber,
lexemeStartPositionInLine
))
);
absolutePosition += 2;
lineNumber += 1;
lexemeStartPositionInLine = 1;
break;
case '\n':
yield return(new NewLineSymbolToken(
absolutePosition,
lineNumber,
lexemeStartPositionInLine
));
absolutePosition += 1;
lineNumber += 1;
lexemeStartPositionInLine = 1;
break;
default:
absolutePosition += 1;
lexemeStartPositionInLine += 1;
break;
}
break;
case '.':
var currentLexeme = currentLexemeBuffer.ToString();
var maybeBeforeToken =
IntegerLiteralToken.FromString(
currentLexeme,
absolutePosition,
lineNumber,
lexemeStartPositionInLine
) ||
IdentifierToken.FromString(
currentLexeme,
absolutePosition,
lineNumber,
lexemeStartPositionInLine
) ||
UnrecognizedToken.FromString(
currentLexeme,
absolutePosition,
lineNumber,
lexemeStartPositionInLine
)
;
var tokes =
source.Peek()
.Some <ImmutableList <IToken> >(c =>
{
var result = ImmutableList <IToken> .Empty;
IToken tokenToAdd = null;
switch (c)
{
case var _ when IsDigit(char.ConvertFromUtf32(c)):
currentLexemeBuffer.Append('.');
return(ImmutableList <IToken> .Empty);
case '.':
absolutePosition += maybeBeforeToken
.Map(t => (uint)t.Lexeme.Length)
.IfNone(0);
lexemeStartPositionInLine += maybeBeforeToken
.Some(t => (uint)t.Lexeme.Length)
.None(0u);
tokenToAdd = new RangeSymbolToken(
absolutePosition,
lineNumber,
lexemeStartPositionInLine
);
result = maybeBeforeToken
.ToImmutableList()
.Add(tokenToAdd);
source.Read();
currentLexemeBuffer.Clear();
lexemeStartPositionInLine += (uint)(tokenToAdd?.Lexeme.Length ?? 0);
absolutePosition += (uint)(tokenToAdd?.Lexeme.Length ?? 0);
return(result);
default:
absolutePosition += maybeBeforeToken
.Map(t => (uint)t.Lexeme.Length)
.IfNone(0);
lexemeStartPositionInLine += maybeBeforeToken
.Some(t => (uint)t.Lexeme.Length)
.None(0u);
tokenToAdd = new DotSymbolToken(
absolutePosition,
lineNumber,
lexemeStartPositionInLine
);
result = maybeBeforeToken
.ToImmutableList()
.Add(tokenToAdd);
currentLexemeBuffer.Clear();
lexemeStartPositionInLine += (uint)(tokenToAdd?.Lexeme.Length ?? 0);
absolutePosition += (uint)(tokenToAdd?.Lexeme.Length ?? 0);
return(result);
}
})
.None(() =>
{
absolutePosition += maybeBeforeToken
.Map(t => (uint)t.Lexeme.Length)
.IfNone(0);
lexemeStartPositionInLine += maybeBeforeToken
.Some(t => (uint)t.Lexeme.Length)
.None(0u);
var tokenToAdd = new DotSymbolToken(
absolutePosition,
lineNumber,
lexemeStartPositionInLine
);
var result = maybeBeforeToken
.ToImmutableList()
.Add(tokenToAdd);
currentLexemeBuffer.Clear();
lexemeStartPositionInLine += (uint)(tokenToAdd?.Lexeme.Length ?? 0);
absolutePosition += (uint)(tokenToAdd?.Lexeme.Length ?? 0);
return(result);
})
;
foreach (var token in tokes)
{
yield return(token);
}
break;
case '/':
maybeToken = FlushBuffer(
currentLexemeBuffer,
ref absolutePosition,
lineNumber,
ref lexemeStartPositionInLine
);
if (maybeToken.IsSome)
{
yield return(maybeToken.ValueUnsafe());
}
yield return(source.Peek()
.Some <IToken>(c =>
{
switch (c)
{
case '/':
var commentContent = source.ReadLine();
var commentToken = new CommentToken(
$"/{commentContent}",
absolutePosition,
lineNumber,
lexemeStartPositionInLine
);
absolutePosition += (uint)commentContent.Length;
lineNumber += 1;
lexemeStartPositionInLine = 0;
return commentToken;
case '=':
var notEqualsToken = new NotEqualsOperatorToken(
absolutePosition,
lineNumber,
lexemeStartPositionInLine
);
source.Read();
absolutePosition += 1;
lexemeStartPositionInLine = 1;
return notEqualsToken;
default:
return new DivideOperatorToken(
(uint)source.BaseStream.Position,
lineNumber,
lexemeStartPositionInLine
);
}
})
.None(() => new DivideOperatorToken(
(uint)source.BaseStream.Position,
lineNumber,
lexemeStartPositionInLine
)));
absolutePosition += 1;
lexemeStartPositionInLine += 1;
break;
case ':':
maybeToken = FlushBuffer(
currentLexemeBuffer,
ref absolutePosition,
lineNumber,
ref lexemeStartPositionInLine
);
if (maybeToken.IsSome)
{
yield return(maybeToken.ValueUnsafe());
}
yield return(source.Peek()
.Filter(c => c == '=')
.Some <IToken>(c =>
{
var result = new AssignmentOperatorToken(
absolutePosition,
lineNumber,
lexemeStartPositionInLine
);
source.Read();
absolutePosition += 1;
lexemeStartPositionInLine += 1;
return result;
})
.None(new ColonSymbolToken(
absolutePosition,
lineNumber,
lexemeStartPositionInLine
)));
absolutePosition += 1;
lexemeStartPositionInLine += 1;
break;
case '>':
maybeToken = FlushBuffer(
currentLexemeBuffer,
ref absolutePosition,
lineNumber,
ref lexemeStartPositionInLine
);
if (maybeToken.IsSome)
{
yield return(maybeToken.ValueUnsafe());
}
yield return(source.Peek()
.Filter(c => c == '=')
.Some <IToken>(_ =>
{
var result = new GeOperatorToken(
absolutePosition,
lineNumber,
lexemeStartPositionInLine
);
source.Read();
absolutePosition += 1;
lexemeStartPositionInLine += 1;
return result;
})
.None(new GtOperatorToken(
(uint)absolutePosition,
lineNumber,
lexemeStartPositionInLine
)));
absolutePosition += 1;
lexemeStartPositionInLine += 1;
break;
case '<':
maybeToken = FlushBuffer(
currentLexemeBuffer,
ref absolutePosition,
lineNumber,
ref lexemeStartPositionInLine
);
if (maybeToken.IsSome)
{
yield return(maybeToken.ValueUnsafe());
}
yield return(source.Peek()
.Filter(c => c == '=')
.Some <IToken>(_ =>
{
var result = new LeOperatorToken(
absolutePosition,
lineNumber,
lexemeStartPositionInLine
);
source.Read();
absolutePosition += 1;
lexemeStartPositionInLine += 1;
return result;
})
.None(new LtOperatorToken(
absolutePosition,
lineNumber,
lexemeStartPositionInLine
)));
absolutePosition += 1;
lexemeStartPositionInLine += 1;
break;
case '*':
case '%':
case '+':
case '-':
case '=':
case ',':
case '[':
case ']':
case '(':
case ')':
case ';':
maybeToken = FlushBuffer(
currentLexemeBuffer,
ref absolutePosition,
lineNumber,
ref lexemeStartPositionInLine
);
if (maybeToken.IsSome)
{
yield return(maybeToken.ValueUnsafe());
}
yield return(SymbolLexemes
.TryGetValue(((char)currentChar).ToString())
.Some(cons => cons(
absolutePosition,
lineNumber,
lexemeStartPositionInLine
))
.None(() => new UnrecognizedToken(
currentChar.ToString(),
absolutePosition,
lineNumber,
lexemeStartPositionInLine
)
));
absolutePosition += 1;
lexemeStartPositionInLine += 1;
break;
default:
currentLexemeBuffer.Append(char.ConvertFromUtf32(currentChar));
break;
}
}
maybeToken = FlushBuffer(
currentLexemeBuffer,
ref absolutePosition,
lineNumber,
ref lexemeStartPositionInLine
);
if (maybeToken.IsSome)
{
yield return(maybeToken.ValueUnsafe());
}
}
/// <summary>
/// Create and initialize a new assignment expression.
/// </summary>
/// <param name="parent">Parent node that defines this expression.</param>
/// <param name="identifier">Identifier token of the assignment target.</param>
/// <param name="token">Token representing the assignment operator.</param>
protected internal AssignmentNode(SemanticNode parent, IdentifierToken identifier, AssignmentOperatorToken token)
: base(parent)
{
#if DEBUG
if (identifier == null)
{
throw new ArgumentNullException("identifier");
}
if (token == null)
{
throw new ArgumentNullException("token");
}
#endif
this.Target = new AssignmentTargetNode(this, identifier);
this.AssignmentOperator = token;
}
